diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d0b356f9..dc833695 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,14 +44,20 @@ env: jobs: build-wallet-sequencer: name: Build wallet-sequencer (${{ matrix.arch }}) - runs-on: ubuntu-24.04 + # Each arch builds natively on its own runner, the way CI builds: the + # sequencer links the pinned emulator's static archive, which the + # cartesi-machine .deb ships only for the host's arch and which links only + # against GCC 14's libstdc++. + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - arch: amd64 + runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu - arch: arm64 + runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu steps: @@ -66,17 +72,27 @@ jobs: - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins + # libclang for the emulator bindings' bindgen; cmake for aws-lc-sys; the + # .deb's libcartesi.a references libslirp. - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y \ ca-certificates \ - lua5.4 \ - liblua5.4-dev \ - libslirp-dev \ build-essential \ - pkg-config \ - libssl-dev + cmake \ + libclang-dev \ + libslirp-dev \ + pkg-config + + # Exports LIBCARTESI_PATH / INCLUDECARTESI_PATH; the binary links the + # archive statically, so the release host needs no emulator install. + - name: Install cartesi-machine + uses: ./.github/actions/install-cartesi-machine + with: + version: ${{ env.CARTESI_MACHINE_VERSION }} + sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} + sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} # rust-toolchain.toml is the only Rust pin. - name: Install Rust toolchain @@ -85,15 +101,26 @@ jobs: rustup toolchain install rustup target add ${{ matrix.target }} - - name: Install cross - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 - with: - tool: cross@${{ env.CROSS_VERSION }} - # `sequencer` is a library crate; the runnable reference binary is the # `wallet-sequencer` example app (sequencer lib + the placeholder wallet). - name: Build (release) - run: cross build -p wallet-sequencer --release --locked --target ${{ matrix.target }} + run: cargo build -p wallet-sequencer --release --locked --target ${{ matrix.target }} + + # The runner has libcartesi.so installed, so loading proves little on its + # own: the binary must not name it, or release hosts would need the .deb. + - name: Check the binary's shared libraries + env: + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + bin="target/${TARGET}/release/wallet-sequencer" + needed="$(readelf -d "${bin}" | grep NEEDED)" + echo "${needed}" + if grep -q libcartesi <<< "${needed}"; then + echo "::error::wallet-sequencer links libcartesi dynamically" + exit 1 + fi + "${bin}" setup --help > /dev/null - name: Package env: @@ -114,22 +141,77 @@ jobs: cat > "package/wallet-sequencer-${TAG}-linux-${ARCH}/RUNNING.md" <<'EOF' ## Running - Required environment variables: + `wallet-sequencer` is the reference sequencer: the sequencer library with the + placeholder wallet. A data directory is set up once with `setup`, then served + with `run`. + + ### Host + + The binary links the Cartesi Machine emulator statically (the version is + `toolchain.cartesi_machine` in RELEASE.json), so the emulator need not be + installed. The binary is built on Ubuntu 24.04 and the emulator's archive + with GCC 14, so the host needs that glibc and GCC 14's libstdc++: Ubuntu + 24.04, Debian 13, or a newer release. At run time it loads libslirp, libgomp + and libstdc++, and its + L1 client reads the system's CA certificates: - - `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` - - `CARTESI_SEQUENCER_BLOCKCHAIN_ID` - - `CARTESI_SEQUENCER_APP_ADDRESS` - - `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE`) + ```bash + apt-get install ca-certificates libgomp1 libslirp0 libstdc++6 + ``` - Example: + ### setup + + `setup` starts an era in an empty data directory. It pins the deployment + identity, flushes the batch submitter's wallet with its key, which pins the + stop block, syncs L1, rebuilds the history from its base through the stop + block, and publishes the era. + + - The base is genesis by default. `--base ` starts from a stored + canonical machine of the deployment's template, stopped between inputs and + stored by the emulator version this binary links. + - `--dry-run` prints the base's counts, the submitter's activity past it and + the rebuild's size, then exits. It needs no key and never touches the data + directory. + - It needs `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, + `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, + `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS` and, unless `--dry-run`, the + submitter's key, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or + `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (whose first line is the key), + which must sign for that address. Chains other than Ethereum mainnet and + Sepolia also need a fee-oracle source, such as + `CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE`. + + Setup refuses a data directory that holds anything (exit 30), so it runs once + per directory; never run it on every boot. Its retryable L1 checks run before + it writes anything, so retry an exit 20 in the same directory; if the retry + is refused because the directory is not empty, or setup failed any other + way, discard the directory and run setup again. Exit 40 means the old + instance's submitter landed an input past the stop block or still has a + transaction pending: stop the old instance, discard this data directory, + run setup again. + + ### run + + `run` serves a directory that setup completed, reading the identity from it. + It needs `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` and the same key. + + ### Example ```bash - CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ + export CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 + export CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE=/run/secrets/submitter-key + export CARTESI_SEQUENCER_DATA_DIR=/var/lib/sequencer + CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ - CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \ - ./wallet-sequencer + CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xa0Ee7A142d267C1f36714E4a8F75612F20a79720 \ + CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE=0 \ + ./wallet-sequencer setup # or: setup --base ./canonical-machine + + ./wallet-sequencer run ``` + + Each subcommand's `--help` lists every option and its environment variable. EOF tar -C package -czf "dist/wallet-sequencer-${TAG}-linux-${ARCH}.tar.gz" "wallet-sequencer-${TAG}-linux-${ARCH}" diff --git a/AGENTS.md b/AGENTS.md index 63753a79..5568627b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,12 +26,12 @@ In order of importance: - **Dispute compatibility** — the design already accounts for rollup dispute resolution. Preserve it. - **Wallet-compatible signing** — users sign with standard wallets via EIP-712. Never introduce custom signing schemes. -- **Deposit availability < 10 minutes** — happy path. The censorship-resistance backstop (`MAX_WAIT_BLOCKS`, ~4h) is the worst case. +- **Deposit availability < 10 minutes** — happy path. The censorship-resistance backstop (the application's `max_wait_blocks`, ~24h for the wallet) is the worst case. ## Design Principles - **App-specific sequencer.** The sequencer may link against the application, enabling validation and execution at ingress time. This is a deliberate design choice. -- **Soft confirmations may be invalidated.** Under adversarial conditions (network, infrastructure, provider, or L1 outages), soft confirmations can be rolled back via recovery. This is by design, not a bug — it is what makes the sequencer sound in the face of liveness failures. +- **Soft confirmations may be invalidated.** Under adversarial conditions (network, infrastructure, provider, or L1 outages), batches can fail to land inside the staleness window; the era then ends, and the cockroach rebuild that starts a new era revokes every soft confirmation that had not landed. This is by design, not a bug — it is what makes the sequencer sound in the face of liveness failures. - **App UX may depend on the sequencer.** Without the sequencer, user experience may degrade substantially. This is an acceptable tradeoff: the on-chain scheduler remains the canonical source of truth; the sequencer only accelerates the UX. - **SQLite-centered local coordination.** Components publish and consume durable local facts through their owned SQLite tables, and the on-chain scheduler remains canonical authority. HTTP ingress ↔ inclusion lane is the one deliberate in-memory exception, not a precedent for a component bus ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)). - **Append-oriented storage.** Avoid mutable status flags for open/closed entities; prefer write-once NULL→value transitions with one owner each, and derive cursors and heads from persisted facts rather than duplicating them as mutable fields. @@ -54,7 +54,8 @@ batches. A frame's `safe_block` tells the scheduler how far to drain directs before the frame's user ops, and commits the sequencer to having accounted for every direct input up to that block. Under honest operation without outages, soft confirmations match the canonical order; if the sequencer goes offline or -freezes `safe_block`, staleness forces recovery and invalidates them. +freezes `safe_block` past the window, staleness ends the era and a cockroach +rebuild revokes them. ### Where the duality lives in code (change one, check all) @@ -65,8 +66,8 @@ This section is the map. Scheduler-acceptance semantics exist in exactly three implementations that must agree: -1. the canonical fold — `Scheduler` ([`sequencer-core/src/scheduler/mod.rs`](sequencer-core/src/scheduler/mod.rs)), the same source compiled into the on-chain machine and driven bare-metal by the recovery fold; -2. the off-chain acceptance predicate — `ProtocolTiming::scheduler_accepts` ([`sequencer-core/src/protocol.rs`](sequencer-core/src/protocol.rs)), which feeds `safe_accepted_batches`; +1. the canonical scheduler — `Scheduler` ([`sequencer-core/src/scheduler/mod.rs`](sequencer-core/src/scheduler/mod.rs)), driven with the executor by `CanonicalRunner` ([`scheduler/driver.rs`](sequencer-core/src/scheduler/driver.rs)): the same source the on-chain machine compiles; +2. the off-chain acceptance predicate — `scheduler_accepts` ([`sequencer-core/src/protocol.rs`](sequencer-core/src/protocol.rs)), whose results `record_accepted_batches` writes to `batches.accepted_input`; 3. the inclusion lane's live prediction (drain + execution order). [I1](docs/invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) @@ -75,25 +76,27 @@ the others. Two mechanical facts the agreement rests on: -- **Drain attribution.** Accumulated newly-safe directs land in the clock-advanced frame — frame K reads "directs ≤ S_K, then user ops validated on top", exactly the scheduler's drain-before-ops rule ([I2](docs/invariants.md)). +- **Drain attribution.** Accumulated newly-safe directs land in the clock-advanced frame — frame K reads "directs ≤ S_K, then the frame's user ops", exactly the scheduler's drain-before-ops rule ([I2](docs/invariants.md)). - **Empty batches are never stale and consume the nonce** (no first frame to measure staleness against). Consistent across all implementations, test-pinned. ## Batch Staleness and Recovery -A batch is **stale** when `inclusion_block - first_frame.safe_block >= MAX_WAIT_BLOCKS` -(1200 blocks, ~4h): the sequencer went offline, or kept submitting while +A batch is **stale** when `inclusion_block - first_frame.safe_block >= max_wait_blocks`, +the application's staleness window (7200 blocks, ~24h, for the wallet): the sequencer went offline, or kept submitting while freezing `safe_block` to hold back directs. The scheduler skips a stale batch without consuming its nonce, and overdue directs are force-drained anyway, which -bounds censorship at ~4h. Every later batch then arrives at an unexpected nonce, -so invalidation cascades through the suffix, open batch included +bounds censorship at the window. Every later batch then arrives at an unexpected nonce ([scheduler semantics](docs/protocol/scheduler-semantics.md)). -- **Automatic recovery** repairs optimistic history after liveness failures: the - danger detector stops the process at a threshold before staleness, and startup - settles outstanding submissions and replaces the invalid suffix - ([automatic recovery](docs/recovery/README.md)). -- **Cockroach recovery** (`setup --recovery`) rebuilds a fresh data directory from - a trusted canonical checkpoint and L1 history, including after a sequencer bug +- **The era ends** when the oldest unaccepted batch can no longer be accepted + (the void history), or when a foreign batch lands at one of the sequencer's + nonces. The sequencer then stops and refuses every start; it repairs nothing + in place ([batch landing and outages](docs/recovery/README.md)). +- **Cockroach recovery**, the only recovery, rebuilds a new data directory from a trusted + canonical machine and L1 history, including after a sequencer bug: + `setup --base ` rebuilds the history through a stop block with the + scheduler, executing each transaction as it is handed out, and publishes a + new era ([cockroach recovery](docs/recovery/cockroach.md)). ## Threat Model (brief) @@ -103,7 +106,7 @@ See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the full mo - **Trusted:** InputBox contract, our own Ethereum node (fail-stop, not byzantine), operator config, batch-submitter key. - **Adversarial:** `POST /tx` callers, direct-input senders, the L1 mempool and block builders (zombie transactions are a first-class threat). - **RPC endpoint:** single (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`), trusted fail-stop, **must be one consistent node** — no fallback tier exists yet ([RPC endpoint](docs/threat-model/README.md#rpc-endpoint)). -- **Self-trust:** normal operation assumes the sequencer's own code is correct; a bug needs a fix and, when state cannot be trusted, cockroach recovery. Automatic recovery does not repair software defects. +- **Self-trust:** normal operation assumes the sequencer's own code is correct; a bug needs a fix and, when state cannot be trusted, cockroach recovery. - **In scope:** correctness bugs *and* exploitation. Under rollup semantics, a correctness bug that causes scheduler/sequencer state divergence is as severe as direct theft. ## Architecture Map @@ -116,11 +119,12 @@ signing, and SSZ batch encoding. ### Workspace - `sequencer/` — sequencer **library** (no binary). App crates compose it into a binary. -- `sequencer-core/` — shared domain types (`Application`, `SignedUserOp`, `SequencedL2Tx`, `Batch`, `Frame`). +- `sequencer-core/` — the canonical scheduler, the executor and shared domain types (`Application`, `SignedUserOp`, `L2Tx`, `Batch`, `Frame`). - `examples/app-core/` — placeholder wallet app implementing the `Application` trait. - `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. The model for what an app author builds (their `Application` impl ≙ `app-core`; their binary crate ≙ this). - `bindings/c-app-engine/` — reusable C ABI adapter implementing `Application` for a native engine. - `bindings/c-app-sequencer/` — optional C-engine CLI host and external-archive binary. +- `bindings/c-scheduler/` — `scheduler.h`, the canonical scheduler's proposed C API, and its reference export: sequencer-core's `Scheduler` behind the C functions, for C guests. Its `tests/conformance.rs` replays the [conformance vectors](docs/protocol/scheduler-semantics.md#conformance) through the header alone. - `examples/c-wallet-engine/` — reference wallet engine exporting the C ABI, plus its genesis tool and conformance tests. - `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. - `examples/canonical-app/` — on-chain scheduler reference implementation. @@ -135,12 +139,13 @@ signing, and SSZ batch encoding. Paths below are relative to `sequencer/src/`: - `lib.rs` and `harness.rs` — public API and shared CLI harness; app binaries supply their genesis-app constructor. -- `commands/` — `setup`, `run` (including worker supervision), and `flush`, with command configuration and exit-code classification. +- `commands/` — `setup` (its base, rebuild and publication under `commands/setup/`), `run` (including worker supervision), and `flush`, with command configuration and exit-code classification. +- `machine.rs` — reads a stored canonical machine's export through the Cartesi Machine emulator, which the crate links statically; setup's base comes through it. - `runtime/` — exclusive process ownership and runtime scope/shutdown. - `ingress/` — public HTTP handlers and the single inclusion lane. - `egress/` — internal HTTP/WS API and DB-backed application-input feed. -- `l1/` — safe-input reader, batch submitter, fee oracle, shared EIP-1559 estimation, and provider access. -- `recovery/` — automatic startup repair, danger detector, and mempool flusher; the manual rebuild lives in `commands/setup/`. +- `l1/` — safe-input reader, batch submitter, mempool flusher, fee oracle, shared EIP-1559 estimation, and provider access. +- `landing.rs` — the worker that watches the terminal facts and reports the landing lag ([code map](docs/recovery/README.md#code-map)). - `storage/` — each writer role's persistence operations, shared queries, and schema. - `http.rs` and `clock.rs` — shared HTTP errors/server setup and the crate-wide wall clock. `L1Config` lives in `l1/`. @@ -149,31 +154,32 @@ Paths below are relative to `sequencer/src/`: - **Chunk** — bounded list of user ops processed and persisted together to amortize SQLite cost. - **Frame** — ordering boundary; commits `safe_block` + user ops. - **Batch** — list of frames posted on-chain as one L1 transaction (SSZ-encoded). -- **Tip** — the open batch at the end of the valid branch; recovery invalidates a suffix and opens a new Tip ([automatic recovery](docs/recovery/README.md#the-state-being-repaired)). +- **Tip** — the open batch: the one batch at the end of the history that still takes user ops; `run`'s start opens one if none is open ([batch landing and outages](docs/recovery/README.md#accepted-and-unaccepted-batches)). - **Inclusion lane** — the single ordering lane ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)); the only writer of open batch/frame state ([I17](docs/invariants.md)) and the system's execution bottleneck. - **Soft confirmation** — sequencer's predicted ordering, emitted before the batch lands on L1. -- **Snapshot** — immutable artifact registered at every batch close; acceptance selects the comparison and recovery checkpoint ([snapshot lifecycle](docs/snapshots/lifecycle.md)). +- **Snapshot** — immutable artifact registered at every batch close; acceptance selects the comparison checkpoint ([snapshot lifecycle](docs/snapshots/lifecycle.md)). +- **Era** — the history one setup publishes: it starts at its base's L2 tx count, holds the history setup rebuilt from L1 through the stop block (`rebuilt_history`), then the live rows; the baseline at the stop block is its first comparison checkpoint ([cockroach recovery](docs/recovery/cockroach.md)). ## Domain Truths - API validates the EIP-712 signature and enqueues a `SignedUserOp`. Method payload decoding happens during application execution, not at ingress. - **Deposits are direct-input-only** (L1 → L2) and must not be represented as user ops. - **InputBox payload classification** is by authenticated sender, never by a tag byte: the batch submitter's inputs are batches; any other sender's are direct inputs, stored verbatim ([scheduler semantics](docs/protocol/scheduler-semantics.md#the-one-stream-the-one-classification)). -- A rejected user op changes no state and is not persisted; the [application contract](docs/protocol/application-contract.md) owns which layer produces each rejection. The charge is a "fee", not "gas": it tracks DA. +- The lane refuses a live user op through `sequencer_core::application::check_live_user_op` (fee cap, nonce, then the application's `admit_user_op`); a refused op changes no state and is not persisted. An included op always executes at its sender's next nonce, even when its sender cannot pay the fee in full ([application contract](docs/protocol/application-contract.md)). The charge is a "fee", not "gas": it tracks DA. - **The user-nonce rule is contract, not wallet detail** ([user nonces](docs/protocol/application-contract.md#user-nonces)); `GET /nonce` relies on it instead of asking the app. - **EIP-712 domain:** both sides build it with `sequencer_core::build_input_domain` ([user-op signatures](docs/protocol/scheduler-semantics.md#user-op-signatures)); setup pins the chain id and app address, validating the chain id against RPC. ## Application Trait Contract -The [application contract](docs/protocol/application-contract.md) owns the `Application` trait; the [C binding guide](docs/protocol/c-application-binding.md) maps it to native engines. Two rules carry the duality: **any input that succeeded live must succeed on replay** (`AppError` is fatal and defines no canonical successor), and the inclusion lane, canonical scheduler, catch-up, and recovery fold execute inputs only through the shared boundary in `sequencer_core::application`, which checks the engine's reported progress. +The [application contract](docs/protocol/application-contract.md) owns the `Application` trait; the [C binding guide](docs/protocol/c-application-binding.md) maps it to native engines. Two rules carry the duality: **any input that succeeded live must succeed on replay** (`AppError` is fatal and defines no canonical successor), and the inclusion lane, the canonical runner, catch-up and setup execute transactions only through `sequencer_core::application::execute`, which decides whether a user op runs (its nonce) and checks the count and nonce the engine reports afterwards. ## Type Boundaries - `SignedUserOp` — ingress/API signature domain (post-validation, pre-execution). -- `ValidUserOp` — application execution domain (after validation boundary). -- `SequencedL2Tx` — application input payload sum (`UserOp | DirectInput`). -- `ExecutedInputCount` — canonical application-history boundary (`X` means the - next input is entry `X`), never a SQLite cursor. Checked arithmetic only. +- `UserOpTx` — a user op as the application executes it: the recovered sender, its signed nonce, the frame's fee and safe block. +- `L2Tx` — what the scheduler hands out and the executor takes (`Direct | UserOp`). +- `L2TxCount` — the L2 tx count, how many L2 transactions have executed (at `X`, + history entry `X` executes next), never a SQLite cursor. Checked arithmetic only. - `ApplicationInputRow` — crate-private pairing of a mandatory application offset and source context. Every row executes; batch envelopes stay in `safe_inputs`. - Keep DB-only helper types private to storage modules; prefer shared domain types at module boundaries. @@ -189,10 +195,13 @@ same patch. CI lints it; run `direnv exec . redocly lint openapi.yaml` locally. ## Command Configuration -Plain `setup` is L1-read-only and never signs; `run` and `flush-mempool` take the -signing key and read the pinned identity from the database. [Running](README.md#running) -owns invocation and why the submitter address must be dedicated (the devnet uses -Anvil account 9). [`commands/config.rs`](sequencer/src/commands/config.rs) owns +Every command takes the signing key: `setup` flushes the submitter's wallet +(its `--dry-run` takes none), and `run` and `flush-mempool` submit. `setup` +runs once, in an empty data directory, from genesis or a stored machine +(`--base`); it is not idempotent and refuses a directory that holds anything. +`run` and `flush-mempool` read the pinned identity from the database. +[Running](README.md#running) owns invocation and why the submitter address must +be dedicated (the devnet uses Anvil account 9). [`commands/config.rs`](sequencer/src/commands/config.rs) owns every variable name, default, and validation; check it before documenting or changing configuration. @@ -200,7 +209,7 @@ changing configuration. - Prefer small, composable functions at module boundaries (`ingress::api` → `ingress::inclusion_lane` → `storage::ingress`; `egress::l2_tx_feed` ← `storage::history`). - Keep application validation and execution deterministic for a given input/state. No `SystemTime::now()`, `HashMap` iteration order, or floating-point in consensus paths. -- Surface user-facing errors via `ApiError` (in `http.rs`); keep internal failures descriptive but safe. An error that wraps a cause ends its Display with `: {source}`, because log sinks print Display only. A dependency error whose Display omits its cause (reqwest names only the request kind) is rendered with `l1::provider::error_chain` where it becomes text, in a `String` variant or a log field; it appends only causes the line does not already hold, so it is also safe on our own wrappers of such an error. A failure logs at its verdict's level: ERROR when the cause is a fault or unknown (terminal, unclassified), WARN when the verdict names a planned procedure (a recovery boot, a retry, `setup --recovery`). +- Surface user-facing errors via `ApiError` (in `http.rs`); keep internal failures descriptive but safe. An error that wraps a cause ends its Display with `: {source}`, because log sinks print Display only. A dependency error whose Display omits its cause (reqwest names only the request kind) is rendered with `l1::provider::error_chain` where it becomes text, in a `String` variant or a log field; it appends only causes the line does not already hold, so it is also safe on our own wrappers of such an error. A failure logs at its verdict's level: ERROR when the cause is a fault or unknown (terminal, unclassified), WARN when the verdict names a planned procedure (a retry, a setup refusal). - Avoid introducing heavy dependencies without strong reason. - **Impossible states fail loud; they are never handled or absorbed.** The [check policy](docs/invariants.md#the-check-policy-fail-loud) owns the decision test and its reasons; [Self-trust](docs/threat-model/README.md#self-trust) owns the trust boundary. @@ -246,15 +255,18 @@ Test placement: unit tests are a child module of the code under test, inline large, never `#[path]`. Cross-module in-process tests go in `sequencer/src/integration_tests/`; devnet, machine and binary tests go in the `tests/*` crates. Shared fixtures live in `crate::test_support`; -`storage::test_helpers` keeps the storage-domain ones (seeding, the recovery -fixture trait, divergence markers); cross-crate signing helpers live in +`storage::test_helpers` keeps the storage-domain ones (the set-up database +`set_up_storage`, seeding, the landing fixture trait, divergence markers, and +the test-only `Storage` reads); cross-crate signing helpers live in `sequencer_core::test_support` behind its `test-support` feature. Some `sequencer` tests use Anvil (Foundry). They run by default and fail with a clear message if `anvil` is not on PATH. Use the configured Nix/direnv environment -or install Foundry. `canonical-test` additionally needs the Cartesi Machine -library named by `LIBCARTESI_PATH`/`INCLUDECARTESI_PATH` (the devshell exports -both) and libslirp. +or install Foundry. The `sequencer` crate links the Cartesi Machine emulator's +static library, so building it, and every crate built on it, needs +`LIBCARTESI_PATH`/`INCLUDECARTESI_PATH` (the devshell exports both) and +libslirp; so does `canonical-test`. Setup's tests supply a base's export +through a seam (`ReadMachineExport`) and never run the emulator. ## Shell and Commands @@ -289,8 +301,8 @@ See [Running](README.md#running) for the two-phase `setup` / `run` workflow. - Preserve API error shape and status code mapping unless intentionally changing the API contract. - Add or update tests when logic changes. - Run at least `cargo check` before finishing. -- Read the relevant recovery guide and both current TLA+ models before touching - recovery code, and the threat model before touching trust-boundary code. +- Read the relevant recovery guide before touching recovery code, and the + threat model before touching trust-boundary code. - Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing (storage changes cross writer boundaries even when the SQL looks local), the owning design for its assumptions, and [`docs/review/register.md`](docs/review/register.md) for unresolved work in the code you're about to touch. Verify review claims against current code. ### Ask First @@ -334,9 +346,9 @@ work reaches another boundary. | Scheduler acceptance, batch nonces, direct-input ordering, or frame clock | [Scheduler semantics](docs/protocol/scheduler-semantics.md) — canonical algorithm and the implementations that must agree. | | Inclusion lane, storage writes, or runtime concurrency | [Invariant register](docs/invariants.md) — enforcement and consumers; [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md) — ownership, acknowledgement, admission, and terminal stop. | | Application implementation, execution, or native integration | [Application contract](docs/protocol/application-contract.md) — determinism, progress, failure, capacity, and checkpoints; [C binding](docs/protocol/c-application-binding.md) for native engines. | -| Automatic recovery or danger detection | [Automatic recovery](docs/recovery/README.md), then [preemptive.tla](docs/recovery/preemptive.tla) and [admission.tla](docs/recovery/admission.tla) — repair ordering and the models' bounded guarantees. | -| Manual rebuild after lost state or a sequencer bug | [Cockroach recovery](docs/recovery/cockroach.md) — trusted checkpoint, fixed input boundary, and fresh baseline. | -| API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior, with [`openapi.yaml`](openapi.yaml) changed alongside it; [application history](docs/protocol/application-history.md) — era, generation, offsets, and recovery boundaries. | +| Batch landing, the void history, or outages | [Batch landing and outages](docs/recovery/README.md) — the rule that ends an era, what `run` does at start and while running, and the operator's playbook. | +| Setup, or a manual rebuild after lost state or a sequencer bug | [Cockroach recovery](docs/recovery/cockroach.md) — one setup from any base, the rebuild, the era it publishes, the playbooks, and a code map; [sequencer recovery plan](docs/plans/2026-11-sequencer-recovery/README.md) for what remains. | +| API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior, with [`openapi.yaml`](openapi.yaml) changed alongside it; [application history](docs/protocol/application-history.md) — era, offsets, and recovery boundaries. | | Snapshots, restart, export, retention, or watchdog checkpoints | [Snapshot lifecycle](docs/snapshots/lifecycle.md) — durable publication, accepted comparison points, leases, and GC; [wallet format](docs/snapshots/format.md) when changing wallet bytes. | | Trust boundaries, provider behavior, or hostile L1 input | [Threat model](docs/threat-model/README.md) — actor assumptions, supported failures, and residual risks. | | Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat model's fee oracle section](docs/threat-model/README.md#fee-oracle) — oracle source and outage assumptions. | diff --git a/Cargo.lock b/Cargo.lock index afbc7d4c..ce535520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1447,6 +1447,19 @@ dependencies = [ "serde", ] +[[package]] +name = "c-scheduler" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "bindgen", + "ethereum_ssz", + "k256", + "sequencer-core", + "serde_json", +] + [[package]] name = "c-wallet-engine" version = "0.1.0" @@ -1475,6 +1488,7 @@ version = "0.1.0" dependencies = [ "alloy-primitives", "app-core", + "ethereum_ssz", "k256", "rollups-types", "sequencer-core", @@ -1490,9 +1504,11 @@ dependencies = [ "ethereum_ssz", "k256", "rollups-types", + "sequencer", "sequencer-core", "serde", "serde_json", + "tempfile", "testsi", ] @@ -4036,11 +4052,9 @@ dependencies = [ "sequencer-rust-client", "serde", "serde_json", - "tar", "tempfile", "tokio", "tokio-tungstenite", - "toml", ] [[package]] @@ -4413,6 +4427,7 @@ dependencies = [ "app-core", "async-trait", "axum", + "cartesi-machine", "cartesi-rollups-contracts", "clap", "ethereum_ssz", diff --git a/Cargo.toml b/Cargo.toml index 00843a57..43ecb207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "sdk/rust-client", "bindings/c-app-engine", "bindings/c-app-sequencer", + "bindings/c-scheduler", "examples/app-core", "examples/canonical-app", "examples/canonical-test", diff --git a/README.md b/README.md index 86e99183..06bbeb4d 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ Rollup applications need fast transaction confirmations. Waiting for L1 finality The protocol objective is that, under supported honest operation, **the off-chain sequencer predicts the same execution order the rollup's on-chain scheduler later produces.** Soft confirmations are optimistic and may be -invalidated by the recovery cases below; L1 remains canonical truth. +revoked by the recovery described below; L1 remains canonical truth. ## Two Chains Synchronizing -The sequencer maintains an optimistic chain of batches — a tree that normally degenerates into a list. Each batch contains frames, and each frame contains user operations plus a `safe_block` reference. The `safe_block` is the synchronization primitive: it tells the on-chain scheduler "drain all direct inputs (deposits) up to this L1 block, then execute these user ops." Both sides follow the rule, producing identical state. +The sequencer maintains an optimistic list of batches, which L1 accepts in order. Each batch contains frames, and each frame contains user operations plus a `safe_block` reference. The `safe_block` is the synchronization primitive: it tells the on-chain scheduler "drain all direct inputs (deposits) up to this L1 block, then execute these user ops." Both sides follow the rule, producing identical state. ``` Sequencer (off-chain) Scheduler (on-chain) @@ -26,9 +26,8 @@ Sequencer (off-chain) Scheduler (on-chain) ``` When things go well, the sequencer's chain and the scheduler's view converge. -When batches risk becoming stale on L1, the sequencer stops serving and startup -determines the required repair. A lost or untrustworthy local state instead -requires an operator rebuild. Both recovery modes are described below. +When a batch can no longer land before it goes stale, or local state is lost or +untrustworthy, an operator rebuilds the sequencer, as described below. ## Trust Model @@ -38,12 +37,13 @@ The sequencer is a **centralized, single-writer** system. It cannot steal funds - **Go offline** — stop providing soft confirmations. - **Diverge** — if batches fail to land on L1 in time, soft confirmations that were issued become invalid. -**Direct inputs** (L1 → L2 messages, used for deposits) bypass the sequencer entirely. They are posted directly to L1 and are **uncensorable** by the sequencer — the scheduler drains them at every `safe_block` boundary. A censoring sequencer can delay when a direct input is executed (up to `MAX_WAIT_BLOCKS`, ~4h), but cannot prevent it. +**Direct inputs** (L1 → L2 messages, used for deposits) bypass the sequencer entirely. They are posted directly to L1 and are **uncensorable** by the sequencer — the scheduler drains them at every `safe_block` boundary. A censoring sequencer can delay when a direct input is executed (up to the application's `max_wait_blocks`, ~24h for the wallet), but cannot prevent it. A direct whose payload is over the application's cap +never executes, on L1 or in the sequencer. During normal operation the sequencer advances logical frame time after five newly-safe blocks have accumulated. That clock tick drains every covered direct -before later user ops and may also create an empty-direct frame to improve the -application-visible clock. Safe-head publication is best effort: if the node +before later user ops and may also create an empty-direct frame, so that later +user ops carry a fresher block: each op executes at its frame's safe block. Safe-head publication is best effort: if the node exposes a multi-block jump, the sequencer creates one frame at the observed tip and never fabricates intermediate frames. @@ -56,19 +56,18 @@ its bounds are recorded in [`docs/invariants.md`](docs/invariants.md) (I9 and I1 ## Recovery -**[Standard recovery](docs/recovery/README.md)** runs automatically at startup -using the existing database. It handles liveness failures such as outages and -extended downtime: reconcile L1 outcomes, invalidate the affected optimistic -suffix, and resume from retained state. Stale batches do not consume the -scheduler's expected nonce, so their successors cannot be accepted until recovery -supplies a replacement at that nonce. +The sequencer has one recovery, **[cockroach recovery](docs/recovery/cockroach.md)**: +an operator-triggered rebuild when the era has ended or the local database is +lost or cannot be trusted, including after a sequencer bug that corrupted state +or emitted malformed batches. The operator +runs `setup --base ` in a new data directory: setup rebuilds the +history from a trusted canonical machine and L1 through a stop block and +publishes a new era, and each client realigns at its own count. -**[Cockroach recovery](docs/recovery/cockroach.md)** is an operator-triggered -rebuild when the local database is lost or cannot be trusted. This includes a -sequencer bug that corrupted state or emitted malformed batches: fix the bug, -choose a trusted canonical application checkpoint, then rebuild in a fresh data -directory. The command processes historical L1 inputs through the canonical -scheduler and prepares a baseline for resuming normal operation. +An era ends when its oldest unaccepted batch can no longer land before it goes +stale, the **void history**: no later batch can then be accepted, so the +sequencer stops and refuses every start +([batch landing and outages](docs/recovery/README.md)). The [threat model](docs/threat-model/README.md#self-trust) explains the boundary between normal operation's self-trust and manual repair after a bug. @@ -77,10 +76,10 @@ between normal operation's self-trust and manual repair after a bug. The sequencer is designed to handle: -- **L1 provider outages** — each worker retries on its next tick; the RPC client resends a rate-limited or temporarily unavailable request at most twice, a second apart or after the server's requested wait when that is 5 s or less. The inclusion lane and API continue operating locally. A wall-clock fallback detects when an outage pushes batches into the danger zone. -- **Undiagnosed interruptions (OOM, SIGKILL, reboot)** — restart can recover automatically: every boot derives any required recovery from SQLite and L1 safe state through startup recovery, never assuming the previous exit was clean. Terminal errors returned through a command bracket best-effort record their cause in `terminal_faults`; terminal runtime aborts leave only process diagnostics. -- **Extended downtime** — startup syncs to the current L1 safe head, flushes if needed, and recovers before admission. A terminal exit requires operator investigation; rebuilding untrustworthy state follows the cockroach recovery procedure above. -- **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. Recovery waits until every covered wallet-nonce slot is consumed at safe depth, whether the original transaction or a flush no-op wins, so delayed "zombie" submissions cannot land later. +- **L1 provider outages** — each worker retries on its next tick; the RPC client resends a rate-limited or temporarily unavailable request at most twice, a second apart or after the server's requested wait when that is 5 s or less. The inclusion lane and API continue operating locally. The sequencer acts only on what L1 shows: it reports the safe head's age for monitoring and makes no wall-clock estimate ([outages](docs/recovery/README.md#when-batches-are-not-landing)). +- **Undiagnosed interruptions (OOM, SIGKILL, reboot)** — restart resumes: every boot syncs L1 and checks the terminal facts from SQLite and L1 safe state, never assuming the previous exit was clean ([what the sequencer does](docs/recovery/README.md#what-the-sequencer-does)). Terminal errors returned through a command bracket best-effort record their cause in `terminal_faults`, except a terminal fact, which the database already holds and every `run` states; terminal runtime aborts leave only process diagnostics. +- **Extended downtime** — startup syncs to the current L1 safe head and resumes. A stop longer than the window ends the era, and every start refuses until cockroach recovery, above. +- **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. No batch nonce is reused inside an era, so a delayed "zombie" submission carries a batch the sequencer still holds. Before a rebuild, setup's flush replaces what the provider's pool holds for the submitter; a submission of the old era that escapes it is refused by setup's escape check or, if it lands later, read as a foreign batch that ends the new era ([the flush and the stop block](docs/recovery/cockroach.md#the-flush-and-the-stop-block)). ## Interfaces @@ -91,8 +90,8 @@ Users submit signed operations via `POST /tx` (JSON). Operations are signed with ### Sequenced Transaction Feed Subscribers restore an HTTP snapshot, then use one WebSocket stream to replay -application inputs and follow the optimistic tip. Recovery can replace that -history; the snapshot's era, generation, and input count bind a resume request +application inputs and follow the optimistic tip. A cockroach rebuild starts +a new era; the snapshot's era and input count bind a resume request to the state the consumer actually holds. The endpoint serves a small number of infrastructure subscribers, which serve users directly. See the [bootstrap workflow](docs/protocol/application-history.md#replica-bootstrap-and-resume) @@ -104,24 +103,29 @@ The batch submitter posts closed batches to L1's InputBox contract. Each batch c ## Running -The sequencer runs in two phases. **`setup`** pins the -deployment identity (including the reviewed fee-oracle source), does the initial L1 sync, and registers the genesis -snapshot — run it once. Plain `setup` is L1-read-only: it takes the batch-submitter -*address*, never the signing key. **`run`** boots the sequencer from the -set-up DB, reading identity from it (so chain id / app address are not `run` -arguments); it holds the signing key because it submits. - -For rebuilding from a trusted checkpoint, follow the -[cockroach recovery procedure](docs/recovery/cockroach.md#run-a-rebuild). -`setup --recovery` also needs the submitter key because it flushes transactions. +The sequencer runs in two phases. **`setup`** creates a data directory: it +pins the deployment identity (including the reviewed fee-oracle source and +the application's staleness window and direct-payload cap), +flushes the submitter's wallet to pin a stop block, syncs L1 through it, +rebuilds the history from a base through it, and publishes the era's baseline. +The base is genesis, or a stored canonical machine with `--base ` +([cockroach recovery](docs/recovery/cockroach.md)). Setup runs once per data +directory: it refuses one that holds anything, a completed setup included, with +exit 30, and never touches it. `--dry-run` prints what setup would rebuild, +takes no key, and never touches the data directory. **`run`** boots the +sequencer from the set-up DB, reading identity from it (so chain id / app +address are not `run` arguments); it refuses, with exit 30, a binary whose +staleness window or direct-payload cap differs from the pinned one. Both take the signing key: +setup's flush and `run`'s submitter sign for the batch-submitter address. ```bash -# Phase A — set up the data dir (run once; idempotent). +# Phase A — set up a new data dir (once; it refuses a non-empty one). CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE=0 \ CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xa0Ee7A142d267C1f36714E4a8F75612F20a79720 \ +CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0x2a87...d409c6 \ cargo run -p wallet-sequencer -- setup # Phase B — run the sequencer. @@ -130,19 +134,23 @@ CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0x2a87...d409c6 \ cargo run -p wallet-sequencer -- run ``` -`setup` exits 20 (retryable) until the application's deployment block is at or below the L1 safe head, so no reorg can move the block it pins; on Ethereum that is a one-time wait of an epoch or two after deployment, and on Anvil `--slots-in-an-epoch` mined blocks (one in the repo's devnet stack, which sets it to 1; Anvil's default is 32). +`setup` exits 20 (retryable) until the application's deployment block is at or below the L1 safe head, so no reorg can move the block it pins; on Ethereum that is a one-time wait of an epoch or two after deployment, and on Anvil `--slots-in-an-epoch` mined blocks (one in the repo's devnet stack, which sets it to 1; Anvil's default is 32). Setup runs that check and its chain-id check before it writes anything, so a retry in the same directory is safe; a setup that fails after it begins writing, whatever its exit code, leaves a directory to discard ([what setup does](docs/recovery/cockroach.md#what-setup-does)). + +The sequencer links the Cartesi Machine emulator statically, to read a base machine: building it needs the emulator's library at `LIBCARTESI_PATH` (the dev shell exports it), and a release binary needs the host libraries its archive's `RUNNING.md` lists, not an emulator install. A third subcommand, **`flush-mempool`**, settles the batch-submitter wallet nonce on demand (keyed operator tool). It is flush-only: it requires a -completed setup and no canonical divergence, and it never performs -Sync/Cascade or launches runtime workers. +completed setup and no canonical divergence, and it launches no runtime +workers. -`setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`. On chains other than Ethereum mainnet and Sepolia it also requires a fee-oracle source (listed below). +`setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`, and the signing key, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE`, which must sign for that address (`--dry-run` takes no key). On chains other than Ethereum mainnet and Sepolia it also requires a fee-oracle source (listed below). The submitter address must equal the `sequencer_address` compiled into the canonical machine; nothing checks this at runtime ([scheduler semantics](docs/protocol/scheduler-semantics.md#the-one-stream-the-one-classification)). -**Use a dedicated submitter address.** Plain `setup` refuses an unsettled wallet -nonce, so sharing a busy contract-deployer address can trip the detection gate -while its deployment transactions are not yet safe. The devnet uses Anvil +**Use a dedicated submitter address.** Setup flushes the submitter's wallet: it +replaces whatever that account has pending with no-op transactions, then +refuses with exit 40 if the wallet is still unsettled. Sharing a busy +contract-deployer address would cancel its pending deployment transactions or +trip that refusal. The devnet uses Anvil account 9 (`DEVNET_SEQUENCER_ADDRESS`), separate from the account-0 deployer. The Sepolia reference reserves [`SEPOLIA_SEQUENCER_ADDRESS`](examples/app-core/src/application/wallet.rs) for this deployment's batch submissions and application fee revenue; do not share @@ -154,9 +162,8 @@ not just a runtime signing-key change. Optional variables, by name; each subcommand's `--help` shows their defaults and validation from [`commands/config.rs`](sequencer/src/commands/config.rs): -- Every subcommand: `CARTESI_SEQUENCER_DATA_DIR` (the SQLite file is `sequencer.db` inside; `setup` creates it, and `run` and `flush-mempool` refuse a missing one), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`, and `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC` (below). -- `setup` and `run`: `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS`, `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS`. -- `setup`: the fee-oracle source it pins, either `CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE` or a Uniswap source (the overrides `CARTESI_SEQUENCER_FEE_TOKEN_ADDRESS`, `CARTESI_SEQUENCER_WETH_ADDRESS` and `CARTESI_SEQUENCER_UNISWAP_V3_POOL`, given together and required on other chains, and `CARTESI_SEQUENCER_FEE_ORACLE_TWAP_WINDOW_SECS`); `CARTESI_SEQUENCER_CHECKPOINT_BLOCK`; and, for a [rebuild](docs/recovery/cockroach.md), `CARTESI_SEQUENCER_RECOVERY`, `CARTESI_SEQUENCER_CHECKPOINT_DUMP_DIR` and the signing key. +- Every subcommand: `CARTESI_SEQUENCER_DATA_DIR` (the SQLite file is `sequencer.db` inside; `setup` creates it and refuses a directory that holds anything but `sequencer.lock`, and `run` and `flush-mempool` refuse a missing one), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`, and `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC` (below). +- `setup`: the fee-oracle source it pins, either `CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE` or a Uniswap source (the overrides `CARTESI_SEQUENCER_FEE_TOKEN_ADDRESS`, `CARTESI_SEQUENCER_WETH_ADDRESS` and `CARTESI_SEQUENCER_UNISWAP_V3_POOL`, given together and required on other chains, and `CARTESI_SEQUENCER_FEE_ORACLE_TWAP_WINDOW_SECS`); and two flags with no variable, `--base ` (a stored canonical machine to rebuild from; genesis without it) and `--dry-run`. - `run`: `CARTESI_SEQUENCER_HTTP_ADDR` (public listener), `CARTESI_SEQUENCER_INTERNAL_HTTP_ADDR` (internal listener), `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH` and `CARTESI_SEQUENCER_FEE_ORACLE_POLL_INTERVAL_MS`. Logs and errors show only the endpoint's scheme, host and port, since its path, query and userinfo can hold an API key or a password, unless the RPC server echoes them inside a JSON-RPC error message or a malformed result, which alloy builds above the redaction layer, or quotes in an error body only part of them or a different URL that carries them, such as a redirect target on another path. An L1 error's text stays on one line, with each cause cut at 256 bytes. The client follows no redirects: an endpoint that answers 3xx fails with that status, so configure the provider's final URL (setup exits 30 at its chain-id check, since a retry gets the same redirect). Debug-level logging for the `alloy_transport_http` target records the full URL, so keep that target at `info` where logs leave the host. @@ -171,7 +178,7 @@ environment: CARTESI_SEQUENCER_ALLOW_INSECURE_RPC: "true" ``` -Process exit codes follow the orchestrator exit-code contract: `0` clean shutdown, `10` restart (expect a recovery boot), `20` transient refusal (retry with backoff), `30` terminal command refusal (operator required — e.g. setup not complete, identity mismatch, canonical divergence, persistent storage/application invariant failure), `40` a previous instance left work past the checkpoint (wipe the data directory and run `setup --recovery`), and `1` for an unclassified operational failure. Diagnosed terminal runtime faults, including worker panics and an undecodable batch landed by the submitter key, immediately call `abort()` (SIGABRT, status 134), without worker drain or database settlement. Supervisors must treat SIGABRT as terminal class. Ordinary shutdown, expected recovery, and transient errors still drain gracefully. Startup panics caught by the command harness are projected to `30`; panics after runtime scope creation abort; `101` remains possible before the command harness starts. The constants live in `sequencer/src/commands/error.rs`; supervisor recipes are in [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md). +Process exit codes follow the orchestrator exit-code contract: `0` clean shutdown, `20` transient refusal (retry with backoff), `30` terminal command refusal (operator required — e.g. setup not complete, setup on a non-empty data directory, identity mismatch, a base machine that disagrees with L1, canonical divergence, a void history, persistent storage/application invariant failure), `40` setup's escape check found the old instance's work past the stop block (do not restart; stop the old instance, discard the data directory, run setup again), and `1` for an unclassified operational failure; `10` is not used. A usage error from argument parsing, such as an unknown flag, exits `2` before any command runs. Diagnosed terminal runtime faults, including worker panics, an undecodable batch landed by the submitter key, and a divergence or void history seen by a running process, immediately call `abort()` (SIGABRT, status 134), without worker drain or database settlement. Supervisors must treat SIGABRT as terminal class. Ordinary shutdown and transient errors still drain gracefully. Startup panics caught by the command harness are projected to `30`; panics after runtime scope creation abort; `101` remains possible before the command harness starts. The constants live in `sequencer/src/commands/error.rs`; supervisor recipes are in [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md). The [EIP-712 domain](docs/protocol/scheduler-semantics.md#user-op-signatures) is fixed except `chainId` and `verifyingContract`, which come from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` and `CARTESI_SEQUENCER_APP_ADDRESS`. @@ -199,10 +206,9 @@ and the health probes use their own shapes. JSON `sender` fields in successful `POST /tx` and `GET /nonce` responses and WebSocket messages, and `verifyingContract` in `GET /domain`, use EIP-55 -checksum casing. Address fields in `/history` and `sender` fields in -`/historical-l1-inputs` use lowercase hex. Clients must compare decoded 20-byte -addresses and use one normalized encoding for account or projection keys across -these routes. +checksum casing. Address fields in `/history` use lowercase hex. Clients must +compare decoded 20-byte addresses and use one normalized encoding for account or +projection keys across these routes. ### `POST /tx` @@ -226,7 +232,7 @@ Notes: - `sender` is required and must match the recovered signer. - `message.data` is SSZ-encoded method payload bytes. - size is bounded at ingress, before the hot path: the raw request body is capped at 4096 bytes (`413 PAYLOAD_TOO_LARGE`), and `message.data` at the application's maximum method payload size (`400 BAD_REQUEST`). -- the inclusion lane checks nonce, `max_fee`, and fee balance. A rejected op returns `422 EXECUTION_REJECTED` and persists nothing. +- the inclusion lane checks, in order, `max_fee` against the open-frame fee, the nonce, then the application's admission policy (the wallet refuses a sender that cannot pay the fee). A refused op returns `422 EXECUTION_REJECTED` with the refusal's text and persists nothing. - overload is enforced at queue admission: if the inclusion-lane queue is full, `POST /tx` returns HTTP `429` with code `OVERLOADED` and message `queue full`. - queue capacity is an internal runtime constant tuned alongside inclusion-lane chunking to absorb short bursts; if this starts triggering persistently, it is a signal to revisit runtime sizing or throughput rather than add another admission layer. - Browser wallets can call `POST /tx`, `GET /fee`, `GET /nonce`, and `GET /domain` from any origin with any request headers; preflight permits GET and POST and is cached for one hour. CORS is applied only to ingress. Egress routes remain operator-only and require network access controls. @@ -254,7 +260,7 @@ Notes: - `fee` is frozen for the lifetime of the open frame (the live inclusion check). - `recommended_fee` is what the next frame will sample at rotation (currently after five newly-safe L1 blocks, best-effort). - `suggested_max_fee` is `max(fee, recommended_fee)` plus 1.5× log-space slack. Wallets can copy this into signed `max_fee`; the user pays the frame fee, not this cap. Clients that want their own policy can ignore it and combine the two facts themselves. -- `200` while an open frame exists (the admitted runtime always has one). Responses carry `Cache-Control: no-store`. +- `200` while an open frame exists (a running sequencer always has one). Responses carry `Cache-Control: no-store`. - `503` with code `UNAVAILABLE` during shutdown, or if no open frame exists. ### `GET /nonce?sender=
` @@ -271,8 +277,8 @@ Notes: - `next_nonce` is one past the sender's latest included op, or 0 if it has none. It counts soft-confirmed ops: an op acknowledged with `200` before this request is counted. (The `nonce` in a `POST /tx` response or WS message is instead the nonce that op consumed.) - Keep one op in flight per sender: sign `next_nonce`, submit, and increment on `200`. Concurrent submits from one sender are unsupported; they can reach the sequencer out of order and be rejected. After a submit times out, a `next_nonce` above the op's nonce means it was included; an unchanged one is inconclusive, since the op may still be queued, so resubmit the same signed op rather than signing a new one at that nonce. - `next_nonce` of 4294967295 (`u32::MAX`) means the account is exhausted: that nonce has no successor, so `POST /tx` can include no further op from it. -- The value is a hint, not a reservation; the application is authoritative. It can go down, because a restart that runs automatic recovery may invalidate soft-confirmed ops. After a `422` bad-nonce rejection, query again rather than incrementing. -- After an operator rebuild from a checkpoint (`setup --recovery`), a sender with no surviving op since the rebuild reads 0 even when its nonce in the rebuilt baseline is higher; this includes a sender whose post-rebuild ops a later recovery invalidated. A `422` bad-nonce rejection names the expected nonce (`bad nonce: expected N, got M`). +- The value is a hint, not a reservation; the application is authoritative. It does not go down within an era; a new era starts its own count (below). After a `422` bad-nonce rejection, which names the expected nonce (`bad nonce: expected N, got M`), query again rather than incrementing. +- It is read from the era's live ops only. After a setup, a sender whose last op lies in the [rebuilt history](docs/protocol/application-history.md#the-rebuilt-history) or below the base count reads 0 until its first live op; sign the nonce the `422` names. - Successful responses carry `Cache-Control: no-store`. `503` with code `UNAVAILABLE` during shutdown. ### `GET /domain` @@ -285,7 +291,7 @@ The EIP-712 domain `POST /tx` verifies signatures against, keyed as `eth_signTyp Clients pin their own domain and assert that it matches this one, the way a wallet checks `eth_chainId`. `chainId` is a JSON number, exact in JavaScript below 2^53. That covers every chain browser wallets accept (MetaMask refuses IDs above 4503599627370476); a larger ID fails closed, because a signature over a rounded `chainId` recovers a different sender at `POST /tx`. Do not sign with a served domain unchecked: `chainId` and `verifyingContract` are what keep a signature from replaying on another deployment, and the name and version are the same everywhere. -### `GET /ws/subscribe?era_id=&recovery_generation=&next_input=` +### `GET /ws/subscribe?era_id=&next_input=` WebSocket stream of the current application history, replaying from the inclusive `next_input` offset and then following the optimistic tip. Fetch and restore @@ -293,14 +299,16 @@ WebSocket stream of the current application history, replaying from the inclusiv After each successfully applied input at offset `X`, persist the claim with `next_input = X + 1` alongside the replica state. -- All three query fields are required. Missing or malformed fields, or missing - or invalid WebSocket upgrade headers, return HTTP `400` with a plain-text body. -- An era or generation mismatch, an unavailable prefix, or a position ahead of - the head returns HTTP `409` before upgrade. The JSON body and `X-History-Error` - header carry the same typed refusal: `ERA_CHANGED`, `STALE_GENERATION`, - `HISTORY_UNAVAILABLE`, or `AHEAD_OF_HEAD`. Rebootstrap on `ERA_CHANGED` or - `HISTORY_UNAVAILABLE`. On `STALE_GENERATION`, a compatible saved checkpoint - can first be selected through `/history` (below). `AHEAD_OF_HEAD` is an +- Both query fields are required. Missing, malformed or unknown fields, or + missing or invalid WebSocket upgrade headers, return HTTP `400` with a + plain-text body. +- An era mismatch, an unavailable prefix, or a position ahead of the head + returns HTTP `409` before upgrade. The JSON body and `X-History-Error` + header carry the same typed refusal: `ERA_CHANGED`, `HISTORY_UNAVAILABLE`, + or `AHEAD_OF_HEAD`. `ERA_CHANGED` names the server's current era, as in + `{"code": "ERA_CHANGED", "era_id": "22222222-2222-4222-8222-222222222222"}`; + adopt it at the saved count when `/history` allows it (below), or + rebootstrap. Rebootstrap on `HISTORY_UNAVAILABLE`. `AHEAD_OF_HEAD` is an invalid claim to correct. - A claim exactly at the head waits for the next input. Replay uses bounded pages and queues, with no total catch-up limit. The subscriber cap is `64`. @@ -311,9 +319,13 @@ After each successfully applied input at offset `X`, persist the claim with - Messages are JSON text frames; binary fields are `0x`-prefixed hex. Direct-input `block_timestamp` values are Unix seconds. - Batch envelopes are absent. Offsets count executed application inputs, - including business failures and malformed-direct no-ops. -- Recovery stops the process and disconnects subscribers. A reconnect must - present its saved claim; offsets alone cannot distinguish a replaced suffix. + including business failures and malformed-direct no-ops. A direct whose + payload is over the application's cap never executes and has no offset. +- The era's history starts at its base count: the rows setup rebuilt from L1 + up to `baseline.rebuilt_history_end`, then the live rows, in one offset + sequence with the same message shapes. +- A stopped process disconnects subscribers. A reconnect must present its + saved claim; offsets alone cannot distinguish another era's history. - Shutdown or a feed read/send failure may disconnect without a WebSocket Close frame. Resume from the saved claim after an unexpected disconnect; a clean close is not required for safe replay. @@ -328,17 +340,18 @@ Message shapes: { "kind": "direct_input", "offset": 11, "sender": "0x...", "block_number": 123, "block_timestamp": 1700000000, "transaction_hash": "0x...", "payload": "0x...", "input_index": 42, "batch_nonce": 4 } ``` -### History metadata and historical L1 inputs (internal only) +### History metadata (internal only) -Readers that maintain additional transfer/order history can reconstruct it from -L1 and then join the application feed. The -[projection replay contract](docs/protocol/projection-replay.md) describes -bootstrap, client checkpoints, pending directs, and terminal drain. +Readers that maintain additional transfer/order history follow the feed from +the era's base count and keep their own checkpoints, which they restore to +realign after a recovery +([the client's playbook](docs/recovery/cockroach.md#the-clients-playbook)). `GET /history` returns one coherent view of the deployment, current application history, immutable era baseline, and latest accepted checkpoint. Optional `era_id=` requires the selected era; a mismatch returns `409 ERA_CHANGED`. -Example immediately after a rebuild: +Any other query parameter returns `400`. +Example immediately after a setup from a machine stopped at the stop block: ```json { @@ -350,110 +363,59 @@ Example immediately after a rebuild: "batch_submitter_address": "0x3333333333333333333333333333333333333333" }, "history": { - "version": { - "era_id": "22222222-2222-4222-8222-222222222222", - "recovery_generation": 0 - }, + "era_id": "22222222-2222-4222-8222-222222222222", "available_from": 7, "head": 7 }, "baseline": { "l1_stop_block": 1240, "l1_end_input_index": 8, - "next_batch_nonce": 2 + "next_batch_nonce": 2, + "rebuilt_history_end": 7 }, - "accepted_checkpoint": null, - "compatibility": null + "accepted_checkpoint": { + "inclusion_block": 1240, + "executed_input_count": 7, + "next_batch_nonce": 2 + } } ``` -- `history.available_from` is baseline application count `K`; entries `[K,head)` - are available through WS. Counts include all executed application inputs. -- `baseline` describes the fixed L1 stopping block `C`, exclusive InputBox end - `R`, and scheduler nonce after recovery's terminal drain. It survives generation - changes and baseline artifact GC. It is distinct from the moving safe head. +- `history.available_from` is the era's base count `K`, the base's L2 tx + count (0 for a genesis base); entries `[K,head)` are available through WS. + Counts include all executed application inputs. +- `baseline` describes where setup's rebuild ended: the stop block `C` it + pinned after flushing the submitter's wallet, the exclusive InputBox end `R` + through `C`, the scheduler's next batch nonce at the end of `C`, and + `rebuilt_history_end` `E`, the count at the end of `C`. The rebuilt history + covers `[K,E)` and the live rows follow. It survives baseline artifact GC, + and it is distinct from the moving safe head. - `accepted_checkpoint`, when available, has `inclusion_block`, - `executed_input_count`, and `next_batch_nonce`, under `history.version`. - Genesis supplies the zero checkpoint; a rebuilt baseline is not itself an - accepted checkpoint. The metadata does not lease or download a native artifact + `executed_input_count`, and `next_batch_nonce`, under `history.era_id`. + Until a batch of the era is accepted, it is the era baseline: block `C`, + count `E` and the baseline's nonce, since the canonical machine reaches the + same state at the end of `C`. The metadata does not lease or download a native artifact and does not certify a client projection. A known divergence returns `503`. -- `compatibility` is `null` unless `from_generation=` is supplied together - with `era_id`. It then contains `from_generation` and `preserved_input_count`: - the prefix that survived every standard recovery since that generation, - bounded by the current head. A future generation or missing era returns - `400 BAD_REQUEST`; an era mismatch takes precedence over the generation bound. - -For example, `GET /history?era_id=&from_generation=0` can return -`"compatibility": {"from_generation": 0, "preserved_input_count": 3}`. -A saved checkpoint from that era/generation is reusable when its count `X` -satisfies `K <= X <= 3`. The boundary is inclusive: the checkpoint has executed -entries before `X`, and resumes at entry `X`. Each checkpoint must be checked -using its own saved generation. With no intervening recovery, the bound is the -current head. The [history contract](docs/protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) -defines the calculation and trust boundary. - -Restore an eligible checkpoint, persist the response's current history version -with it, and subscribe using that version and its actual count. A recovery -between lookup and subscription still returns `STALE_GENERATION`; repeat the -lookup using the version associated with the restored state. Compatibility does -not certify the client's application or projection implementation, and cannot -cross a cockroach recovery's new era. - -`GET /historical-l1-inputs` requires `era_id` and exactly one starting selector: - -- `next_input_index=`: inclusive per-application InputBox index, starting at 0. -- `after_block=`: initially seek to the first input strictly after that block; - continue using the returned `next_input_index`. - -The endpoint serves only `[0,R)` through the selected era's `C`. A response to -`next_input_index=5&limit=1` can be: -```json -{ - "era_id": "22222222-2222-4222-8222-222222222222", - "l1_stop_block": 1240, - "end_input_index": 8, - "next_input_index": 6, - "items": [{ - "input_index": 5, - "sender": "0x3333333333333333333333333333333333333333", - "payload": "0x00", - "block_number": 1230, - "block_timestamp": 1700014760, - "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" - }] -} -``` - -Records preserve original inner payloads and authenticated senders, including -malformed/rejected batches; they are not complete `EvmAdvance` envelopes. Indices -are contiguous and ordered. Binary values are hex; timestamps are Unix seconds. -Clients must preserve integer precision. A page may split a block. - -Optional `limit` defaults to 256 and accepts 1–256. Pages target 1 MiB of raw -payloads; a larger first input is returned alone, intact. Hex encoding increases -wire size, so this is not a hard response-size limit. Eight historical responses -can be in flight; a permit remains held through body delivery or cancellation. -SQLite read transactions end before network delivery. These limits bound memory -by the page target or largest single input, not total history length. - -Only `next_input_index == end_input_index` means EOF; a short page does not. -Requesting `next_input_index=R` or `after_block=C` returns an empty completed page. -Generation changes do not invalidate historical pages; an era change does. - -Malformed/unknown query fields, invalid selectors/limits, or positions above -`R`/`C` return the existing `400 BAD_REQUEST` JSON shape. An era mismatch returns -the existing `409 ERA_CHANGED` history-policy body before semantic position -checks. Capacity exhaustion returns `429 OVERLOADED`; shutdown or an operational -read failure returns `503 UNAVAILABLE`. Interrupted bodies are failed pages. -Missing durable rows or other storage invariant failures follow the process's -terminal fault policy, never a successful partial page. - -The Rust SDK exposes `history(expected_era, from_generation)` and -`historical_l1_inputs(era, start, limit)` with typed metadata and era refusals. -Both use the configured request deadline, including body transfer; callers may -increase it for large historical inputs. The client owns replay, persistence, -checkpoint selection, and subscription. +Inside an era the history only appends, so a saved state at any count `X` +with `K <= X <= head` resumes with the claim it was saved with: its era id and +`next_input = X`. The [history contract](docs/protocol/application-history.md#identity-and-available-history) +owns this rule. + +A client of an older era (its claim refused with `ERA_CHANGED`) holding state at +count `X` adopts the current era when `K <= X <= E`: it subscribes with the +current `history.era_id` and `next_input = X`. Below `K`, the era's base is +newer than its state; above `E`, its state holds rows the old instance served +that never landed by `C`, even when `head` is past `X`. Both need another +starting state. The check compares counts only: a state that is wrong below +its count passes it, and only comparing the feed's rows from `K` with its own +exposes that. The [history contract](docs/protocol/application-history.md#adopting-a-new-era) +defines the rule; the server adds no check of its own. + +The Rust SDK exposes `history(expected_era)` and +`adopt_current_era(count)` with typed metadata, adoption and era refusals. +Both use the configured request deadline, including body transfer. The client +owns persistence, checkpoint selection, and subscription. ### Operator snapshot endpoints (internal only) @@ -471,24 +433,25 @@ they are served only on the internal listener, behind network controls. (`application/octet-stream`), with `X-Inclusion-Block`, `X-Executed-Input-Count`, and `ETag: "block-"` (`If-None-Match` supports `304`). The watchdog downloads it only as divergence evidence. -- `GET /latest_snapshot` — streams a restorable tar archive of the newest valid +- `GET /latest_snapshot` — streams a restorable tar archive of the newest batch-close snapshot, or the era baseline. Includes immutable `info.toml` and the application's opaque `state` file or directory. - `GET /finalized_snapshot` — streams the accepted snapshot as a tar archive, adding a coherent `checkpoint.toml` receipt with its L1 inclusion block and - next batch nonce for trusted recovery. + next batch nonce: an operator backup, which no command consumes. -Successful state/archive downloads include `X-History-Era`, `X-Recovery-Generation`, -and `X-Executed-Input-Count`, selected atomically with the artifact lease. +Successful state/archive downloads include `X-History-Era` and +`X-Executed-Input-Count`, selected in the read that takes the artifact lease. Streaming holds the lease until the response ends or the client disconnects. Bodies stream without `Content-Length`; a failure after the headers aborts the transfer, so treat an incomplete body as a failed download. -The accepted endpoints return `404` until a comparable checkpoint exists: -genesis is comparable at block zero; a rebuilt baseline is restorable but only -a later accepted batch establishes a comparison point. Known divergence makes +A set-up sequencer always has a comparable checkpoint: until a batch of the era +is accepted, it is the era baseline at the stop block, genesis base included, +so a `404` from the accepted endpoints means the wrong listener or a database +setup never completed. Known divergence makes all four finalized endpoints return `503 UNAVAILABLE`, including conditional -state requests. The check shares the checkpoint-selection transaction, before -any lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecycle.md). +state requests. The check shares the checkpoint-selection read, before any +lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecycle.md). ### Health probes (internal only) @@ -496,24 +459,34 @@ any lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecyc - `GET /readyz` returns `200` while the inclusion-lane receiver is open and shutdown has not been requested; otherwise `503`. Its body is empty. - `GET /healthz` uses the same status as `/readyz` and returns JSON: - `{ "status": "ok", "inclusion_lane": "ok" }`. `status` becomes `"degraded"` + `{ "status": "ok", "inclusion_lane": "ok", "landing_lag_blocks": 40, "max_wait_blocks": 7200, "safe_head_age_seconds": 780 }`. + `status` becomes `"degraded"` for either failure condition; `inclusion_lane` becomes `"stopped"` once its receiver is closed. The lane closes it soon after shutdown is requested, so it can still read `"ok"` early in a shutdown. - -These probes cover process reachability, the lane channel, and shutdown state. -They do not certify L1 freshness, submitter balance, or canonical agreement. +- `landing_lag_blocks`, `max_wait_blocks` and `safe_head_age_seconds` are the + [landing lag](docs/recovery/README.md#what-the-sequencer-does), the + application's window, and the safe block's age by the wall clock. + The history is void when `landing_lag_blocks` reaches `max_wait_blocks`. + `landing_lag_blocks` and `safe_head_age_seconds` are `null` until the first + reading after start, about two seconds. They do not change the status code. + +These probes cover process reachability, the lane channel, and shutdown state, +and report the landing lag and the safe head's age for monitoring to alert on; +they set no threshold of their own. They do not certify submitter balance or +canonical agreement. ## Storage Model -- `batches`: batch metadata +- `batches`: one row per batch, keyed by its nonce, with the safe input that carried its accepted landing (`accepted_input`) - `frames`: frame boundaries within each batch - `frames.fee`: committed fee for each frame - `user_ops`: included user operations - `application_inputs`: current application sequence keyed by mandatory pre-execution offset; each row references a user op or an external direct input and its owning batch/frame - `safe_inputs`: every raw InputBox observation, including batch envelopes -- `history_state`: immutable era baseline (application count and accounted L1 block) plus recovery generation -- `snapshots` and `dumps`: immutable batch-close/baseline artifacts and streaming leases; accepted status is derived from `safe_accepted_batches` +- `era`: one immutable row (base count, stop block, rebuilt history end, first queued input, anchor batch nonce); it exists exactly when setup has completed +- `rebuilt_history`: the transactions setup executed from the base through the stop block, each with its count; the feed serves them before the live rows +- `snapshots`: one immutable row per batch-close or baseline artifact (its directory and count); accepted status is read from `batches.accepted_input`, and download leases live in the serving process - `batch_policy`: singleton fee and batch-sizing knobs; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target` ([formula](sequencer/src/storage/migrations/0001_schema.sql)). A batch closes when its `batch_size_target` byte budget fills or `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS` elapses, whichever comes first. The [threat model](docs/threat-model/README.md#fee-oracle) owns how the oracle prices and refreshes. ## Project Layout @@ -522,7 +495,10 @@ The sequencer is a library: an application crate composes it with its `Application` implementation into a binary, as [`examples/wallet-sequencer/`](examples/wallet-sequencer/) does with the placeholder wallet in [`examples/app-core/`](examples/app-core/). Native engines -bind through the [C ABI adapter](bindings/c-app-engine/README.md). The +bind through the [C ABI adapter](bindings/c-app-engine/README.md). A C or C++ +guest reaches the canonical scheduler through the proposed +[`scheduler.h`](bindings/c-scheduler/README.md), and a port of it is held to the +[conformance vectors](docs/protocol/scheduler-semantics.md#conformance). The [architecture map](AGENTS.md#architecture-map) covers the workspace and the sequencer's modules. @@ -532,7 +508,7 @@ image names and how to run it. ## Prototype Limits -- The `Application` trait defines dump/load behavior ([format](docs/snapshots/format.md)). Every batch close registers a durable snapshot atomically with the seal. Restart restores the latest valid snapshot and replays application inputs from its count. Acceptance determines the recovery checkpoint and garbage-collection frontier without mutating artifact metadata. [Snapshot lifecycle](docs/snapshots/lifecycle.md) documents leases and crash ordering. +- The `Application` trait defines dump/load behavior ([format](docs/snapshots/format.md)). Every batch close registers a durable snapshot atomically with the seal. Restart restores the latest valid snapshot and replays application inputs from its count. Acceptance determines the comparison checkpoint and garbage-collection frontier without mutating artifact metadata. [Snapshot lifecycle](docs/snapshots/lifecycle.md) documents leases and crash ordering. - Schema and migrations are still in prototype mode and may change. ## Development diff --git a/bindings/c-app-engine/README.md b/bindings/c-app-engine/README.md index 3113c8d5..23c5029c 100644 --- a/bindings/c-app-engine/README.md +++ b/bindings/c-app-engine/README.md @@ -10,12 +10,14 @@ The bridge is `Send`, without `Clone` or `Sync`. The [binding guide](../../docs/protocol/c-application-binding.md) maps the ABI to the Application and scheduler contracts. -The native engine owns application state and its execution count/safe-block -clock. Successful execution advances that progress, including counted no-ops; -protocol rejection leaves it unchanged. The shared Rust execution functions -check these transitions. Fatal validation, execution, or output-drain failures -return `AppError`; callers must discard that instance. Exceptions must not cross -the C ABI. +The native engine owns application state, each sender's nonce and the L2 tx +count, and holds no clock. Execution is total, the shared executor compares +nonces, admission is lane-only policy, and `EngineApp` checks the engine's ABI +version before any other call; the binding guide's +[execution and ownership](../../docs/protocol/c-application-binding.md#execution-and-ownership) +section maps each call. Fatal admission, nonce-read, execution, or +output-drain failures return `AppError`; callers must discard that instance. +Exceptions must not cross the C ABI. `NOT_FOUND` and `INVALID_DUMP` retain the distinction between missing/corrupt checkpoint artifacts and other operational `IO_ERROR` failures. Error strings @@ -24,14 +26,16 @@ bindings must use the same header, including these status declarations. A dump prefix may be a file or a directory. Opening it produces independently mutable state without changing the source; checkpoint creation may mutate the -engine's backing arrangement, while preserving logical state and progress. +engine's backing arrangement, while preserving logical state and its count. Successful checkpoints are durable and immutable under subsequent execution. All checkpoint artifacts reside at or below the prefix; the sequencer disposes of them with ordinary recursive filesystem deletion. Restored engines remain usable after source deletion. `state_file_in_dump` names the one canonical comparison file, which can be the -whole dump or a projection alongside richer restoration artifacts. `EngineApp` -does not implement the optional Rust `CanonicalState` inspection trait. +whole dump or a projection alongside richer restoration artifacts. +`application_engine_from_canonical_state` opens an engine over such a file at a +canonical machine's count. `EngineApp` does not implement the optional Rust +`CanonicalState` inspection trait. ## Reference wallet @@ -46,10 +50,10 @@ cargo run -p c-wallet-sequencer -- run cargo test -p c-wallet-engine --test conformance ``` -The ordinary setup/run environment configuration is still required. The genesis -path is required only when plain `setup` needs its first snapshot; completed -setup, warm startup, `flush-mempool`, and `setup --recovery` use the sequencer's -durable checkpoints. +The ordinary setup/run environment configuration is still required. Only a +setup without `--base` reads the genesis path; `setup --base` builds the +engine from the machine's state, and completed setup, warm startup, and +`flush-mempool` use the sequencer's durable checkpoints. The host adds `--state-file` through its own parser and passes the parsed command to `sequencer::run_command`. This shares `run_main`'s command lifecycle and exit @@ -72,10 +76,13 @@ APPLICATION_ENGINE_HEADER=/absolute/path/application-engine.h \ cargo build -p c-app-sequencer ``` -The linked engine reports its stable payload bound through -`application_engine_max_method_payload_bytes()`; zero permits only empty method -payloads. Bindgen generates the Rust records from the header, so a build needs -libclang. The engine also supplies its own genesis tool; configuration does not +The linked engine reports its ABI version through +`application_engine_abi_version()`, its stable payload bound through +`application_engine_max_method_payload_bytes()`, where zero permits only empty +method payloads, and its scheduler constants (the staleness window, the fridge +bound and the direct payload cap) through +`application_engine_scheduler_limits()`. Bindgen generates the Rust records +from the header, so a build needs libclang. The engine also supplies its own genesis tool; configuration does not cross this ABI. With no external archive configured, the generic binary reports that no engine was linked, and `c-wallet-sequencer` supplies the reference implementation through Cargo. @@ -103,18 +110,20 @@ and `sequencer` at the same Git revision and compose `EngineApp` with `sequencer::run_command` directly. No wallet crate is required in either case. The conformance suite compares native and ABI execution over mixed inputs, -notices and vouchers, rejection/no-op progress, dump round trips, independent -instances, and fatal/error classification. +notices and vouchers, nonce skips, admission refusals, underfunded ops that +still execute, no-op counting, dump round trips, the canonical-state projection, +independent instances, and fatal/error classification. `cargo test -p c-app-engine --lib` also checks mixed-output ordering, copying -reused engine buffers, and full-width voucher values with a small ABI fixture. +reused engine buffers, full-width voucher values, the version check, and +admission's reason mapping with a small ABI fixture. The `c_host_` scenarios in `rollups-e2e` launch the reference C host with generated devnet genesis, delete that source before startup, and exercise ordinary execution, -clean restart, stale recovery, and checkpoint-based rebuild. Independent -`EngineApp` replicas restore HTTP archives, discard the downloaded sources, -follow backlog and live WS inputs, and check history identity across recovery. -Ordinary execution and both recovery paths also compare the host against the -canonical machine. Run them with: +clean restart, and setup's rebuild through the engine, from genesis and from a +stored canonical machine. Independent `EngineApp` replicas restore HTTP +archives, discard the downloaded sources, and follow backlog and live WS +inputs, from a snapshot to the live feed. Ordinary execution also compares the +host against the canonical machine. Run them with: ```sh just setup @@ -125,5 +134,5 @@ target/debug/rollups-e2e c_host_ --nocapture `just test-rollups-e2e` includes these scenarios in CI. This covers the reference wallet across the C ABI; private engines still need their own integration and -canonical comparison. Rebuilding from a native recovery archive does not test -the application's canonical-machine-to-native exporter. +canonical comparison. No scenario starts a sequencer from a canonical machine; +the conformance suite checks `application_engine_from_canonical_state` alone. diff --git a/bindings/c-app-engine/include/application-engine.h b/bindings/c-app-engine/include/application-engine.h index f1f951f7..2077dc94 100644 --- a/bindings/c-app-engine/include/application-engine.h +++ b/bindings/c-app-engine/include/application-engine.h @@ -8,7 +8,7 @@ /// @file /// The C API of an application engine, and the only surface an engine exports. It mirrors the -/// Cartesi sequencer's Application contract, so the host owning the fold stays application +/// Cartesi sequencer's Application contract, so the host running the engine stays application /// agnostic and an engine is swappable behind this header. Plain C so any host can consume it. /// /// An application implements these declarations into a static archive, which the `c-app-engine` @@ -33,6 +33,12 @@ /// a host is entitled to refuse a value it does not know rather than to have it renumbered /// underneath. /// +/// The header is versioned by APPLICATION_ENGINE_ABI_VERSION, and the engine reports the version +/// it was built against through application_engine_abi_version. A host checks it before any other +/// call: a function whose meaning changed, or a record that grew, links against an older archive +/// without complaint, and only the version tells them apart. A header without the version is +/// version 1. +/// /// Widths are fixed, nothing crosses as size_t. A C enum's underlying type is implementation /// defined, so an engine must static_assert each one's width against int32_t, which is what lets /// a host mirror them as a plain 32-bit integer. @@ -43,24 +49,33 @@ /// would hand any caller a process kill, since it is fatal-no-resume. /// /// No exception may cross. Fallible entry points report errors through status codes; a fatal -/// validation or execution error defines no successor and the caller must discard the instance. -/// IO_ERROR is legal from every status-returning entry point. In validation, execution, and -/// output draining it is fatal for the instance; the host may retry with a fresh instance after -/// an operational failure. NOT_FOUND and INVALID_DUMP describe lifecycle failures only. -/// Validation acceptance, state transitions, progress, and outputs must agree with the canonical -/// application. Rejection diagnostics and error messages need not be identical across builds. +/// admission, nonce read, or execution error defines no successor and the caller must discard the +/// instance. IO_ERROR is legal from every status-returning entry point. In admission, nonce reads, +/// execution, and output draining it is fatal for the instance; the host may retry with a fresh +/// instance after an operational failure. NOT_FOUND and INVALID_DUMP describe lifecycle failures +/// only. +/// +/// Execution is consensus and admission is not. State transitions, nonces, the L2 tx count, and +/// outputs must agree with the canonical application. Admission is the operator's policy, asked +/// only by the sequencer before it takes an op in, and the canonical machine never calls it, so a +/// canonical build need not implement it. Refusal diagnostics and error messages need not be +/// identical across builds. /// -/// Nonces follow one rule, which the host relies on to serve each sender's next nonce from the -/// ops it persisted: a genesis state starts every account at 0, validation accepts only the -/// sender's expected nonce, each executed user op advances its sender's nonce by exactly one, and -/// nothing else changes a nonce. UINT32_MAX has no successor, so the caller rejects an op carrying -/// it even when validation accepts it. See docs/protocol/application-contract.md. +/// Nonces follow one rule, which the caller relies on to decide which user ops run: a genesis +/// state starts every account at 0, application_engine_user_nonce reports the sender's next nonce, +/// each executed user op advances its sender's nonce by exactly one, and nothing else changes a +/// nonce. The engine stores nonces and never compares them. The caller reads the sender's next +/// nonce, skips an op whose nonce differs from it or equals UINT32_MAX, which has no successor, and +/// executes every other op, so an engine decides what an op does but never whether it runs. See +/// docs/protocol/application-contract.md. /// /// Fees are uint16_t exponents with base 129/128, denominated in the fee token's smallest unit. /// The conversion is defined by sequencer-core/src/fee.rs and its build.rs-generated table: /// integer fixed-point arithmetic with 64 fractional bits, including its rounding and exponent -/// bound. Native and canonical implementations must agree on this conversion; the max-fee guard -/// compares exponents, while balance validation and execution use the converted amount. +/// bound. Native and canonical implementations must agree on this conversion. The max-fee guard +/// compares exponents and belongs to the caller. Admission and execution use the converted amount: +/// admission may refuse an op its sender cannot pay in full, while execution charges what the +/// sender has, up to the fee, and never fails over it. /// /// Errors are errno style. A fallible entry point returns a status (or a null pointer for /// application_engine_state_file_in_dump) and leaves the reason for @@ -94,16 +109,47 @@ /// raw big-endian bytes rather than as a number a host may not be able to spell. #define APPLICATION_ENGINE_VALUE_SIZE 32 +/// @brief The version of this header, which the engine reports back to the host. +/// @details Raised whenever a record's layout or a function's meaning changes. Version 2 +/// replaced validation with admission and made user-op execution total. Version 3 added the +/// staleness window to the scheduler limits. +#define APPLICATION_ENGINE_ABI_VERSION 3 + #ifdef __cplusplus extern "C" { #endif +/// @brief The header version the engine was built against. +/// @returns APPLICATION_ENGINE_ABI_VERSION as the engine's build saw it. +/// @details Pure and infallible. The host calls it before anything else and refuses an engine +/// reporting another version, since a changed meaning is not a link error. +APPLICATION_ENGINE_API uint32_t application_engine_abi_version(void) APPLICATION_ENGINE_NOEXCEPT; + /// @brief The largest user-op method payload, in bytes. /// @details Pure, infallible, and constant for the linked engine implementation, independent of /// any loaded deployment. Zero permits only empty method payloads. The host uses this value for /// ingress admission and batch sizing; it does not bound direct inputs from L1. APPLICATION_ENGINE_API uint64_t application_engine_max_method_payload_bytes(void) APPLICATION_ENGINE_NOEXCEPT; +/// @brief The scheduler constants a template compiles in for this application. +/// @details They belong to the application: the window is how long it lets a sequencer hold a +/// direct back, and the other two bound what its directs may cost the canonical machine. All +/// three are consensus values: a template's scheduler and every host must read the same ones. +typedef struct ApplicationEngineSchedulerLimits { + /// The staleness window in L1 blocks, at least 1: a direct queued this long is executed, and + /// a batch whose first frame is this old is skipped. + uint64_t max_wait_blocks; + uint64_t fridge_bound; ///< How many admitted directs the fridge may hold. + uint64_t max_direct_payload_bytes; ///< A direct with a longer payload never executes. +} ApplicationEngineSchedulerLimits; + +/// @brief Read the scheduler constants for this application. +/// @param out_limits Writable record, written whole before returning. +/// @details Pure, infallible, and constant for the linked engine implementation, independent of +/// any loaded deployment, like the method payload bound. +APPLICATION_ENGINE_API void application_engine_scheduler_limits( + ApplicationEngineSchedulerLimits *out_limits) APPLICATION_ENGINE_NOEXCEPT; + /// @brief An account address, raw bytes and no encoding. /// @details A named type rather than a loose buffer, so an address and an amount cannot be /// passed for one another. @@ -131,58 +177,66 @@ typedef struct ApplicationEngineByteSpan { /// @details Append only. The status is the contract, the accompanying message is a diagnostic. typedef enum ApplicationEngineStatus { APPLICATION_ENGINE_STATUS_OK = 0, ///< Call succeeded. - APPLICATION_ENGINE_STATUS_INVALID = -1, ///< Refused by protocol or configuration validation. + APPLICATION_ENGINE_STATUS_INVALID = -1, ///< Refused by admission. APPLICATION_ENGINE_STATUS_INTERNAL_ERROR = -2, ///< Internal engine failure, fatal-no-resume. APPLICATION_ENGINE_STATUS_IO_ERROR = -3, ///< An operational filesystem or mapping failure. APPLICATION_ENGINE_STATUS_NOT_FOUND = -4, ///< A required dump artifact is missing. APPLICATION_ENGINE_STATUS_INVALID_DUMP = -5, ///< A dump is malformed or truncated. } ApplicationEngineStatus; -/// @brief A user op as its sender signed it. -/// @details The engine reads the nonce and the data and carries max_fee without checking it, -/// that guard belongs to the caller, so an op arrives whole. +/// @brief A user op as its sender signed it, the record admission reads. +/// @details The engine carries the nonce and max_fee without checking either, those guards +/// belong to the caller, so an op arrives whole. typedef struct ApplicationEngineUserOp { uint32_t nonce; ///< Sender replay protection nonce. uint16_t max_fee; ///< Highest frame fee exponent the sender accepts, base 129/128. ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. } ApplicationEngineUserOp; -/// @brief A user op that already passed validation, as the caller sequenced it. -/// @details Not the signed op: execution consumes the nonce the state expects, advancing it by -/// exactly one, and the max fee went with the guard the caller already settled, leaving the fee -/// the frame charges. -typedef struct ApplicationEngineValidUserOp { +/// @brief A user op as the scheduler hands it out, the record execution reads. +/// @details The caller found its nonce equal to the sender's next one before handing it over, so +/// execution advances that nonce by exactly one without comparing. Its max fee already covered +/// the frame's price, leaving the fee the frame charges. +typedef struct ApplicationEngineUserOpTx { ApplicationEngineEthereumAddress sender; ///< The recovered signer. + uint32_t nonce; ///< The nonce the sender signed, the sender's next. uint16_t fee; ///< The charged frame fee exponent, base 129/128. + uint64_t safe_block; ///< The safe block of the frame carrying the op. ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. -} ApplicationEngineValidUserOp; +} ApplicationEngineUserOpTx; /// @brief An input taken straight from the L1 input box. /// @details Its sender is authenticated by the chain rather than recovered from a signature. -/// Its payload is still untrusted application input and has no method-payload bound at this ABI. +/// Its payload is still untrusted application input; the caller never hands over one longer +/// than the scheduler limits' max_direct_payload_bytes. typedef struct ApplicationEngineDirectInput { ApplicationEngineEthereumAddress sender; ///< The L1 authenticated sender. + uint64_t input_index; ///< The application's input box index of the input. uint64_t block_number; ///< The L1 inclusion block number. + uint64_t timestamp; ///< The L1 inclusion block's Unix time, in seconds. ApplicationEngineByteSpan payload; ///< The raw input payload. } ApplicationEngineDirectInput; -/// @brief Why an engine refused a user op, mirroring the sequencer's rejection reasons. +/// @brief Why admission refused a user op. /// @details Append only. Written only on APPLICATION_ENGINE_STATUS_INVALID, and it selects which -/// member of ApplicationEngineInvalidValues carries the diagnostics. +/// member of ApplicationEngineInvalidValues carries the diagnostics. Admission produces only +/// INSUFFICIENT_FEE_BALANCE and REFUSED_BY_POLICY; the others are retired, since their checks +/// belong to the caller, and keep their numbers so none is ever reused. typedef enum ApplicationEngineInvalidReason { - APPLICATION_ENGINE_INVALID_NONCE = 0, ///< Nonce or account binding, read `nonce`. - APPLICATION_ENGINE_INVALID_MAX_FEE = 1, ///< The caller-owned max fee guard, read `max_fee`. + APPLICATION_ENGINE_INVALID_NONCE = 0, ///< Retired, the caller owns the nonce check. + APPLICATION_ENGINE_INVALID_MAX_FEE = 1, ///< Retired, the caller owns the max fee guard. APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE = 2, ///< Cannot cover the frame fee, read `fee_balance`. - APPLICATION_ENGINE_NONCE_EXHAUSTED = 3, ///< The caller-owned UINT32_MAX nonce guard, no values. + APPLICATION_ENGINE_NONCE_EXHAUSTED = 3, ///< Retired, the caller owns the UINT32_MAX guard. + APPLICATION_ENGINE_REFUSED_BY_POLICY = 4, ///< Any other operator policy, no values. } ApplicationEngineInvalidReason; -/// @brief Diagnostics for APPLICATION_ENGINE_INVALID_NONCE. +/// @brief Diagnostics for the retired APPLICATION_ENGINE_INVALID_NONCE. typedef struct ApplicationEngineInvalidNonce { uint32_t expected; ///< The nonce the account expects next. uint32_t got; ///< The nonce the op carried. } ApplicationEngineInvalidNonce; -/// @brief Diagnostics for APPLICATION_ENGINE_INVALID_MAX_FEE. +/// @brief Diagnostics for the retired APPLICATION_ENGINE_INVALID_MAX_FEE. /// @details Both values are log space exponents, base 129/128. No engine produces this reason, /// the guard belongs to the caller, it is carried so the reason vocabulary stays whole. typedef struct ApplicationEngineInvalidMaxFee { @@ -200,9 +254,11 @@ typedef struct ApplicationEngineInsufficientFeeBalance { } ApplicationEngineInsufficientFeeBalance; /// @brief The diagnostics of a refusal, read through the member its reason selects. +/// @details APPLICATION_ENGINE_REFUSED_BY_POLICY has no member: its explanation is text, which +/// travels in the last error message. typedef union ApplicationEngineInvalidValues { - ApplicationEngineInvalidNonce nonce; ///< APPLICATION_ENGINE_INVALID_NONCE. - ApplicationEngineInvalidMaxFee max_fee; ///< APPLICATION_ENGINE_INVALID_MAX_FEE. + ApplicationEngineInvalidNonce nonce; ///< The retired APPLICATION_ENGINE_INVALID_NONCE. + ApplicationEngineInvalidMaxFee max_fee; ///< The retired APPLICATION_ENGINE_INVALID_MAX_FEE. ApplicationEngineInsufficientFeeBalance fee_balance; ///< APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE. } ApplicationEngineInvalidValues; @@ -246,12 +302,6 @@ typedef struct ApplicationEngineOutput { /// @brief The engine instance behind the handle, opaque to every caller. typedef struct ApplicationEngine ApplicationEngine; -/// @brief Progress embedded in the engine's logical state and every checkpoint. -typedef struct ApplicationEngineProgress { - uint64_t executed_input_count; ///< Next input offset; included no-ops count, rejections do not. - uint64_t last_executed_safe_block; ///< Maximum block carried by any executed input. -} ApplicationEngineProgress; - /// @brief Get the message describing the most recent failure. /// @returns A NUL terminated string, never null, empty when the last fallible call succeeded. /// @details Read it after a negative status or a null handle. Every fallible entry point clears @@ -285,50 +335,80 @@ APPLICATION_ENGINE_API const char *application_engine_get_last_error_message(voi APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_from_dump(const char *prefix, ApplicationEngine **out_engine) APPLICATION_ENGINE_NOEXCEPT; +/// @brief Open an engine over a canonical machine's comparison bytes, at the machine's count. +/// @param state_file The file holding the bytes the canonical machine yields for its state: its +/// whole labeled state range, or its `state` inspect report. +/// @param l2_tx_count The canonical machine's L2 tx count, which the new instance starts at +/// whether or not the bytes hold one. +/// @param out_engine The engine handle, written only on OK and left untouched otherwise. +/// @returns OK, NOT_FOUND for a missing file, INVALID_DUMP for bytes that do not decode or that +/// contradict l2_tx_count, IO_ERROR for other filesystem failures, or INTERNAL_ERROR. +/// @details How a host starts from a canonical machine rather than from one of its own dumps. A +/// dump of the result must hold the same comparison bytes in the file +/// application_engine_state_file_in_dump names. Like application_engine_from_dump it never +/// changes the source file, and the instance stays usable after the file is deleted. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_from_canonical_state(const char *state_file, + uint64_t l2_tx_count, ApplicationEngine **out_engine) APPLICATION_ENGINE_NOEXCEPT; + /// @brief Destroy an engine instance. /// @param engine The engine handle. APPLICATION_ENGINE_API void application_engine_destroy(ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; -/// @brief Validate a user op against current state, pure and read-only. +/// @brief Read a sender's next nonce, pure and read-only. +/// @param engine The engine handle. +/// @param sender The account whose nonce to read. +/// @param out_nonce The nonce the sender's next executed op carries, written only on OK. +/// @returns OK, IO_ERROR, or INTERNAL_ERROR. An error requires discarding the instance. +/// @details Zero for an address that has sent nothing. The caller compares it with each user op +/// before executing, and checks after an execution that it advanced by exactly one. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_user_nonce(const ApplicationEngine *engine, + const ApplicationEngineEthereumAddress *sender, uint32_t *out_nonce) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Decide whether the operator wants a user op in a frame at this fee, pure and read-only. /// @param engine The engine handle. /// @param sender The recovered signer. -/// @param user_op The op to validate, as its sender signed it. -/// @param current_fee The frame fee exponent, base 129/128. +/// @param user_op The op to admit, as its sender signed it. +/// @param fee The frame fee exponent, base 129/128. /// @param out_invalid Why the op was refused, written whole and only on INVALID. -/// @returns OK, INVALID with diagnostics, IO_ERROR, or INTERNAL_ERROR. -/// @details A rejection reports itself through out_invalid and leaves the last error message -/// empty; IO_ERROR and INTERNAL_ERROR carry an error message. The max-fee and exhausted-nonce -/// guards belong to the caller and are never checked here, so APPLICATION_ENGINE_INVALID_MAX_FEE -/// and APPLICATION_ENGINE_NONCE_EXHAUSTED never come back. Queued -/// outputs are left alone, only an execution touches them. -APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_validate_user_op(const ApplicationEngine *engine, - const ApplicationEngineEthereumAddress *sender, const ApplicationEngineUserOp *user_op, uint16_t current_fee, +/// @returns OK to admit, INVALID with a reason, IO_ERROR, or INTERNAL_ERROR. +/// @details Policy, never consensus: only the sequencer asks, before it takes an op in, and the +/// answer never changes what an included op does. The nonce, the max-fee and exhausted-nonce +/// guards belong to the caller and are never checked here, so only +/// APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE and APPLICATION_ENGINE_REFUSED_BY_POLICY come back. +/// +/// An INSUFFICIENT_FEE_BALANCE refusal reports itself through out_invalid alone and leaves the +/// last error message empty. A REFUSED_BY_POLICY refusal leaves its explanation in the last error +/// message, which the sequencer hands to the client verbatim. IO_ERROR and INTERNAL_ERROR carry an +/// error message. Queued outputs are left alone, only an execution touches them. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_admit_user_op(const ApplicationEngine *engine, + const ApplicationEngineEthereumAddress *sender, const ApplicationEngineUserOp *user_op, uint16_t fee, ApplicationEngineInvalid *out_invalid) APPLICATION_ENGINE_NOEXCEPT; -/// @brief Execute a validated user op, advancing its sender's expected nonce by exactly one. +/// @brief Execute a user op the caller found at its sender's next nonce. /// @param engine The engine handle. -/// @param user_op The validated op to execute. -/// @param safe_block The covering frame safe block, folded into the clock as max(clock, it). +/// @param user_op The op to execute. /// @param out_output_count How many outputs this op left waiting, written only on OK. /// @returns OK, IO_ERROR, or INTERNAL_ERROR. An error requires discarding the instance. -/// @details An op the method rejects still executed and still counts, so it reports OK. Only -/// the resulting state, progress, and outputs cross the seam; the application's business-failure -/// diagnostics are not returned separately. +/// @details Total: whatever the op does or pays, it reports OK, advances its sender's nonce by +/// exactly one, and advances the L2 tx count by exactly one. A fee the sender cannot cover is no +/// failure: the engine charges what the sender has, up to the fee, and runs the method on what is +/// left. An op the method rejects still executed and still counts. Only the resulting state, +/// count, and outputs cross the seam; business-failure diagnostics are not returned separately. /// /// An execution refuses to run while an earlier execution's outputs are still queued, reporting /// INTERNAL_ERROR without executing anything rather than discarding outputs meant to reach the /// chain. That refusal is what makes the count reported this op's own. -APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_valid_user_op(ApplicationEngine *engine, - const ApplicationEngineValidUserOp *user_op, uint64_t safe_block, - uint64_t *out_output_count) APPLICATION_ENGINE_NOEXCEPT; +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_user_op(ApplicationEngine *engine, + const ApplicationEngineUserOpTx *user_op, uint64_t *out_output_count) APPLICATION_ENGINE_NOEXCEPT; /// @brief Execute a direct input from the L1 input box. /// @param engine The engine handle. -/// @param input The input to execute, its L1 block folded into the clock as max(clock, it). +/// @param input The input to execute. /// @param out_output_count How many outputs this input left waiting, written only on OK. /// @returns OK, IO_ERROR, or INTERNAL_ERROR. An error requires discarding the instance. -/// @details An input the engine rejects is a counted no-op and still reports OK, the same way a -/// rejected user op does. Outputs behave as they do for a user op. It never changes a nonce. +/// @details An input the engine ignores or cannot parse is a counted no-op and still reports OK. +/// It advances the L2 tx count by exactly one and never changes a nonce. Outputs behave as they +/// do for a user op. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct_input(ApplicationEngine *engine, const ApplicationEngineDirectInput *input, uint64_t *out_output_count) APPLICATION_ENGINE_NOEXCEPT; @@ -345,13 +425,14 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_drain_output(ApplicationEngine *engine, ApplicationEngineOutput *out_output) APPLICATION_ENGINE_NOEXCEPT; -/// @brief Read the engine's current progress without changing state. +/// @brief Read the engine's L2 tx count without changing state. /// @param engine The engine handle. -/// @param out_progress Writable record, written whole before returning. -/// @details Count zero implies clock zero. Every successful execution advances the count once -/// with checked arithmetic and takes max(previous clock, input block). Both fields survive dumps. -APPLICATION_ENGINE_API void application_engine_progress(const ApplicationEngine *engine, - ApplicationEngineProgress *out_progress) APPLICATION_ENGINE_NOEXCEPT; +/// @returns How many L2 transactions the state has executed, which is the history offset of the +/// next one. +/// @details Infallible. Zero at genesis; every successful execution, a counted no-op included, +/// advances it by exactly one with checked arithmetic. It survives dumps. +APPLICATION_ENGINE_API uint64_t application_engine_l2_tx_count( + const ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; /// @brief Create a crash durable dump of the engine state (write, fsync). /// @param engine The engine handle. @@ -365,7 +446,7 @@ APPLICATION_ENGINE_API void application_engine_progress(const ApplicationEngine /// host to remove with the unreferenced checkpoint directory. /// /// The dump contains the current state. The engine may change backing files or reopen internal -/// handles while checkpointing, but application state and progress stay unchanged. Subsequent +/// handles while checkpointing, but application state and its count stay unchanged. Subsequent /// execution must leave the completed dump immutable. /// All checkpoint-owned artifacts reside at or beneath prefix as ordinary files/directories. /// The host discards a checkpoint by recursive filesystem deletion, with no engine callback. diff --git a/bindings/c-app-engine/src/lib.rs b/bindings/c-app-engine/src/lib.rs index 2167c64e..0417f02f 100644 --- a/bindings/c-app-engine/src/lib.rs +++ b/bindings/c-app-engine/src/lib.rs @@ -11,13 +11,14 @@ mod tests; use std::ffi::{CStr, CString, OsStr}; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; +use std::sync::OnceLock; use alloy_primitives::{Address, U256}; use sequencer_core::application::{ - AppError, AppOutput, AppOutputs, ApplicationProgress, InvalidReason, ValidationOutcome, + Admission, AppError, AppOutput, AppOutputs, PolicyRefusal, SchedulerLimits, }; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; pub use sequencer_core::application::Application; @@ -67,14 +68,76 @@ fn internal(reason: impl Into) -> AppError { } } -/// Validation, execution and draining. NOT_FOUND and INVALID_DUMP describe -/// lifecycle failures only, so here they break the contract and are fatal. +/// Refuse an engine built against another header. A changed meaning, such as +/// an execution that must no longer fail, links without complaint, so only the +/// reported version tells the two apart. +fn abi_version_matches(reported: u32) -> Result<(), AppError> { + if reported == sys::APPLICATION_ENGINE_ABI_VERSION { + return Ok(()); + } + Err(internal(format!( + "linked engine reports ABI version {reported}, but this host speaks version {}", + sys::APPLICATION_ENGINE_ABI_VERSION + ))) +} + +/// Asked once per process, before any other engine call: every handle comes +/// from a constructor that checks it, and the handle-free getters check it too. +fn check_abi_version() -> Result<(), AppError> { + static CHECKED: OnceLock> = OnceLock::new(); + CHECKED + .get_or_init(|| { + abi_version_matches(unsafe { sys::application_engine_abi_version() }) + .map_err(|err| err.to_string()) + }) + .clone() + .map_err(internal) +} + +/// The infallible getters have no error to return, and an engine of another +/// version must not be asked anything. +fn assert_abi_version() { + if let Err(err) = check_abi_version() { + panic!("{err}"); + } +} + +/// Admission's answer. Only the fee-balance and policy reasons are admission's +/// to give; the retired ones, or one this host does not know, break the contract. +fn admission_from( + status: i32, + invalid: &sys::ApplicationEngineInvalid, +) -> Result { + if status != sys::APPLICATION_ENGINE_STATUS_INVALID { + check(status, "admit_user_op")?; + return Ok(Admission::Admit); + } + let refusal = match invalid.reason { + sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE => { + let balance = unsafe { invalid.values.fee_balance }; + PolicyRefusal::Underfunded { + required: U256::from_be_bytes(balance.required.bytes), + available: U256::from_be_bytes(balance.available.bytes), + } + } + sys::APPLICATION_ENGINE_REFUSED_BY_POLICY => PolicyRefusal::Other(last_error_message()), + other => { + return Err(internal(format!( + "engine admission reported unsupported invalid reason {other}" + ))); + } + }; + Ok(Admission::Refuse(refusal)) +} + +/// Admission, nonce reads, execution and draining. NOT_FOUND and INVALID_DUMP +/// describe lifecycle failures only, so here they break the contract and are fatal. fn check(status: i32, operation: &str) -> Result<(), AppError> { check_status(status, operation, false) } -/// `from_dump` and `create_dump`, where a missing or invalid dump is an -/// operational I/O failure. +/// `from_dump`, `from_canonical_state` and `create_dump`, where a missing or +/// invalid dump is an operational I/O failure. fn check_lifecycle(status: i32, operation: &str) -> Result<(), AppError> { check_status(status, operation, true) } @@ -146,138 +209,127 @@ impl EngineApp { } Ok(outputs) } + + /// Take the handle a successful constructor wrote. + fn from_handle(engine: *mut sys::ApplicationEngine) -> Self { + assert!( + !engine.is_null(), + "engine reported successful load without a handle" + ); + Self { engine } + } + + /// Run one execution entry point and take the outputs it queued. + fn execute_with( + &mut self, + operation: &str, + run: impl FnOnce(*mut sys::ApplicationEngine, &mut u64) -> i32, + ) -> Result { + let mut count = 0; + check(run(self.engine, &mut count), operation)?; + self.drain_outputs(count) + } } impl Application for EngineApp { + fn user_nonce(&self, sender: Address) -> Result { + let mut nonce = 0; + check( + unsafe { + sys::application_engine_user_nonce(self.engine, &abi_address(sender), &mut nonce) + }, + "user_nonce", + )?; + Ok(nonce) + } + + fn apply_user_op(&mut self, op: &UserOpTx) -> Result { + let op = sys::ApplicationEngineUserOpTx { + sender: abi_address(op.sender), + nonce: op.nonce, + fee: op.fee, + safe_block: op.block, + data: abi_span(&op.data), + }; + self.execute_with("execute_user_op", |engine, count| unsafe { + sys::application_engine_execute_user_op(engine, &op, count) + }) + } + + fn apply_direct(&mut self, direct: &Direct) -> Result { + let direct = sys::ApplicationEngineDirectInput { + sender: abi_address(direct.sender), + input_index: direct.input_index, + block_number: direct.block, + timestamp: direct.timestamp, + payload: abi_span(&direct.payload), + }; + self.execute_with("execute_direct_input", |engine, count| unsafe { + sys::application_engine_execute_direct_input(engine, &direct, count) + }) + } + + fn l2_tx_count(&self) -> L2TxCount { + L2TxCount::new(unsafe { sys::application_engine_l2_tx_count(self.engine) }) + } + fn max_method_payload_bytes() -> usize { + assert_abi_version(); usize::try_from(unsafe { sys::application_engine_max_method_payload_bytes() }) .expect("engine payload bound exceeds usize") } - fn validate_user_op( - &self, - sender: Address, - user_op: &UserOp, - current_fee: u16, - ) -> Result { + fn scheduler_limits() -> SchedulerLimits { + assert_abi_version(); + let mut limits = sys::ApplicationEngineSchedulerLimits { + max_wait_blocks: 0, + fridge_bound: 0, + max_direct_payload_bytes: 0, + }; + unsafe { sys::application_engine_scheduler_limits(&mut limits) }; + SchedulerLimits { + max_wait_blocks: limits.max_wait_blocks, + fridge_bound: limits.fridge_bound, + max_direct_payload_bytes: limits.max_direct_payload_bytes, + } + } + + fn admit_user_op(&self, sender: Address, op: &UserOp, fee: u16) -> Result { let mut invalid = sys::ApplicationEngineInvalid { reason: 0, values: sys::ApplicationEngineInvalidValues { - nonce: sys::ApplicationEngineInvalidNonce { - expected: 0, - got: 0, + fee_balance: sys::ApplicationEngineInsufficientFeeBalance { + required: sys::ApplicationEngineUint256 { bytes: [0; 32] }, + available: sys::ApplicationEngineUint256 { bytes: [0; 32] }, }, }, }; let op = sys::ApplicationEngineUserOp { - nonce: user_op.nonce, - max_fee: user_op.max_fee, - data: abi_span(user_op.data.as_ref()), + nonce: op.nonce, + max_fee: op.max_fee, + data: abi_span(op.data.as_ref()), }; let status = unsafe { - sys::application_engine_validate_user_op( + sys::application_engine_admit_user_op( self.engine, &abi_address(sender), &op, - current_fee, + fee, &mut invalid, ) }; - if status != sys::APPLICATION_ENGINE_STATUS_INVALID { - check(status, "validate_user_op")?; - return Ok(ValidationOutcome::Accept); - } - let reason = match invalid.reason { - sys::APPLICATION_ENGINE_INVALID_NONCE => { - let nonce = unsafe { invalid.values.nonce }; - InvalidReason::InvalidNonce { - expected: nonce.expected, - got: nonce.got, - } - } - sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE => { - let balance = unsafe { invalid.values.fee_balance }; - InvalidReason::InsufficientFeeBalance { - required: U256::from_be_bytes(balance.required.bytes), - available: U256::from_be_bytes(balance.available.bytes), - } - } - // Max-fee and exhausted-nonce rejections belong to the shared execution boundary. - other => { - return Err(internal(format!( - "engine reported unsupported invalid reason {other}" - ))); - } - }; - Ok(ValidationOutcome::Reject(reason)) - } - - fn apply_valid_user_op( - &mut self, - user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - let op = sys::ApplicationEngineValidUserOp { - sender: abi_address(user_op.sender), - fee: user_op.fee, - data: abi_span(&user_op.data), - }; - let mut count = 0; - check( - unsafe { - sys::application_engine_execute_valid_user_op( - self.engine, - &op, - safe_block, - &mut count, - ) - }, - "execute_valid_user_op", - )?; - self.drain_outputs(count) - } - - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { - let direct = sys::ApplicationEngineDirectInput { - sender: abi_address(input.sender), - block_number: input.block_number, - payload: abi_span(&input.payload), - }; - let mut count = 0; - check( - unsafe { - sys::application_engine_execute_direct_input(self.engine, &direct, &mut count) - }, - "execute_direct_input", - )?; - self.drain_outputs(count) - } - - fn progress(&self) -> ApplicationProgress { - let mut progress = sys::ApplicationEngineProgress { - executed_input_count: 0, - last_executed_safe_block: 0, - }; - unsafe { sys::application_engine_progress(self.engine, &mut progress) }; - ApplicationProgress::try_new( - ExecutedInputCount::new(progress.executed_input_count), - progress.last_executed_safe_block, - ) - .expect("engine returned incoherent application progress") + admission_from(status, &invalid) } fn from_dump(prefix: &Path) -> Result { + check_abi_version()?; let prefix = path_to_cstring(prefix); let mut engine = std::ptr::null_mut(); check_lifecycle( unsafe { sys::application_engine_from_dump(prefix.as_ptr(), &mut engine) }, "from_dump", )?; - assert!( - !engine.is_null(), - "engine reported successful load without a handle" - ); - Ok(Self { engine }) + Ok(Self::from_handle(engine)) } fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { @@ -289,6 +341,7 @@ impl Application for EngineApp { } fn state_file_in_dump(prefix: &Path) -> PathBuf { + assert_abi_version(); let prefix = path_to_cstring(prefix); let state_file = unsafe { sys::application_engine_state_file_in_dump(prefix.as_ptr()) }; assert!( @@ -300,4 +353,21 @@ impl Application for EngineApp { unsafe { CStr::from_ptr(state_file) }.to_bytes(), )) } + + fn from_canonical_state(state_file: &Path, l2_tx_count: L2TxCount) -> Result { + check_abi_version()?; + let state_file = path_to_cstring(state_file); + let mut engine = std::ptr::null_mut(); + check_lifecycle( + unsafe { + sys::application_engine_from_canonical_state( + state_file.as_ptr(), + l2_tx_count.get(), + &mut engine, + ) + }, + "from_canonical_state", + )?; + Ok(Self::from_handle(engine)) + } } diff --git a/bindings/c-app-engine/src/tests.rs b/bindings/c-app-engine/src/tests.rs index a3934548..6f10c3b8 100644 --- a/bindings/c-app-engine/src/tests.rs +++ b/bindings/c-app-engine/src/tests.rs @@ -3,12 +3,21 @@ use super::*; -// This fixture covers output borrowing and a zero payload bound; wallet tests cover execution/dumps. +// This fixture covers output borrowing, the handle-free getters, admission's reason mapping and +// the records execution hands across; wallet tests cover what execution does, and dumps. // Every non-empty output borrows the same allocation, overwritten by the next drain. #[derive(Default)] struct OutputEngine { drained: usize, payload: [u8; 4], + /// The last records the execution entry points received, read back field by field. + direct: Option, + user_op: Option, +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_abi_version() -> u32 { + sys::APPLICATION_ENGINE_ABI_VERSION } #[unsafe(no_mangle)] @@ -16,11 +25,49 @@ extern "C" fn application_engine_max_method_payload_bytes() -> u64 { 0 } +#[unsafe(no_mangle)] +unsafe extern "C" fn application_engine_scheduler_limits( + out_limits: *mut sys::ApplicationEngineSchedulerLimits, +) { + // SAFETY: EngineApp supplies a writable record. + unsafe { + *out_limits = sys::ApplicationEngineSchedulerLimits { + max_wait_blocks: u64::MAX - 1, + fridge_bound: 3, + max_direct_payload_bytes: u64::MAX, + } + }; +} + #[test] fn empty_method_payload_bound_comes_from_the_engine() { assert_eq!(EngineApp::max_method_payload_bytes(), 0); } +#[test] +fn scheduler_limits_come_from_the_engine_at_full_width() { + assert_eq!( + EngineApp::scheduler_limits(), + SchedulerLimits { + max_wait_blocks: u64::MAX - 1, + fridge_bound: 3, + max_direct_payload_bytes: u64::MAX, + } + ); +} + +#[test] +fn an_engine_of_another_abi_version_is_refused() { + abi_version_matches(sys::APPLICATION_ENGINE_ABI_VERSION).unwrap(); + for reported in [0, 1, sys::APPLICATION_ENGINE_ABI_VERSION + 1, u32::MAX] { + assert!(matches!( + abi_version_matches(reported), + Err(AppError::Internal { reason }) if reason.contains(&format!("version {reported}")) + )); + } + check_abi_version().unwrap(); +} + #[unsafe(no_mangle)] unsafe extern "C" fn application_engine_drain_output( engine: *mut sys::ApplicationEngine, @@ -72,6 +119,76 @@ unsafe extern "C" fn application_engine_drain_output( sys::APPLICATION_ENGINE_STATUS_OK } +#[unsafe(no_mangle)] +unsafe extern "C" fn application_engine_execute_direct_input( + engine: *mut sys::ApplicationEngine, + input: *const sys::ApplicationEngineDirectInput, + out_output_count: *mut u64, +) -> i32 { + // SAFETY: as for drain; EngineApp passes a live record and a writable count. + let engine = unsafe { &mut *engine.cast::() }; + let input = unsafe { &*input }; + engine.direct = Some(Direct { + input_index: input.input_index, + sender: Address::from(input.sender.bytes), + block: input.block_number, + timestamp: input.timestamp, + payload: payload_from(input.payload), + }); + unsafe { *out_output_count = 0 }; + sys::APPLICATION_ENGINE_STATUS_OK +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn application_engine_execute_user_op( + engine: *mut sys::ApplicationEngine, + user_op: *const sys::ApplicationEngineUserOpTx, + out_output_count: *mut u64, +) -> i32 { + // SAFETY: as for drain; EngineApp passes a live record and a writable count. + let engine = unsafe { &mut *engine.cast::() }; + let user_op = unsafe { &*user_op }; + engine.user_op = Some(UserOpTx { + sender: Address::from(user_op.sender.bytes), + nonce: user_op.nonce, + fee: user_op.fee, + block: user_op.safe_block, + data: payload_from(user_op.data), + }); + unsafe { *out_output_count = 0 }; + sys::APPLICATION_ENGINE_STATUS_OK +} + +/// The wallet ignores a direct's index, block and timestamp and a user op's nonce and safe block, +/// so only an engine that records them shows a dropped or swapped field. +#[test] +fn execution_hands_every_field_to_the_engine() { + let mut app = EngineApp { + engine: Box::into_raw(Box::::default()).cast(), + }; + let direct = Direct { + input_index: 3, + sender: Address::repeat_byte(0x41), + block: 7, + timestamp: 1_700_000_011, + payload: vec![1, 2, 3], + }; + let op = UserOpTx { + sender: Address::repeat_byte(0x42), + nonce: 5, + fee: 300, + block: 9, + data: vec![4, 5], + }; + assert_eq!(app.apply_direct(&direct).unwrap(), AppOutputs::new()); + assert_eq!(app.apply_user_op(&op).unwrap(), AppOutputs::new()); + + // SAFETY: the handle is the OutputEngine boxed above, alive until `app` drops. + let engine = unsafe { &*app.engine.cast::() }; + assert_eq!(engine.direct.as_ref(), Some(&direct)); + assert_eq!(engine.user_op.as_ref(), Some(&op)); +} + #[unsafe(no_mangle)] unsafe extern "C" fn application_engine_destroy(engine: *mut sys::ApplicationEngine) { // SAFETY: EngineApp calls this once for the allocation transferred by the test. @@ -80,7 +197,7 @@ unsafe extern "C" fn application_engine_destroy(engine: *mut sys::ApplicationEng #[unsafe(no_mangle)] extern "C" fn application_engine_get_last_error_message() -> *const std::ffi::c_char { - c"output fixture exhausted".as_ptr() + c"fixture refuses: quota spent".as_ptr() } #[test] @@ -131,4 +248,86 @@ fn lifecycle_statuses_are_fatal_outside_dump_lifecycle_calls() { check(sys::APPLICATION_ENGINE_STATUS_IO_ERROR, "drain_output"), Err(AppError::Io(_)) )); + // An execution refused over queued outputs reports INTERNAL_ERROR, which is fatal. + assert!(matches!( + check( + sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR, + "execute_user_op" + ), + Err(AppError::Internal { .. }) + )); +} + +fn refusal(reason: sys::ApplicationEngineInvalidReason) -> sys::ApplicationEngineInvalid { + let mut required = [0; 32]; + required[0] = 0x80; + required[31] = 7; + sys::ApplicationEngineInvalid { + reason, + values: sys::ApplicationEngineInvalidValues { + fee_balance: sys::ApplicationEngineInsufficientFeeBalance { + required: sys::ApplicationEngineUint256 { bytes: required }, + available: sys::ApplicationEngineUint256 { bytes: [0xff; 32] }, + }, + }, + } +} + +#[test] +fn admission_maps_the_reasons_admission_may_give() { + let ignored = refusal(sys::APPLICATION_ENGINE_INVALID_NONCE); + assert_eq!( + admission_from(sys::APPLICATION_ENGINE_STATUS_OK, &ignored).unwrap(), + Admission::Admit + ); + assert_eq!( + admission_from( + sys::APPLICATION_ENGINE_STATUS_INVALID, + &refusal(sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE) + ) + .unwrap(), + Admission::Refuse(PolicyRefusal::Underfunded { + required: (U256::from(1) << 255) + U256::from(7), + available: U256::MAX, + }) + ); + // The policy text is whatever the engine left in its last error message. + assert_eq!( + admission_from( + sys::APPLICATION_ENGINE_STATUS_INVALID, + &refusal(sys::APPLICATION_ENGINE_REFUSED_BY_POLICY) + ) + .unwrap(), + Admission::Refuse(PolicyRefusal::Other("fixture refuses: quota spent".into())) + ); +} + +#[test] +fn admission_refuses_retired_and_unknown_reasons_and_lifecycle_statuses() { + for reason in [ + sys::APPLICATION_ENGINE_INVALID_NONCE, + sys::APPLICATION_ENGINE_INVALID_MAX_FEE, + sys::APPLICATION_ENGINE_NONCE_EXHAUSTED, + sys::APPLICATION_ENGINE_REFUSED_BY_POLICY + 1, + ] { + assert!(matches!( + admission_from(sys::APPLICATION_ENGINE_STATUS_INVALID, &refusal(reason)), + Err(AppError::Internal { reason: text }) if text.contains("unsupported invalid reason") + )); + } + let ignored = refusal(sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE); + for status in [ + sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR, + sys::APPLICATION_ENGINE_STATUS_NOT_FOUND, + sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP, + ] { + assert!(matches!( + admission_from(status, &ignored), + Err(AppError::Internal { .. }) + )); + } + assert!(matches!( + admission_from(sys::APPLICATION_ENGINE_STATUS_IO_ERROR, &ignored), + Err(AppError::Io(_)) + )); } diff --git a/bindings/c-app-engine/tests/abi_version_mismatch.rs b/bindings/c-app-engine/tests/abi_version_mismatch.rs new file mode 100644 index 00000000..07890210 --- /dev/null +++ b/bindings/c-app-engine/tests/abi_version_mismatch.rs @@ -0,0 +1,102 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! An engine built against another header is refused before it is asked anything else. A binary +//! of its own, because the host caches the version check for the life of the process. + +use std::ffi::c_char; +use std::path::Path; + +use c_app_engine::{Application, EngineApp, sys}; +use sequencer_core::application::AppError; +use sequencer_core::history::L2TxCount; + +const REPORTED: u32 = 1; + +#[unsafe(no_mangle)] +extern "C" fn application_engine_abi_version() -> u32 { + REPORTED +} + +// Every other entry point aborts the test binary if reached: a panic cannot unwind out of +// an extern "C" function. + +#[unsafe(no_mangle)] +extern "C" fn application_engine_max_method_payload_bytes() -> u64 { + panic!("max_method_payload_bytes reached past the version check") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_scheduler_limits(_: *mut sys::ApplicationEngineSchedulerLimits) { + panic!("scheduler_limits reached past the version check") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_state_file_in_dump(_: *const c_char) -> *const c_char { + panic!("state_file_in_dump reached past the version check") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_from_dump( + _: *const c_char, + _: *mut *mut sys::ApplicationEngine, +) -> i32 { + panic!("from_dump reached past the version check") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_from_canonical_state( + _: *const c_char, + _: u64, + _: *mut *mut sys::ApplicationEngine, +) -> i32 { + panic!("from_canonical_state reached past the version check") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_destroy(_: *mut sys::ApplicationEngine) { + panic!("destroy reached without a handle") +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_get_last_error_message() -> *const c_char { + panic!("get_last_error_message reached past the version check") +} + +fn names_the_version(reason: &str) -> bool { + reason.contains(&format!("ABI version {REPORTED}")) + && reason.contains(&format!("version {}", sys::APPLICATION_ENGINE_ABI_VERSION)) +} + +#[test] +fn an_engine_of_another_abi_version_is_asked_nothing_else() { + // The constructors return the refusal; the infallible getters can only panic with it. + for result in [ + EngineApp::from_dump(Path::new("/nonexistent/dump")), + EngineApp::from_canonical_state(Path::new("/nonexistent/state"), L2TxCount::ZERO), + ] { + assert!(matches!( + result, + Err(AppError::Internal { reason }) if names_the_version(&reason) + )); + } + + let getters: [fn(); 3] = [ + || { + let _ = EngineApp::scheduler_limits(); + }, + || { + let _ = EngineApp::max_method_payload_bytes(); + }, + || { + let _ = EngineApp::state_file_in_dump(Path::new("/nonexistent/dump")); + }, + ]; + for getter in getters { + let payload = std::panic::catch_unwind(getter).expect_err("a getter must refuse"); + let message = payload + .downcast_ref::() + .expect("the refusal is a formatted message"); + assert!(names_the_version(message), "{message}"); + } +} diff --git a/bindings/c-app-sequencer/src/lib.rs b/bindings/c-app-sequencer/src/lib.rs index 590fcea3..d9bc8369 100644 --- a/bindings/c-app-sequencer/src/lib.rs +++ b/bindings/c-app-sequencer/src/lib.rs @@ -20,12 +20,16 @@ use tracing_subscriber::EnvFilter; version, about = "Rollup sequencer host for a C application.\n\n\ Runs the application engine linked in at build time, the one implementing the \ - application-engine C API. The subcommands come from the sequencer library.\n\n\ + application-engine C API. The subcommands come from the sequencer library. \ + `--state-file` gives the genesis state, which only a setup without `--base` \ + reads.\n\n\ All options can also be set via environment variables (shown in brackets)." )] struct Cli { - /// Genesis dump used only when plain setup needs its initial snapshot. - /// Created by the application's genesis tool + /// The application's genesis state, written by its genesis tool. Read only + /// by a setup without `--base`, which starts the history from it; a setup + /// from a stored machine takes the state from the machine, and `run` never + /// reads this. #[arg(long, env = "CARTESI_SEQUENCER_STATE_FILE", value_name = "PATH")] state_file: Option, #[command(subcommand)] @@ -47,11 +51,12 @@ pub async fn run() -> ExitCode { ) .init(); + // The library calls this only for a setup from genesis. sequencer::run_command(command, move || { let state_file = state_file.ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::InvalidInput, - "plain setup requires --state-file or CARTESI_SEQUENCER_STATE_FILE", + "setup without --base requires --state-file or CARTESI_SEQUENCER_STATE_FILE", ) })?; EngineApp::from_dump(&state_file) diff --git a/bindings/c-scheduler/Cargo.toml b/bindings/c-scheduler/Cargo.toml new file mode 100644 index 00000000..a922d25d --- /dev/null +++ b/bindings/c-scheduler/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "c-scheduler" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "The reference scheduler exported through the proposed scheduler.h C API" +homepage.workspace = true +repository.workspace = true +readme = "README.md" +authors.workspace = true + +[lib] +# `staticlib` is what a C or C++ guest links; `rlib` is what this crate's tests link, calling the +# exported functions through the header's own declarations. +crate-type = ["staticlib", "rlib"] + +[build-dependencies] +# Generates the records from scheduler.h, the same pin as c-app-engine so both read one layout. +bindgen = "0.72" + +[dependencies] +sequencer-core = { path = "../../sequencer-core" } +alloy-primitives = { workspace = true } + +[dev-dependencies] +sequencer-core = { path = "../../sequencer-core", features = ["test-support"] } +alloy-sol-types = { workspace = true } +k256 = { workspace = true } +ssz = { workspace = true } +# The conformance test reads the vector file. +serde_json = { workspace = true } diff --git a/bindings/c-scheduler/README.md b/bindings/c-scheduler/README.md new file mode 100644 index 00000000..d71db1d7 --- /dev/null +++ b/bindings/c-scheduler/README.md @@ -0,0 +1,28 @@ +# C scheduler export + +[`include/scheduler.h`](include/scheduler.h) is the canonical scheduler's C +API, and this crate is its reference implementation: sequencer-core's +`Scheduler` behind the C functions, built as `libc_scheduler.a` (and an rlib +for its own tests). The transaction records are +[`application-engine.h`](../c-app-engine/include/application-engine.h)'s, so a +C or C++ guest hands what the scheduler gives straight to its engine; +[`c/feed_loop.c`](c/feed_loop.c) is that loop without an engine. + +An application's author may implement the header with their own scheduler, +to compile into their guest and for setup to run natively in a rebuild. No +Rust adapter over a foreign scheduler exists yet: the sequencer and setup +call the Rust `Scheduler` directly +([todo](../../docs/plans/2026-11-sequencer-recovery/todo.md#build)). + +The reference panics on impossible states; the export turns each into a +negative status and a dead handle, so no panic unwinds into C. That needs the +default `panic = "unwind"`: a build with `panic = "abort"` aborts instead. + +`cargo test -p c-scheduler` replays the +[conformance vectors](../../docs/protocol/scheduler-semantics.md#conformance) +through the header's declarations alone in +[`tests/conformance.rs`](tests/conformance.rs), the check a port of the header +is held to. It also compares the export with the Rust `Scheduler` input by +input, and builds and runs the C example with the system compiler (`CC`, else +`cc`), linked against the archive Cargo built for that test run (in +`target//deps`), so no separate `cargo build` comes first. diff --git a/bindings/c-scheduler/build.rs b/bindings/c-scheduler/build.rs new file mode 100644 index 00000000..d8c02aae --- /dev/null +++ b/bindings/c-scheduler/build.rs @@ -0,0 +1,43 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Generates the C records and declarations from `include/scheduler.h`, so the export and the +//! header cannot drift: a header change either reaches the export or fails this build. + +use std::env; +use std::path::PathBuf; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let header = manifest_dir.join("include").join("scheduler.h"); + // The transaction records are the engine's own, from the engine header beside this crate. + let engine_include = manifest_dir.join("../c-app-engine/include"); + println!("cargo::rerun-if-changed={}", header.display()); + println!( + "cargo::rerun-if-changed={}", + engine_include.join("application-engine.h").display() + ); + + let out_path = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")).join("bindings.rs"); + bindgen::Builder::default() + .header( + header + .to_str() + .expect("scheduler.h path is not valid UTF-8"), + ) + // The C arm of the header; its C++ arm only spells noexcept. + .clang_args(["-x", "c", "-std=c11"]) + .clang_arg(format!("-I{}", engine_include.display())) + // The scheduler's surface, and the engine records it reaches, never stdint.h's. + .allowlist_item("^(scheduler_|Scheduler|SCHEDULER_).*") + // Integer constants, never Rust enums: a value the header adds later must be refusable. + .default_enum_style(bindgen::EnumVariation::Consts) + .prepend_enum_name(false) + // Records are written whole, so they start from zeroes. + .derive_default(true) + .rust_edition(bindgen::RustEdition::Edition2024) + .generate() + .expect("failed to generate bindings from scheduler.h") + .write_to_file(&out_path) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", out_path.display())); +} diff --git a/bindings/c-scheduler/c/feed_loop.c b/bindings/c-scheduler/c/feed_loop.c new file mode 100644 index 00000000..035b1e1a --- /dev/null +++ b/bindings/c-scheduler/c/feed_loop.c @@ -0,0 +1,123 @@ +/* (c) Cartesi and individual authors (see AUTHORS) */ +/* SPDX-License-Identifier: Apache-2.0 (see LICENSE) */ + +/* The canonical feed/next loop over scheduler.h, without an application: it prints what the + * scheduler hands out instead of executing it. tests/c_example.rs builds it against + * libc_scheduler.a and compares its output with the Rust reference on the same inputs. */ + +#include +#include +#include +#include + +#include "scheduler.h" + +/* The header's enums cross as 32-bit integers, which a C compiler need not pick on its own. */ +_Static_assert(sizeof(SchedulerStatus) == sizeof(int32_t), "SchedulerStatus is not 32 bits"); +_Static_assert(sizeof(SchedulerTxKind) == sizeof(int32_t), "SchedulerTxKind is not 32 bits"); +_Static_assert(sizeof(SchedulerInputOutcomeKind) == sizeof(int32_t), + "SchedulerInputOutcomeKind is not 32 bits"); +_Static_assert(sizeof(SchedulerBatchRejectReason) == sizeof(int32_t), + "SchedulerBatchRejectReason is not 32 bits"); + +#define SEQUENCER_BYTE 0x11 +#define USER_BYTE 0x22 +#define CHAIN_ID 1 + +/* ssz(Batch { nonce: 0, frames: [Frame { user_ops: [], safe_block: 2, fee_price: 0 }] }) */ +static const uint8_t BATCH[] = { + 0, 0, 0, 0, 0, 0, 0, 0, /* nonce */ + 12, 0, 0, 0, /* offset of frames */ + 4, 0, 0, 0, /* offset of frame 0 within the list */ + 14, 0, 0, 0, /* offset of user_ops within the frame */ + 2, 0, 0, 0, 0, 0, 0, 0, /* safe_block */ + 0, 0, /* fee_price */ +}; + +static SchedulerInput input_at(uint64_t index, uint8_t sender, uint64_t block, const uint8_t *payload, + uint64_t size) { + SchedulerInput input; + memset(&input, 0, sizeof input); + input.index = index; + memset(input.sender.bytes, sender, sizeof input.sender.bytes); + input.block_number = block; + input.timestamp = block * 12; + input.chain_id = CHAIN_ID; + input.payload.data = payload; + input.payload.size = size; + return input; +} + +static int fail(const char *what, SchedulerStatus status) { + fprintf(stderr, "%s failed with %d: %s\n", what, (int)status, scheduler_get_last_error_message()); + return 1; +} + +int main(void) { + if (scheduler_abi_version() != SCHEDULER_ABI_VERSION) { + fprintf(stderr, "scheduler ABI version %" PRIu32 ", expected %d\n", scheduler_abi_version(), + SCHEDULER_ABI_VERSION); + return 1; + } + + SchedulerConfig config; + memset(&config, 0, sizeof config); + memset(config.sequencer_address.bytes, SEQUENCER_BYTE, sizeof config.sequencer_address.bytes); + config.max_wait_blocks = 10; + config.fridge_bound = 16; + config.max_direct_payload_bytes = 4; + + Scheduler *scheduler = NULL; + SchedulerStatus status = scheduler_create(&config, &scheduler); + if (status != SCHEDULER_STATUS_OK) { + return fail("scheduler_create", status); + } + + static const uint8_t A[] = {'a'}, B[] = {'b'}, C[] = {'c'}, LONG[] = {'l', 'o', 'n', 'g', '!'}; + const SchedulerInput inputs[] = { + input_at(0, USER_BYTE, 1, A, sizeof A), + input_at(1, USER_BYTE, 2, B, sizeof B), + input_at(2, USER_BYTE, 3, C, sizeof C), + input_at(3, USER_BYTE, 4, LONG, sizeof LONG), /* over the payload cap: dropped */ + input_at(4, SEQUENCER_BYTE, 5, BATCH, sizeof BATCH), + input_at(5, USER_BYTE, 20, NULL, 0), /* the backstop drains the direct at block 3 */ + }; + + for (size_t i = 0; i < sizeof inputs / sizeof inputs[0]; i++) { + SchedulerInputOutcome outcome; + status = scheduler_feed(scheduler, &inputs[i], &outcome); + if (status != SCHEDULER_STATUS_OK) { + scheduler_destroy(scheduler); + return fail("scheduler_feed", status); + } + printf("input %" PRIu64 " outcome %d\n", inputs[i].index, (int)outcome.kind); + + SchedulerTx tx; + while ((status = scheduler_next_tx(scheduler, &tx)) == SCHEDULER_STATUS_OK) { + /* A guest would execute here: a direct always, a user op at its sender's next nonce. */ + if (tx.kind == SCHEDULER_TX_DIRECT) { + printf("direct %" PRIu64 " block %" PRIu64 " bytes %" PRIu64 " batch %" PRIu64 "\n", + tx.values.direct.input_index, tx.values.direct.block_number, + tx.values.direct.payload.size, tx.batch_nonce); + } else { + printf("user_op nonce %" PRIu32 " block %" PRIu64 " batch %" PRIu64 "\n", + tx.values.user_op.nonce, tx.values.user_op.safe_block, tx.batch_nonce); + } + } + if (status != SCHEDULER_STATUS_DONE) { + scheduler_destroy(scheduler); + return fail("scheduler_next_tx", status); + } + } + + SchedulerState state; + status = scheduler_state(scheduler, &state); + if (status != SCHEDULER_STATUS_OK) { + scheduler_destroy(scheduler); + return fail("scheduler_state", status); + } + printf("state %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n", state.l1_input_count, + state.next_batch_nonce, state.clock, state.first_queued_input, state.fridge_len); + scheduler_destroy(scheduler); + return 0; +} diff --git a/bindings/c-scheduler/include/scheduler.h b/bindings/c-scheduler/include/scheduler.h new file mode 100644 index 00000000..0952baa4 --- /dev/null +++ b/bindings/c-scheduler/include/scheduler.h @@ -0,0 +1,273 @@ +/* (c) Cartesi and individual authors (see AUTHORS) */ +/* SPDX-License-Identifier: Apache-2.0 (see LICENSE) */ + +#ifndef SCHEDULER_H +#define SCHEDULER_H + +#include + +#include "application-engine.h" + +/// @file +/// The C API of the canonical scheduler, a proposal to settle with the C port's author. It +/// mirrors sequencer-core's Scheduler: it turns an application's L1 inputs, in InputBox order, +/// into the one sequence of L2 transactions the application executes. The rules it carries out +/// are docs/protocol/scheduler-semantics.md; this header states only the calls. +/// +/// The scheduler never calls the application and reads nothing from it. The fridge admits a +/// direct by its sender and payload length, both from the configuration, and a user op is handed +/// out once its signer recovers and its max fee covers the frame's price. Whether a handed-out op +/// executes, its nonce, is the caller's check against the application (application-engine.h). +/// No callback crosses: the caller feeds one input, then pulls its transactions one at a time. +/// +/// The transaction records are application-engine.h's own, so a guest hands them straight to +/// the engine. The conventions are that header's: fixed widths and nothing as size_t, plain C +/// layout, append-only vocabularies whose values are never reused, a C enum mirrored as a 32-bit +/// integer, no exception crossing (SCHEDULER_NOEXCEPT), and errno-style failures whose text +/// travels in scheduler_get_last_error_message. Statuses are the contract, messages are +/// diagnostics, never branch on their text. +/// +/// Ownership. A handle belongs to its caller from create or resume until scheduler_destroy. It +/// may move between threads, but two threads never use it at once. Every record a caller passes +/// in is borrowed for the call alone: the scheduler copies what it keeps. Every record it writes +/// out is the caller's, except the byte spans inside a SchedulerTx, which borrow the handle's +/// storage until the next scheduler_next_tx, scheduler_feed or scheduler_destroy on it. +/// +/// Failure. The reference halts on an impossible state: an input out of order, a feed while the +/// previous input still has transactions, or a counter with no successor. Here such a call +/// returns a negative status instead, and a negative status from scheduler_feed or +/// scheduler_next_tx leaves the handle dead: every later call on it returns +/// SCHEDULER_STATUS_MISUSE, and only scheduler_destroy is still meaningful. Any failure kills +/// the handle, so no implementation has to prove which failures left its state whole. + +/// @brief Marks the public C API, exported even when the consumer builds with +/// -fvisibility=hidden. Carried by declarations only, definitions inherit it. +#if defined(__GNUC__) || defined(__clang__) +#define SCHEDULER_API __attribute__((visibility("default"))) +#else +#define SCHEDULER_API +#endif + +/// @brief Spells the non-throwing guarantee for a C++ consumer, C has no equivalent. +#ifdef __cplusplus +#define SCHEDULER_NOEXCEPT noexcept +#else +#define SCHEDULER_NOEXCEPT +#endif + +/// @brief The version of this header, which the implementation reports back to its caller. +/// @details Raised whenever a record's layout or a function's meaning changes. It moves apart +/// from APPLICATION_ENGINE_ABI_VERSION, though this header reuses that header's records. +#define SCHEDULER_ABI_VERSION 1 + +#ifdef __cplusplus +extern "C" { +#endif + +/// @brief Status returned by every fallible entry point. Zero is success, DONE is the one other +/// non-failure, and every failure is negative, so `status < 0` is the failure test. +/// @details Append only. +typedef enum SchedulerStatus { + /// Call succeeded. + SCHEDULER_STATUS_OK = 0, + /// scheduler_next_tx: the input has no more transactions. + SCHEDULER_STATUS_DONE = 1, + /// A call against the contract: out of order, a null or oversized argument, a bad + /// configuration, or a dead handle. + SCHEDULER_STATUS_MISUSE = -1, + /// The L1 input count or the batch nonce has no successor. The reference halts here too, so + /// the caller halts. + SCHEDULER_STATUS_NO_SUCCESSOR = -2, + /// scheduler_resume: the state and the queued range disagree. The message names the numbers. + SCHEDULER_STATUS_RESUME_REFUSED = -3, + /// A failure the reference scheduler has no name for, an impossible state. + SCHEDULER_STATUS_INTERNAL_ERROR = -4, +} SchedulerStatus; + +/// @brief Deployment constants, compiled into the template. None has a default: a build that +/// forgot one would run another deployment's rules. +typedef struct SchedulerConfig { + /// The batch submitter: its inputs are batches, every other sender's are directs. + ApplicationEngineEthereumAddress sequencer_address; + uint64_t max_wait_blocks; ///< The staleness window, at least 1. + uint64_t fridge_bound; ///< The fridge is full at this many directs. + uint64_t max_direct_payload_bytes; ///< A direct with a longer payload never joins the fridge. +} SchedulerConfig; + +/// @brief One InputBox input, as EvmAdvance delivers it. +typedef struct SchedulerInput { + uint64_t index; ///< The application's InputBox index. + ApplicationEngineEthereumAddress sender; ///< The L1 authenticated sender. + uint64_t block_number; ///< The L1 inclusion block number. + uint64_t timestamp; ///< The inclusion block's Unix time, in seconds. + uint64_t chain_id; ///< With app_contract, the ops' signing domain. + ApplicationEngineEthereumAddress app_contract; ///< The application contract. + ApplicationEngineByteSpan payload; ///< The raw input payload. +} SchedulerInput; + +/// @brief All that carries over between inputs, except the queued directs' bytes, which L1 +/// holds. With the application's L2 tx count, it is the machine's `scheduler` export. +typedef struct SchedulerState { + uint64_t l1_input_count; ///< Inputs consumed: the position in the InputBox. + uint64_t next_batch_nonce; ///< The nonce the next accepted batch carries. + uint64_t clock; ///< The latest block the scheduler has executed at. + uint64_t first_queued_input; ///< The oldest queued direct; l1_input_count when none is. + uint64_t fridge_len; ///< How many directs are queued. +} SchedulerState; + +/// @brief What became of one input. +/// @details Append only. Every rejected or stale batch has the same effect, none; the reject +/// reason is a diagnostic. +typedef enum SchedulerInputOutcomeKind { + SCHEDULER_DIRECT_QUEUED = 0, ///< The direct joined the fridge. + SCHEDULER_DIRECT_DROPPED = 1, ///< Its payload is over the cap: it never executes. + SCHEDULER_BATCH_ACCEPTED = 2, ///< The batch's frames follow, and its nonce is spent. + SCHEDULER_BATCH_STALE = 3, ///< Its first frame is older than the staleness window. + SCHEDULER_BATCH_REJECTED = 4, ///< A gate refused it, read `reject_reason`. +} SchedulerInputOutcomeKind; + +/// @brief Why a batch was rejected. +/// @details Append only. It selects which member of SchedulerBatchRejectValues carries values. +typedef enum SchedulerBatchRejectReason { + SCHEDULER_REJECT_DECODE_FAILED = 0, ///< Not an SSZ batch, no values. + SCHEDULER_REJECT_WRONG_NONCE = 1, ///< Read `wrong_nonce`. + SCHEDULER_REJECT_SAFE_BLOCK_NOT_BELOW_INCLUSION_BLOCK = 2, ///< No values. + SCHEDULER_REJECT_NON_MONOTONIC_SAFE_BLOCKS = 3, ///< No values. + SCHEDULER_REJECT_FEE_PRICE_OUT_OF_RANGE = 4, ///< No values. + SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK = 5, ///< Read `first_frame_below_clock`. +} SchedulerBatchRejectReason; + +/// @brief Values for SCHEDULER_REJECT_WRONG_NONCE. +typedef struct SchedulerWrongNonce { + uint64_t expected; ///< The scheduler's next batch nonce. + uint64_t got; ///< The nonce the batch carried. +} SchedulerWrongNonce; + +/// @brief Values for SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK. +typedef struct SchedulerFirstFrameBelowClock { + uint64_t safe_block; ///< The first frame's safe block. + uint64_t clock; ///< The scheduler's clock, above it. +} SchedulerFirstFrameBelowClock; + +/// @brief A rejection's values, read through the member its reason selects. +typedef union SchedulerBatchRejectValues { + SchedulerWrongNonce wrong_nonce; ///< SCHEDULER_REJECT_WRONG_NONCE. + SchedulerFirstFrameBelowClock first_frame_below_clock; ///< SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK. +} SchedulerBatchRejectValues; + +/// @brief An input's outcome, written whole and only when scheduler_feed returns OK. +/// @details `reject_reason` and `reject_values` mean something only for +/// SCHEDULER_BATCH_REJECTED; for any other kind they are zero. +typedef struct SchedulerInputOutcome { + SchedulerInputOutcomeKind kind; ///< What became of the input. + SchedulerBatchRejectReason reject_reason; ///< Why a rejected batch was rejected. + SchedulerBatchRejectValues reject_values; ///< The values that reason selects. +} SchedulerInputOutcome; + +/// @brief The two kinds of L2 transaction. +/// @details Append only. It selects which member of SchedulerTxValues is written. +typedef enum SchedulerTxKind { + SCHEDULER_TX_DIRECT = 0, ///< A direct input, read `direct`. + SCHEDULER_TX_USER_OP = 1, ///< A user op, read `user_op`. +} SchedulerTxKind; + +/// @brief A transaction's body, the engine's own record for its kind. +typedef union SchedulerTxValues { + /// SCHEDULER_TX_DIRECT, which always executes. + ApplicationEngineDirectInput direct; + /// SCHEDULER_TX_USER_OP, which executes only at its sender's next nonce, the caller's check. + ApplicationEngineUserOpTx user_op; +} SchedulerTxValues; + +/// @brief One handed-out transaction, written whole and only when scheduler_next_tx returns OK. +/// @details Its byte spans borrow the handle until the next scheduler_next_tx, scheduler_feed or +/// scheduler_destroy on it, so a caller that keeps the bytes copies them first. +typedef struct SchedulerTx { + SchedulerTxKind kind; ///< Which member of `values` is written. + /// The scheduler's next batch nonce when it handed this out: the batch whose frame holds or + /// drained it, or the next batch to come. + uint64_t batch_nonce; + SchedulerTxValues values; ///< The transaction. +} SchedulerTx; + +/// @brief The scheduler behind the handle, opaque to every caller. +typedef struct Scheduler Scheduler; + +/// @brief The header version the implementation was built against. +/// @returns SCHEDULER_ABI_VERSION as the implementation's build saw it. +/// @details Pure and infallible. A caller checks it before anything else, since a changed +/// meaning is not a link error. +SCHEDULER_API uint32_t scheduler_abi_version(void) SCHEDULER_NOEXCEPT; + +/// @brief The message describing the most recent failure on this thread. +/// @returns A NUL terminated string, never null, empty when the last fallible call succeeded. +/// @details Every fallible entry point clears it on entry and the next one overwrites it, so +/// copy rather than retain the pointer. The storage is thread local. +SCHEDULER_API const char *scheduler_get_last_error_message(void) SCHEDULER_NOEXCEPT; + +/// @brief Create a scheduler at genesis: nothing consumed, nothing queued, the clock at 0. +/// @param config The deployment constants, copied. +/// @param out_scheduler The new handle, written only on OK and left untouched otherwise. +/// @returns OK, or MISUSE for a null argument or a max_wait_blocks of 0. +SCHEDULER_API SchedulerStatus scheduler_create(const SchedulerConfig *config, + Scheduler **out_scheduler) SCHEDULER_NOEXCEPT; + +/// @brief Resume a scheduler from an exported state. +/// @param config The deployment constants, copied. +/// @param state The exported state. +/// @param queued Every input from state->first_queued_input up to state->l1_input_count, in +/// order, batches and dropped directs included. Null only when queued_count is 0. +/// @param queued_count How many records `queued` holds. +/// @param out_scheduler The new handle, written only on OK and left untouched otherwise. +/// @returns OK; RESUME_REFUSED for a state whose first_queued_input is past its +/// l1_input_count, or for a range out of sequence, short or long, whose blocks decrease, whose +/// first input is not an admitted direct, with a queued direct at or below the clock, or that +/// admits another number of directs than fridge_len; or MISUSE as for create, or for a null span +/// or a count this host cannot address. +/// @details One call, so a refused range never yields a handle. The scheduler re-admits the +/// directs among the range by the same rule and copies them; the caller keeps the range. +SCHEDULER_API SchedulerStatus scheduler_resume(const SchedulerConfig *config, + const SchedulerState *state, const SchedulerInput *queued, uint64_t queued_count, + Scheduler **out_scheduler) SCHEDULER_NOEXCEPT; + +/// @brief Destroy a scheduler, live or dead. Null is a no-op. +SCHEDULER_API void scheduler_destroy(Scheduler *scheduler) SCHEDULER_NOEXCEPT; + +/// @brief Read the scheduler's state. +/// @param out_state Written only on OK. +/// @returns OK, or MISUSE for a null argument or a dead handle, which this call leaves as it was. +/// @details A feed decides everything at once (gates, drains, the clock, the nonce), so the +/// state already describes the scheduler after the last input, even while that input still has +/// transactions to hand out. +SCHEDULER_API SchedulerStatus scheduler_state(const Scheduler *scheduler, + SchedulerState *out_state) SCHEDULER_NOEXCEPT; + +/// @brief Take the next input and say what became of it. Its transactions follow from +/// scheduler_next_tx. +/// @param input Its index must be the state's l1_input_count. Borrowed for the call. +/// @param out_outcome Written whole and only on OK. +/// @returns OK; MISUSE for an index other than l1_input_count, for a feed before +/// scheduler_next_tx has returned DONE for the previous input, or for a null argument; +/// NO_SUCCESSOR when the input count, or the nonce of a batch that would be accepted, is +/// UINT64_MAX; or INTERNAL_ERROR. Every negative status leaves the handle dead. +/// @details The reference lets a feed follow an input whose transactions ran out unseen; this +/// API asks for the DONE, which the canonical loop always reads. +SCHEDULER_API SchedulerStatus scheduler_feed(Scheduler *scheduler, const SchedulerInput *input, + SchedulerInputOutcome *out_outcome) SCHEDULER_NOEXCEPT; + +/// @brief Hand out the current input's next transaction, in execution order. +/// @param out_tx Written whole and only on OK. Its spans borrow the handle until the next +/// scheduler_next_tx, scheduler_feed or scheduler_destroy on it. +/// @returns OK with a transaction; DONE when the input has no more, which it keeps returning +/// until the next feed, and before the first; MISUSE for a null argument; or INTERNAL_ERROR. +/// Every negative status leaves the handle dead. +/// @details A user op whose signature does not recover, or whose max fee is below its frame's +/// price, is never handed out; every other op is, whatever its nonce. +SCHEDULER_API SchedulerStatus scheduler_next_tx(Scheduler *scheduler, + SchedulerTx *out_tx) SCHEDULER_NOEXCEPT; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SCHEDULER_H */ diff --git a/bindings/c-scheduler/src/lib.rs b/bindings/c-scheduler/src/lib.rs new file mode 100644 index 00000000..1c9e28ba --- /dev/null +++ b/bindings/c-scheduler/src/lib.rs @@ -0,0 +1,558 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! sequencer-core's reference [`Scheduler`](sequencer_core::scheduler::Scheduler) exported +//! through `include/scheduler.h`, building to `libc_scheduler.a`. The header states the +//! contract; this file only maps it onto the reference. +//! +//! The reference panics on impossible states. Here every call checks the ones it can name +//! before calling in (an input out of order, a feed before DONE, an exhausted counter) and +//! catches any other panic, so none unwinds into C. A build with `panic = "abort"` cannot +//! catch: there the batch-nonce halt aborts the process, as the reference would. + +pub mod sys; + +use std::any::Any; +use std::cell::RefCell; +use std::ffi::{CString, c_char}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use alloy_primitives::Address; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::{ + BatchRejectReason, InputOutcome, L1Input, Scheduled, Scheduler as Reference, SchedulerConfig, + SchedulerState, +}; + +/// The exported functions must have exactly the header's signatures: the bindgen declarations +/// and these definitions coerce to one function-pointer type each, or this fails to compile. +const _: () = { + let _: [unsafe extern "C" fn() -> u32; 2] = [sys::scheduler_abi_version, scheduler_abi_version]; + let _: [unsafe extern "C" fn() -> *const c_char; 2] = [ + sys::scheduler_get_last_error_message, + scheduler_get_last_error_message, + ]; + let _: [unsafe extern "C" fn( + *const sys::SchedulerConfig, + *mut *mut sys::Scheduler, + ) -> sys::SchedulerStatus; 2] = [sys::scheduler_create, scheduler_create]; + let _: [unsafe extern "C" fn( + *const sys::SchedulerConfig, + *const sys::SchedulerState, + *const sys::SchedulerInput, + u64, + *mut *mut sys::Scheduler, + ) -> sys::SchedulerStatus; 2] = [sys::scheduler_resume, scheduler_resume]; + let _: [unsafe extern "C" fn(*mut sys::Scheduler); 2] = + [sys::scheduler_destroy, scheduler_destroy]; + let _: [unsafe extern "C" fn( + *const sys::Scheduler, + *mut sys::SchedulerState, + ) -> sys::SchedulerStatus; 2] = [sys::scheduler_state, scheduler_state]; + let _: [unsafe extern "C" fn( + *mut sys::Scheduler, + *const sys::SchedulerInput, + *mut sys::SchedulerInputOutcome, + ) -> sys::SchedulerStatus; 2] = [sys::scheduler_feed, scheduler_feed]; + let _: [unsafe extern "C" fn( + *mut sys::Scheduler, + *mut sys::SchedulerTx, + ) -> sys::SchedulerStatus; 2] = [sys::scheduler_next_tx, scheduler_next_tx]; +}; + +thread_local! { + /// The last failure's message, thread local as the header promises. + static LAST_ERROR: RefCell = RefCell::new(CString::default()); +} + +fn clear_error() { + LAST_ERROR.with(|slot| *slot.borrow_mut() = CString::default()); +} + +fn set_error(message: &str) { + // A NUL inside a diagnostic is not worth failing over: truncate at it. + let bytes = message.split('\0').next().unwrap_or_default().as_bytes(); + LAST_ERROR.with(|slot| *slot.borrow_mut() = CString::new(bytes).unwrap_or_default()); +} + +/// A failed call: the status the header names for it and the diagnostic behind it. +struct Failure { + status: sys::SchedulerStatus, + message: String, +} + +impl Failure { + fn misuse(message: impl Into) -> Self { + Self { + status: sys::SCHEDULER_STATUS_MISUSE, + message: message.into(), + } + } + + fn no_successor(message: impl Into) -> Self { + Self { + status: sys::SCHEDULER_STATUS_NO_SUCCESSOR, + message: message.into(), + } + } + + fn internal(message: impl Into) -> Self { + Self { + status: sys::SCHEDULER_STATUS_INTERNAL_ERROR, + message: message.into(), + } + } + + fn report(self) -> sys::SchedulerStatus { + set_error(&self.message); + self.status + } +} + +/// Run a call into the reference, turning a panic into its message so it never reaches C. +fn catch(call: impl FnOnce() -> T) -> Result { + catch_unwind(AssertUnwindSafe(call)).map_err(|payload| panic_message(payload.as_ref())) +} + +fn panic_message(payload: &(dyn Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_owned() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "the reference scheduler panicked without a message".to_owned() + } +} + +/// The scheduler behind an opaque `Scheduler *`. +struct Handle { + reference: Reference, + /// Every transaction of the last fed input has been handed out: `next_tx` said DONE. + pulled_all: bool, + /// A call failed; the reference may be torn, so nothing reaches it again. + dead: bool, + /// The transaction the last `next_tx` handed out, which its spans borrow. + current: Option, +} + +impl Handle { + fn new(reference: Reference) -> Self { + Self { + reference, + pulled_all: true, + dead: false, + current: None, + } + } + + fn into_raw(self) -> *mut sys::Scheduler { + Box::into_raw(Box::new(self)).cast() + } +} + +/// # Safety +/// `scheduler` is null or a handle from `scheduler_create` or `scheduler_resume`, not yet +/// destroyed, that no other thread is using. +unsafe fn live<'a>(scheduler: *const sys::Scheduler) -> Result<&'a Handle, Failure> { + let handle = unsafe { scheduler.cast::().as_ref() } + .ok_or_else(|| Failure::misuse("null scheduler handle"))?; + if handle.dead { + return Err(Failure::misuse("the scheduler failed earlier and is dead")); + } + Ok(handle) +} + +/// Run a mutating call on a live handle; any failure leaves the handle dead. +/// +/// # Safety +/// As [`live`], and no other reference to the handle is live. +unsafe fn on_live_handle( + scheduler: *mut sys::Scheduler, + call: impl FnOnce(&mut Handle) -> Result, +) -> sys::SchedulerStatus { + clear_error(); + if let Err(failure) = unsafe { live(scheduler) } { + return failure.report(); + } + // Live just checked it is non-null and not dead. + let handle = unsafe { &mut *scheduler.cast::() }; + match call(handle) { + Ok(status) => status, + Err(failure) => { + handle.dead = true; + handle.current = None; + failure.report() + } + } +} + +fn address_from(address: &sys::ApplicationEngineEthereumAddress) -> Address { + Address::from(address.bytes) +} + +fn address_record(address: Address) -> sys::ApplicationEngineEthereumAddress { + sys::ApplicationEngineEthereumAddress { + bytes: address.into_array(), + } +} + +/// # Safety +/// `config` is null or readable. +unsafe fn config_from(config: *const sys::SchedulerConfig) -> Result { + let config = unsafe { config.as_ref() }.ok_or_else(|| Failure::misuse("null config"))?; + // The reference asserts this; checked here so a bad build reads as a status, not a panic. + if config.max_wait_blocks == 0 { + return Err(Failure::misuse("max_wait_blocks must be at least 1")); + } + Ok(SchedulerConfig { + sequencer_address: address_from(&config.sequencer_address), + max_wait_blocks: config.max_wait_blocks, + fridge_bound: config.fridge_bound, + max_direct_payload_bytes: config.max_direct_payload_bytes, + }) +} + +fn state_from(state: &sys::SchedulerState) -> SchedulerState { + SchedulerState { + l1_input_count: state.l1_input_count, + next_batch_nonce: state.next_batch_nonce, + clock: state.clock, + first_queued_input: state.first_queued_input, + fridge_len: state.fridge_len, + } +} + +fn state_record(state: SchedulerState) -> sys::SchedulerState { + sys::SchedulerState { + l1_input_count: state.l1_input_count, + next_batch_nonce: state.next_batch_nonce, + clock: state.clock, + first_queued_input: state.first_queued_input, + fridge_len: state.fridge_len, + } +} + +/// `count` records of `size` bytes as a slice length, if the host can address them: +/// `from_raw_parts` requires the whole range to fit `isize::MAX` bytes. +fn addressable(count: u64, size: usize) -> Option { + let len = usize::try_from(count).ok()?; + let bytes = len.checked_mul(size)?; + (bytes <= isize::MAX as usize).then_some(len) +} + +/// Borrow an input's payload, refusing a span that cannot describe readable memory. +/// +/// # Safety +/// A non-empty span with a non-null pointer describes a readable range that outlives the call. +unsafe fn payload_from<'a>(span: &sys::ApplicationEngineByteSpan) -> Result<&'a [u8], Failure> { + if span.size == 0 { + // An empty span may carry a null pointer, which a Rust slice rejects even when empty. + return Ok(&[]); + } + if span.data.is_null() { + return Err(Failure::misuse( + "non-empty payload span with a null pointer", + )); + } + let len = addressable(span.size, 1) + .ok_or_else(|| Failure::misuse("payload length exceeds this host's address space"))?; + Ok(unsafe { std::slice::from_raw_parts(span.data, len) }) +} + +/// Copy an input record into the reference's own. +/// +/// # Safety +/// As [`payload_from`] for the input's payload. +unsafe fn input_from(input: &sys::SchedulerInput) -> Result { + Ok(L1Input { + index: input.index, + sender: address_from(&input.sender), + block: input.block_number, + timestamp: input.timestamp, + chain_id: input.chain_id, + app_contract: address_from(&input.app_contract), + payload: unsafe { payload_from(&input.payload) }?.to_vec(), + }) +} + +fn outcome_record(outcome: InputOutcome) -> sys::SchedulerInputOutcome { + let mut record = sys::SchedulerInputOutcome::default(); + match outcome { + InputOutcome::DirectQueued => record.kind = sys::SCHEDULER_DIRECT_QUEUED, + InputOutcome::DirectDropped => record.kind = sys::SCHEDULER_DIRECT_DROPPED, + InputOutcome::BatchAccepted => record.kind = sys::SCHEDULER_BATCH_ACCEPTED, + InputOutcome::BatchStale => record.kind = sys::SCHEDULER_BATCH_STALE, + InputOutcome::BatchRejected(reason) => { + record.kind = sys::SCHEDULER_BATCH_REJECTED; + record.reject_reason = match reason { + BatchRejectReason::DecodeFailed => sys::SCHEDULER_REJECT_DECODE_FAILED, + BatchRejectReason::WrongNonce { expected, got } => { + record.reject_values.wrong_nonce = sys::SchedulerWrongNonce { expected, got }; + sys::SCHEDULER_REJECT_WRONG_NONCE + } + BatchRejectReason::SafeBlockNotBelowInclusionBlock => { + sys::SCHEDULER_REJECT_SAFE_BLOCK_NOT_BELOW_INCLUSION_BLOCK + } + BatchRejectReason::NonMonotonicSafeBlocks => { + sys::SCHEDULER_REJECT_NON_MONOTONIC_SAFE_BLOCKS + } + BatchRejectReason::FeePriceOutOfRange => { + sys::SCHEDULER_REJECT_FEE_PRICE_OUT_OF_RANGE + } + BatchRejectReason::FirstFrameBelowClock { safe_block, clock } => { + record.reject_values.first_frame_below_clock = + sys::SchedulerFirstFrameBelowClock { safe_block, clock }; + sys::SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK + } + }; + } + } + record +} + +/// A span over bytes the handle keeps until its next `next_tx`, `feed` or destroy. +fn span(bytes: &[u8]) -> sys::ApplicationEngineByteSpan { + sys::ApplicationEngineByteSpan { + data: if bytes.is_empty() { + std::ptr::null() + } else { + bytes.as_ptr() + }, + size: u64::try_from(bytes.len()).expect("a slice length fits in u64 on supported hosts"), + } +} + +fn tx_record(scheduled: &Scheduled) -> sys::SchedulerTx { + let mut record = sys::SchedulerTx { + batch_nonce: scheduled.batch_nonce, + ..Default::default() + }; + match &scheduled.tx { + L2Tx::Direct(direct) => { + record.kind = sys::SCHEDULER_TX_DIRECT; + record.values.direct = sys::ApplicationEngineDirectInput { + sender: address_record(direct.sender), + input_index: direct.input_index, + block_number: direct.block, + timestamp: direct.timestamp, + payload: span(&direct.payload), + }; + } + L2Tx::UserOp(op) => { + record.kind = sys::SCHEDULER_TX_USER_OP; + record.values.user_op = sys::ApplicationEngineUserOpTx { + sender: address_record(op.sender), + nonce: op.nonce, + fee: op.fee, + safe_block: op.block, + data: span(&op.data), + }; + } + } + record +} + +#[unsafe(no_mangle)] +pub extern "C" fn scheduler_abi_version() -> u32 { + sys::SCHEDULER_ABI_VERSION +} + +#[unsafe(no_mangle)] +pub extern "C" fn scheduler_get_last_error_message() -> *const c_char { + LAST_ERROR.with(|slot| slot.borrow().as_ptr()) +} + +/// Hand a new handle out, or report why there is none; `out_scheduler` is written only on OK. +/// +/// # Safety +/// `out_scheduler` is null or writable. +unsafe fn hand_out( + out_scheduler: *mut *mut sys::Scheduler, + made: impl FnOnce() -> Result, +) -> sys::SchedulerStatus { + clear_error(); + if out_scheduler.is_null() { + return Failure::misuse("null out_scheduler").report(); + } + match made() { + Ok(reference) => { + unsafe { *out_scheduler = Handle::new(reference).into_raw() }; + sys::SCHEDULER_STATUS_OK + } + Err(failure) => failure.report(), + } +} + +/// # Safety +/// `config` is null or readable, and `out_scheduler` is null or writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_create( + config: *const sys::SchedulerConfig, + out_scheduler: *mut *mut sys::Scheduler, +) -> sys::SchedulerStatus { + unsafe { + hand_out(out_scheduler, || { + let config = config_from(config)?; + catch(|| Reference::genesis(config)).map_err(Failure::internal) + }) + } +} + +/// # Safety +/// `config` and `state` are null or readable; `queued` is null or holds `queued_count` readable +/// records whose payload spans are readable; `out_scheduler` is null or writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_resume( + config: *const sys::SchedulerConfig, + state: *const sys::SchedulerState, + queued: *const sys::SchedulerInput, + queued_count: u64, + out_scheduler: *mut *mut sys::Scheduler, +) -> sys::SchedulerStatus { + unsafe { + hand_out(out_scheduler, || { + let config = config_from(config)?; + let state = state + .as_ref() + .ok_or_else(|| Failure::misuse("null state"))?; + let queued: &[sys::SchedulerInput] = if queued_count == 0 { + &[] + } else { + if queued.is_null() { + return Err(Failure::misuse("null queued with a non-zero queued_count")); + } + let len = addressable(queued_count, size_of::()).ok_or_else( + || Failure::misuse("queued_count exceeds this host's address space"), + )?; + std::slice::from_raw_parts(queued, len) + }; + // Check every span first, so the reference copies each input once, as it reads it. + for input in queued { + payload_from(&input.payload)?; + } + let inputs = queued.iter().map(|input| { + input_from(input).unwrap_or_else(|_| unreachable!("every span was checked")) + }); + match catch(|| Reference::resume(config, state_from(state), inputs)) { + Ok(Ok(reference)) => Ok(reference), + Ok(Err(refused)) => Err(Failure { + status: sys::SCHEDULER_STATUS_RESUME_REFUSED, + message: refused.to_string(), + }), + Err(message) => Err(Failure::internal(message)), + } + }) + } +} + +/// # Safety +/// `scheduler` is null or a handle from `scheduler_create` or `scheduler_resume`, not yet +/// destroyed and not in use on another thread. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_destroy(scheduler: *mut sys::Scheduler) { + if !scheduler.is_null() { + drop(unsafe { Box::from_raw(scheduler.cast::()) }); + } +} + +/// # Safety +/// As [`scheduler_destroy`] for `scheduler`, and `out_state` is null or writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_state( + scheduler: *const sys::Scheduler, + out_state: *mut sys::SchedulerState, +) -> sys::SchedulerStatus { + clear_error(); + let handle = match unsafe { live(scheduler) } { + Ok(handle) => handle, + Err(failure) => return failure.report(), + }; + if out_state.is_null() { + return Failure::misuse("null out_state").report(); + } + unsafe { *out_state = state_record(handle.reference.state()) }; + sys::SCHEDULER_STATUS_OK +} + +/// # Safety +/// As [`scheduler_destroy`] for `scheduler`; `input` is null or readable with a readable +/// payload span; `out_outcome` is null or writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_feed( + scheduler: *mut sys::Scheduler, + input: *const sys::SchedulerInput, + out_outcome: *mut sys::SchedulerInputOutcome, +) -> sys::SchedulerStatus { + unsafe { + on_live_handle(scheduler, |handle| { + let input = input + .as_ref() + .ok_or_else(|| Failure::misuse("null input"))?; + if out_outcome.is_null() { + return Err(Failure::misuse("null out_outcome")); + } + if !handle.pulled_all { + return Err(Failure::misuse( + "the previous input still has transactions: pull until DONE before feeding", + )); + } + let state = handle.reference.state(); + if input.index != state.l1_input_count { + return Err(Failure::misuse(format!( + "input index {} is not the next, {}", + input.index, state.l1_input_count + ))); + } + if state.l1_input_count == u64::MAX { + return Err(Failure::no_successor("the L1 input count has no successor")); + } + let input = input_from(input)?; + // The one panic left: an accepted batch whose nonce has no successor. Only the + // submitter's input at the last nonce can reach it. + let nonce_exhausted = input.sender == handle.reference.config().sequencer_address + && state.next_batch_nonce == u64::MAX; + handle.current = None; + let outcome = catch(|| handle.reference.feed(input)).map_err(|message| { + if nonce_exhausted { + Failure::no_successor(message) + } else { + Failure::internal(message) + } + })?; + handle.pulled_all = false; + *out_outcome = outcome_record(outcome); + Ok(sys::SCHEDULER_STATUS_OK) + }) + } +} + +/// # Safety +/// As [`scheduler_destroy`] for `scheduler`, and `out_tx` is null or writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn scheduler_next_tx( + scheduler: *mut sys::Scheduler, + out_tx: *mut sys::SchedulerTx, +) -> sys::SchedulerStatus { + unsafe { + on_live_handle(scheduler, |handle| { + if out_tx.is_null() { + return Err(Failure::misuse("null out_tx")); + } + handle.current = None; + if handle.pulled_all { + return Ok(sys::SCHEDULER_STATUS_DONE); + } + match catch(|| handle.reference.next_tx()).map_err(Failure::internal)? { + None => { + handle.pulled_all = true; + Ok(sys::SCHEDULER_STATUS_DONE) + } + Some(scheduled) => { + let scheduled = handle.current.insert(scheduled); + *out_tx = tx_record(scheduled); + Ok(sys::SCHEDULER_STATUS_OK) + } + } + }) + } +} diff --git a/bindings/c-scheduler/src/sys.rs b/bindings/c-scheduler/src/sys.rs new file mode 100644 index 00000000..09fc6a68 --- /dev/null +++ b/bindings/c-scheduler/src/sys.rs @@ -0,0 +1,16 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The scheduler C ABI, generated by bindgen from `include/scheduler.h`: change the header, not +//! this file. Enums arrive as integer constants; records carry generated layout assertions. + +// Generated code follows C's naming and carries the whole surface, the declarations of the +// functions this crate defines included. +#![allow( + non_upper_case_globals, + non_camel_case_types, + non_snake_case, + dead_code +)] + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/bindings/c-scheduler/tests/c_example.rs b/bindings/c-scheduler/tests/c_example.rs new file mode 100644 index 00000000..52eadcd2 --- /dev/null +++ b/bindings/c-scheduler/tests/c_example.rs @@ -0,0 +1,204 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Builds `c/feed_loop.c` against `libc_scheduler.a` with the system C compiler, runs it, and +//! compares what it prints with the Rust reference on the same inputs. It also parses the header +//! as C++, where its noexcept spelling lives. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use alloy_primitives::Address; +use c_scheduler::sys; +use sequencer_core::batch::{Batch, Frame}; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::{InputOutcome, L1Input, Scheduler, SchedulerConfig}; + +const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +fn include_args() -> [String; 2] { + [ + format!("-I{MANIFEST_DIR}/include"), + format!("-I{MANIFEST_DIR}/../c-app-engine/include"), + ] +} + +/// The newest `libc_scheduler-*.a` beside this test in `target//deps`. Cargo builds the +/// staticlib there whenever it builds the library for this test, but copies it up to +/// `target/` only for a `cargo build` of the library itself, so the copy can be stale or +/// missing. Before the tests run, Cargo has rebuilt this test's own archive if a source changed, +/// so the newest one is built from the current sources. +fn static_library() -> PathBuf { + let exe = std::env::current_exe().expect("test executable path"); + let deps = exe + .parent() + .expect("the test runs from target//deps"); + let newest = std::fs::read_dir(deps) + .expect("read the deps directory") + .filter_map(|entry| { + let path = entry.ok()?.path(); + let name = path.file_name()?.to_str()?; + let archive = name.starts_with("libc_scheduler-") && name.ends_with(".a"); + let modified = path.metadata().ok()?.modified().ok()?; + archive.then_some((modified, path)) + }) + .max_by_key(|(modified, _)| *modified); + newest + .unwrap_or_else(|| panic!("no libc_scheduler-*.a in {}", deps.display())) + .1 +} + +/// The system libraries a Rust staticlib needs, as `rustc --print native-static-libs` lists them. +fn native_libraries() -> &'static [&'static str] { + if cfg!(target_os = "macos") { + &["-liconv", "-lSystem", "-lc", "-lm"] + } else { + &[ + "-lgcc_s", + "-lutil", + "-lrt", + "-lpthread", + "-lm", + "-ldl", + "-lc", + ] + } +} + +fn compiler(variable: &str, fallback: &str) -> String { + std::env::var(variable).unwrap_or_else(|_| fallback.to_owned()) +} + +fn run(command: &mut Command) -> String { + let output = command + .output() + .unwrap_or_else(|err| panic!("cannot run {command:?}: {err}")); + assert!( + output.status.success(), + "{command:?} failed with {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("UTF-8 output") +} + +/// What `feed_loop.c` prints, computed by the reference over the inputs it feeds. +fn expected_output() -> String { + let sequencer = Address::repeat_byte(0x11); + let user = Address::repeat_byte(0x22); + let mut scheduler = Scheduler::genesis(SchedulerConfig { + sequencer_address: sequencer, + max_wait_blocks: 10, + fridge_bound: 16, + max_direct_payload_bytes: 4, + }); + let batch = ssz::Encode::as_ssz_bytes(&Batch { + nonce: 0, + frames: vec![Frame { + user_ops: Vec::new(), + safe_block: 2, + fee_price: 0, + }], + }); + let inputs = [ + (user, 1, b"a".to_vec()), + (user, 2, b"b".to_vec()), + (user, 3, b"c".to_vec()), + (user, 4, b"long!".to_vec()), + (sequencer, 5, batch), + (user, 20, Vec::new()), + ]; + + let mut lines = Vec::new(); + for (index, (sender, block, payload)) in (0_u64..).zip(inputs) { + let outcome = scheduler.feed(L1Input { + index, + sender, + block, + timestamp: block * 12, + chain_id: 1, + app_contract: Address::ZERO, + payload, + }); + let kind = match outcome { + InputOutcome::DirectQueued => sys::SCHEDULER_DIRECT_QUEUED, + InputOutcome::DirectDropped => sys::SCHEDULER_DIRECT_DROPPED, + InputOutcome::BatchAccepted => sys::SCHEDULER_BATCH_ACCEPTED, + InputOutcome::BatchStale => sys::SCHEDULER_BATCH_STALE, + InputOutcome::BatchRejected(_) => sys::SCHEDULER_BATCH_REJECTED, + }; + lines.push(format!("input {index} outcome {kind}")); + while let Some(scheduled) = scheduler.next_tx() { + let batch = scheduled.batch_nonce; + lines.push(match scheduled.tx { + L2Tx::Direct(direct) => format!( + "direct {} block {} bytes {} batch {batch}", + direct.input_index, + direct.block, + direct.payload.len() + ), + L2Tx::UserOp(op) => { + format!( + "user_op nonce {} block {} batch {batch}", + op.nonce, op.block + ) + } + }); + } + } + let state = scheduler.state(); + lines.push(format!( + "state {} {} {} {} {}", + state.l1_input_count, + state.next_batch_nonce, + state.clock, + state.first_queued_input, + state.fridge_len + )); + lines.iter().map(|line| format!("{line}\n")).collect() +} + +#[test] +fn c_feed_loop_matches_the_reference() { + let out_dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("c-scheduler-example"); + std::fs::create_dir_all(&out_dir).expect("create the example's directory"); + let binary = out_dir.join("feed_loop"); + + run(Command::new(compiler("CC", "cc")) + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-pedantic"]) + .args(include_args()) + .arg(format!("{MANIFEST_DIR}/c/feed_loop.c")) + .arg(static_library()) + .args(native_libraries()) + .arg("-o") + .arg(&binary)); + let printed = run(&mut Command::new(&binary)); + + let expected = expected_output(); + // The run must reach every path the example is there to show. + assert!(expected.contains("outcome 1\n"), "a dropped direct"); + assert!(expected.contains("outcome 2\n"), "an accepted batch"); + assert!( + expected.contains("direct 2 block 3 bytes 1 batch 1\n"), + "a backstop drain" + ); + assert_eq!(printed, expected); +} + +#[test] +fn header_parses_as_cpp() { + let probe = Path::new(env!("CARGO_TARGET_TMPDIR")).join("c-scheduler-header-probe.cpp"); + std::fs::write( + &probe, + "#include \"scheduler.h\"\n\ + static_assert(noexcept(scheduler_abi_version()), \"the API is noexcept\");\n", + ) + .expect("write the C++ probe"); + run(Command::new(compiler("CXX", "c++")) + .args(["-std=c++17", "-c", "-Wall", "-Wextra", "-Werror"]) + .args(include_args()) + .arg(&probe) + .arg("-o") + .arg(probe.with_extension("o"))); +} diff --git a/bindings/c-scheduler/tests/conformance.rs b/bindings/c-scheduler/tests/conformance.rs new file mode 100644 index 00000000..2e72a265 --- /dev/null +++ b/bindings/c-scheduler/tests/conformance.rs @@ -0,0 +1,414 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Replays the guest and scheduler-only scenarios of `tests/fixtures/consensus_vectors_v2.json` +//! through the C functions of scheduler.h and nothing else: each input's outcome, every +//! transaction it hands out and the state after it must match the file, and each resume case +//! must resume or be refused as pinned. The file was computed once from the reference, so this is +//! the check a C port of the header is held to. It links this crate's export; a port replays the +//! file the same way in its own harness until the check is packaged to link another. + +use std::collections::BTreeMap; +use std::ffi::CStr; + +use alloy_primitives::{Address, hex}; +use c_scheduler::sys; +use serde_json::{Value, json}; + +const CONSENSUS_VECTORS: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/fixtures/consensus_vectors_v2.json" +)); + +fn text(value: &Value) -> &str { + value.as_str().expect("pinned string") +} + +fn number(value: &Value) -> u64 { + value.as_u64().expect("pinned number") +} + +fn bytes(value: &Value) -> Vec { + hex::decode(text(value)).expect("pinned hex bytes") +} + +fn address(value: &Value) -> Address { + text(value).parse().expect("pinned address") +} + +fn hex_of(bytes: &[u8]) -> String { + format!("0x{}", hex::encode(bytes)) +} + +fn address_record(address: Address) -> sys::ApplicationEngineEthereumAddress { + sys::ApplicationEngineEthereumAddress { + bytes: address.into_array(), + } +} + +/// Panics with the implementation's message on a failure status. +fn check(status: sys::SchedulerStatus, call: &str) -> sys::SchedulerStatus { + if status < 0 { + let message = unsafe { CStr::from_ptr(sys::scheduler_get_last_error_message()) }; + panic!("{call} failed with {status}: {}", message.to_string_lossy()); + } + status +} + +/// Copy a span the implementation handed out; it borrows the handle until the next call. +fn span_bytes(span: &sys::ApplicationEngineByteSpan) -> Vec { + if span.size == 0 { + return Vec::new(); + } + let len = usize::try_from(span.size).expect("a span fits this host"); + unsafe { std::slice::from_raw_parts(span.data, len) }.to_vec() +} + +/// One input of a scenario, kept alive while its record borrows the payload. +struct Input { + sender: Address, + block: u64, + timestamp: u64, + payload: Vec, +} + +/// The outcome in the file's form: `outcome`, and `reject` for a rejected batch. +fn rendered_outcome(record: &sys::SchedulerInputOutcome) -> Value { + let outcome = match record.kind { + sys::SCHEDULER_DIRECT_QUEUED => "direct_queued", + sys::SCHEDULER_DIRECT_DROPPED => "direct_dropped", + sys::SCHEDULER_BATCH_ACCEPTED => "batch_accepted", + sys::SCHEDULER_BATCH_STALE => "batch_stale", + sys::SCHEDULER_BATCH_REJECTED => "batch_rejected", + other => panic!("unknown outcome kind {other}"), + }; + if record.kind != sys::SCHEDULER_BATCH_REJECTED { + return json!({"outcome": outcome}); + } + let reject = match record.reject_reason { + sys::SCHEDULER_REJECT_DECODE_FAILED => json!({"reason": "decode_failed"}), + sys::SCHEDULER_REJECT_WRONG_NONCE => { + let values = unsafe { record.reject_values.wrong_nonce }; + json!({"reason": "wrong_nonce", "expected": values.expected, "got": values.got}) + } + sys::SCHEDULER_REJECT_SAFE_BLOCK_NOT_BELOW_INCLUSION_BLOCK => { + json!({"reason": "safe_block_not_below_inclusion_block"}) + } + sys::SCHEDULER_REJECT_NON_MONOTONIC_SAFE_BLOCKS => { + json!({"reason": "non_monotonic_safe_blocks"}) + } + sys::SCHEDULER_REJECT_FEE_PRICE_OUT_OF_RANGE => json!({"reason": "fee_price_out_of_range"}), + sys::SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK => { + let values = unsafe { record.reject_values.first_frame_below_clock }; + json!({ + "reason": "first_frame_below_clock", + "safe_block": values.safe_block, + "clock": values.clock, + }) + } + other => panic!("unknown reject reason {other}"), + }; + json!({"outcome": outcome, "reject": reject}) +} + +/// A handed-out transaction in the file's form, without its fate, which the executor decides. +/// A direct must carry back the input it was admitted from, whole. +fn rendered_tx(record: &sys::SchedulerTx, inputs: &BTreeMap) -> Value { + match record.kind { + sys::SCHEDULER_TX_DIRECT => { + let direct = unsafe { record.values.direct }; + let index = direct.input_index; + let input = inputs + .get(&index) + .unwrap_or_else(|| panic!("direct {index} names no input given")); + assert_eq!( + Address::from(direct.sender.bytes), + input.sender, + "direct {index}" + ); + assert_eq!(direct.block_number, input.block, "direct {index}"); + assert_eq!(direct.timestamp, input.timestamp, "direct {index}"); + assert_eq!(span_bytes(&direct.payload), input.payload, "direct {index}"); + json!({"direct": direct.input_index, "batch_nonce": record.batch_nonce}) + } + sys::SCHEDULER_TX_USER_OP => { + let op = unsafe { record.values.user_op }; + json!({ + "user_op": { + "sender": Address::from(op.sender.bytes).to_string(), + "nonce": op.nonce, + "fee": op.fee, + "block": op.safe_block, + "data": hex_of(&span_bytes(&op.data)), + }, + "batch_nonce": record.batch_nonce, + }) + } + other => panic!("unknown transaction kind {other}"), + } +} + +/// A handle from the implementation, destroyed on every exit. +struct Handle(*mut sys::Scheduler); + +impl Drop for Handle { + fn drop(&mut self) { + unsafe { sys::scheduler_destroy(self.0) }; + } +} + +/// The file's inputs: senders and payloads by name from the guest section. +struct Pinned<'a> { + guest: &'a Value, + chain_id: u64, + app_contract: Address, +} + +impl Pinned<'_> { + fn input(&self, record: &Value) -> Input { + Input { + sender: address(&self.guest["senders"][text(&record["from"])]), + block: number(&record["block"]), + timestamp: number(&record["timestamp"]), + payload: bytes(&self.guest["payloads"][text(&record["payload"])]), + } + } + + /// The header's record of `input`, borrowing its payload. + fn record(&self, index: u64, input: &Input) -> sys::SchedulerInput { + sys::SchedulerInput { + index, + sender: address_record(input.sender), + block_number: input.block, + timestamp: input.timestamp, + chain_id: self.chain_id, + app_contract: address_record(self.app_contract), + payload: sys::ApplicationEngineByteSpan { + data: input.payload.as_ptr(), + size: u64::try_from(input.payload.len()).expect("fits"), + }, + } + } +} + +fn pinned_config(pinned: &Value) -> sys::SchedulerConfig { + sys::SchedulerConfig { + sequencer_address: address_record(address(&pinned["sequencer_address"])), + max_wait_blocks: number(&pinned["max_wait_blocks"]), + fridge_bound: number(&pinned["fridge_bound"]), + max_direct_payload_bytes: number(&pinned["max_direct_payload_bytes"]), + } +} + +fn state_fields(state: &sys::SchedulerState) -> [u64; 5] { + [ + state.l1_input_count, + state.next_batch_nonce, + state.clock, + state.first_queued_input, + state.fridge_len, + ] +} + +fn pinned_state_fields(pinned: &Value) -> [u64; 5] { + [ + number(&pinned["l1_input_count"]), + number(&pinned["next_batch_nonce"]), + number(&pinned["clock"]), + number(&pinned["first_queued_input"]), + number(&pinned["fridge_len"]), + ] +} + +fn read_state(handle: &Handle) -> sys::SchedulerState { + let mut state = sys::SchedulerState::default(); + check( + unsafe { sys::scheduler_state(handle.0, &mut state) }, + "scheduler_state", + ); + state +} + +/// Feeds the step as input `index` and compares its outcome, every transaction it hands out +/// and the state after it with the file. +fn replay_step( + pinned: &Pinned, + handle: &Handle, + inputs: &mut BTreeMap, + index: u64, + step: &Value, + label: &str, +) { + let input = inputs.entry(index).or_insert(pinned.input(step)); + let record = pinned.record(index, input); + let mut outcome = sys::SchedulerInputOutcome::default(); + check( + unsafe { sys::scheduler_feed(handle.0, &record, &mut outcome) }, + "scheduler_feed", + ); + let mut expected = json!({"outcome": step["outcome"].clone()}); + if let Some(reject) = step.get("reject") { + expected["reject"] = reject.clone(); + } + assert_eq!( + rendered_outcome(&outcome), + expected, + "{label}, step {index}" + ); + + let mut handed_out = Vec::new(); + loop { + let mut tx = sys::SchedulerTx::default(); + let status = check( + unsafe { sys::scheduler_next_tx(handle.0, &mut tx) }, + "scheduler_next_tx", + ); + match status { + sys::SCHEDULER_STATUS_OK => handed_out.push(rendered_tx(&tx, inputs)), + sys::SCHEDULER_STATUS_DONE => break, + other => panic!("unknown status {other}"), + } + } + let pinned_txs: Vec = step["txs"] + .as_array() + .expect("pinned txs") + .iter() + .map(|tx| { + let mut tx = tx.clone(); + tx.as_object_mut().expect("pinned tx").remove("fate"); + tx + }) + .collect(); + assert_eq!(handed_out, pinned_txs, "{label}, step {index}"); + + assert_eq!( + state_fields(&read_state(handle)), + pinned_state_fields(&step["scheduler"]), + "{label}, step {index}: l1_input_count, next_batch_nonce, clock, \ + first_queued_input, fridge_len" + ); +} + +/// Replays `scenario` on a scheduler created at genesis under `config`. +fn replay_scenario(pinned: &Pinned, config: &sys::SchedulerConfig, scenario: &Value) { + let name = text(&scenario["name"]); + let mut raw = std::ptr::null_mut(); + check( + unsafe { sys::scheduler_create(config, &mut raw) }, + "scheduler_create", + ); + let handle = Handle(raw); + let mut inputs = BTreeMap::new(); + for (index, step) in (0_u64..).zip(scenario["steps"].as_array().expect("pinned steps")) { + replay_step(pinned, &handle, &mut inputs, index, step, name); + } +} + +fn parsed_vectors() -> Value { + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json") +} + +fn pinned_inputs(vectors: &Value) -> Pinned<'_> { + let guest = &vectors["guest"]; + Pinned { + guest, + chain_id: number(&guest["chain_id"]), + app_contract: address(&guest["app_contract"]), + } +} + +#[test] +fn every_guest_scenario_replays_through_the_header() { + assert_eq!( + unsafe { sys::scheduler_abi_version() }, + sys::SCHEDULER_ABI_VERSION + ); + let vectors = parsed_vectors(); + let pinned = pinned_inputs(&vectors); + let config = pinned_config(&vectors["guest"]["config"]); + let scenarios = vectors["guest"]["scenarios"] + .as_array() + .expect("pinned scenarios"); + assert!(!scenarios.is_empty()); + for scenario in scenarios { + replay_scenario(&pinned, &config, scenario); + } +} + +/// The overflow drain at a small fridge bound, which the guest scenarios never reach. +#[test] +fn every_scheduler_only_scenario_replays_through_the_header() { + let vectors = parsed_vectors(); + let pinned = pinned_inputs(&vectors); + let section = &vectors["scheduler_only"]; + let config = pinned_config(§ion["config"]); + let scenarios = section["scenarios"].as_array().expect("pinned scenarios"); + assert!(!scenarios.is_empty()); + for scenario in scenarios { + replay_scenario(&pinned, &config, scenario); + } +} + +/// scheduler_resume takes or refuses each pinned state and range; a resumed scheduler then +/// continues as pinned. The header reports one refusal status, so the reason is not compared. +#[test] +fn every_resume_case_replays_through_the_header() { + let vectors = parsed_vectors(); + let pinned = pinned_inputs(&vectors); + let section = &vectors["scheduler_only"]; + let config = pinned_config(§ion["config"]); + let cases = section["resume"].as_array().expect("pinned resume cases"); + assert!(!cases.is_empty()); + for case in cases { + let name = text(&case["name"]); + let fields = pinned_state_fields(&case["state"]); + let state = sys::SchedulerState { + l1_input_count: fields[0], + next_batch_nonce: fields[1], + clock: fields[2], + first_queued_input: fields[3], + fridge_len: fields[4], + }; + let mut inputs = BTreeMap::new(); + for record in case["queued"].as_array().expect("pinned range") { + inputs.insert(number(&record["index"]), pinned.input(record)); + } + let records: Vec = case["queued"] + .as_array() + .expect("pinned range") + .iter() + .map(|record| { + let index = number(&record["index"]); + pinned.record(index, &inputs[&index]) + }) + .collect(); + let mut raw = std::ptr::null_mut(); + let status = unsafe { + sys::scheduler_resume( + &config, + &state, + if records.is_empty() { + std::ptr::null() + } else { + records.as_ptr() + }, + u64::try_from(records.len()).expect("fits"), + &mut raw, + ) + }; + if text(&case["result"]) == "refused" { + assert_eq!(status, sys::SCHEDULER_STATUS_RESUME_REFUSED, "{name}"); + assert!(raw.is_null(), "{name}: a refused resume yields no handle"); + continue; + } + let handle = Handle(raw); + check(status, "scheduler_resume"); + assert_eq!(state_fields(&read_state(&handle)), fields, "{name}"); + let then = case["then"] + .as_array() + .expect("pinned steps after a resume"); + for (index, step) in (state.l1_input_count..).zip(then) { + replay_step(&pinned, &handle, &mut inputs, index, step, name); + } + } +} diff --git a/bindings/c-scheduler/tests/differential.rs b/bindings/c-scheduler/tests/differential.rs new file mode 100644 index 00000000..4f4b896c --- /dev/null +++ b/bindings/c-scheduler/tests/differential.rs @@ -0,0 +1,930 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Drives the exported functions through the header's own declarations and compares them, +//! transaction by transaction and state by state, with the Rust reference on the same inputs. + +use std::ffi::CStr; + +use alloy_primitives::Address; +use alloy_sol_types::Eip712Domain; +use c_scheduler::sys; +use k256::ecdsa::SigningKey; +use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; +use sequencer_core::scheduler::{ + BatchRejectReason, InputOutcome, L1Input, Scheduled, Scheduler, SchedulerConfig, SchedulerState, +}; +use sequencer_core::user_op::UserOp; + +const SEQUENCER: Address = Address::repeat_byte(0x11); +const USER: Address = Address::repeat_byte(0x22); +const CHAIN_ID: u64 = 1; +const APP_CONTRACT: Address = Address::repeat_byte(0x33); + +fn config( + max_wait_blocks: u64, + fridge_bound: u64, + max_direct_payload_bytes: u64, +) -> SchedulerConfig { + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks, + fridge_bound, + max_direct_payload_bytes, + } +} + +fn domain() -> Eip712Domain { + sequencer_core::build_input_domain(CHAIN_ID, APP_CONTRACT) +} + +fn key(seed: u8) -> SigningKey { + SigningKey::from_bytes((&[seed; 32]).into()).expect("signing key") +} + +fn op(key: &SigningKey, nonce: u32, max_fee: u16, data: &[u8]) -> WireUserOp { + let user_op = UserOp { + nonce, + max_fee, + data: data.to_vec().into(), + }; + let signature = sequencer_core::test_support::sign_user_op(key, &domain(), &user_op); + WireUserOp { + nonce, + max_fee, + data: data.to_vec(), + signature: signature.as_bytes().to_vec(), + } +} + +fn frame(safe_block: u64, fee_price: u16, user_ops: Vec) -> Frame { + Frame { + user_ops, + safe_block, + fee_price, + } +} + +/// Inputs in InputBox order from index `base`: each builder takes the next index. +struct Script { + base: u64, + inputs: Vec, +} + +impl Script { + fn new() -> Self { + Self::from_index(0) + } + + fn from_index(base: u64) -> Self { + Self { + base, + inputs: Vec::new(), + } + } + + fn push(&mut self, sender: Address, block: u64, payload: Vec) -> &mut Self { + let index = self.base + u64::try_from(self.inputs.len()).expect("fits"); + self.inputs.push(L1Input { + index, + sender, + block, + timestamp: block * 12, + chain_id: CHAIN_ID, + app_contract: APP_CONTRACT, + payload, + }); + self + } + + fn direct(&mut self, block: u64, payload: &[u8]) -> &mut Self { + self.push(USER, block, payload.to_vec()) + } + + fn batch(&mut self, block: u64, nonce: u64, frames: Vec) -> &mut Self { + self.push( + SEQUENCER, + block, + ssz::Encode::as_ssz_bytes(&Batch { nonce, frames }), + ) + } + + fn raw_batch(&mut self, block: u64, payload: &[u8]) -> &mut Self { + self.push(SEQUENCER, block, payload.to_vec()) + } + + fn inputs(&self) -> &[L1Input] { + &self.inputs + } +} + +type CallError = (sys::SchedulerStatus, String); + +fn last_error() -> String { + let message = unsafe { CStr::from_ptr(sys::scheduler_get_last_error_message()) }; + message.to_string_lossy().into_owned() +} + +fn check(status: sys::SchedulerStatus) -> Result { + if status < 0 { + Err((status, last_error())) + } else { + Ok(status) + } +} + +fn address_record(address: Address) -> sys::ApplicationEngineEthereumAddress { + sys::ApplicationEngineEthereumAddress { + bytes: address.into_array(), + } +} + +fn config_record(config: &SchedulerConfig) -> sys::SchedulerConfig { + sys::SchedulerConfig { + sequencer_address: address_record(config.sequencer_address), + max_wait_blocks: config.max_wait_blocks, + fridge_bound: config.fridge_bound, + max_direct_payload_bytes: config.max_direct_payload_bytes, + } +} + +fn state_record(state: &SchedulerState) -> sys::SchedulerState { + sys::SchedulerState { + l1_input_count: state.l1_input_count, + next_batch_nonce: state.next_batch_nonce, + clock: state.clock, + first_queued_input: state.first_queued_input, + fridge_len: state.fridge_len, + } +} + +/// An input record borrowing `input`'s payload, which must outlive the call it is passed to. +fn input_record(input: &L1Input) -> sys::SchedulerInput { + sys::SchedulerInput { + index: input.index, + sender: address_record(input.sender), + block_number: input.block, + timestamp: input.timestamp, + chain_id: input.chain_id, + app_contract: address_record(input.app_contract), + payload: sys::ApplicationEngineByteSpan { + data: input.payload.as_ptr(), + size: u64::try_from(input.payload.len()).expect("fits"), + }, + } +} + +/// Copy a span the export handed out; it borrows the handle until the next call. +fn bytes(span: &sys::ApplicationEngineByteSpan) -> Vec { + if span.size == 0 { + return Vec::new(); + } + let len = usize::try_from(span.size).expect("fits"); + unsafe { std::slice::from_raw_parts(span.data, len) }.to_vec() +} + +fn outcome_from(record: &sys::SchedulerInputOutcome) -> InputOutcome { + match record.kind { + sys::SCHEDULER_DIRECT_QUEUED => InputOutcome::DirectQueued, + sys::SCHEDULER_DIRECT_DROPPED => InputOutcome::DirectDropped, + sys::SCHEDULER_BATCH_ACCEPTED => InputOutcome::BatchAccepted, + sys::SCHEDULER_BATCH_STALE => InputOutcome::BatchStale, + sys::SCHEDULER_BATCH_REJECTED => InputOutcome::BatchRejected(match record.reject_reason { + sys::SCHEDULER_REJECT_DECODE_FAILED => BatchRejectReason::DecodeFailed, + sys::SCHEDULER_REJECT_WRONG_NONCE => { + let values = unsafe { record.reject_values.wrong_nonce }; + BatchRejectReason::WrongNonce { + expected: values.expected, + got: values.got, + } + } + sys::SCHEDULER_REJECT_SAFE_BLOCK_NOT_BELOW_INCLUSION_BLOCK => { + BatchRejectReason::SafeBlockNotBelowInclusionBlock + } + sys::SCHEDULER_REJECT_NON_MONOTONIC_SAFE_BLOCKS => { + BatchRejectReason::NonMonotonicSafeBlocks + } + sys::SCHEDULER_REJECT_FEE_PRICE_OUT_OF_RANGE => BatchRejectReason::FeePriceOutOfRange, + sys::SCHEDULER_REJECT_FIRST_FRAME_BELOW_CLOCK => { + let values = unsafe { record.reject_values.first_frame_below_clock }; + BatchRejectReason::FirstFrameBelowClock { + safe_block: values.safe_block, + clock: values.clock, + } + } + other => panic!("unknown reject reason {other}"), + }), + other => panic!("unknown outcome kind {other}"), + } +} + +fn scheduled_from(record: &sys::SchedulerTx) -> Scheduled { + let tx = match record.kind { + sys::SCHEDULER_TX_DIRECT => { + let direct = unsafe { record.values.direct }; + L2Tx::Direct(Direct { + input_index: direct.input_index, + sender: Address::from(direct.sender.bytes), + block: direct.block_number, + timestamp: direct.timestamp, + payload: bytes(&direct.payload), + }) + } + sys::SCHEDULER_TX_USER_OP => { + let op = unsafe { record.values.user_op }; + L2Tx::UserOp(UserOpTx { + sender: Address::from(op.sender.bytes), + nonce: op.nonce, + fee: op.fee, + block: op.safe_block, + data: bytes(&op.data), + }) + } + other => panic!("unknown transaction kind {other}"), + }; + Scheduled { + batch_nonce: record.batch_nonce, + tx, + } +} + +/// A handle from the export, used only through the header's declarations. +struct CScheduler(*mut sys::Scheduler); + +impl CScheduler { + fn create(config: &SchedulerConfig) -> Result { + let mut handle = std::ptr::null_mut(); + check(unsafe { sys::scheduler_create(&config_record(config), &mut handle) })?; + assert!(!handle.is_null()); + Ok(Self(handle)) + } + + fn resume( + config: &SchedulerConfig, + state: &SchedulerState, + queued: &[L1Input], + ) -> Result { + let records: Vec<_> = queued.iter().map(input_record).collect(); + let mut handle = std::ptr::null_mut(); + check(unsafe { + sys::scheduler_resume( + &config_record(config), + &state_record(state), + if records.is_empty() { + std::ptr::null() + } else { + records.as_ptr() + }, + u64::try_from(records.len()).expect("fits"), + &mut handle, + ) + })?; + assert!(!handle.is_null()); + Ok(Self(handle)) + } + + fn state(&self) -> Result { + let mut record = sys::SchedulerState::default(); + check(unsafe { sys::scheduler_state(self.0, &mut record) })?; + Ok(SchedulerState { + l1_input_count: record.l1_input_count, + next_batch_nonce: record.next_batch_nonce, + clock: record.clock, + first_queued_input: record.first_queued_input, + fridge_len: record.fridge_len, + }) + } + + fn feed(&mut self, input: &L1Input) -> Result { + let mut record = sys::SchedulerInputOutcome::default(); + check(unsafe { sys::scheduler_feed(self.0, &input_record(input), &mut record) })?; + Ok(outcome_from(&record)) + } + + fn next_tx(&mut self) -> Result, CallError> { + let mut record = sys::SchedulerTx::default(); + match check(unsafe { sys::scheduler_next_tx(self.0, &mut record) })? { + sys::SCHEDULER_STATUS_OK => Ok(Some(scheduled_from(&record))), + sys::SCHEDULER_STATUS_DONE => Ok(None), + other => panic!("unknown status {other}"), + } + } +} + +impl Drop for CScheduler { + fn drop(&mut self) { + unsafe { sys::scheduler_destroy(self.0) }; + } +} + +/// Feed both, comparing every outcome, transaction and state; returns the outcomes and the +/// transactions handed out, for the scenario's own assertions. +fn run_both( + reference: &mut Scheduler, + exported: &mut CScheduler, + inputs: &[L1Input], +) -> (Vec, Vec) { + let mut outcomes = Vec::new(); + let mut handed_out = Vec::new(); + for input in inputs { + let expected = reference.feed(input.clone()); + let got = exported.feed(input).expect("feed"); + assert_eq!(got, expected, "outcome of input {}", input.index); + outcomes.push(expected); + loop { + let expected = reference.next_tx(); + let got = exported.next_tx().expect("next_tx"); + assert_eq!(got, expected, "transaction of input {}", input.index); + let Some(scheduled) = expected else { break }; + handed_out.push(scheduled); + } + assert_eq!( + exported.state().expect("state"), + reference.state(), + "state after input {}", + input.index + ); + } + (outcomes, handed_out) +} + +fn run_from_genesis( + config: &SchedulerConfig, + inputs: &[L1Input], +) -> (Vec, Vec) { + let mut reference = Scheduler::genesis(*config); + let mut exported = CScheduler::create(config).expect("create"); + assert_eq!(exported.state().expect("state"), reference.state()); + run_both(&mut reference, &mut exported, inputs) +} + +fn user_op_data(scheduled: &[Scheduled]) -> Vec> { + scheduled + .iter() + .filter_map(|scheduled| match &scheduled.tx { + L2Tx::UserOp(op) => Some(op.data.clone()), + L2Tx::Direct(_) => None, + }) + .collect() +} + +#[test] +fn abi_version_is_the_headers() { + assert_eq!( + unsafe { sys::scheduler_abi_version() }, + sys::SCHEDULER_ABI_VERSION + ); +} + +#[test] +fn accepted_batches_hand_out_the_same_transactions() { + let alice = key(1); + let bob = key(2); + let mut script = Script::new(); + script + .direct(1, b"d1") + .direct(2, b"d2") + .direct(4, b"d4") + .batch( + 6, + 0, + vec![ + frame(2, 3, vec![op(&alice, 0, 3, b"a0"), op(&bob, 0, 9, b"b0")]), + frame(4, 5, vec![op(&alice, 1, 5, b"a1")]), + ], + ) + .direct(7, b"d7") + .batch(8, 1, Vec::new()) + .batch(9, 2, vec![frame(7, 0, vec![op(&bob, 7, 0, b"b7")])]); + let (outcomes, handed_out) = run_from_genesis(&config(100, 100, 100), script.inputs()); + + assert_eq!( + outcomes + .iter() + .filter(|outcome| **outcome == InputOutcome::BatchAccepted) + .count(), + 3 + ); + // Every op is handed out, whatever its nonce: that is the executor's check. + assert_eq!( + user_op_data(&handed_out), + [b"a0".to_vec(), b"b0".into(), b"a1".into(), b"b7".into()] + ); + assert_eq!(handed_out.len(), 8); +} + +#[test] +fn fee_cap_and_signature_failures_are_never_handed_out() { + let alice = key(1); + // A zero r recovers no signer. + let mut bad_signature = op(&alice, 1, 9, b"bad-sig"); + bad_signature.signature[..32].fill(0); + let mut short_signature = op(&alice, 1, 9, b"short-sig"); + short_signature.signature.pop(); + let mut script = Script::new(); + script.batch( + 5, + 0, + vec![frame( + 1, + 4, + vec![ + op(&alice, 0, 3, b"under-cap"), + op(&alice, 0, 4, b"at-cap"), + bad_signature, + short_signature, + op(&alice, 1, 5, b"over-cap"), + ], + )], + ); + let (_, handed_out) = run_from_genesis(&config(100, 100, 100), script.inputs()); + assert_eq!( + user_op_data(&handed_out), + [b"at-cap".to_vec(), b"over-cap".into()] + ); +} + +#[test] +fn rejected_and_stale_batches_change_nothing_alike() { + let alice = key(1); + let mut script = Script::new(); + script + .direct(1, b"d1") + .raw_batch(2, b"not ssz") + .batch(3, 5, Vec::new()) + .batch(4, 0, vec![frame(4, 0, Vec::new())]) + .batch(5, 0, vec![frame(3, 0, Vec::new()), frame(2, 0, Vec::new())]) + .batch(6, 0, vec![frame(1, u16::MAX, Vec::new())]) + .batch(8, 0, vec![frame(3, 0, vec![op(&alice, 0, 0, b"a0")])]) + .batch(9, 1, vec![frame(2, 0, Vec::new())]) + .direct(30, b"d30") + .batch(40, 1, vec![frame(20, 0, Vec::new())]); + let (outcomes, _) = run_from_genesis(&config(10, 100, 100), script.inputs()); + assert_eq!( + outcomes, + [ + InputOutcome::DirectQueued, + InputOutcome::BatchRejected(BatchRejectReason::DecodeFailed), + InputOutcome::BatchRejected(BatchRejectReason::WrongNonce { + expected: 0, + got: 5 + }), + InputOutcome::BatchRejected(BatchRejectReason::SafeBlockNotBelowInclusionBlock), + InputOutcome::BatchRejected(BatchRejectReason::NonMonotonicSafeBlocks), + InputOutcome::BatchRejected(BatchRejectReason::FeePriceOutOfRange), + InputOutcome::BatchAccepted, + InputOutcome::BatchRejected(BatchRejectReason::FirstFrameBelowClock { + safe_block: 2, + clock: 3 + }), + InputOutcome::DirectQueued, + InputOutcome::BatchStale, + ] + ); +} + +#[test] +fn a_full_fridge_drains_its_oldest_blocks() { + let mut script = Script::new(); + script + .direct(1, b"a") + .direct(1, b"b") + .direct(2, b"c") + .direct(3, b"d") + .direct(3, b"e") + .direct(3, b"f") + .direct(4, b"g") + .direct(5, &[0; 9]) + .direct(5, b"h"); + let (outcomes, handed_out) = run_from_genesis(&config(1_000, 2, 8), script.inputs()); + assert!(outcomes.contains(&InputOutcome::DirectDropped)); + // The bound is soft by one block: the arriving block's directs never drain. + assert!(!handed_out.is_empty()); + assert!(handed_out.iter().all(|scheduled| match &scheduled.tx { + L2Tx::Direct(direct) => direct.block < 5, + L2Tx::UserOp(_) => false, + })); +} + +/// The inputs `[state.first_queued_input, state.l1_input_count)` from a run starting at 0. +fn queued_range<'a>(inputs: &'a [L1Input], state: &SchedulerState) -> &'a [L1Input] { + let start = usize::try_from(state.first_queued_input).expect("fits"); + let end = usize::try_from(state.l1_input_count).expect("fits"); + &inputs[start..end] +} + +#[test] +fn resume_with_queued_directs_continues_alike() { + let alice = key(1); + let config = config(20, 100, 4); + let mut script = Script::new(); + script + .direct(1, b"d1") + .batch(3, 0, vec![frame(1, 0, vec![op(&alice, 0, 0, b"a0")])]) + .direct(4, b"d4") + .direct(5, b"long!") + .batch(6, 7, Vec::new()) + .direct(6, b"d6") + .batch(7, 1, vec![frame(5, 0, Vec::new())]) + // The cut: d6 is still queued, behind a batch and a dropped direct. + .direct(8, b"d8") + .batch(9, 2, vec![frame(8, 0, vec![op(&alice, 1, 0, b"a1")])]) + .direct(40, b"d40"); + let inputs = script.inputs(); + let cut = 7; + + let mut uninterrupted = Scheduler::genesis(config); + for input in &inputs[..cut] { + uninterrupted.feed(input.clone()); + while uninterrupted.next_tx().is_some() {} + } + let state = uninterrupted.state(); + assert!(state.fridge_len > 0, "the cut must leave directs queued"); + let queued = queued_range(inputs, &state); + assert!(queued.len() > usize::try_from(state.fridge_len).expect("fits")); + + let mut reference = Scheduler::resume(config, state, queued.to_vec()).expect("resume"); + let mut exported = CScheduler::resume(&config, &state, queued).expect("resume"); + assert_eq!(exported.state().expect("state"), state); + let (_, resumed) = run_both(&mut reference, &mut exported, &inputs[cut..]); + + // And the resumed export continues exactly as the uninterrupted reference. + let mut continued = Vec::new(); + for input in &inputs[cut..] { + uninterrupted.feed(input.clone()); + continued.extend(std::iter::from_fn(|| uninterrupted.next_tx())); + } + assert_eq!(resumed, continued); + assert_eq!(reference.state(), uninterrupted.state()); +} + +#[test] +fn refused_resume_writes_no_handle() { + let config = config(20, 100, 100); + let mut script = Script::new(); + script.direct(1, b"d1").direct(2, b"d2"); + let state = SchedulerState { + l1_input_count: 2, + next_batch_nonce: 0, + clock: 0, + first_queued_input: 0, + fridge_len: 3, + }; + let expected = Scheduler::resume(config, state, script.inputs().to_vec()) + .expect_err("the reference refuses it"); + + let sentinel = std::ptr::dangling_mut::(); + let mut handle = sentinel; + let records: Vec<_> = script.inputs().iter().map(input_record).collect(); + let status = unsafe { + sys::scheduler_resume( + &config_record(&config), + &state_record(&state), + records.as_ptr(), + 2, + &mut handle, + ) + }; + assert_eq!(status, sys::SCHEDULER_STATUS_RESUME_REFUSED); + assert_eq!(handle, sentinel); + assert_eq!(last_error(), expected.to_string()); + + // A short range is refused the same way. + let error = CScheduler::resume( + &config, + &SchedulerState { + fridge_len: 2, + ..state + }, + &script.inputs()[..1], + ) + .err() + .expect("refused"); + assert_eq!(error.0, sys::SCHEDULER_STATUS_RESUME_REFUSED); +} + +#[test] +fn bad_arguments_are_misuse_and_write_no_handle() { + let sentinel = std::ptr::dangling_mut::(); + let mut handle = sentinel; + + let zero_window = config_record(&config(0, 1, 1)); + assert_eq!( + unsafe { sys::scheduler_create(&zero_window, &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + assert!(last_error().contains("max_wait_blocks")); + assert_eq!( + unsafe { sys::scheduler_create(std::ptr::null(), &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + let good = config_record(&config(1, 1, 1)); + assert_eq!( + unsafe { sys::scheduler_create(&good, std::ptr::null_mut()) }, + sys::SCHEDULER_STATUS_MISUSE + ); + let state = state_record(&SchedulerState { + l1_input_count: 1, + next_batch_nonce: 0, + clock: 0, + first_queued_input: 0, + fridge_len: 1, + }); + assert_eq!( + unsafe { sys::scheduler_resume(&good, &state, std::ptr::null(), 1, &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + let null_payload = sys::SchedulerInput { + payload: sys::ApplicationEngineByteSpan { + data: std::ptr::null(), + size: 1, + }, + ..Default::default() + }; + assert_eq!( + unsafe { sys::scheduler_resume(&good, &state, &null_payload, 1, &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + // Counts no host can address are refused before any memory is read. + let records = std::ptr::NonNull::::dangling().as_ptr(); + assert_eq!( + unsafe { sys::scheduler_resume(&good, &state, records, u64::MAX, &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + assert!(last_error().contains("address space")); + let huge_payload = sys::SchedulerInput { + payload: sys::ApplicationEngineByteSpan { + data: std::ptr::NonNull::::dangling().as_ptr(), + size: u64::MAX, + }, + ..Default::default() + }; + assert_eq!( + unsafe { sys::scheduler_resume(&good, &state, &huge_payload, 1, &mut handle) }, + sys::SCHEDULER_STATUS_MISUSE + ); + assert!(last_error().contains("address space")); + assert_eq!(handle, sentinel); + + // A success clears the message. + let exported = CScheduler::create(&config(1, 1, 1)).expect("create"); + assert_eq!(last_error(), ""); + // Reading the state with nowhere to write it leaves the handle alive. + assert_eq!( + unsafe { sys::scheduler_state(exported.0, std::ptr::null_mut()) }, + sys::SCHEDULER_STATUS_MISUSE + ); + assert!(exported.state().is_ok()); + unsafe { sys::scheduler_destroy(std::ptr::null_mut()) }; +} + +/// After a failure every call but destroy is MISUSE. +fn assert_dead(exported: &mut CScheduler, next: &L1Input) { + assert_eq!( + exported.state().err().map(|error| error.0), + Some(sys::SCHEDULER_STATUS_MISUSE) + ); + assert_eq!( + exported.next_tx().err().map(|error| error.0), + Some(sys::SCHEDULER_STATUS_MISUSE) + ); + assert_eq!( + exported.feed(next).err().map(|error| error.0), + Some(sys::SCHEDULER_STATUS_MISUSE) + ); + assert!(last_error().contains("dead")); +} + +#[test] +fn an_input_out_of_order_kills_the_handle() { + let mut script = Script::new(); + script.direct(1, b"d0").direct(2, b"d1"); + let mut exported = CScheduler::create(&config(10, 10, 10)).expect("create"); + let error = exported + .feed(&script.inputs()[1]) + .expect_err("out of order"); + assert_eq!(error.0, sys::SCHEDULER_STATUS_MISUSE); + assert!(error.1.contains("not the next"), "{}", error.1); + assert_dead(&mut exported, &script.inputs()[0]); +} + +#[test] +fn a_feed_before_done_kills_the_handle() { + let alice = key(1); + let mut script = Script::new(); + script + .batch( + 3, + 0, + vec![frame( + 1, + 0, + vec![op(&alice, 0, 0, b"a0"), op(&alice, 1, 0, b"a1")], + )], + ) + .direct(4, b"d4"); + let mut exported = CScheduler::create(&config(10, 10, 10)).expect("create"); + exported.feed(&script.inputs()[0]).expect("feed"); + assert!(exported.next_tx().expect("next_tx").is_some()); + let error = exported + .feed(&script.inputs()[1]) + .expect_err("transactions left"); + assert_eq!(error.0, sys::SCHEDULER_STATUS_MISUSE); + assert_dead(&mut exported, &script.inputs()[1]); +} + +#[test] +fn done_comes_before_the_first_feed_and_repeats() { + let mut script = Script::new(); + script.direct(1, b"d1"); + let mut exported = CScheduler::create(&config(10, 10, 10)).expect("create"); + assert_eq!(exported.next_tx().expect("next_tx"), None); + exported.feed(&script.inputs()[0]).expect("feed"); + assert_eq!(exported.next_tx().expect("next_tx"), None); + assert_eq!(exported.next_tx().expect("next_tx"), None); +} + +#[test] +fn an_exhausted_batch_nonce_halts_like_the_reference() { + let config = config(10, 10, 10); + let state = SchedulerState { + l1_input_count: 0, + next_batch_nonce: u64::MAX, + clock: 0, + first_queued_input: 0, + fridge_len: 0, + }; + let mut script = Script::new(); + script + .batch(3, 0, Vec::new()) + .batch(4, u64::MAX, Vec::new()); + + // A rejected batch at the last nonce changes nothing and halts nothing. + let mut reference = Scheduler::resume(config, state, Vec::new()).expect("resume"); + let mut exported = CScheduler::resume(&config, &state, &[]).expect("resume"); + run_both(&mut reference, &mut exported, &script.inputs()[..1]); + + let halted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + reference.feed(script.inputs()[1].clone()) + })); + assert!( + halted.is_err(), + "the reference halts on an accepted batch at u64::MAX" + ); + let error = exported + .feed(&script.inputs()[1]) + .expect_err("no successor"); + assert_eq!(error.0, sys::SCHEDULER_STATUS_NO_SUCCESSOR); + assert_dead(&mut exported, &script.inputs()[1]); +} + +#[test] +fn an_exhausted_input_count_halts() { + let config = config(10, 10, 10); + let state = SchedulerState { + l1_input_count: u64::MAX, + next_batch_nonce: 0, + clock: 0, + first_queued_input: u64::MAX, + fridge_len: 0, + }; + let mut script = Script::from_index(u64::MAX); + script.direct(1, b"d"); + let mut exported = CScheduler::resume(&config, &state, &[]).expect("resume"); + let error = exported + .feed(&script.inputs()[0]) + .expect_err("no successor"); + assert_eq!(error.0, sys::SCHEDULER_STATUS_NO_SUCCESSOR); + assert_dead(&mut exported, &script.inputs()[0]); +} + +#[test] +fn spans_stay_valid_across_a_state_read() { + let mut script = Script::new(); + script + .direct(1, b"payload") + .batch(3, 0, vec![frame(2, 0, Vec::new())]); + let mut exported = CScheduler::create(&config(10, 10, 10)).expect("create"); + exported.feed(&script.inputs()[0]).expect("feed"); + assert_eq!(exported.next_tx().expect("next_tx"), None); + exported.feed(&script.inputs()[1]).expect("feed"); + let mut record = sys::SchedulerTx::default(); + assert_eq!( + unsafe { sys::scheduler_next_tx(exported.0, &mut record) }, + sys::SCHEDULER_STATUS_OK + ); + exported.state().expect("state"); + assert_eq!(bytes(&unsafe { record.values.direct }.payload), b"payload"); +} + +/// A small deterministic generator, so a failing run replays from its seed. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + // SplitMix64. + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } +} + +/// Random inputs that exercise every outcome: directs over and under the cap, batches at the +/// right nonce and not, frames old and new, ops over and under the price, bad signatures. +fn random_inputs(rng: &mut Rng, keys: &[SigningKey], count: usize) -> Vec { + let mut script = Script::new(); + let mut block = 1; + let mut nonce = 0; + for _ in 0..count { + block += rng.below(4); + if rng.chance(60) { + let len = usize::try_from(rng.below(10)).expect("fits"); + script.direct( + block, + &vec![u8::try_from(rng.below(256)).expect("fits"); len], + ); + continue; + } + let batch_nonce = if rng.chance(85) { + nonce + } else { + nonce + 1 + rng.below(2) + }; + let mut frames = Vec::new(); + let mut safe_block = block.saturating_sub(1 + rng.below(30)); + for _ in 0..rng.below(3) { + let price = u16::try_from(rng.below(6)).expect("fits"); + let ops = (0..rng.below(3)) + .map(|_| { + let signer = &keys[usize::try_from(rng.below(3)).expect("fits")]; + let mut wire = op( + signer, + u32::try_from(rng.below(4)).expect("fits"), + u16::try_from(rng.below(8)).expect("fits"), + &rng.next().to_le_bytes(), + ); + // A flipped bit recovers another signer; a zero r recovers none. + if rng.chance(10) { + wire.signature[10] ^= 1; + } else if rng.chance(5) { + wire.signature[..32].fill(0); + } + wire + }) + .collect(); + frames.push(frame(safe_block, price, ops)); + safe_block = (safe_block + rng.below(3)).min(block - 1); + } + if rng.chance(5) { + script.raw_batch(block, &rng.next().to_le_bytes()); + continue; + } + script.batch(block, batch_nonce, frames); + if batch_nonce == nonce { + // Accepted or not, the generator guesses; a wrong guess only means more rejections. + nonce += u64::from(rng.chance(70)); + } + } + script.inputs().to_vec() +} + +#[test] +fn random_runs_match_the_reference_from_genesis_and_from_every_resume() { + let keys = [key(1), key(2), key(3)]; + for seed in 0..8 { + let mut rng = Rng(seed); + let config = config(1 + rng.below(25), 1 + rng.below(6), rng.below(9)); + let inputs = random_inputs(&mut rng, &keys, 80); + let (outcomes, _) = run_from_genesis(&config, &inputs); + assert!( + outcomes.contains(&InputOutcome::DirectQueued), + "seed {seed}" + ); + + // Resume the export at several cuts and compare it with the reference resumed there. + let mut uninterrupted = Scheduler::genesis(config); + for (cut, input) in inputs.iter().enumerate() { + if cut % 11 == 0 { + let state = uninterrupted.state(); + let queued = queued_range(&inputs, &state); + let mut reference = + Scheduler::resume(config, state, queued.to_vec()).expect("resume"); + let mut exported = CScheduler::resume(&config, &state, queued).expect("resume"); + run_both(&mut reference, &mut exported, &inputs[cut..]); + } + uninterrupted.feed(input.clone()); + while uninterrupted.next_tx().is_some() {} + } + } +} diff --git a/docs/cartesi-machine.md b/docs/cartesi-machine.md index bef11873..59710c78 100644 --- a/docs/cartesi-machine.md +++ b/docs/cartesi-machine.md @@ -1,8 +1,11 @@ # Cartesi Machine Facts We Rely On The canonical application runs inside the Cartesi Machine (CM). The watchdog -drives it, the canonical guest is built for it, and cockroach recovery exports -from it. This page records the CM behavior those components depend on, +drives it, the canonical guest is built for it, and setup reads a base +machine's `state` and `scheduler` inspect reports when it rebuilds from one +([cockroach recovery](recovery/cockroach.md); +[reading a stored machine's export](#reading-a-stored-machines-export)). +This page records the CM behavior those components depend on, especially behavior that is easy to assume wrongly. Everything here holds for the pinned version (`CARTESI_MACHINE_VERSION` in [`toolchain-pins.env`](../toolchain-pins.env), currently v0.22.0) and was @@ -70,12 +73,12 @@ not: The checkout path does not enter the hash, and `just canonical-print-build-hashes` prints all four inputs to compare; compare cold builds, since a warm `target/` can hold crates compiled before `rust-src` arrived. A move that is meant to preserve behavior is -checked against [`tests/fixtures/consensus_vectors_v1.json`](../tests/fixtures/consensus_vectors_v1.json), +checked against [`tests/fixtures/consensus_vectors_v2.json`](../tests/fixtures/consensus_vectors_v2.json), which pins the consensus bytes (EIP-712 hashes, signature recovery, SSZ, fee -table, output payloads) natively and, through `canonical-test`'s golden cases, -in the built image, where each signature variant is checked for running as the -signer or not. The fixture is never regenerated: a mismatch is a behavior -change to explain. A release that moves the hash says so, and deployed +table, output payloads) natively, and whose guest scenarios `canonical-test`'s +golden cases replay in the built image, comparing every output and inspect +report ([conformance](protocol/scheduler-semantics.md#conformance)). The +fixture is never regenerated: a mismatch is a behavior change to explain. A release that moves the hash says so, and deployed applications keep their old image, which the watchdog checks against the on-chain hash. @@ -165,6 +168,21 @@ a cheaper throughput flood breaks Dave settlement first, so the gap is accepted on cost; revisit it before deploying on a cheap L2, where such floods cost about a thousand times less. +An epoch's inputs are an InputBox index range, not a block range. +`DaveConsensus` seals each epoch at the input count read in the transaction +that seals it (its constructor for epoch 0, `acceptStagedTournamentResult` for +every later one), so an epoch can end inside an L1 block. The last finalized +machine, whose root `getLastFinalizedMachineMerkleRoot` returns (zero before +the first acceptance), has consumed exactly the inputs below the current sealed +epoch's `inputIndexLowerBound`, read when the previous epoch was sealed. Take +that position from the chain. The machine keeps only the last input it took, +in the receive buffer ([the guest side](#the-guest-side)), so for a guest that +never rejects, as ours (a batch the scheduler drops is still accepted), that +input's `index` plus one cross-checks the position. After an `RX_REJECTED` +input or at a fixed point the buffer lags, and no root locates a state either, +since both repeat roots. Sling keeps only the newest finalized machine, ingests +only L1-finalized blocks, and serves stored machines only as files. + ## Storage, sharing, and snapshots - **Loading** defaults to `SHARING_NONE`: every backing file is mapped @@ -204,6 +222,53 @@ Every revert in that run restored exactly the recorded pre-input root hash. On a filesystem without reflinks, each clone is a full sparse copy of the machine. +## Reading a stored machine's export + +Setup reads a base through `sequencer::machine::read_machine_export` +([`sequencer/src/machine.rs`](../sequencer/src/machine.rs)), which links the +emulator through the same `cartesi-machine` bindings as `sdk/guest/testsi`. +`canonical-test`'s `export` cases run it against the built image and against +a machine advanced and stored by testsi. + +- **Load privately, and the directory stays as it was.** `Machine::load` is + `SHARING_NONE`: the queries run on a copy-on-write image in memory, which is + dropped. A machine that another process runs in place (`SHARING_ALL`, as + sling does) holds an exclusive lock and cannot be loaded meanwhile; read a + clone. +- **A stored machine is between inputs only at an `RX_ACCEPTED` manual + yield.** The image recipe stores one there, and so does a node; not every + machine the watchdog stores does + ([which ones](watchdog/README.md#machines-a-recovery-can-start-from)). Check it as + the step does (`iflags.H`, `iflags.Y`, then tohost's device, command and + reason) before sending anything. +- **What to send.** An inspect is `send_cmio_response` with the + inspect-state reason, the query bytes, and no revert root hash: the + emulator refuses one on any response but an advance. An undeliverable + response is a silent no-op, so check that `iflags.Y` cleared. +- **What the machine yields.** Each report is an automatic `TX_REPORT` + yield, and `run` returns at each one; the next `run` resumes. The guest + then ends at `RX_ACCEPTED` again, so one load answers both queries in turn. + The query moves the in-memory root hash (the RX buffer now holds it), which + matters only to a host that keeps the machine. Our guest answers each query + with exactly one report; it emits no outputs and never rejects. +- **The bindings panic on some yields.** `receive_cmio_request` panics on an + automatic yield with an unknown reason, so classify a yield from the tohost + registers first and read its data only for a report or an exception. +- **The loader bounds each query itself.** It runs to a fixed mcycle target, + 2^36 cycles past the query, and fails there, so a guest that never answers + fails setup instead of hanging it. +- **A report is at most 2 MiB.** libcmt copies a report's bytes raw into the + TX buffer (2^21 bytes) and refuses a longer one with `-ENOBUFS`; the guest + halts on that error, so the loader sees a halt, not a state. The wallet's + `state` report holds 52 bytes per balance and 24 per nonce, so it fits up + to roughly 27,000 accounts that have both. A larger application state + needs another state source, such as a labeled memory range read with + `read_memory` ([memory ranges](#memory-ranges-for-application-state)), or a + guest that splits its answer over several reports. +- **The `state` report is whatever the guest answered.** A guest whose + encode fails answers `inspect failed: …` instead, which the loader writes + as the state; the application's `from_canonical_state` refuses those bytes. + ## Memory ranges for application state An application can keep canonical state in a labeled memory range instead of @@ -242,6 +307,11 @@ answering an inspect query: rootfs. Without `xhalt`, a failing entrypoint halts with payload 0. The watchdog test guest ships musl stand-ins for both; the canonical wallet image does not, so its panics halt with exit code 0. +- **The canonical guest answers two inspect queries.** `state` (or an empty + query) reports the application's comparison bytes, and `scheduler` reports + the scheduler's export: six little-endian `u64` (`SchedulerExport` in + [`sequencer-core/src/scheduler/mod.rs`](../sequencer-core/src/scheduler/mod.rs)). + Any other query gets the report `unsupported inspect query`, not a halt. - **trolley has no exception call.** A guest that must raise a CMIO exception does it through `libcmt-sys` directly (see `watchdog/test-guest`). - **The last input stays in the machine.** The CMIO receive buffer keeps it, @@ -261,6 +331,18 @@ answering an inspect query: ceil_log2(length))` returns a range's Merkle root at the cost of rehashing dirty pages only, and `get_proof` ties it to the machine root hash. Any party comparing against that root must reproduce the same tree. +- Loading a stored machine takes its root hash from the stored hash tree + (`hash_tree.sht` and each range's `.dht`), rehashing only pages marked + dirty; the load checks only the sparse tree's shape. A root that matches an + on-chain hash therefore authenticates a stored machine only from a source we + trust. +- `verify_hash_tree` compares each page not marked dirty with its hash in the + range's `.dht` and recomputes nothing above the page: it catches a page + changed under a tree that holds it clean, such as in a corrupt copy, but not + hash files rewritten to match. In v0.22.0 it serves the emulator's internal + tests, is unoptimized, and the emulator team asked that nothing call it, so + Dave's node keeps its own check disabled (Dave's `docs/todo.md`). An + upcoming release brings a mechanism to verify an untrusted load. ## The CLI as a test oracle diff --git a/docs/invariants.md b/docs/invariants.md index ffe0069e..9548f6a9 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -29,9 +29,9 @@ When you change anything listed under *enforced by*, re-check every line under - **Never absorb silently.** No `INSERT OR IGNORE`, saturating decode, or `unwrap_or_default` on data the contracts make impossible; use the loud variant of the same operation. -- **Admission, terminal faults, the maintenance flush, and startup repair** +- **Terminal faults, the terminal facts, and the maintenance flush** follow the [authority-boundary ADR](plans/2026-08-authority-boundary-adr.md) - (mechanisms 1–3) and the [recovery design](recovery/README.md). Raw worker + (mechanisms 1–3) and [batch landing and outages](recovery/README.md). Raw worker and HTTP launch surfaces are crate-private, and public low-level storage helpers are not an authority API: mutation and output authorization stay role-local at the durable boundaries this register lists. @@ -49,41 +49,42 @@ don't. ### Writer roles -One writer role per fact. Reads over batch data go through the `valid_*` -views (`valid_batches`, `valid_closed_batches`, `valid_open_batch`), which encapsulate the "exclude invalidated rows" -filter; writers target the base tables. The batch lifecycle columns partition -by writer and are write-once (`0001_schema.sql`). +One writer role per fact. The batch lifecycle columns partition by writer and +are write-once (`trg_batch_lifecycle`, `0001_schema.sql`). The sealed batches +are the `closed_batches` view and the one unsealed batch is `open_batch`; +writers target the base tables. | Writer | Writes | |---|---| -| inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | -| input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | -| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion and replacement direct-input rows | -| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline and generation; `history_generation_cuts` — immutable preserved-prefix cuts written with non-empty standard-recovery cascades | +| inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `snapshots` (batch close) | +| input reader | `safe_inputs`, `l1_safe_head`, `batches.accepted_input`, `canonical_divergence` (the divergence poison marker) | +| `run`'s start | an open batch when none is open, which is how the era's first batch is created | +| history metadata (setup) | `era` — one immutable row (base count, stop block, rebuilt history end, first queued input, anchor nonce), written once by setup's publish | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | -| egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | -| setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows and rebuild root `batches`/`frames` (atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | -| snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | obsolete `snapshots` and unreferenced `dumps` row deletion (`gc_unreferenced_dumps`), including a superseded baseline artifact | +| setup | `deployment_identity` (pinned once, the direct-payload cap with it), `rebuilt_history` (its only writer, and only before it publishes the era), the `era` row (whose existence is setup's completion), the baseline `snapshots` row (atomic with the era; no `batches` or `frames`), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | +| snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | obsolete, unleased `snapshots` row deletion (`collect_snapshots`), including a superseded baseline artifact | | command brackets (run, setup, flush) | `terminal_faults` (append-only, best-effort at settlement) | | fee oracle | `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (Uniswap mode only; stamps on every successful refresh) | ### I1. Scheduler-acceptance semantics agree across all implementations - **Authoritative prose:** [`docs/protocol/scheduler-semantics.md`](protocol/scheduler-semantics.md). -- **Holds:** the canonical fold (`Scheduler`, - `sequencer-core/src/scheduler/mod.rs`), the off-chain predicate - (`ProtocolTiming::scheduler_accepts`, `sequencer-core/src/protocol.rs`), and +- **Holds:** the canonical scheduler (`Scheduler`, + `sequencer-core/src/scheduler/mod.rs`, which `CanonicalRunner` in + `scheduler/driver.rs` drives with the executor), the off-chain predicate + (`scheduler_accepts`, `sequencer-core/src/protocol.rs`), and the inclusion lane's live prediction produce the same - accept/reject/ordering decisions for every input. The recovery fold - (`fold_replay`) drives `Scheduler` itself, not a copy. (Known, documented - exception: the predicate omits the three structural rejections — self-trust, - since the simulator only runs over the sequencer's own well-formed batches; - the reason is recorded at `scheduler_accepts`, and it is test-pinned by - the I1 duality test in `sequencer-core/src/scheduler/mod.rs`, which asserts - the canonical fold and the predicate diverge exactly and only there.) + accept/reject/ordering decisions for every input. (Known, documented + exception: the predicate omits gate d's three structural rejections and + gate f — self-trust, since the simulator only runs over the sequencer's own + well-formed batches, opened in order at safe heads it has seen; the reason + is recorded at `scheduler_accepts`, and it is test-pinned by the I1 duality + test in `sequencer-core/src/scheduler/mod.rs`, which asserts the scheduler + and the predicate diverge exactly and only there.) - **Enforced by:** review + the duality test. No structural mechanism. -- **Depended on by:** everything — the gold frontier, recovery's cascade pivot, - checkpoint selection, soft-confirmation honesty. +- **Depended on by:** everything — the gold frontier, the void history's + choice of the oldest unaccepted batch (I4), checkpoint selection, + soft-confirmation honesty. - **Breaks:** silent permanent scheduler/sequencer divergence. ### I2. Drain attribution: accumulated directs land in the clock-advanced frame @@ -93,170 +94,216 @@ by writer and are write-once (`0001_schema.sql`). **new** frame, which is stamped with the observed safe head (`close_frame_in`, `storage/ingress.rs`). Directs may have accumulated across several below-threshold observations. Frame K's wire content is therefore - "directs ≤ S_K, then ops validated on top"; a clock tick with no directs is + "directs ≤ S_K, then the frame's ops"; a clock tick with no directs is an empty-prefix instance of the same rule. That leading direct prefix is recoverable from `application_inputs` plus `frames.safe_block` alone. - **Enforced by:** `close_frame_in` ordering; lane convention. - **Depended on by:** the duality (scheduler's drain-before-ops equals the flattened replay order); catch-up; the feed. -- **Breaks:** ops validated against a state the scheduler won't reproduce — +- **Breaks:** ops executed against a state the scheduler won't reproduce — divergence. ### I3. Frame `safe_block`s are non-decreasing along the spine -- **Holds:** frame `safe_block`s never decrease along the valid spine; equal +- **Holds:** frame `safe_block`s never decrease along the batches; equal adjacent clocks are valid. The [frame-clock policy](protocol/scheduler-semantics.md#sequencer-frame-clock-policy) says when the lane advances the clock, including the anchoring exception - for bootstrap and recovery. + for bootstrap. - **Enforced by:** lane flow + `append_ingested_safe_inputs_with_timestamp`'s monotonicity asserts (`storage/l1_inputs.rs`) + - `ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS` + `FRAME_CLOCK_INTERVAL_SAFE_BLOCKS` (`sequencer-core/src/protocol.rs`). The lane's frontier time gate bounds SQLite observation load and is not part of the clock semantics. -- **Depended on by:** `check_danger`'s arm ordering (see I4); the scheduler's - within-batch monotonicity check; "if the frontier batch is fresh, all are". -- **Breaks:** I4's guarantee evaporates; danger detection mis-orders. - -### I4. Closed-frontier danger takes precedence over Tip danger - -- **Holds:** `check_danger` checks `ClosedBatchInDanger` before `TipInDanger`. - With monotonic frame clocks, the closed frontier is at least as old as the Tip. -- **Enforced by:** arm order in `storage/recovery.rs` and I3. -- **Depended on by:** dispatch: closed-frontier danger selects flush before - a Tip-only repair can be selected. The Tip itself has no L1 footprint. - -### I5. Recovery removes exactly the invalidated application suffix - -- **Holds:** invalidating a batch deletes its `application_inputs` through the - schema trigger. The cascade, generation cut/increment, and replacement Tip commit - together. Original source records and immutable snapshots remain; snapshot - selection excludes invalidated batches and GC retires their unleased artifacts. -- **Enforced by:** `cascade_and_reopen`, application-input constraints, valid views. -- **Breaks:** loading invalidated state or leaving a hole in current history. +- **Depended on by:** the void history (I4), which judges the oldest + unaccepted batch's first frame: with monotonic clocks no later batch is + older; the scheduler's within-batch monotonicity check. +- **Breaks:** the void history judges the wrong batch and fires late. + +### I4. A void history ends the era + +- **Holds:** with every input through the persisted safe head `H` judged, the + oldest batch L1 has not accepted, closed or open, opened at first-frame safe + block `s`, is void once `H + 1 - s >= max_wait_blocks`; no later batch can + then be accepted. It stays true, so it is derived again on every read and + never stored, and while it holds `run` refuses and a running process stops + ([the void history](recovery/README.md#the-void-history)). +- **Enforced by:** `Storage::terminal_fact`, read at `run`'s start before any + L1 call and again inside the transaction that opens a batch, and by the + worker that watches the terminal facts while the process runs + ([what the sequencer does](recovery/README.md#what-the-sequencer-does)). +- **Depended on by:** soft-confirmation honesty: the process stops within one + poll of that worker, two seconds, of the safe head showing the history void, + and the ops confirmed in that interval are lost with the rest; cockroach + recovery as the remedy. +- **Breaks:** the sequencer keeps confirming user ops that can never land. + +### I5. Batches and application history only append + +- **Holds:** a batch is created once, sealed once with its hash, accepted + once, and never deleted or replaced; a frame, a user op and an application + input are written once and never deleted. Inside an era the history + therefore only grows, what the feed served stays served, and a sealed batch + always re-encodes to the bytes its hash was taken from. +- **Enforced by:** `trg_batch_lifecycle` (a batch row changes only by sealing + once or by being accepted once); the triggers that refuse every delete on + `batches`, `frames`, `user_ops` and `application_inputs`, and every update + on `frames` and `user_ops`; and `PRAGMA recursive_triggers = ON` on every + connection, without which an `INSERT OR REPLACE` deletes a conflicting row + past those triggers. +- **Depended on by:** a client resuming at any count it saved in this era; + the void history (I4), which judges a batch nothing replaces; the content + identity check (I9) and the nonce rule (I16), which need one batch per + nonce. +- **Breaks:** a client's saved state stops matching the feed it resumes from, + or a nonce names two batches. ### I6. The frame clock accounts for a complete L1 interval -- **Holds:** the surviving latest frame clock, floored by baseline block `C`, - identifies the completely accounted L1 prefix. Reconciliation executes all - external directs in the newly safe interval before committing its next frame - and application rows. Envelopes-only intervals advance the clock with no rows. +- **Holds:** the latest frame clock, floored by the era's stop block + `C`, identifies the completely accounted L1 prefix. Reconciliation executes all + external directs within the payload cap in the newly safe interval before + committing its next frame and application rows. Envelopes-only intervals advance the clock with no rows. + With no frame in the era, the next undrained input is the era's first + queued input: the era's baseline accounts for every input through `C` + except the directs still queued there, so the root frame drains from it + (`next_undrained_safe_input_index_in`, `storage/ingress.rs`). - **Enforced by:** complete-block ingestion: every committed `safe_inputs` / `l1_safe_head` pair holds every input through its head, witnessed by index contiguity plus the InputBox count at that head (`l1/reader.rs`), above a scan floor setup pins only once the deployment is safe; the lane's indivisible reconciliation turn; storage range and execution-receipt checks. Batch closure preserves the current frame clock. -- **Breaks:** skipping or double-applying directs after restart/recovery. +- **Breaks:** skipping or double-applying directs after restart. ### I7. Every committed batch close has an immutable snapshot - **Holds:** file creation precedes the transaction sealing the batch, opening - the next Tip, and registering its snapshot by local `batch_index` and count. + the next Tip, and registering its snapshot by batch nonce and count. - **Enforced by:** `close_batch_with_snapshot` and `close_frame_and_batch_with_snapshot`. Selection requires the exact expected batch's snapshot; a missing artifact fails loud. -- **Breaks:** losing the accepted recovery/watchdog checkpoint. - -### I8. A rollback-safe snapshot and valid Tip exist before the lane starts - -- **Holds:** setup publishes the complete durable baseline before completion; - recovery retains the newest accepted snapshot, or that baseline until first - acceptance. A rebuilt baseline is a restore point, not a comparison at `C`. -- **Enforced by:** `complete_baseline_setup`, recovery admission, atomic Tip - reopen, and `PreparedRuntime::prepare` artifact checks. +- **Breaks:** losing the accepted comparison checkpoint. + +### I8. An accepted snapshot and an open batch exist before the lane starts + +- **Holds:** setup publishes the durable era baseline, at the rebuilt history + end `E`, in the transaction that inserts the era; snapshot collection + retains the newest accepted snapshot, or that baseline until first + acceptance, for the comparison endpoints and the lane's restart. The baseline is the application's + state at the end of stop block `C`, which the canonical machine reaches too, + so until a batch of the era is accepted it is the comparison checkpoint + (inclusion block `C`, count `E`, the anchor nonce), genesis base or not. + Setup opens no Tip; `run`'s start opens the era's first batch before the + lane starts. +- **Enforced by:** `publish_era`, `finalized_dump_in`'s assertion that the + baseline sits at `E`, `run`'s guarded open-a-batch transaction, and + `PreparedRuntime::prepare` artifact checks. - **Depended on by:** unconditional snapshot restore and catch-up. - **Breaks:** startup fails loud instead of inventing state. -### I9. Acceptance identity: "accepted nonce N" means "our valid batch N" +### I9. Acceptance identity: "accepted nonce N" means "our batch N" - **Holds:** by nonce **and content** — the **content-identity check**: every landing strictly after baseline block `C` that the off-chain - `scheduler_accepts` simulation accepts is compared against the local valid - closed batch at that nonce — `keccak256(landed bytes)` vs the hash stamped at + `scheduler_accepts` simulation accepts is compared against the local closed + batch at that nonce — `keccak256(landed bytes)` vs the hash stamped at seal by the same encode path the submitter broadcasts. The exhaustive local - outcomes are `Match`, `Foreign` (no local valid closed batch), and `Mismatch` - (different bytes); the last two record divergence. + outcomes are `Match`, `Foreign` (no local closed batch), and `Mismatch` + (different bytes); the last two record divergence. A `Match` is recorded on + the batch itself, as `batches.accepted_input`: the input reader's one + write-once update, which the schema admits only on a sealed batch + (`trg_batch_lifecycle`) whose nonce follows the greatest accepted batch's + (the anchor before any), at a later input + (`trg_batch_acceptance_extends_the_prefix`). A batch is inserted unaccepted + (`trg_batch_insert`). The accepted batches are a prefix of the list. - **Why content, not identity, suffices:** batches deliberately carry no identifier because content-equal copies are *effect-equal* — an accepted batch's application effects depend on its inclusion block only through the overdue force-drain, and for any fresh copy that force-executed prefix is a subset of the first frame's drain, in the same queue order. Which physical L1 transaction landed carries no semantic weight. -- **Enforced by:** prevention — the flush resolving every wallet-nonce slot - before a cascade reuses a nonce, anchored by the persisted watermark (I14); +- **Enforced by:** prevention — no batch nonce is reused inside an era; + across eras setup's flush is best effort, and its escape check refuses what + it sees ([the flush and the stop block](recovery/cockroach.md#the-flush-and-the-stop-block)); detection — the content-identity check in - `populate_safe_accepted_batches`, which on violation persists the + `record_accepted_batches`, which on violation persists the `canonical_divergence` marker and freezes the frontier (I15). -- **Depended on by:** the gold frontier, cascade pivot selection, checkpoint selection, +- **Depended on by:** the gold frontier, the void history's oldest unaccepted batch, checkpoint selection, local-state ↔ canonical-state agreement. -- **Breaks:** would be silent divergence (a zombie replay of our own stale tx - winning a wallet-nonce slot; a power-loss re-seal at the same batch nonce +- **Breaks:** would be silent divergence (a foreign landing at one of our + nonces; a power-loss re-seal at the same batch nonce with different content); instead it is a detected `CanonicalDivergence` refusal whose remedy is cockroach recovery. - **Completeness boundary:** the check completely enforces the accepted-batch identity predicate above; it is intentionally not a general canonical/application - divergence oracle. The entire L1 prefix through baseline block `C` is opaque, - including previously rejected future-nonce batches; the check trusts the - checkpoint state and continuation nonce instead of reinterpreting that prefix. + divergence oracle. The entire L1 prefix through stop block `C` is opaque to + it, including previously rejected future-nonce batches: setup's rebuild + judged the inputs past its base with the scheduler, and the check trusts the + base, that rebuild and the anchor nonce instead of reinterpreting them. It shares `scheduler_accepts` (including its documented self-trust omissions), and does not independently detect bugs in - direct-input/user-op execution. A wrong-high cockroach checkpoint nonce is a - known example that can escape it. Absence of the marker therefore does not + direct-input/user-op execution. Absence of the marker therefore does not prove global agreement. Conversely, a structurally malformed foreign landing may conservatively record divergence even when the on-chain algorithm would reject it — an accepted false positive under self-trust. Detection is automatic once the landing is safe and successfully ingested; repair is - manual cockroach recovery, never standard - recovery. Detection latency is inherent to the optimistic model: the check + manual cockroach recovery. Detection latency is inherent to the optimistic model: the check fires when the divergent landing reaches safe depth and is ingested, so soft confirmations issued inside that window are built on already-diverged - state — bounded, and those confirmations are rollbackable by design. + state — bounded, and those confirmations are revocable by design. ### I10. Replay uses an inclusive application-history boundary -- **Holds:** `ExecutedInputCount = X` means input `X` executes next. Current - rows cover `[K, H)`; snapshots at `X` replay with `offset >= X`. `H` waits. -- **Enforced by:** integer primary key, contiguous insertion, coherent versioned - pages, and pre-execution catch-up count checks. +- **Holds:** `L2TxCount = X` means entry `X` executes next. The + rebuilt rows cover `[K, E)` and the live rows `[E, H)`; a snapshot + is never below `E`, so native restart replays only live rows with + `offset >= X`. `H` waits. +- **Enforced by:** integer primary key, contiguous insertion from `E` + (`trg_application_inputs_contiguous`), contiguous rebuilt counts + (`trg_rebuilt_history_insert`, `trg_era_fits`), coherent + era-checked pages spanning the seam, and pre-execution catch-up count checks. - **Depended on by:** restart and replica resume without skipping or duplication. ### I11. Batch envelopes remain outside application history - **Holds:** `safe_inputs` retains every InputBox observation. The storage/lane - boundary selects external directs by the setup-pinned submitter address; + boundary selects external directs by the setup-pinned submitter address and + payload cap ([I20](#i20-application-history-is-committed-with-its-execution-receipts)); only those inputs and included user ops enter `application_inputs`. - **Enforced by:** append-only `safe_inputs` triggers; classified direct reads - and complete receipt validation at append. Standard startup recovery - attributes undrained directs to the new Tip; lane catch-up executes them - before processing queued user operations. Manual - rebuild represents the folded prefix through `C` in its baseline snapshot, - with no application-history rows for that prefix. Replay and WS need no - envelope filter because every row executes. + and complete receipt validation at append. The era's root frame takes the + directs still queued at `C`; lane catch-up executes them + before processing queued user operations. Setup's rebuild selects the same + directs (`rebuilt_history` refuses a submitter input as a direct), and the + era's baseline holds their effects through `C`, with no live rows for that + prefix. Replay and WS need no + envelope filter because every served row executes. - **Depended on by:** application replay and replicated state correctness. -### I12. Safe head advances only on real observation; `synced_at_ms` is genuine progress time +### I12. Safe head advances only on real observation - **Holds:** the reader early-returns when the fetched head doesn't advance; - `append_ingested_safe_inputs_with_timestamp` asserts monotonicity and stamps - `synced_at_ms` only on commit. + the append asserts the safe head never decreases and commits it with every + input through it. - **Enforced by:** reader floor check + asserts (`storage/l1_inputs.rs`). -- **Depended on by:** the wall-clock danger arms (`L1ViewStale`, - `EstimatedBatchInDanger`) — their baseline must be true progress time, or - outages are masked. -- **Breaks:** danger detection silently late during exactly the outages it - exists for. +- **Depended on by:** the void history (I4): `H` only rises, so a void history + stays void; the landing lag and the safe head's age that `/healthz` reports. +- **Breaks:** a void history read from a head L1 never reached, or a lag that + hides an outage. -### I13. No `dumps` row points at a missing directory +### I13. No `snapshots` row points at a missing directory - **Holds:** file create (fsync'd) before row insert; row delete before file delete; orphan *files* are acceptable and swept at startup. -- **Enforced by:** ordering split between `storage/snapshot_dumps.rs` +- **Enforced by:** ordering split between `storage/snapshots.rs` (SQLite-only) and the FS halves outside it (the lane's `inclusion_lane/snapshot.rs`; the startup sweep in `commands/run/startup_hygiene.rs`) — the module boundary *is* the ordering - guarantee. Startup and egress classify a missing or structurally corrupt - DB-referenced artifact as terminal; generic filesystem availability errors - remain operational. + guarantee. GC never collects a snapshot a download has leased + ([lease protocol](snapshots/lifecycle.md#retention-leases-and-crash-safety)). + Startup and egress classify a missing or structurally corrupt DB-referenced + artifact as terminal; generic filesystem availability errors remain + operational. - **Depended on by:** `from_dump` at catch-up; the serving endpoints. - **Breaks:** terminal startup refusal (or a terminal egress fault if detected while serving), requiring inspection or cockroach recovery rather than an @@ -268,37 +315,42 @@ by writer and are write-once (`0001_schema.sql`). durably (`synchronous=FULL`) before any broadcast at a new nonce, uniformly for batch txs and flush no-ops. A crash between commit and send only over-covers (the flush later no-ops a never-used slot — harmless). -- **Enforced by:** write-before-broadcast — `EthereumBatchPoster::submit_batches` - raises through `WalletNonceWatermarkSink` before its first send; - `MempoolFlusher::flush_and_wait` likewise before its no-ops, and refuses to - complete until `safe >= watermark + 1`. -- **Depended on by:** [flush completeness](recovery/README.md#closed-batches-flush-sync-cascade) - and cascade soundness (I9). -- **Breaks:** zombie txs evade the flush — a dropped-locally but - network-surviving batch tx re-lands at a slot the recovery batch reuses, - and the scheduler executes invalidated content. +- **Enforced by:** the `Covered` witness (`l1/watermark.rs`): only `cover` + mints it, after `WalletNonceWatermarkSink::raise_to` succeeds, and the + poster's `send_batch_at_wallet_nonce` and the flusher's `submit_noops` + require it and assert each wallet nonce they send is at or below it. + `EthereumBatchPoster::submit_batches` covers its whole consecutive range; + `MempoolFlusher::flush_and_wait` covers every no-op range it sends, and + refuses to complete until `safe >= watermark + 1`. Pinned by a + raise-before-send test on each side. +- **Depended on by:** `flush-mempool`'s completeness. Setup's flush runs on a + new database, which has no watermark, so it does not rely on this + ([the flush and the stop block](recovery/cockroach.md#the-flush-and-the-stop-block)). +- **Breaks:** zombie txs evade the flush — a batch tx dropped locally but + surviving in the network lands after the flush reports the wallet settled. ### I15. Divergence marker present ⇒ acceptance frontier frozen - **Holds:** a fully accepted landing that fails the content-identity check writes the `canonical_divergence` singleton **in the detecting sync's - transaction**, and `populate_safe_accepted_batches` returns early while the - marker exists, so no acceptance row or gold-frontier advance follows a + transaction**, and `record_accepted_batches` returns early while the + marker exists, so no acceptance or gold-frontier advance follows a detected divergence. - **Enforced by:** the `trg_*_frozen_on_divergence` triggers - (`0001_schema.sql`) make batch-tree writes and snapshot collection RAISE - while the marker exists: an immediate, persisted freeze of those tables, not - a user-op hot-path barrier. The accepted frontier has no trigger; its single - writer's guard refuses. `check_danger`'s first arm is `CanonicalDivergence`. - Startup inspection refuses before any provider query, and each repair - transaction reasserts its preconditions and the marker's absence before - writing. A clean worker drain waits for in-flight reader appends; `run` then - re-reads the marker (`refuse_divergence_on_clean_exit`) and exits terminal, + (`0001_schema.sql`) make batch-tree writes, acceptance + (`batches.accepted_input`) included, and snapshot collection RAISE while the + marker exists: an immediate, persisted freeze of those tables, not a user-op + hot-path barrier. Created after every other BEFORE trigger on `batches`, the + freeze answers first. `Storage::terminal_fact` reads the marker before the + void history. `run`'s start refuses before any provider query, and its + open-a-batch transaction reasserts the marker's absence before writing. A + clean worker drain waits for in-flight reader appends; `run` then + re-reads the terminal facts (`refuse_terminal_fact`) and exits terminal, not 0, the one code that would break the supervisor's restart-then-refuse - rediscovery. `admission.tla` checks the controller ordering - (`LocalTerminalDominance`); `preemptive.tla` checks slot and batch safety. -- **Runtime reaction:** the danger detector owns the prompt process-wide - reaction, polling `check_danger` every `DANGER_DETECTOR_POLL_INTERVAL`. + rediscovery. +- **Runtime reaction:** the worker that watches the terminal facts owns the + prompt process-wide reaction, reading `Storage::terminal_fact` every two + seconds. Independently, the lane's existing time-gated SQLite read returns `SafeFrontierState::CanonicalDivergence` instead of `Open`, and the lane exits terminal, so the supervisor aborts, before direct execution or the @@ -310,60 +362,59 @@ by writer and are write-once (`0001_schema.sql`). - **Race bound:** a lane turn that already read `Open` may finish while the reader commits divergence; preventing that would need a lock or transaction spanning application execution. The freeze triggers stop conflicting - batch-tree writes, and the detector or the next typed read stops the + batch-tree writes, and the watching worker or the next typed read stops the process, so a chunk committed before either may acknowledge and later roll back. - **Watchdog boundary:** accepted-checkpoint selection checks the marker in - the same transaction as selection and any download lease, and refuses while - it is present. A matching batch before a divergent acceptance in the same L1 + the same read as selection, under the lease registry's lock for a download, + and refuses while it is present. A matching batch before a divergent acceptance in the same L1 block is not that block's final state, and the watchdog skips replay when the finalized inclusion block is unchanged, so it does not subsume this wire-identity detector. Nor does the check subsume the watchdog's broader, independent application-state comparison. -- **Depended on by:** standard recovery never running on a diverged frontier - (a flush and cascade there would compound the divergence); egress never - publishing a diverged landing; cockroach recovery as the only remedy. +- **Depended on by:** egress never publishing a diverged landing; cockroach + recovery as the only remedy. - **Breaks:** silent permanent scheduler/sequencer divergence, the theft-equivalent failure. - **Baseline-aware frontier:** accepted-batch scanning starts strictly after - the immutable baseline L1 block `C`, seeded with anchor nonce `N'`; the - prefix through `C` is opaque - ([I9](#i9-acceptance-identity-accepted-nonce-n-means-our-valid-batch-n)). - Later scans resume after the last acceptance. Rebuild defers the projection - until the complete baseline and anchor are published. - -### I16. The batch tree has exactly one valid parentless root, carrying the deployment's anchor nonce - -- **Holds:** every batch's nonce is - `parent.nonce + 1`, except the single parentless root, which carries the - `batch_tree_anchor` nonce — `0` for a genesis deployment, `N'` for a - cockroach-recovered one (`setup --recovery` writes the anchor before the - `setup_complete` marker). The first Tip is that root (there is no separate sentinel batch): plain - setup leaves its creation to startup recovery; rebuild creates it at `C` in - the baseline transaction. A fully-torn cascade re-roots parentless at the - same anchor via `open_fresh_tip_in_tx`'s `parent = None` path, after - invalidating the old root — so only one *valid* parentless root ever exists, - invalidated ones coexisting. -- **Enforced by:** the parent foreign key (enabled on every writer) rejects - dangling parents; `trg_enforce_nonce_contiguity` checks nonce succession. - Its parentless arm is an - *exact* match `nonce == (SELECT nonce FROM batch_tree_anchor)` (tighter - than a bare "must be 0"), plus an at-most-one-valid-parentless-root guard - scoped to `invalidated_at_ms IS NULL`; `compute_next_nonce(None)` reads the - same anchor; `trg_batch_tree_anchor_write_once` freezes the anchor once - `setup_complete` exists. + the era's stop block `C`, seeded with anchor nonce `N'`; the prefix through + `C` is opaque + ([I9](#i9-acceptance-identity-accepted-nonce-n-means-our-batch-n)). + Later scans resume after the last acceptance. Storage maintains the + accepted frontier exactly when the era row exists, reading the batch + submitter from the pinned deployment identity: before the era (every setup + sync, genesis included) there is no stop block or anchor, and the old + instance's batches would read as `Foreign` and falsely freeze the frontier. + The first append after `publish_era`, `run`'s first sync, judges every + submitter input past `C`, whichever sync ingested it. + +### I16. Batches carry contiguous, unique nonces from the era's anchor + +- **Holds:** a batch's key is its nonce, and the batches carry the nonces + `N'`, `N' + 1`, … where `N'` is the era's `anchor_batch_nonce` — the + scheduler's next batch nonce at the end of the era's stop block, which + setup's rebuild ends with and publishes in the era row. A new batch carries + the last nonce + 1, or the anchor when there is no batch: the era's first + batch (setup leaves its creation to `run`'s start; there is no sentinel + batch). Each nonce names one batch, so no nonce is reused inside an era. No + batch exists before the era does. +- **Enforced by:** `trg_batch_insert`, a BEFORE INSERT check that + `nonce IS COALESCE(last nonce + 1, era anchor)` (with neither, it + compares against NULL and aborts); `compute_next_nonce` computes the same + rule and fails when the era is absent; `batches.nonce` as the primary key; + `trg_batch_lifecycle`, which keeps a nonce fixed + once inserted; and the era row's immutability (`trg_era_immutable`, `trg_era_not_deletable`, + `trg_era_single_insert`), so a live deployment is never re-anchored. - **Depended on by:** the submitter resuming at the right nonce — `run` submits - `valid_closed_batches` with `nonce >= frontier_nonce`, where `frontier_nonce` - defaults to the anchor (`= N'`) while `safe_accepted_batches` is still empty - after recovery, so the submitter starts at `N'` rather than 0; the recovery - fill roots the rebuilt tree at `N'` without replaying history. (`N'` is fold-derived from trusted - checkpoint nonce `N`; wrong-low and wrong-high `N` are outside the supported - checkpoint model — see - [`docs/recovery/cockroach.md`](recovery/cockroach.md#data-dictionary).) -- **Breaks:** a tree mis-anchored at the wrong nonce ⇒ `run`'s first batch - carries a nonce the scheduler rejects ⇒ the sequencer is wedged (never - submits), or — worse, if defenses were absent — a recovered tree silently - diverging from canonical L1 state. + `closed_batches` with `nonce >= frontier_nonce`, where `frontier_nonce` + defaults to the anchor while no batch is accepted, so the + submitter starts at the anchor; nonce lookups such as the content-identity + check (I9), which expect one batch per nonce. +- **Breaks:** batches mis-anchored at the wrong nonce ⇒ `run`'s first + batch carries a nonce the scheduler rejects ⇒ the sequencer is wedged (never + submits), or — worse, if defenses were absent — a recovered deployment + silently diverging from canonical L1 state; two batches at one nonce ⇒ + a nonce lookup picks an arbitrary one. ### I17. `WriteHead` is a coherent cache of the durable open Tip/frame @@ -375,11 +426,14 @@ by writer and are write-once (`0001_schema.sql`). discards it and reloads from SQLite. - **Enforced by:** the lane being the only open-state writer; `load_current_write_head` being the single constructor for persisted state; - the `Storage::append_executed_user_ops_chunk`/attributed `close_*` update - ordering; and the Tip, - frame-position, FK, and PK constraints that fail loud on dangerous stale - cache writes. Direct-input uniqueness is enforced in the current application sequence; - invalidation removes the old row before recovery can reuse its source. + the head being opaque outside storage (private fields, read accessors, + storage-private mutators); the + `Storage::append_executed_user_ops_chunk`/attributed `close_*` update + ordering; storage reloading the Tip and comparing it with the head (batch, + nonce, frame, counts, fee, safe block) at every frame rotation and batch + close; and the Tip, frame-position, FK, and PK constraints that fail loud on + dangerous stale cache writes. Direct-input uniqueness is enforced in the + current application sequence. - **Depended on by:** the hot path avoiding a redundant SQLite re-read on every chunk; batch-size/frame counters; safe-block drain attribution; every storage method that trusts the passed head. @@ -391,49 +445,60 @@ by writer and are write-once (`0001_schema.sql`). simplify the lane, but is an independent benchmarked change rather than part of the lane-reconciliation cutover. -### I18. History identity is published with the complete baseline - -- **Holds:** file-first setup publishes `(EraId, generation=0, K, C, N')`, the - baseline snapshot, any recovery root, and setup completion in one FULL - transaction. The history row is absent before this boundary. `K` and `C` - remain immutable even after baseline artifact GC or recovery-root invalidation. -- **Standard recovery:** one generation increment iff a valid batch is - invalidated, with an immutable cut at the count after suffix deletion and - before replacement directs. The entire transition commits in the cascade - transaction. Clean restart changes neither token. Every intervening cut is - required to authorize reusing a checkpoint from an older generation. -- **Enforced by:** `complete_baseline_setup`, immutable baseline and - `history_generation_cuts` triggers, the exact-`+1` generation trigger requiring - its cut, and `cascade_and_reopen`. `preserved_input_count_in` asserts that - every intervening generation has a cut before computing compatibility. -- **Depended on by:** mandatory snapshot-derived WS claims and `/history` - checkpoint compatibility across standard recoveries. Identity is validated - before the requested count, including for empty history. Cuts remain available - for the era's lifetime; their absence must never authorize a partial minimum. -- **Breaks:** a client silently resumes a replaced suffix or inaccessible prefix. -- **Operational boundary:** rebuilding uses a fresh/wiped data directory. - Checkpoint state, inclusion block, and next nonce are trusted operator inputs; - neither clone detection nor distributed fencing is implied. +### I18. History identity is published with the era + +- **Holds:** file-first setup publishes the era row + `(EraId, published at, K, E, C, first queued input, anchor N')` and the + baseline snapshot at `E` in one `IMMEDIATE` transaction (`publish_era`), + with no batch in the database; the row's existence is setup's completion. + The era row is absent before this boundary. Its fields remain immutable even + after baseline artifact GC. The era id is the history's whole identity: + inside an era the history only appends (I5), so a claim is an era id and a + count, every count from `K` to the head resumes, and a restart changes + nothing. +- **Rebuilt history:** `rebuilt_history` gains one row per executed + transaction, written once with its count, each count the last one plus one + and each direct after every earlier direct in input order, only while the + era row is absent; no row is ever updated or deleted, and a skipped user op + has none. The era's `K` and `E` are exactly the span of the rows, no rebuilt + direct lies at or past its first queued input, which is at most the input + count through `C`, and the live rows continue at `E`. A client of an older era + adopts this one only at a count in `[K, E]` + ([adopting a new era](protocol/application-history.md#adopting-a-new-era)). +- **Enforced by:** `publish_era`, the era triggers (`trg_era_fits`, + `trg_era_single_insert`, `trg_era_immutable`, `trg_era_not_deletable`), the + rebuilt-history triggers (`trg_rebuilt_history_insert`, + `trg_rebuilt_history_immutable`, `trg_rebuilt_history_not_deletable`). +- **Depended on by:** mandatory snapshot-derived WS claims, the adopt rule, + and the feed's pages across the rebuilt/live seam. Identity is validated + before the requested count, including for empty history. +- **Breaks:** a client silently resumes another era's history or an + inaccessible prefix. +- **Operational boundary:** setup refuses a data directory that holds anything + but its lock file (exit 30) before writing anything, and runs every L1 check + that can fail retryably first, so a retry finds the directory empty; a setup + that fails later leaves a directory to discard. `publish_era` refuses a + completed database or one holding a batch. The base it starts from is + trusted; neither clone detection nor distributed fencing is implied. ### I19. Application progress follows the shared execution contract -- **Holds:** `ApplicationProgress` is the pair - `(ExecutedInputCount, last_executed_safe_block)`. Count zero implies clock - zero. A successful canonical application input returns its pre-execution - count as the offset and commits exactly `(count + 1, max(clock, - input_clock))`; rejection changes neither field. `AppError` is fatal and - defines no canonical successor. -- **Enforced by:** the application owns and persists the progress pair and - returns it by value. All execution consumers use the shared boundary, which - preflights count overflow and asserts the exact expected successor after a - successful hook. Validation purity and native-engine mutation remain - self-trusted; progress ownership does not require a Rust-side mirror. -- **Depended on by:** the canonical scheduler, inclusion lane, catch-up, - recovery fold, cockroach base `K`, durable execution attribution, and the - versioned replica protocol. -- **Breaks:** an input can be applied without advancing history, an offset can - advance twice, or recovery can derive the wrong checkpoint clock — silent - application-history divergence. +- **Holds:** the application's progress is its L2 tx count (`L2TxCount`) alone. + An executed transaction returns its pre-execution count as its index + (`Executed::Done { index, .. }`) and leaves the count at exactly + `count + 1`; an executed user op also leaves its sender's nonce at exactly + `nonce + 1`. A skipped op (wrong nonce) changes nothing. `AppError` is fatal + and defines no canonical successor. +- **Enforced by:** the application owns and persists the count and nonces. + Every execution consumer calls `sequencer_core::application::execute`, which + compares the op's nonce, preflights count overflow, and asserts both + successors after a successful hook. Admission purity and native-engine + mutation remain self-trusted; the count does not need a Rust-side mirror. +- **Depended on by:** the canonical runner, inclusion lane, catch-up, durable + execution attribution, and the era-checked replica protocol. +- **Breaks:** a transaction can be applied without advancing history, an + offset can advance twice, or an op can run at a nonce the canonical machine + skips — silent application-history divergence. - **Scope:** application-specific mutation and determinism remain self-trusted. A failing hook is not rolled back; every production caller terminates that path and discards the instance. @@ -441,47 +506,50 @@ by writer and are write-once (`0001_schema.sql`). ### I20. Application history is committed with its execution receipts - **Holds:** `application_inputs` contains every current included user op and - external direct exactly once, keyed by mandatory pre-execution count. Its - source reference, owning batch/frame, and payload tables reconstruct replay. - There are no entries for batch envelopes or the opaque baseline prefix. + every external direct within the application's payload cap + (`scheduler_limits().max_direct_payload_bytes`) exactly once, keyed by + mandatory pre-execution count. A direct over the cap never executes and has + no row; one SQL predicate (`DIRECT_EXECUTES_SQL`, `storage/queries.rs`) + selects the directs to execute in every query, and it reads both its + operands, the submitter and the cap, from the pinned `deployment_identity` + row. Setup pins the cap, and the staleness window with it; `run` refuses a + binary whose cap or window differs. A + direct's source reference, owning batch/frame, and payload tables + reconstruct replay. + There are no entries for batch envelopes, and none below the rebuilt + history end `E`: the era's rows before it are `rebuilt_history`'s + ([I18](#i18-history-identity-is-published-with-the-era)). - **Creation atomicity:** user-op source rows and application rows commit in the same FULL chunk transaction that authorizes acknowledgements. Direct inputs commit with the complete frame rotation, after checking all execution receipts. - Startup/recovery create leading direct rows before restoring the engine; + Startup creates leading direct rows before restoring the engine; successful catch-up is required before admitting that sequence. -- **Recovery:** source records remain, while invalidation deletes the current - suffix. Replacement rows reuse counts under the incremented generation. +- **Append-only:** no application input is ever deleted (I5). - **Snapshot/replay agreement:** a batch-close snapshot records storage-derived `H`; the restored engine must report the same count. Each replay row must - match the engine's next count and executes with its persisted frame fee/clock - (or the direct input's source block). Missing rows or count mismatches fail - loud; they are never repaired or backfilled. + match the engine's next count and executes with its persisted frame fee and + safe block (or the direct input's index, block and timestamp). Missing rows + or count mismatches fail loud; they are never repaired or backfilled. - **Enforced by:** shared execution receipts, storage append APIs, PK/FK/XOR/ uniqueness and contiguous-offset triggers, coherent canonical pages, and catch-up checks. - **Performance boundary:** head discovery uses the integer primary-key maximum; - replay seeks directly by offset and joins bounded source rows. Neither scans - invalidated history. Chunk insertion adds no durability transaction or actor. + replay seeks directly by offset and joins bounded source rows. Chunk + insertion adds no durability transaction or actor. ## Do-not-simplify (deliberate shapes that look like cleanup targets) The refactorer-facing mirror of the register above — each of these *looks* like a simplification and would break a registered invariant: -- **Don't move filesystem work into `storage/snapshot_dumps.rs`** — the +- **Don't move filesystem work into `storage/snapshots.rs`** — the module boundary *is* the GC crash-ordering guarantee (I13). -- **Don't reorder `check_danger`'s arms** or merge its two `find_*` helpers - into one that consults the Tip first — the closed-frontier-first order is - the dispatch table's meaning (I4). - **Don't derive application order from L1 positions** — optimistic user ops precede their envelope and have no general one-to-one L1 mapping. - **Don't retain a numeric resume offset without its history identity** — - recovery deliberately reuses suffix counts (I18). -- **Don't discard a valid snapshot beyond the accepted frontier** — that exact - batch can become the next required recovery checkpoint (I7). + a new era reuses counts (I18). +- **Don't discard a snapshot beyond the accepted frontier** — that exact + batch can become the next comparison checkpoint (I7). - **Don't add internal retry loops to the flusher/submitter for provider errors** — the orchestrator respawn is the retry mechanism; internal - retries mask exactly the failures the danger machinery routes on. -- **Don't unify the two staleness references** (inclusion-relative vs - current-relative) — deliberately different formulas for different - questions. + retries mask the failures the exit codes and the landing lag report. diff --git a/docs/l1-fee-policy.md b/docs/l1-fee-policy.md index 0602c721..87210785 100644 --- a/docs/l1-fee-policy.md +++ b/docs/l1-fee-policy.md @@ -35,13 +35,12 @@ estimate: base-fee changes are protocol rules, while tips reflect the fee market and builder choices. Clearing within minutes is an operational expectation to measure, not a bound this policy establishes. -The danger detector stops normal operation when the configured danger -threshold is reached. That bounds continued soft-confirmation issuance under -the detector's assumptions; it does not bound inclusion or recovery duration. -[Recovery](recovery/README.md#closed-batches-flush-sync-cascade) requires all covered -wallet slots to resolve at safe depth before a cascade can proceed. The -flusher's fixed headroom can also fail to replace an unmineable original. -The sequencer remains offline until recovery succeeds or the operator acts. +Nothing in the sequencer stops intake while batches wait to land. If no batch +lands inside the staleness window, the era ends and the sequencer stops for +good ([batch landing and outages](recovery/README.md#the-void-history)). This +policy bounds neither the time to inclusion nor that outcome. The flusher's +fixed headroom can also fail to replace an unmineable original, which keeps a +flush, and the setup that runs one, waiting until the operator acts. The poster warns after a nonce has been unresolved for five minutes and at most once per further interval. Accepted re-broadcasts do not reset its age. @@ -59,10 +58,10 @@ funding pressure, and recovery incompatibility. These failures motivate the current choice; they do not prove that every bounded replacement policy would be unsound. -Revisit with evidence of sustained batch delays, recovery frequency or -duration, and submission cost. Record the logs and affected L1 block range. -Any future escalation policy needs an explicit urgency trigger, spending -budget, and compatible recovery pricing. A rejected replacement alone is +Revisit with evidence of sustained batch delays, landing lag near the window, +and submission cost. Record the logs and affected L1 block range. +Any future escalation policy needs an explicit urgency trigger and a spending +budget. A rejected replacement alone is insufficient motivation. Fee-policy tests must distinguish [geth's two-component replacement diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index a2b43f9e..fafbe806 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -12,40 +12,36 @@ freely at this stage — no backward-compatibility constraints. | # | Track | Owner | Status | |---|-------|-------|--------| -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented; client adoption and deployment work remain; frozen until the shared recovery tool and a new client sync** — [scope](#track-3--feed--replay-protocol-redesign) | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented; client adoption and deployment work remain; frozen until a new client sync** — [scope](#track-3--feed--replay-protocol-redesign) | | 5 | Fee exponentiation LUT | us | **unblocked, needs Bart** — fees stay in log space; the batch-level alternative is deferred with [batch-scoped replay protection](2026-10-batch-scoped-replay-protection.md). Shape decided: exact-floor table, algorithm-free; a consensus change agreed with Bart before implementation; the log-additive `batch_policy` design is frozen until then | | 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | **Current campaign order:** -1. Bart's client integrates the feed and treats cockroach recoveries as out of scope; we fix bugs it finds, and other feedback is input to the new sync. A shared recovery tool, still to be designed, will produce both the sequencer's and the client's startup state from a checkpoint. Until it lands, `setup --recovery`, `fold.rs` and the cockroach reader seams get bug fixes only. -2. Application integrators/operators validate the private engine, canonical-to-native exporter and recovery drill, and representative capacity before production use. Reference C-host lifecycle coverage is part of repository CI. +1. Bart's client integrates the feed and treats cockroach recoveries as out of scope; we fix bugs it finds, and other feedback is input to the new sync. [Sequencer recovery](2026-11-sequencer-recovery/README.md) is built: `setup --base ` rebuilds the sequencer from a canonical machine's exported state through `from_canonical_state` ([cockroach recovery](../recovery/cockroach.md#what-setup-does)), and each client realigns through the feed from its own count. Its devnet round-trip drills run in CI, and the rest is in its [to do](2026-11-sequencer-recovery/todo.md). +2. Application integrators/operators validate the private engine, its `from_canonical_state` and the recovery drill, and representative capacity before production use. Reference C-host lifecycle coverage is part of repository CI. 3. Track 5 (fee LUT) after a sync with Bart, batched with other guest-image changes. -Full restore archives now support file and directory application prefixes. +Full restore archives support file and directory application prefixes. Additional snapshot retention or transport mechanisms require a measured consumer need. ## Track 3 — Feed & replay protocol redesign -The egress API is implemented. The [history contract](../protocol/application-history.md), -the [projection contract](../protocol/projection-replay.md) and the -[API contract](../../README.md#api) own its behavior; the -[integration plan](2026-07-track3-feed-replay-design.md) owns deployment follow-ups. - -A shared recovery tool will produce the sequencer's and a client's startup -state from one checkpoint, and a new client sync follows its design. Until -then, what they replace gets bug fixes only, and its docs are not polished: - -- Deleted when the tool lands: projection replay across a rebuild - (`/historical-l1-inputs`, the SDK's `historical_l1_inputs`, the main - `historical_bootstrap` test, the projection contract, and the integration - plan). First move the projection contract's seed/replay rules and worked - boundary into the tool's design and tests, move the integration plan's - deployment follow-ups to the register, and repoint the links to both files. +The egress API is implemented. The [history contract](../protocol/application-history.md) +and the [API contract](../../README.md#api) own its behavior; the +[register](../review/register.md#integration-work-owned-elsewhere) owns deployment follow-ups. + +[Cockroach recovery](../recovery/cockroach.md) rebuilds the sequencer from a +canonical machine's exported state, and each client realigns through the feed +from its own count, so the sequencer serves no raw L1 inputs for a client to +replay. A new client sync follows that design. Until then, what it +replaces gets bug fixes only, and its docs are not polished: + - Reshaped or retired with the new sync: the WS wire format, the SDK snapshot-then-subscribe bootstrap, `/latest_snapshot` and its history headers, - `/history`'s baseline, accepted checkpoint and compatibility query, and the - era/generation identity types. In-era generation signalling survives in some form. + and `/history`'s baseline and accepted checkpoint. Within an era the + history only appends + ([history identity](../protocol/application-history.md#identity-and-available-history)). Not frozen: the ingress routes, `/finalized_state` and its probes, `/finalized_snapshot`, download leases, and the canonical page reader with `HistoryClaim`. @@ -73,24 +69,21 @@ table with Bart and ship it with the other guest-image changes. The [Application contract](../protocol/application-contract.md) owns execution, engine progress, and checkpoint semantics. The [C binding guide](../protocol/c-application-binding.md) maps that contract to native engines. Its reference conformance suite and C-host -process scenarios cover snapshot-to-live bootstrap, restart, standard recovery, -and fresh-era rebuild with canonical comparison. Reference bridge conformance -cannot establish private-engine correctness. +process scenarios cover snapshot-to-live bootstrap, restart, and the rebuild +from genesis and from a stored machine. +Reference bridge conformance cannot establish private-engine correctness. Remaining checks need the actual consumer: -- Supply the application's versioned canonical-machine-to-native recovery - exporter and completed operator runbook. Require the - [non-genesis recovery drill](../recovery/cockroach.md#recovery-readiness-before-deployment) - for production readiness: the old native state is unavailable, the exported - bundle restores correctly, and execution after rebuild matches the canonical - machine. For the DEX, pin the designated state drive/memory region and derive - resume metadata from canonical execution. The exporter must check - [pending-direct eligibility](../recovery/cockroach.md#checkpoint-eligibility) - and the drill must exercise refusal and earlier-checkpoint fallback, alongside - eligible pending-queue recovery. Add the integration check to the - release validation once the actual artifacts are available; no generic trait - or deployment gate currently enforces this requirement. +- Supply the application's `from_canonical_state` and its `state` inspect + answer, and the completed operator runbook. Require a recovery drill from a + stored machine for production readiness: the old native state is + unavailable, `setup --base` builds the engine from the machine's state, and + execution after the rebuild matches the canonical machine, with directs + queued at the stop block. For the DEX, pin the designated state drive/memory + region. Add the integration check to the release validation once the actual + artifacts are available; no generic trait or deployment gate currently + enforces this requirement. - Repeat snapshot-to-live bootstrap and canonical comparison with the private engine's genesis and meaningful accepted/rejected inputs. The reference `c_host_` scenarios supply the lifecycle pattern; compare canonical state @@ -108,7 +101,7 @@ Remaining checks need the actual consumer: Proposed: an `Application` nonce query plus an enumeration method. Setup already holds the baseline engine when it publishes the baseline, so it can enumerate it into a write-once `baseline_nonces` table in the same - transaction, the user-nonce counterpart of `batch_tree_anchor`. The point + transaction, the user-nonce counterpart of the era's `anchor_batch_nonce`. The point query also lets the shared execution boundary check the one-step nonce advance per op, as it checks progress. Both need C ABI exports and the engine owner's agreement. diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md deleted file mode 100644 index a4ed71df..00000000 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ /dev/null @@ -1,120 +0,0 @@ -# Feed and replay integration (Track 3) - -The application-history protocol is implemented. Its current contracts live in: - -- [Application history and replay](../protocol/application-history.md): coordinates, - storage/recovery boundaries, snapshot bootstrap, and consumer resume. -- [README API](../../README.md#api): routes, wire messages, refusal codes, and limits. -- [Snapshot lifecycle](../snapshots/lifecycle.md): durable artifacts, accepted - comparisons, recovery exports, and leases. - -## Merge scope and follow-up ownership - -The repository delivery includes the egress contracts, storage, SDK, and reference -replay/recovery tests. Its merge criteria are review and the relevant repository -checks. Private-engine integration, operational rehearsal, and representative -capacity measurements are follow-up work; they do not block merging this API. -Merge the implementation and let consumers integrate. Bugs they find are fixed; -other feedback is input to the new client sync -([Track 3](2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign)). -There are no live deployments requiring compatibility. - -| Follow-up | Owner | When it is needed | -|---|---|---| -| Private DEX indexer and database backups | Bart / application integration | While adopting the API. Report bugs for fixing; missing fields or awkward workflow feed the new client sync. | -| Canonical-to-native export and incident rehearsal | Application integration and operators, under Track 6 | Before relying on that application's recovery procedure in production. | -| Ingress latency, indexing headroom, and recovery capacity | Sequencer/application maintainers and deployment operators | Before claiming support for the target deployment workload; measure historical serving alongside ordinary traffic. | - -The [validation record](../review/2026-09-16-track3-validation.md) records the -wallet's nonempty cold bootstrap, concurrent replay/live delivery, recovery and -rebootstrap, canonical-machine gates, and local latency measurements. Those -results do not establish private-engine conformance or target-deployment capacity. - -The reference C host has process coverage in the `c_host_` rollups E2E scenarios: -generated genesis, source-independent `EngineApp` snapshot restore, concurrent -backlog/live replay, clean restart, stale recovery, and a fresh-era rebuild. -Ordinary execution and recovery compare against the canonical machine. The -[C binding guide](../../bindings/c-app-engine/README.md#reference-wallet) owns -the commands and scope. This closes the reference-host follow-up; the private -engine and canonical-to-native exporter still require their own evidence. - -## Application projections and recovery - -Historical bootstrap and standard-recovery checkpoint reuse are implemented. -Bounded internal readers can keep additional -application-specific transfers, orders, deals, and portfolio history outside the -sequencer's application state. The client owns indexing, complete checkpoints, -and replay. The sequencer owns optimistic ordering; the scheduler remains the -canonical authority. - -Current contracts live in the [projection replay guide](../protocol/projection-replay.md) -and [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only). -`/history` supplies deployment/baseline/current-generation metadata and a coherent -accepted checkpoint receipt. `/historical-l1-inputs` serves the immutable raw -prefix through the era's stop block, with block seek, bounded pages, and typed -era refusal. `/history` also checks a saved generation against every intervening -recovery cut. The SDK exposes both reads. Standard recovery records cuts in its -existing transaction before replacement directs are inserted. - -The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) -restores a complete wallet/projection checkpoint, replays one-record HTTP pages -through the scheduler, exercises a malformed-batch overdue drain and terminal -drain, and subscribes at nonzero `K`. It checks application state and explicit -projection order, including a same-block pending direct and the first live -input. Storage/API/SDK tests cover limits, oversized single inputs, fixed prefix -boundaries, era/generation behavior, errors, and response deadlines. This is -reference evidence, not private-engine conformance or a deployment benchmark. - -The [checkpoint compatibility test](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) -uses guarded recovery, complete wallet/projection backups, HTTP compatibility -lookups, and WS replay. It distinguishes equal counts from different generations, -restores the nearest eligible checkpoint after missed recoveries, and retries a -recovery between lookup and subscription. Storage tests cover cuts before -replacement directs, empty/no-op invalidation, nonzero baselines, and rollback. - -### Goals and acceptance criteria - -| Goal | Status / remaining outcome | -|---|---| -| Historical bootstrap | Implemented: replay the complete fixed prefix and join the feed at `K`, including after a nonzero rebuild. | -| Standard recovery | Implemented: choose the nearest retained checkpoint whose prefix survived every intervening generation change, then restore/resubscribe. | -| Cockroach reader recovery | Core replay/metadata workflow and reference checkpoint restore implemented. Superseded for clients by the planned shared recovery tool; bug fixes only, with no client follow-up. | -| Stable coordinates | Existing era/generation/application count for claims; separate InputBox indices for raw paging. Counts alone do not certify cross-era compatibility. | -| Bounded serving cost | Page/item/response bounds implemented; representative bootstrap must preserve the ingress latency target and demonstrate catch-up headroom. | -| Operational readiness | Each production application supplies its canonical-to-native exporter, runbook, and non-genesis recovery drill under Track 6. | - -### Follow-up sequence - -1. **Consumer adoption.** Bart's client follows the feed and treats cockroach - recoveries as out of scope. A planned shared recovery tool will produce both - the sequencer's and the client's startup state from a checkpoint, so clients - never resume across a rebuild; new client work should not build on the - projection-replay path, which receives bug fixes only. -2. **Deployment readiness.** Validate the private native engine and measure - latency, historical serving cost, projection throughput, - catch-up headroom, and recovery time. Track 6 independently owns the versioned - canonical-machine exporter and native-state-unavailable drill. - -### Scope boundaries - -Keep the existing recovery terminal drain: the first accepted resumed frame -accounts for old pending directs before its user ops. Replacing that mechanism -would require carrying pending work across the baseline and is not justified by -this consumer requirement. - -Keep manual cross-era trust selection. Execution-prefix hash chains can identify -an input trace but cannot prove that an engine or indexer computed correct state; -revisit only if automated matching or measured reconstruction costs justify them. -Server-side scheduler replay producing a flattened execution archive and -submitter/key rotation remain separate future work. No client checkpoint -registration, server-side projection storage, or historical execution archive is -required by this design. - -## Revisit only with a consumer need - -- Resumable snapshot transfer: when artifact size makes interrupted downloads costly. -- Session fencing: if history can mutate within an admitted process or multiple - local writers become supported. - -These are triggers for design work, not promised APIs. The -[coordination plan](2026-07-coordination-tracks.md) owns cross-track priorities. diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index bc1b4ce5..e25e7f7b 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -1,7 +1,7 @@ # ADR: The Runtime Authority Boundary The architecture decision record for how authority — over speculative state, -promises, process lifetime, and recovery admission — is owned in the +promises, process lifetime, and admission to run — is owned in the sequencer. The mechanisms and their reasons below describe the current design; Git preserves the earlier proposals and review history. @@ -15,8 +15,8 @@ The policy separates into three guarantees: 1. **G1 — terminal stop:** a diagnosed terminal runtime fault aborts the process without worker drain or database settlement. -2. **G2 — no silent fast-path:** every boot inspects durable facts and runs - recovery, regardless of the previous process's verdict. +2. **G2 — no silent fast-path:** every boot inspects durable facts and syncs + L1, regardless of the previous process's verdict. 3. **G3 — scoped divergence freeze:** a committed canonical-divergence fact freezes its persisted acceptance domain immediately. The mechanism, its runtime reaction, its race bound, and the watchdog boundary are stated @@ -34,8 +34,8 @@ may still complete remotely; the zombie-transaction model accounts for them. Every command acquires the OS-held exclusive data-directory lock before inspection (`runtime/process_lock.rs`). For `run`, ownership transfers into a `RuntimeScope` containing the lock and a `ShutdownSignal`. Workers may hold -the scope or the lock and signal separately. Operator shutdown, expected -recovery, and transient exits signal all workers and drain them normally. +the scope or the lock and signal separately. Operator shutdown and transient +exits signal all workers and drain them normally. Terminal runtime errors call `abort_terminal`: emit a diagnostic and call `std::process::abort`. They do not signal a graceful drain, write a terminal @@ -64,64 +64,64 @@ The reader can cancel a pending RPC read, but awaits any started SQLite append before joining, so the final clean-exit divergence check sees every committed sync. -Runtime construction is prepare → admit → launch: fallible or awaited -dependency preparation happens before workers launch; final admission checks one -consistent fact set; launch spawns every worker in one infallible, -non-yielding block, consuming the single-use `RuntimeAdmission` witness. A -preparation failure cannot leave a partially launched runtime, and no -refusal or retry can mint the witness. This boundary does not promise that -worker initialization succeeds: application restore and catch-up run inside -the launched lane. +Runtime construction is prepare → launch: fallible or awaited dependency +preparation happens before workers launch, and launch spawns every worker in +one infallible, non-yielding block. A preparation failure cannot leave a +partially launched runtime. Under the process lock, with no worker running, +nothing changes the facts startup last checked before launch. This boundary +does not promise that worker initialization succeeds: application restore and +catch-up run inside the launched lane. ### 2. Fact-derived admission and the terminal-fault black box -Admission is governed by three facts, each with one owner: the kernel -process lock (concurrent owners), two-sided `setup_complete` (command -ordering), and `canonical_divergence` (the one absorbing refusal — only a -fresh-directory cockroach rebuild proceeds); baseline schema and history -creation and setup completion are each one `synchronous=FULL` transaction. +Admission is governed by four facts, each with one owner: the kernel +process lock (concurrent owners), the era's existence, two-sided (command +ordering), and the two terminal facts, `canonical_divergence` and the void +history, derived from the batches and the safe head (absorbing refusals — +only a fresh-directory cockroach rebuild proceeds); baseline schema creation and +the era's publication (which is setup's completion) are each one +`synchronous=FULL` transaction. There is no lifecycle admission state machine or operator acknowledgement. -Standard recovery is automatic, and restart policy after a terminal fault -is the exit contract (30 or SIGABRT = do not restart, page), which the +Restart policy after a terminal fault is the exit contract (30 or SIGABRT = do not restart, page), which the supervisor is expected to honor. `terminal_faults` stores append-only causes for terminal errors returned through a command bracket, best-effort and verdict-neutrally. Runtime aborts -do not pass through that bracket and leave only process diagnostics. Nothing -reads the black box for decisions. +do not pass through that bracket and leave only process diagnostics. It keeps +causes that would otherwise be lost, so a terminal fact is not copied into +it: the database holds the fact and every `run` states it. Nothing reads the +black box for decisions. A known-terminal fault refuses at re-detection rather than at a boot gate. Every fault whose evidence the boot path reads re-refuses before the first soft confirmation; the residual window is recorded in the threat model. -The honesty backstops (rollbackable soft confirmations, the watchdog +The honesty backstops (revocable soft confirmations, the watchdog byte-compare, and the divergence freeze) do not depend on a boot gate. A durable gate on the previous verdict would require operator acknowledgement without adding evidence about the current facts. A full integrity sweep would still miss semantic faults outside its read set. Revisit admission checks for a specific detectable fault, rather than treating a previous verdict as proof. -### 3. Ordered startup recovery - -Normal `run` startup inspects local terminal facts, syncs L1, selects a repair -from current facts, and checks the result. The flush branch orders flush → -sync through the returned safe block → cascade explicitly. There is no -phase driver or progress ledger; the flush witness is a local value. A restart -must obtain fresh flush/sync evidence; persisting a phase would let it skip -work based on an earlier attempt's observation. -Setup/rebuild, maintenance flush, and normal-run recovery retain distinct -typed controllers because they establish different facts; a universal -controller would represent combinations none of those commands needs. -The dispatch table, boot-local witnesses, and final -admission check are owned by -[`docs/recovery/README.md`](../recovery/README.md); -[`admission.tla`](../recovery/admission.tla) verifies the controller ordering. +### 3. Ordered startup + +Normal `run` startup checks the terminal facts on the database as it is, +before any L1 call; syncs L1; checks them again; and opens a batch if none is +open, in one guarded transaction. It repairs, flushes and replaces nothing, +and keeps no phase driver or progress ledger: every start derives the same +answer from the same facts. +Setup/rebuild, maintenance flush, and `run`'s start retain distinct typed +controllers because they establish different facts; a universal controller +would represent combinations none of those commands needs. +[What the sequencer does](../recovery/README.md#what-the-sequencer-does) owns +the sequence and what each check refuses. ### 4. SQLite-centered runtime and the two-regime inclusion lane SQLite is the durable coordination boundary between components. The input reader atomically commits `safe_inputs`, `l1_safe_head`, -`safe_accepted_batches`, and any `canonical_divergence` fact in one sync -transaction (a `setup --recovery` interim sync defers the frontier half); +`batches.accepted_input`, and any `canonical_divergence` fact in one sync +transaction (every setup sync defers the frontier half until the era's +anchor is published); the lane reads that durable projection and receives no in-memory cursor from the reader. The one deliberate exception is HTTP ingress ↔ inclusion lane (bounded MPSC + oneshot), because @@ -151,9 +151,9 @@ deployment assumes the application promptly digests the whole range only if production measurements disprove that). Authority remains role-local and auditable: a FULL-committed user-op chunk -authorizes its acknowledgement; a valid sealed batch plus the durable +authorizes its acknowledgement; a sealed batch plus the durable write-before-broadcast watermark authorizes an L1 submission; committed -version-checked application-input rows authorize the feed output. Effects handed to the network before process termination may still +era-checked application-input rows authorize the feed output. Effects handed to the network before process termination may still complete remotely. A global effect mutex or actor would duplicate these owners and put authority @@ -168,18 +168,17 @@ cannot establish, or the supported guarantee changes. ## External history ```text -HistoryVersion = (EraId, RecoveryGeneration) -HistoryPosition = (HistoryVersion, ExecutedInputCount) +HistoryClaim = (EraId, L2TxCount) ``` -`EraId` (UUIDv4, minted write-once in the baseline transaction) identifies a -setup/rebuild era; `RecoveryGeneration` increments exactly once in the -standard-recovery transaction iff it invalidates at least one valid batch; a -clean restart changes neither. The pair is an equality/discontinuity token, -not an ordered counter. Snapshot headers and mandatory WS claims expose these coordinates. Every -application row has its pre-execution count; recovery replaces only the current -suffix. See the [history contract](../protocol/application-history.md) and -[follow-up plan](2026-07-track3-feed-replay-design.md). +`EraId` (UUIDv4, minted write-once in the baseline transaction) is the +history's identity: it identifies a setup/rebuild era, inside which the +history only appends, and a clean restart leaves it unchanged. It is an +equality/discontinuity token, not an ordered counter. Snapshot headers and +mandatory WS claims expose the era id and the count. Every application row +has its pre-execution count. See the +[history contract](../protocol/application-history.md) and +[register](../review/register.md#integration-work-owned-elsewhere). ## Performance posture @@ -190,4 +189,4 @@ records exact revisions, workload, and same-host ACK/WS measurements. They are regression evidence: client/host contention and excluded startup or backlog work prevent interpreting them as deployment capacity. Representative latency, including checkpoint and L1-reconciliation overlap, remains a -[deployment follow-up](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership). +[deployment follow-up](../review/register.md#integration-work-owned-elsewhere). diff --git a/docs/plans/2026-09-watchdog-v2.md b/docs/plans/2026-09-watchdog-v2.md index 343c7f31..a49434b9 100644 --- a/docs/plans/2026-09-watchdog-v2.md +++ b/docs/plans/2026-09-watchdog-v2.md @@ -31,6 +31,19 @@ file keeps the remaining work. exit code 0. The test guest's `xhalt` can be shared. - **trolley exceptions.** Add an exception call to `sdk/guest/trolley`; the test guest raises its exception through `libcmt-sys` meanwhile. +- **Stop calling `verify_hash_tree`.** PR #56 runs it on the starting machine + of `init` and `replay`, but the emulator team asked that nothing call it on + v0.22.0 (Dave's `docs/todo.md`), and it catches corruption only, not hash + files rewritten to match ([hashing](../cartesi-machine.md#hashing)). Replace + it with the untrusted-load verification an upcoming emulator release brings. +- **A tie to the validators**, outside the current scope. The watchdog checks + the sequencer against its own replay from the operator's bootstrap, not + against what Dave's validators settled. At each settled epoch boundary it + could compare its own machine's root with `getLastFinalizedMachineMerkleRoot`. + The root is its own, so no untrusted load is involved, but the watchdog + would have to read the consensus contract and take a root at an input index, + often inside a block + ([Dave epochs](../cartesi-machine.md#rollup-host-semantics)). ## Open questions for the DEX integration diff --git a/docs/plans/2026-10-batch-scoped-replay-protection.md b/docs/plans/2026-10-batch-scoped-replay-protection.md index 89510eed..79e0d269 100644 --- a/docs/plans/2026-10-batch-scoped-replay-protection.md +++ b/docs/plans/2026-10-batch-scoped-replay-protection.md @@ -3,7 +3,9 @@ **Status:** deferred design, not implemented. The current rule is the per-sender nonce in the [application contract](../protocol/application-contract.md#user-nonces). Revisit when replay state must become prunable, or when the fee moves from the -frame to the batch. +frame to the batch. The first holds for libdex, which deletes session wallets +with their nonces, so the choice is open +([register](../review/register.md#canonical-rules-open-for-the-c-port)). ## Why @@ -20,7 +22,7 @@ batch-scoped design fixes that race too. **Signed coordinates.** The user signs `(b, i)` in place of a nonce: - `b` is the scheduler batch nonce the op targets: the canonical batch sequence - number, not the sequencer's local `batch_index`. + number. - `i` orders the sender's ops within `b`. **Look-back.** The sequencer adds an unsigned field `k` (2 bits, so `k` ∈ 0..=3) @@ -75,14 +77,14 @@ The per-sender `(b, i)` entry rejects the second inclusion. ## Recovery and invalidation -A stale batch does not consume its nonce, so a cascade rolls the open batch -back and replacements reuse the nonces. An op from an invalidated batch never -executed, but its signature stays executable while canonical `c <= b + 3`, -including when its `b` is above the rolled-back open batch. That breaks -re-signing, which today's nonce makes safe: +A stale batch does not consume its nonce, so a cockroach rebuild whose stop +block falls past it starts a new era whose batches reuse the nonces after it. +An op from a batch the rebuild revoked never executed, but its signature stays +executable while canonical `c <= b + 3`, including when its `b` is above the +new era's open batch. That breaks re-signing, which today's nonce makes safe: ``` -user signs X with (15, 0); batches 10..15 are invalidated; open batch is 10 +user signs X with (15, 0); batches 10..15 never land; the new era opens batch 10 user re-signs X' with (10, 0); it executes and stores (10, 0) canonical c reaches 15: X arrives with k=0, (15, 0) > (10, 0), accepted again ``` @@ -132,6 +134,7 @@ This is one coordinated contract change, agreed with the engine owner: ## Why deferred -The per-sender nonce works, and the engine owner no longer needs to prune -replay state. The change spans signing, wire format, scheduler, ABI and API at -once, so it waits for a concrete trigger. +The change spans signing, wire format, scheduler, ABI and API at once, so it +waits for a decision. The per-sender nonce works only for an application that +keeps every signer's nonce forever, which libdex does not +([libdex's accounts](../review/2026-11-12-libdex-answers.md#accounts-and-nonces)). diff --git a/docs/plans/2026-11-sequencer-recovery/README.md b/docs/plans/2026-11-sequencer-recovery/README.md new file mode 100644 index 00000000..c8a41597 --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/README.md @@ -0,0 +1,73 @@ +# Sequencer recovery + +**Status:** implemented in this tree: the free-standing scheduler and its C +export, the `Application` trait, the state export, +`sequencer setup [--base ] [--dry-run]`, the rebuilt history and +the feed's realignment. [todo.md](todo.md) holds what remains. It supersedes +two [archived plans](../../recovery/history/README.md#superseded-cockroach-recovery-plans). + +## The scenario + +The sequencer is gone. It had a bug, it served things that never landed, its +SQLite is corrupt, or its disk failed. We start from zero, with the old +instance kept down. + +1. **Ask each client for a state it trusts.** Bart names an L2 tx count: the + count of a backup he knows is right. A count is the only name a client has + for a state. +2. **Pick a base machine**, a Cartesi machine whose L2 tx count is at or below + every client's. By default it comes from our validator node at a settled + epoch, which the operator can check on chain. +3. **Run `sequencer setup --base `**, then `sequencer run`. +4. **Each client realigns at its own count**, on the feed it already uses. + +## The ideas + +- **The machine exports its state.** Setup reads the application's state and + the scheduler's: its counts, its batch nonce, its clock and where its queue + of directs starts. L1 holds the queued directs themselves. Nothing below the + machine is rebuilt. +- **The scheduler never reads the application.** From the machine's scheduler + state and the L1 inputs after it, the scheduler decides which transactions + are handed out, in order. Setup feeds it one input at a time and the + application executes each transaction as it is handed out. +- **Setup is one procedure from any base.** A new deployment starts from + genesis, a recovery from a machine. Recovery is needed exactly when L1 shows + the old submitter's activity past the base. +- **Clients realign through the feed.** The new era serves rows from the base + count: the rebuilt history first, then the live rows. There is no file, no + separate tool and no special client mode. +- **A client's count is checked where it is used.** Setup never sees it. + +## What to read + +The real docs hold what the code does: + +| For | Read | +|---|---| +| Whoever writes the guest or ports the scheduler | [Scheduler semantics](../../protocol/scheduler-semantics.md), every check and gate and the state export; the [application contract](../../protocol/application-contract.md) and its [C binding](../../protocol/c-application-binding.md); [`scheduler.h`](../../../bindings/c-scheduler/include/scheduler.h) | +| Whoever runs the sequencer or a client | [Cockroach recovery](../../recovery/cockroach.md), the procedure and the playbooks; the [incident runbook](../../watchdog/incident-runbook.md) | +| Whoever builds or reviews the sequencer's side | The [history contract](../../protocol/application-history.md#era-baseline), the era, the rebuilt history and the adopt rule; the [invariants](../../invariants.md) | + +This directory keeps the design's reasons and what remains: +[scheduler.md](scheduler.md), [application.md](application.md) and +[interfaces.md](interfaces.md) for the shared software; +[operations.md](operations.md), [sequencer.md](sequencer.md) and +[client-backups.md](client-backups.md) for the rest; [todo.md](todo.md) for +the work left. This is the sequencer's only recovery: it repairs nothing by +itself, and stops for good when its batches can no longer land +([batch landing and outages](../../recovery/README.md)). + +## Names + +| Name | Meaning | +|---|---| +| L2 tx count | How many L2 transactions a state has executed; it names the state ([scheduler](scheduler.md#what-counts)) | +| base | The state a setup starts from: a machine, or genesis | +| base count | The base's L2 tx count, where the new era's history starts | +| stop block | The safe L1 block setup pins after the flush; the rebuilt history ends with it | +| rebuilt history | The L2 transactions from the base count through the stop block, rebuilt from L1 | +| scheduler state | The next batch nonce, the scheduler clock and the fridge | +| scheduled transaction | A transaction the scheduler hands the application; a user op may still be skipped | +| executed transaction | A scheduled transaction the application executed; it has an L2 tx count | +| first queued input | The InputBox index of the oldest direct still queued, at the base and again at the end of the stop block | diff --git a/docs/plans/2026-11-sequencer-recovery/application.md b/docs/plans/2026-11-sequencer-recovery/application.md new file mode 100644 index 00000000..480cfdf6 --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/application.md @@ -0,0 +1,131 @@ +# The application and its machine + +**Status:** implemented in this tree, except that setup reads the +application's state only through the `state` inspect report, not the labeled +range ([register](../../review/register.md#sequencer-recovery-follow-ups)). +The [questions for Bart](todo.md#questions-for-bart) stay open. **New** marks +what changed from the earlier template. + +An application lives in two places: natively, inside the sequencer and its +clients, and in the canonical machine, the template every validator runs. A +recovery reads a machine's state and builds the native side from it, so both +sides carry obligations. The [scheduler](scheduler.md) owns the rules the +machine runs; this page owns what surrounds them: the machine's state export, +the `Application` trait's obligations, and the guest's duties. +[Interfaces](interfaces.md) outlines the signatures, in Rust and in C. + +## The state export + +**New.** Setup reads two things from a stopped machine. The canonical harness +answers the scheduler's part for every application, since it owns that state, +and the application answers only for its own state, in an encoding it chooses. + +| What | How setup reads it | Answered by | +|---|---|---| +| The application's state | The `state` inspect report; a read through the labeled range, as the watchdog can, is not built ([todo](todo.md#build)) | The application | +| The scheduler's state and the counts | The `scheduler` inspect query | The harness | + +The `scheduler` report is six little-endian `u64`s, in this order: + +```rust +pub struct SchedulerExport { + /// InputBox inputs the machine has consumed: its position. + pub l1_input_count: u64, + /// L2 transactions it has executed: the base count of a recovery from it. + pub l2_tx_count: u64, + pub next_batch_nonce: u64, + /// The latest block the scheduler has executed at. + pub clock: u64, + /// Index of the oldest queued direct; the L1 input count when none is. + pub first_queued_input: u64, + /// How many directs are queued. + pub fridge_len: u64, +} +``` + +- **The application's state is the comparison bytes**, the same bytes the + watchdog compares ([state sources](../../watchdog/README.md#state-sources)). + Setup reads them only through the `state` report, so today they must fit one + 2 MiB report. +- **The fridge is named, not exported.** It holds the admitted directs from the + first queued input up to the L1 input count + ([scheduler](scheduler.md#the-schedulers-state)). Setup reads them from its + own input table and checks the export against it + ([what setup does](../../recovery/cockroach.md#what-setup-does)). +- **The application chooses its encoding.** The wallet exports its SSZ + snapshot; libdex could export the flat buffer its drive holds, as it is. +- **The `state` query keeps its meaning.** The literal bytes or an empty + payload return the comparison bytes whole, as today + ([`inspect_state`](../../../sequencer-core/src/scheduler/mod.rs)). The + literal `scheduler` is the one query added. +- **An unknown query gets a report saying so**, never a halt, as today. + +## The `Application` trait + +What the sequencer calls, in +[`sequencer-core/src/application/mod.rs`](../../../sequencer-core/src/application/mod.rs). +[Interfaces](interfaces.md#the-application) outlines the trait as proposed. +The [C binding](../../protocol/c-application-binding.md) maps it to native +engines and gets the same changes. + +| Method | Obligation | +|---|---| +| `max_method_payload_bytes` | A fixed cap on a user op's payload, which only the lane enforces | +| **New:** `user_nonce` | Report a sender's next nonce, which the shared executor compares with the op's | +| `admit_user_op` | Say, without changing state, whether the operator wants an op; **New:** it no longer checks the nonce, and only the lane calls it | +| `apply_user_op` | Advance the sender's nonce and the L2 tx count by exactly one, business failures included; **New:** never fail, even on a fee the sender cannot pay | +| `apply_direct` | Apply a direct, advancing the L2 tx count and never a nonce; **New:** it receives the input index and timestamp | +| `l2_tx_count` | Report the L2 tx count; **New:** the count alone, without the clock | +| `create_dump`, `from_dump`, `state_file_in_dump` | Write a durable dump, load it back to the same state, and name the file that holds the comparison bytes | +| `CanonicalState::canonical_snapshot_bytes` | The comparison bytes, which the `state` query returns whole | +| **New:** `from_canonical_state` | Build a native instance from a machine's comparison bytes, at the machine's L2 tx count | + +- **Nonces.** The application stores each sender's next nonce and reports it. + It no longer decides whether an op's nonce is right: the shared executor + compares before an op and checks the one-step advance after it, so the + [nonce rule](../../protocol/application-contract.md#user-nonces) is checked + the same way for every application. The same getter is what + [Track 6](../2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) + proposes for an exact `GET /nonce`. +- **What an op pays.** The scheduler executes every op its sender authorized, + funded or not ([scheduler](scheduler.md#a-user-ops-fate)). How much + `apply_user_op` charges and whether the effect runs are the application's + choice. Charging what the sender has, a saturating subtraction, is the + recommendation. +- **Admission.** The lane asks before it includes an op, and refuses an + underfunded one, since it would pay less than the L1 data it costs. It is + policy: the canonical machine never calls it, a guest build need not link + it, and its answer never changes what an included op does. An operator's + exemptions, such as a sponsored role, are answered here. +- **The projection.** The instance it builds must equal the machine's: the + same comparison bytes, at the L2 tx count it is given. For the wallet that + means decoding its SSZ snapshot; for libdex, loading the flat buffer. + +The rules behind the trait still hold. Execution is deterministic and reads +nothing but the state and the transaction +([§1](../../protocol/application-contract.md#1-determinism--purity)); a +skipped op changes nothing, and every executed op follows the nonce rule +([§2](../../protocol/application-contract.md#2-replay-safety--rejection-inclusion-and-failure)); +an `AppError` is fatal and defines no successor state. + +## The canonical machine + +What the guest must do, for every validator and for recovery: + +- **Run the scheduler, then the application, input by input**, matching the + [scheduler](scheduler.md) bit for bit, as the conformance vectors test. +- **Never answer an input with `RX_REJECTED`**, and halt only on the + [fatal conditions](scheduler.md#no-reverts-and-when-to-halt). +- **Keep each input's index.** The guest records how many inputs it has + consumed, and each queued direct keeps its own index. +- **Answer the [export](#the-state-export)**: its state through the state + source, and the `scheduler` query. +- **Compile in the deployment's constants**: the pinned submitter as + `sequencer_address`, the fridge's bound and the directs' payload cap. + +## Where this gets written down + +The trait's rows live in the +[application contract](../../protocol/application-contract.md). The export, +the projection and the changed rows go into it and into the C binding when the +template lands ([todo](todo.md#promote-to-real-docs)). diff --git a/docs/plans/2026-11-sequencer-recovery/client-backups.md b/docs/plans/2026-11-sequencer-recovery/client-backups.md new file mode 100644 index 00000000..0d9ac190 --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/client-backups.md @@ -0,0 +1,55 @@ +# Client backups + +Recovery asks one thing of a client: that it can restore a state at a count it +trusts. How it keeps its storage is its own business. Two rules make that easy +for a client that keeps its state in SQLite, as Bart's does, at any size: + +1. **Every commit is a boundary and keeps its rows.** Apply each feed row, or + each run of rows, in one SQLite transaction that also writes the L2 tx + count and era, and stores the rows themselves. Any consistent + copy of the database, taken at any time, is then at a transaction boundary, + says which, and holds every row before it. +2. **A new era discards later snapshots.** Inside an era the history only + appends; only a new era replaces it, past the count the client realigned + at, and reuses those counts. So once realigned, the client deletes every + snapshot past that count and stops snapshotting its old live database. + +To reach a count, restore the newest snapshot at or below it, replay the rows +after it from the live database or a newer snapshot through the client's +normal apply, and check the count. Never restore a snapshot past the count: +its state holds rows past it. Replay is exact only if applying a row reads +nothing but the database and the row. + +Retention sets recovery time, not correctness: a pruned snapshot costs a longer +replay from an older one. Keep recent snapshots and a thinning tail that +outlasts the oldest base machine an operator would pick, and stop pruning once +an incident opens. + +## Snapshotting a large SQLite database + +From the SQLite, Litestream, OpenZFS and AWS documentation as of 2026-10-29: + +- Volume or filesystem snapshots (ZFS, LVM thin, EBS) are the cheap choice at + terabyte scale: atomic and incremental. A ZFS snapshot holds every completed + write, so SQLite recovers it to its last committed transaction. An EBS + snapshot holds only what reached the volume, as a power cut would: with + `synchronous` NORMAL or FULL that only moves its count back, but with OFF it + can be corrupt. The database and its `-wal` file must be in the same + snapshot. Restore by cloning a snapshot, never by rolling back the live + volume, whose rows the replay reads. +- Litestream suits gigabytes rather than terabytes: by default it uploads a + full snapshot every 24 hours and keeps 24 hours. It restores only to the end + of one of its own files (`-txid`, `-timestamp`), which can batch many + transactions, so pick one at or below the count and let the rows in the live + database cover the rest. It deletes only by age, so it cannot discard the + restore points past the realignment count (rule 2): on a new era, + replicate to a new path, and restore from the old path only at or below + that count. +- The backup API and `VACUUM INTO` copy every page on every run. On a busy + database a copy either restarts after each write from another connection or + holds one read transaction, which stops WAL checkpoints until it ends. +- Every SQLite engine that opens a WAL database needs 3.51.3 or later (or the + 3.44.6 or 3.50.7 backport) once two connections, in threads or processes, can + write or checkpoint at once: earlier versions can corrupt it. Litestream + checkpoints with its own embedded engine, fixed from v0.5.17. WAL mode fixes + the page size, so choose it at creation. diff --git a/docs/plans/2026-11-sequencer-recovery/interfaces.md b/docs/plans/2026-11-sequencer-recovery/interfaces.md new file mode 100644 index 00000000..478b7f9e --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/interfaces.md @@ -0,0 +1,455 @@ +# The interfaces + +**Status:** the Rust side, `application-engine.h` and `scheduler.h` with its +reference export are code in this tree. An application's author may implement +`scheduler.h` with their own scheduler; the Rust adapter that would link it +is not built ([todo](todo.md#build)). The signatures of the two +components, in Rust and in C. The [scheduler](scheduler.md) owns the rules +they carry out, and [application](application.md) owns the trait's +obligations and the machine's export. Where this page and the code differ, +the code is right. + +## The model + +```text +L1 inputs ──▶ scheduler ──▶ L2 transactions ──▶ application ──▶ state, outputs +``` + +- **The scheduler turns inputs into transactions.** It reads nothing from the + application: not its state, not its code. +- **The application executes every transaction it is handed.** It never + refuses one. +- **One check needs both sides.** A user op executes only at its sender's next + nonce. The application stores the nonce, and shared code compares it. + +There are four interfaces, each component in Rust and in C: + +| Interface | Implemented by | Called by | +|---|---|---| +| `Scheduler`, a Rust struct in `sequencer-core` | The reference scheduler | The guest harness, through `CanonicalRunner`, and setup | +| `scheduler.h` | The reference scheduler's C export, or an author's own scheduler | A C or C++ guest; no Rust adapter exists yet | +| `Application`, a Rust trait | An application, and the adapter over `application-engine.h` | The executor, the lane and setup | +| `application-engine.h` | A native engine | The Rust adapter | + +One record crosses from the scheduler to the application, the +[L2 transaction](#l2-transactions), and it is the same record in the guest, in +setup's rebuild, in the lane's replay and on the feed. + +## Who decides a user op's fate + +The [scheduler](scheduler.md#a-user-ops-fate) owns the rule: an op its sender +did not authorize is skipped, an op the operator does not want is refused at +`POST /tx`, and every other op executes. Each verdict has one place here: + +| Verdict | Decided in | +|---|---| +| Skipped for its signature or its fee cap | The [scheduler](#the-scheduler), which never hands the op out | +| Skipped for its nonce | The [executor](#the-executor) | +| Refused | `admit_user_op`, which only the lane calls | +| What an executed op pays and does | `apply_user_op` | + +## L2 transactions + +```rust +/// What the scheduler hands out and the application executes. +pub enum L2Tx { + Direct(Direct), + UserOp(UserOpTx), +} + +pub struct Direct { + pub input_index: u64, + pub sender: Address, + /// Its L1 inclusion block. + pub block: u64, + pub timestamp: u64, + pub payload: Vec, +} + +pub struct UserOpTx { + /// The signer the scheduler recovered. + pub sender: Address, + /// The nonce the sender signed. + pub nonce: u32, + /// The frame's fee price, which the op's cap covers. + pub fee: u16, + /// The frame's safe block. + pub block: u64, + pub data: Vec, +} +``` + +The op keeps its nonce, so every caller runs the same nonce check, a replay +included. `UserOpTx::new(sender, &op, fee, block)` builds the record from a +signed op and its frame, the same way in every caller. + +## The scheduler + +```rust +/// Deployment constants, compiled into the template. No default: a build +/// that forgot one would run another deployment's rules. +pub struct SchedulerConfig { + pub sequencer_address: Address, + /// At least 1; genesis and resume assert it. + pub max_wait_blocks: u64, + pub fridge_bound: u64, + /// A direct with a longer payload never joins the fridge. + pub max_direct_payload_bytes: u64, +} + +impl SchedulerConfig { + /// The submitter and the application's limits. + pub fn for_application(sequencer_address: Address) -> Self; +} + +/// One InputBox input, as `EvmAdvance` delivers it. +pub struct L1Input { + pub index: u64, + pub sender: Address, + pub block: u64, + pub timestamp: u64, + pub chain_id: u64, + pub app_contract: Address, + pub payload: Vec, +} + +/// All that carries over between inputs, except the queued directs' bytes, +/// which L1 holds. +pub struct SchedulerState { + /// Inputs consumed: the scheduler's position in the InputBox. + pub l1_input_count: u64, + pub next_batch_nonce: u64, + pub clock: u64, + /// Index of the oldest queued direct; `l1_input_count` when none is queued. + pub first_queued_input: u64, + /// How many directs are queued, to check the re-supplied range against. + pub fridge_len: u64, +} + +pub enum InputOutcome { + DirectQueued, + DirectDropped, + BatchAccepted, + BatchStale, + BatchRejected(BatchRejectReason), +} + +pub struct Scheduled { + /// The scheduler's next batch nonce when it handed this out: the batch + /// whose frame holds or drained it, or the next batch to come. + pub batch_nonce: u64, + pub tx: L2Tx, +} + +pub struct Scheduler { /* private */ } + +impl Scheduler { + pub fn genesis(config: SchedulerConfig) -> Self; + + /// Resume from an exported state. `queued` re-supplies every input from + /// `first_queued_input` up to `l1_input_count`, in order. The scheduler + /// re-admits the directs among them by the same rule, and refuses a + /// range that does not match the state. + pub fn resume( + config: SchedulerConfig, + state: SchedulerState, + queued: impl IntoIterator, + ) -> Result; + + pub fn config(&self) -> &SchedulerConfig; + pub fn state(&self) -> SchedulerState; + + /// Take the next input, whose index must be `l1_input_count`, and say + /// what became of it. Panics on misuse: another index, or a previous + /// input with transactions left. + pub fn feed(&mut self, input: L1Input) -> InputOutcome; + + /// The current input's next transaction, in execution order. + pub fn next_tx(&mut self) -> Option; +} + +pub enum ResumeError { + FirstQueuedAfterCount { first_queued_input: u64, l1_input_count: u64 }, + OutOfSequence { expected: u64, got: u64 }, + BeyondInputCount { index: u64, l1_input_count: u64 }, + MissingInputs { next: u64, l1_input_count: u64 }, + BlockDecreases { index: u64, block: u64, previous_block: u64 }, + FirstQueuedNotADirect { index: u64 }, + DirectNotAboveClock { index: u64, block: u64, clock: u64 }, + FridgeLenMismatch { expected: u64, admitted: u64 }, +} + +/// The machine's `scheduler` inspect report: six little-endian `u64`, in +/// field order (48 bytes). +pub struct SchedulerExport { + pub l1_input_count: u64, + pub l2_tx_count: u64, + pub next_batch_nonce: u64, + pub clock: u64, + pub first_queued_input: u64, + pub fridge_len: u64, +} +``` + +- **It pulls, one transaction at a time.** The guest executes each transaction + as it comes, a batch's signatures are recovered only as its ops are reached, + and no callback crosses the C boundary. +- **It needs no application.** The fridge admits by sender and by payload + length, both from the configuration. A user op is handed out once its sender + is recovered and its cap covers the price; the nonce is the executor's check. +- **Resuming needs four numbers and a range of L1.** The fridge is a queue + ([scheduler](scheduler.md#the-schedulers-state)), so `queued` is the inputs + from `first_queued_input` up to `l1_input_count`, which setup's input table + already has. Resume refuses a range out of sequence, short or long, whose + blocks decrease, whose first input is not an admitted direct, with a queued + direct at or below the clock, or that admits another number of directs. +- **Its state is the machine's export.** `CanonicalRunner` + (`scheduler/driver.rs`) holds the scheduler and the application, feeds each + input and executes its transactions, and answers the `state` and + `scheduler` queries, the latter from `state()` and the application's count + ([application](application.md#the-state-export)). A runner whose input + failed with an `AppError` refuses further use. +- **It halts where the reference halts.** A batch nonce without a successor + panics in Rust and returns a fatal status in C. + +In C, the transaction records are the application's own, so a guest passes +them straight on. The header and its reference export are +[`bindings/c-scheduler`](../../../bindings/c-scheduler/include/scheduler.h), +which states each call's statuses and lifetimes: + +```c +typedef struct Scheduler Scheduler; + +typedef enum SchedulerStatus { + SCHEDULER_STATUS_OK = 0, + SCHEDULER_STATUS_DONE = 1, /* next_tx: the input has no more */ + SCHEDULER_STATUS_MISUSE = -1, /* out of order, a bad argument, a dead handle */ + SCHEDULER_STATUS_NO_SUCCESSOR = -2, /* a counter is exhausted: halt */ + SCHEDULER_STATUS_RESUME_REFUSED = -3, /* the state and the queued range disagree */ + SCHEDULER_STATUS_INTERNAL_ERROR = -4, /* an impossible state */ +} SchedulerStatus; + +typedef enum SchedulerTxKind { + SCHEDULER_TX_DIRECT = 0, + SCHEDULER_TX_USER_OP = 1, +} SchedulerTxKind; + +typedef union SchedulerTxValues { + ApplicationEngineDirectInput direct; + ApplicationEngineUserOpTx user_op; +} SchedulerTxValues; + +typedef struct SchedulerTx { + SchedulerTxKind kind; + uint64_t batch_nonce; + SchedulerTxValues values; /* spans borrowed until the next next_tx, feed or destroy */ +} SchedulerTx; + +uint32_t scheduler_abi_version(void); +const char *scheduler_get_last_error_message(void); +SchedulerStatus scheduler_create(const SchedulerConfig *config, Scheduler **out); +/* The queued range, from first_queued_input up to l1_input_count, in one call. */ +SchedulerStatus scheduler_resume(const SchedulerConfig *config, + const SchedulerState *state, const SchedulerInput *queued, uint64_t queued_count, + Scheduler **out); +void scheduler_destroy(Scheduler *scheduler); +SchedulerStatus scheduler_state(const Scheduler *scheduler, SchedulerState *out); +SchedulerStatus scheduler_feed(Scheduler *scheduler, const SchedulerInput *input, + SchedulerInputOutcome *out); +SchedulerStatus scheduler_next_tx(Scheduler *scheduler, SchedulerTx *out); +``` + +`SchedulerConfig`, `SchedulerInput`, `SchedulerState` and +`SchedulerInputOutcome` mirror the Rust records, with fixed widths and no +`size_t`, as the engine's header does. + +- **Resume is one call.** It takes the whole queued range, as the Rust + `resume` does, so a refused range never yields a handle and no handle is + ever half resumed. It refuses a state whose `first_queued_input` is past its + `l1_input_count` before it reads any input, and a range that does not match + the state. +- **A failure kills the handle.** Where Rust panics, C returns a negative + status, and after a negative status from `feed` or `next_tx` every call but + `destroy` is a misuse. No port then has to prove which failures left its + state whole, which is also why `scheduler_state` returns a status. +- **A feed waits for `DONE`.** Rust lets a feed follow an input whose + transactions ran out unseen; C asks for the `DONE` the canonical loop always + reads, so the export names every misuse before calling in. + +## The executor + +The one way a transaction reaches the application. The guest, setup's rebuild, +the lane and its replay all call it, and it is the whole of the canonical +user-op check that needs the application: + +```rust +pub enum Executed { + /// It executed; `index` is the L2 tx count before it, its index in the history. + Done { index: L2TxCount, outputs: AppOutputs }, + /// A user op whose nonce is not its sender's next. Nothing changed. + Skipped(SkipReason), +} + +pub enum SkipReason { + WrongNonce { expected: u32, got: u32 }, + /// The sender's next nonce is `u32::MAX`, which has no successor. + NonceExhausted, +} + +pub fn execute(app: &mut A, tx: &L2Tx) -> Result; +``` + +A direct always executes. For a user op, the executor reads +`app.user_nonce(sender)`, skips on a mismatch, then on `u32::MAX`, and +otherwise applies the op. The mismatch comes first, so a stale op that carries +`u32::MAX` still reads as a wrong nonce. After either kind it checks +that the L2 tx count advanced by exactly one, and after a user op that the +sender's nonce did. + +A canonical guest is then one loop, here in C: + +```c +scheduler_feed(scheduler, &input, &outcome); +while (scheduler_next_tx(scheduler, &tx) == SCHEDULER_STATUS_OK) { + if (tx.kind == SCHEDULER_TX_DIRECT) { + application_engine_execute_direct_input(engine, &tx.values.direct, &outputs); + } else { + application_engine_user_nonce(engine, &tx.values.user_op.sender, &nonce); + if (nonce != tx.values.user_op.nonce || nonce == UINT32_MAX) continue; + application_engine_execute_user_op(engine, &tx.values.user_op, &outputs); + } + emit_outputs(engine, outputs); +} +``` + +The lane checks a live op with `check_live_user_op`, then calls `execute`. The +check runs the fee cap, the executor's own nonce rule, then admission, so an +op it lets through is one `execute` runs; a skip after it is a terminal lane +fault: + +```rust +pub fn check_live_user_op( + app: &A, + sender: Address, + op: &UserOp, + frame_fee: u16, +) -> Result, AppError>; + +pub enum Refusal { + FeeCapBelowPrice { max_fee: u16, price: u16 }, + Skip(SkipReason), + Policy(PolicyRefusal), +} +``` + +A refusal reaches the client as a 422 whose text is the refusal's `Display`. The README +teaches clients to read the expected nonce from `bad nonce: expected N, got M`, +and the tests match the fee-balance text. + +## The application + +```rust +pub trait Application: Send + Sized { + // Execution: the canonical machine, the sequencer and every replica. + + /// The sender's next nonce; zero for an address that has sent nothing. + fn user_nonce(&self, sender: Address) -> Result; + + /// Execute a user op. It never refuses: whatever the op does or pays, + /// the sender's nonce and the L2 tx count each advance by one. + fn apply_user_op(&mut self, op: &UserOpTx) -> Result; + + /// Execute a direct. The L2 tx count advances by one; no nonce moves. + fn apply_direct(&mut self, direct: &Direct) -> Result; + + /// The L2 tx count. + fn l2_tx_count(&self) -> L2TxCount; + + // Admission: the sequencer's lane only. Policy, never consensus. + + fn max_method_payload_bytes() -> usize; + + /// The template's fridge bound and direct payload cap. + fn scheduler_limits() -> SchedulerLimits; + + /// Does the operator want this op in a frame at this fee? The canonical + /// machine never asks, and the answer never changes what an included + /// op does. + fn admit_user_op( + &self, + sender: Address, + op: &UserOp, + fee: u16, + ) -> Result; + + // Durable state: the sequencer and replicas. + + fn from_dump(prefix: &Path) -> Result; + fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError>; + fn state_file_in_dump(prefix: &Path) -> PathBuf; + + /// Build an instance from a canonical machine's comparison bytes, at the + /// machine's L2 tx count. A dump of the result holds the same bytes. + fn from_canonical_state( + state_file: &Path, + l2_tx_count: L2TxCount, + ) -> Result; +} + +pub enum Admission { + Admit, + Refuse(PolicyRefusal), +} + +pub struct SchedulerLimits { + /// The staleness window, at least 1. + pub max_wait_blocks: u64, + pub fridge_bound: u64, + pub max_direct_payload_bytes: u64, +} + +pub enum PolicyRefusal { + /// The sender cannot pay the fee. + Underfunded { required: U256, available: U256 }, + Other(String), +} +``` + +- **The application has no say in which ops execute.** It stores nonces and + answers `user_nonce`; the comparison is shared code. An application bug can + then change what a transaction does, but not which transactions execute or + what their counts are. +- **Admission is the old validation without the nonce.** A guest build needs + only the first group of methods. +- **The projection takes the count.** Whether the comparison bytes hold the + L2 tx count or not, the instance starts at the machine's. +- **`user_nonce` may need a "no such sender" answer (Open).** An engine with a + bounded account table cannot store a nonce for an address it has never seen. + Its ops would then be skipped, as a wrong nonce is + ([todo](todo.md#questions-for-bart)). + +[Application](application.md#the-application-trait) states each method's +obligation and what changes from today. + +`application-engine.h` matches it. Its ABI is version 3: version 2 made the +changes below, and version 3 added the staleness window to the limits. + +| Version 1 | Version 2 | +|---|---| +| `application_engine_validate_user_op` | `application_engine_admit_user_op`: the same call without the nonce check, refusing with the fee-balance reason or a new policy reason | +| No nonce getter | `application_engine_user_nonce(engine, sender, out_nonce)` | +| `ApplicationEngineValidUserOp` and a `safe_block` argument | `ApplicationEngineUserOpTx`: sender, nonce, fee, safe block and data | +| `application_engine_execute_valid_user_op` fails on an unpayable fee | `application_engine_execute_user_op` never fails on one | +| `ApplicationEngineDirectInput`: sender, block and payload | The same with the input index and timestamp | +| `ApplicationEngineProgress`: count and clock | `application_engine_l2_tx_count` | +| No projection | `application_engine_from_canonical_state(state_file, l2_tx_count, out_engine)` | +| No version | `application_engine_abi_version()`, which the adapter checks before anything else | +| No limits | `application_engine_scheduler_limits(out_limits)` | + +A new function fails at link time against an old archive, but a changed +meaning does not: an engine that still fails on an unpayable fee would halt +the canonical machine on the first underfunded op a batch carries. The version +is the only check on that. The reason values the engine stops producing keep +their numbers, since the header's vocabularies are append only; admission +adds `APPLICATION_ENGINE_REFUSED_BY_POLICY` (4), whose text travels in the +last error message. diff --git a/docs/plans/2026-11-sequencer-recovery/operations.md b/docs/plans/2026-11-sequencer-recovery/operations.md new file mode 100644 index 00000000..5231576f --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/operations.md @@ -0,0 +1,25 @@ +# Recovering the sequencer: what remains + +The procedure is built. [Cockroach recovery](../../recovery/cockroach.md) is +its home: one setup from any base, its inputs and outputs, what setup does, +the operator's and the client's playbooks, and what can go wrong. This page +keeps only the operational work that is not built yet. + +- **Finding a trusted machine.** Setup takes a machine; finding one at or + below a given L2 tx count, checking it, and backing machines up is the + operator's job, with no tooling yet + ([machine sourcing](todo.md#machine-sourcing)). +- **Drills that are not tests yet.** No test clears a watchdog latch around a + rebuild, or realigns a client other than the devnet's toy wallet + ([drills](../../watchdog/incident-runbook.md#drills)). +- **Bart's client.** Its backups and its realignment at a count it trusts are + his to build ([client backups](client-backups.md), + [questions for Bart](todo.md#questions-for-bart)). +- **Refusing a client past the rebuilt history end.** The feed checks a claim + only against the era's start and its head, so the check is the client's; + whether the sequencer refuses such a client itself is part of the + [egress redesign](todo.md#redesign-the-egress-api). +- **Recovery time on a real deployment.** The rebuild's time follows the user + ops past the base ([how long a recovery takes](../../recovery/cockroach.md#how-long-a-recovery-takes)); + the L1 sync through a deployment's provider and libdex's execution are not + measured yet ([measure](todo.md#measure)). diff --git a/docs/plans/2026-11-sequencer-recovery/scheduler.md b/docs/plans/2026-11-sequencer-recovery/scheduler.md new file mode 100644 index 00000000..627ffbc2 --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/scheduler.md @@ -0,0 +1,490 @@ +# The scheduler, for the next template + +**Status:** implemented. The reference +[`Scheduler`](../../../sequencer-core/src/scheduler/mod.rs) runs these rules, +and [scheduler semantics](../../protocol/scheduler-semantics.md) is their +contract; read that page for the rules in force. This page stays until its +**Open** items settle with the port's author +([register](../../review/register.md#canonical-rules-open-for-the-c-port)), +then shrinks to what remains ([todo](todo.md#promote-to-real-docs)). **New** +marks a rule that changed from the earlier template. + +The canonical scheduler turns an application's L1 inputs, in L1 order, into +one sequence of L2 transactions, and the application executes them. Every +validator runs it inside the canonical machine. The sequencer predicts it +while it sequences, and runs it on its own during a recovery. The reference is +the concrete [`Scheduler`](../../../sequencer-core/src/scheduler/mod.rs), with +[`application::execute`](../../../sequencer-core/src/application/mod.rs) running +each transaction it hands out. This text is the contract, and the reference can +have bugs, so report any disagreement. + +## What changes + +| Change | Why | Does recovery need it? | +|---|---|---| +| [State export](#state-export) | Setup starts a sequencer from a machine's whole state | Yes, and nothing else on this list | +| [The fridge's bound and payload cap](#the-fridge) | An unbounded fridge can exhaust the machine's memory, which halts the application for good | No; a recovery is only the remedy after an overflow | +| [Monotone blocks](#the-scheduler-clock) | A transaction's block never goes backwards, so applications may read blocks as time | No | +| [An underfunded op executes](#a-user-ops-fate) | Whether a sender can pay protects only the operator, so the lane checks it and consensus does not | No | +| [Shared code compares the nonce](#frame-execution) | No application code decides which ops execute | No | +| [Direct context](#direct-context) | The application learns a direct's input index and timestamp | No | +| [Progress is the L2 tx count alone](#outputs-and-progress) | The scheduler keeps the clock now, so the application need not | No | + +Each change ships with the next template, ours and the port's, and with new +[conformance vectors](#conformance). A deployed application keeps its +template for life +([Cartesi Machine facts](../../cartesi-machine.md#versions-are-all-or-nothing)). + +## The model + +### The L2 transaction sequence + +An **L2 transaction** is a direct input or a user op. The scheduler builds the +sequence one L1 input at a time: + +1. **Every input first runs the backstop.** Directs queued for at least + `max_wait_blocks` execute, oldest first, even when the input is a batch the + gates then reject. +2. **A direct input**, from any sender but the sequencer, joins the fridge, + the queue of directs not yet executed. **New:** one whose payload is over + the cap never joins, and a full fridge drains its oldest blocks to make + room. +3. **A batch**, the sequencer's input, is accepted or changes nothing + ([batch validity](#batch-validity)). +4. **Each frame of an accepted batch** first executes every queued direct up + to its safe block, oldest first, then offers its user ops in order. +5. **A user op is executed or skipped** ([a user op's fate](#a-user-ops-fate)). + +### What counts + +A state's **L2 tx count** is how many L2 transactions it has executed. It +names the state, and it names the transaction that state executes next, so +the feed numbers its rows with it. It counts transactions, not L1 inputs. + +A user op has three outcomes, and only one of them leaves no trace: + +| Outcome | What happens | Counts | +|---|---|---| +| **Skipped** | Nothing: no nonce, no fee, no effect | No | +| **Executed, succeeded** | The sender's nonce advances, the fee is charged, the effect applies | Yes | +| **Executed, failed** | The sender's nonce advances and the fee is charged, but the effect fails, such as a transfer larger than the balance | Yes | + +An op whose sender cannot pay the whole fee executes too (**New**, +[a user op's fate](#a-user-ops-fate)). Every executed direct counts, whatever +the application does with it. +Nothing else counts: not a queued or dropped direct, not a skipped op, not a +batch envelope, not anything in a rejected or stale batch. This page avoids +"no-op": today's docs use it both for an op that counts and for an empty batch. + +### The scheduler's state + +| Field | Meaning | +|---|---| +| `next_batch_nonce` | The nonce the next accepted batch must carry; `u64`, 0 at genesis | +| `clock` | **New.** The latest block the scheduler has executed at; 0 at genesis ([scheduler clock](#the-scheduler-clock)) | +| `fridge` | The admitted directs not yet executed, oldest first; each keeps its input index (**New**) | + +Nothing else carries over between inputs. **The scheduler's state never reads +the application's.** The nonce, the clock and the fridge depend only on the +inputs; the application affects them only by halting the machine. Whether the +application executes a user op changes none of them. The scheduler runs no +application code either: the fridge admits by sender and payload length. A +port must keep this property: it is what lets the sequencer run the scheduler +apart from the application during a recovery. + +**The fridge is a queue.** Directs join at the back in input order and leave +only from the front, whichever rule drains them. So the fridge always holds +exactly the admitted directs from one input index, the *first queued input*, +up to the L1 input count. + +### Two counts + +| Count | Names | Who knows it | +|---|---|---| +| **L2 tx count** | A state, and the next L2 transaction | Every application state; a client knows only this one | +| **L1 input count** | A machine's position: how many InputBox inputs it has consumed | The machine | + +They differ, because a batch envelope or a queued direct is an L1 input but +not an L2 transaction. The machine exports both +([state export](#state-export)). + +### The invariant + +**New.** The [scheduler clock](#the-scheduler-clock) gives two properties +after every input: + +- **Blocks never go backwards** along the L2 sequence. Gate f gives this. +- **The clock splits the directs.** Every admitted direct at or below the + clock has executed, and every one above it is still queued. Three more rules + give this: strict frames, the backstop's raise and whole-block drains. + +The split gives the clock a meaning an application can use: at clock `c`, it +has applied every direct up to block `c` and none after. Recovery does not +depend on it, since setup takes the first queued input from the machine +([state export](#state-export)). Two readers keep it true: + +- **Setup checks it.** Among the inputs a base machine consumed, the admitted + directs above its clock must be exactly the queued ones its export names + ([setup](../../recovery/cockroach.md#what-setup-does)). +- **The conformance vectors assert it** after every input, through the + `scheduler` report ([conformance](#conformance)). + +## Inputs + +Each input is a Cartesi Rollups `EvmAdvance` from the InputBox. The scheduler +reads its `msgSender`, its `blockNumber` (the *inclusion block*), its `chainId` +and `appContract` (the signature domain), its `payload`, and, for a direct, +its `index` and `timestamp` (**New**, [direct context](#direct-context)). It +classifies by sender only, never by payload content: + +- `msgSender == sequencer_address`: the payload is a **batch**. +- Any other sender: a **direct input**. + +`sequencer_address` is deployment data compiled into the canonical machine. It +must equal the batch-submitter address the sequencer's setup pins, and nothing +checks this at runtime (a setup check is **Open**). If they differ, the machine +treats every batch as a direct while the sequencer reports its batches +accepted, so only a watchdog comparison reveals it. + +## Wire format + +A batch payload is the SSZ encoding of `Batch` +([`batch.rs`](../../../sequencer-core/src/batch.rs)), fields in this order: + +```text +Batch { nonce: uint64, frames: List[Frame] } +Frame { user_ops: List[WireUserOp], safe_block: uint64, fee_price: uint16 } +WireUserOp { nonce: uint32, max_fee: uint16, data: List[uint8], signature: List[uint8] } +``` + +Integers are little-endian. Each variable-size field has a 4-byte offset in its +container's fixed part, and the variable parts follow in field order. A list of +containers starts with one 4-byte offset per element. Lists have no length +limit. Decoding is strict: offsets must start right after the fixed part, never +decrease and stay in bounds, so only the canonical encoding decodes, and +re-encoding returns the same bytes. `max_fee` and `fee_price` are fee exponents +([constants](#constants)). A `signature` of any length decodes, but only 65 +bytes can recover a sender; a fixed 65-byte field is **Open**. + +## User-op signatures + +A user op signs the EIP-712 struct `UserOp(uint32 nonce,uint16 max_fee,bytes data)` +([`user_op.rs`](../../../sequencer-core/src/user_op.rs)) under the domain +`EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)`: +name `CartesiAppSequencer`, version `1`, the input's `chainId`, its +`appContract` as `verifyingContract`, and no salt. The sender is not signed; it +is recovered from `r ‖ s ‖ v`, and any failure skips the op: + +1. The signature must be 65 bytes, and `v` must be 0, 1, 27, 28 or at least 35. + The y-parity is `v` for 0 and 1; for `v ≥ 27` it is 1 when `v` is even, + else 0. +2. `r` and `s` must lie in `[1, n − 1]`, where `n` is the secp256k1 order. +3. If `s > n / 2`, `s` becomes `n − s` and the parity flips, so a high-`s` + signature and its low-`s` twin of opposite parity recover the same sender. +4. The sender is the address of the public key recovered from `(r, s, parity)` + and the EIP-712 signing hash. + +This acceptance set is alloy's. Narrowing it to `v ∈ {27, 28}` and low `s` is +**Open**. + +## Processing an input + +1. **The backstop.** Pop and execute directs from the front of the fridge + while `input.inclusion_block − front.inclusion_block ≥ max_wait_blocks`, + raising the clock to each one's block (**New**). This bounds how long a + sequencer that stops submitting, or freezes `safe_block`, can hold a direct + back. +2. **A direct** joins the fridge, unless its payload is over the cap, and once + there is room ([the fridge](#the-fridge)). The cap comes first: a direct + that never joins drains nothing. +3. **A batch** runs the gates below. + +### Batch validity + +The first gate that applies decides the outcome: + +| # | Gate | Outcome | `next_batch_nonce` | +|---|---|---|---| +| a | The payload does not SSZ-decode as `Batch` | Rejected | Kept | +| b | `batch.nonce != next_batch_nonce` | Rejected | Kept | +| c | `frames` is empty | Accepted; executes nothing | +1 | +| d | The frame structure is invalid (below) | Rejected | Kept | +| e | `inclusion_block − frames[0].safe_block ≥ max_wait_blocks` | Skipped as stale | Kept | +| f | **New.** `frames[0].safe_block < clock` | Rejected | Kept | +| g | Otherwise: execute every frame | Accepted | +1 | + +Gate d requires, for every frame: + +- its `safe_block` below `inclusion_block` (**New**: today, at or below); +- its `safe_block` at or above the previous frame's; +- its `fee_price` at most `MAX_EXPONENT`. + +Gate d checks every frame before any frame executes; the backstop has already +run. Ages saturate at zero. An empty batch is accepted at any age, since it has +no first frame; rejecting empty batches is **Open**. A rejected batch and a +stale one have the same effect: no state change, nonce kept. So every later +batch fails gate b until one reuses that nonce, as only a new era's batches +do. The reject reason the reference reports is diagnostic. + +### Frame execution + +For each frame of an accepted batch, in order: + +1. **Drain covered directs.** Pop and execute directs from the front of the + fridge while their `inclusion_block ≤ frame.safe_block`. The `safe_block` is + the sequencer's claim that it accounted for every direct up to that block; + the scheduler enforces the drain whatever the sequencer did. Then raise the + clock to `frame.safe_block` (**New**). +2. **Offer the user ops** in order. Skip an op, checked in this order, when: + - its sender cannot be recovered ([signatures](#user-op-signatures)); + - its `max_fee` is below `frame.fee_price`; + - its nonce is not its sender's next; + - its sender's next nonce is `u32::MAX`, which has no successor. + + Otherwise the op executes with fee `frame.fee_price` and block + `frame.safe_block`: the application advances the sender's nonce by one and + applies the op, and the L2 tx count advances by one. + +**New: shared code compares the nonce.** The application stores each sender's +next nonce and reports it; the shared executor compares it with the op's +([interfaces](interfaces.md#the-executor)). Today the application's own +validation decides +([`validate_and_execute_user_op`](../../../sequencer-core/src/application/mod.rs)). +The first two checks need no application, so the scheduler makes them alone. + +### A user op's fate + +Two parties decide, and they ask different questions: + +- **The canonical scheduler asks whether the sender authorized the op.** A + check that protects the user is consensus: the signature, the nonce and the + fee cap. An op that fails one is **skipped**, with no trace. +- **The sequencer's lane asks whether the operator wants the op.** A check that + protects the operator is admission: can the sender pay, does the payload fit + the batch, does the operator's policy allow it. An op that fails one is + **refused** at `POST /tx`. The canonical scheduler never asks: the fee is + owed to the sequencer, and only the sequencer includes ops. + +| An op that… | Checked by | Canonical scheduler | Sequencer's lane | +|---|---|---|---| +| has no recoverable sender | The scheduler | Skipped | Refused | +| caps its fee below the frame's price | The scheduler | Skipped | Refused | +| carries a nonce that is not its sender's next | The executor, from the application's stored nonce | Skipped | Refused | +| comes from a sender whose next nonce is `u32::MAX` | The executor | Skipped | Refused | +| comes from a sender who cannot pay the fee (**New**) | The application's admission | Executed | Refused | +| exceeds the application's declared payload bound | The lane | Executed | Refused | +| breaks an operator policy | The application's admission, or the lane | Executed | Refused | +| fails in the application, such as a transfer above the balance | The application's execution | Executed | Admitted | +| is sponsored: the application charges it nothing | The application's execution | Executed | Admitted | +| is none of the above | | Executed | Admitted | + +- **Skipped** means nothing changes: no nonce, no count, no fee. +- **Executed** means the sender's nonce and the L2 tx count each advance by + one. What the op pays and whether its effect applies are the application's + choice ([application](application.md#the-application-trait)). The + application must not fail, since an application error halts the machine. +- **The lane refuses everything the scheduler skips**, by running the same + checks first. No op is admitted and then skipped. + +**New: an underfunded op executes.** Today the scheduler also skips an op whose +sender cannot pay the frame fee. That check protects only the sequencer from +its own inclusion, so it moves to the lane. With it gone, which transactions +execute depends on signatures, fee caps and nonces alone, never on a balance. +A fault in the application's logic can then change what a transaction does, +but not which transactions execute or what their counts are. + +**A fee cap below the price skips the op.** The cap is part of what the sender +signed: execute this op only at a price up to the cap. Charging the price would +take more than the sender authorized. Executing at the cap instead would make +a frame's price mean something else for one op. The sequencer loses nothing by +the skip: it sets each frame's price, so it can put the op in a frame it fits. +A skipped op stays valid until its sender's nonce moves past it, as a pending +transaction does anywhere. + +**A sponsored op needs no rule of its own.** Whether an executed op pays is +the application's choice, and whether an unpaid op is wanted is admission's. +An operator can run its own price feed for nothing in three ways, none of +which the scheduler sees: + +- fund the feed's account: where the application credits fees to the operator, + as the wallet does, the operator pays itself; +- exempt the role in the application: admission accepts its ops and execution + charges them nothing; +- put its ops in a frame priced at exponent 0, one smallest unit. + +A sponsored sender still signs a cap at or above the frame's price, such as +`MAX_EXPONENT`. An exemption must hang on something the signature proves, the +sender's role, or anyone can use it. + +## The scheduler clock + +**New.** The clock is the latest block the scheduler has executed at: an +accepted frame's safe block, or an executed direct's inclusion block. Every +frame of an accepted batch raises it, and so does every executed direct, +whether a frame, the backstop or an overflow drained it. It only rises: a +raise takes the later of the clock and the new block. Gate f rejects a batch +whose first frame lies below it, and gate d orders the rest. + +**Gate f alone keeps blocks from going backwards.** Every frame lies at or +above the clock, so a frame's directs come from the clock or above and at or +below its safe block, and its user ops see the safe block. The backstop needs +no help: a batch whose first frame lies below a block the backstop drained is +already stale by gate e. + +Three more rules keep the clock's split of the directs +([the invariant](#the-invariant)): + +- **Strict frames** (gate d) let a frame drain every direct of its own safe + block, because all of them landed before its batch. With equality allowed, a + direct from that block landing after the batch would stay queued at the + clock. Equality is also a claim no sequencer can make honestly: the block + was not finished when the batch was sealed. +- **The backstop raises the clock**, as every drain does, so no direct it + executed sits above the clock. The raise changes no outcome: gate e already + skips any batch it would reject. +- **Overflow drains whole blocks**, so no direct at or below the clock stays + queued ([the fridge](#the-fridge)). Draining only as many entries as needed + would hand out the same L2 sequence under a hard bound, but it would leave + part of a block queued at the clock. + +Unless an overflow overtakes it, a correct sequencer never trips gate f or the +strict rule: it never opens a frame below an earlier one, and it opens frames +only at safe heads it has already seen, so its batch lands in a later block. +Its own frame ordering ([I3](../../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine)) +becomes consensus-critical, though: a bug there now gets batches rejected. + +## The fridge + +- **Admission (New).** A direct whose payload is longer than + `MAX_DIRECT_PAYLOAD_BYTES` never joins: it never executes and never counts. + Every other direct joins, whatever it holds, and executes as today, where + the application ignores what it cannot read. The cap is a constant of the + template, set from the longest payload the application accepts. The + sequencer's lane drops the same directs by the same constant, or the two + diverge. +- **Why a constant.** A count bounds memory only if each entry's size is + bounded too: portals accept zero-value deposits with extra data up to the + InputBox's 64 KiB cap. A constant bounds the size without the scheduler + running application code. A filter written by the application would also + drop small garbage, but a zero-value portal deposit takes a slot for about + the same L1 gas, so it would buy little. +- **Entries.** Each entry keeps the direct as delivered (sender, block, + payload) with its input index (**New**), and the [direct context](#direct-context) + the application receives. +- **The bound (New).** The fridge is full when it holds `FRIDGE_BOUND` entries + or more. When a direct arrives at a full fridge, the scheduler executes whole + blocks from the front, oldest first, until there is room, raising the clock + to the last drained block. It never drains the arriving direct's own block: + if only that block's directs remain, the direct joins over the bound. The + bound is then soft by at most one block's directs, and memory stays bounded + by what one L1 block can carry. +- **An overflow can reject sealed batches.** The sequencer cannot predict an + overflow, and a batch it sealed earlier then lands in one of two ways. If + its first frame is at or above the last drained block, it is accepted, and + the L2 sequence is the one the sequencer predicted: the drained directs are + the first ones that frame would have drained. If it is below, gate f rejects + the batch, and every later one fails the nonce gate. The sequencer's + acceptance check does not see the clock, so it counts those batches as + accepted and diverges. A manual recovery answers that, and the watchdog's + divergence latch reports it. An overflow also moves which input executes the + drained directs, which the watchdog's comparison at that block can see + ([register](../../review/register.md#bound-the-canonical-direct-input-fridge)). + +The bound's value, the cap's value, and the largest drain the guest can afford +are **Open** +([register](../../review/register.md#bound-the-canonical-direct-input-fridge)). + +## Direct context + +A direct executes with its inclusion block as its block. **New:** the +application also receives its input index and timestamp. Both cost nothing: the +sequencer's input table and the feed already carry them. prevRandao comes only +on a stated need, since it needs a column and a feed field, and the block +proposer can bias it. Which of them libdex reads is a question for Bart. A user +op never sees its batch's context; it sees its frame's safe block and fee. + +## Outputs and progress + +An input's outputs are the application's notices and vouchers in execution +order: the backstop's directs first, then, for a direct, any overflow drain, +or, for a batch, each frame's drained directs and executed ops. Reports are diagnostics outside the outputs tree: the +reference reports a rejected batch with one fixed string and a stale skip not +at all. + +The application reports its progress: its L2 tx count, and today also the +latest block it executed at. **New:** progress is the L2 tx count alone; the +clock moves to the scheduler. + +## No reverts, and when to halt + +The guest never answers an input with `RX_REJECTED`. A batch a gate rejects is +still accepted at the machine level. A revert would undo the whole input and +roll the scheduler's state back with the application's, so the scheduler's +state would depend on the application. It would also drop the backstop's +executions, which the reference keeps. + +The reference halts the machine when the application returns an error; when +the batch nonce, the L2 tx count or the L1 input count has no `u64` successor; +and when an input's `index`, `blockNumber`, `timestamp` or `chainId` does not +fit `u64`. A halt is a permanent fixed point, +not a reject ([rollup host semantics](../../cartesi-machine.md#rollup-host-semantics)), +so a port must halt on the same inputs. No canonical successor state exists. + +## State export + +**New, and the only rule recovery needs.** The machine exports its whole state, +so the sequencer's setup can start from it: + +- its L1 input count and its L2 tx count; +- the scheduler's state: the next batch nonce, the clock, and the first queued + input, where the fridge starts; +- the application's state. + +The queued directs themselves are not exported. +[The fridge is a queue](#the-schedulers-state), so the first queued input and +the L1 input count name them, and L1 holds their bytes. +[Application](application.md#the-state-export) specifies how the machine +answers. The harness answers the scheduler's part for every application; the +application supplies only its own state. + +## The scheduler's interface + +During a recovery the sequencer runs the scheduler apart from the +application: it feeds the scheduler one input, then executes each +transaction the scheduler hands out for it. The scheduler never reads the +application. That needs a free-standing scheduler with an interface of its +own, as the application has its trait and its C binding. +[Interfaces](interfaces.md#the-scheduler) outlines it, in Rust and in C; an +application's author may implement the C header with their own scheduler. +This page fixes the scheduler's behavior whatever the implementation. + +## Constants + +| Constant | Value | Home | +|---|---|---| +| `max_wait_blocks` | 7200 L1 blocks, about 24 h, for the wallet | the application's template | +| `MAX_EXPONENT` | 17101 | [`fee.rs`](../../../sequencer-core/src/fee.rs) (`build.rs` generates it) | +| `FRIDGE_BOUND` | **New**, value **Open** | the template | +| `MAX_DIRECT_PAYLOAD_BYTES` | **New**, value **Open** | the template | +| `sequencer_address` | per deployment | compiled into the canonical machine | + +The window `max_wait_blocks` is at least one block. With zero, the backstop +could drain the arriving input's own block and leave a direct from that block +queued at the clock. + +A fee exponent `n` stands for `(129/128)^n` smallest token units. The +application converts it; [`fee_to_linear`](../../../sequencer-core/src/fee.rs) +owns the bit-exact conversion. A checked-in lookup table in its place is +**Open** ([Track 5](../2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut)). + +## Conformance + +[`consensus_vectors_v2.json`](../../../tests/fixtures/consensus_vectors_v2.json) +is the new template's vector set, written once and never regenerated. It +restates the earlier set's guest scenarios under strict frames, which change 18 +of the 19, and adds scenarios for the rules this plan changes, including the +overflow drain and resume at a small fridge bound. A native test, the +in-machine golden tests and a test through `scheduler.h` hold it, and the last +is what a C port runs. +[Scheduler semantics](../../protocol/scheduler-semantics.md#conformance) owns +what the set holds and how each check runs. diff --git a/docs/plans/2026-11-sequencer-recovery/sequencer.md b/docs/plans/2026-11-sequencer-recovery/sequencer.md new file mode 100644 index 00000000..f1ff4ac2 --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/sequencer.md @@ -0,0 +1,24 @@ +# Inside the sequencer: what remains + +The sequencer's side of this plan is built: loading a machine, the rebuild +with the free-standing scheduler, the rebuilt history table, the era, the feed across +the rebuilt and live rows, and the lane draining from the first queued input. +[Cockroach recovery](../../recovery/cockroach.md) describes them, with a code +map. This page keeps only what is not built yet. + +- **A large application state.** Setup reads the base's state as one `state` + inspect report, which holds at most 2 MiB: 27,593 wallet accounts that hold + a balance and a nonce, or 40,328 that hold only a balance. That size, not the loader's + 2^36-cycle bound per query, is the limit that binds + ([measurements](../../review/2026-11-10-recovery-measurements.md#b-reading-a-base-machine)). + A larger state needs another state source, such as a labeled memory range, + or a guest that splits its answer + ([the loader's limits](../../cartesi-machine.md#reading-a-stored-machines-export)). +- **A foreign scheduler.** `bindings/c-scheduler` exports the Rust scheduler + through `scheduler.h`, and an application's author may implement that header + with their own. Setup still calls the Rust scheduler directly: the trait and + the adapter that would link a foreign one are not built + ([todo](todo.md#build)). +- **Names.** The storage columns call the L2 tx count `l2_tx_count`; the HTTP + API still calls it `executed_input_count`, which changes with the + [egress redesign](todo.md#redesign-the-egress-api). diff --git a/docs/plans/2026-11-sequencer-recovery/todo.md b/docs/plans/2026-11-sequencer-recovery/todo.md new file mode 100644 index 00000000..a7fea95d --- /dev/null +++ b/docs/plans/2026-11-sequencer-recovery/todo.md @@ -0,0 +1,151 @@ +# To do + +What this plan still needs. The scheduler, the shared executor, the +`Application` trait, the state export, `setup [--base] [--dry-run]`, the +rebuilt history and the feed's realignment are code in this tree. +[Cockroach recovery](../../recovery/cockroach.md) states the procedure and the +playbooks, and the +[history contract](../../protocol/application-history.md#era-baseline) the +era, the rebuilt history and the adopt rule. The +[register](../../review/register.md#sequencer-recovery-follow-ups) holds the +limits this work accepts and the paths it left unverified. + +## Promote to real docs + +- **[scheduler.md](scheduler.md)**: [scheduler semantics](../../protocol/scheduler-semantics.md) + states the rules the reference runs, so this page shrinks to its open items, + as [operations](operations.md) and [sequencer](sequencer.md) have. +- **[application.md](application.md)**: the + [application contract](../../protocol/application-contract.md) and the + [C binding](../../protocol/c-application-binding.md) state the trait, the + export and the projection; this page shrinks to the guest's open duties and + the questions below. +- **[interfaces.md](interfaces.md)** is code, except the outline of the + scheduler's C header, which stays until a foreign scheduler links through + it ([build](#build)). + +## Build + +- **A foreign scheduler in the rebuild.** An application's author may bring + their own scheduler: one library, compiled into their guest, and natively + behind [`scheduler.h`](../../../bindings/c-scheduler/include/scheduler.h) + for setup's rebuild, as libdex will. The reference scheduler is then the + specification's implementation and the wallet's scheduler, and its export + is the header's reference. Setup calls the Rust `Scheduler` directly today + ([`rebuild.rs`](../../../sequencer/src/commands/setup/rebuild.rs)); it needs + a `Scheduler` trait and an adapter over the header, linked as an engine's + archive is ([C engine](../../../bindings/c-app-engine/README.md)). The lane + still predicts the canonical scheduler in Rust, so a foreign scheduler is + held to the [conformance vectors](../../protocol/scheduler-semantics.md#conformance) + through `c-scheduler`'s `tests/conformance.rs`, which today links only the + reference export; package it so an author links their own archive in one + command. +- **A base machine's state through the range source.** Setup reads only the + `state` inspect report, which must fit 2 MiB, and libdex's state is a 512 MiB + range, so setup cannot start from a libdex machine. Add the labeled-range + read the watchdog uses, and measure it against the loader's cycle cap + ([register](../../review/register.md#sequencer-recovery-follow-ups)). +- **Close a batch on its real size.** The lane closes a batch on its op count + times the largest op the application accepts. libdex's largest op is 6,971 + bytes and its usual op under 50, so its batches would close at 2 ops where + about 110 fit ([issue 58](https://github.com/cartesi/sequencer/issues/58), + [libdex's sizes](../../review/2026-11-12-libdex-answers.md#ops-and-their-sizes)). + Count the encoded bytes instead: it is the lane's own choice and changes no + image. The fee still charges every op as 126 bytes; a per-byte price changes + what a signed op means, and waits. +- **Drills that are not tests yet.** The round trips from genesis and from a + stored machine run on the devnet for both hosts + ([drills](../../watchdog/incident-runbook.md#drills)). No test clears a + watchdog latch around a rebuild, or realigns a reader other than the toy + wallet. A path run once in a deployment's life rots. + +## Redesign the egress API + +Later, in light of this plan: + +- keep the WS subscription; +- consider a paginated HTTP endpoint for history; +- remove the methods nothing uses, starting with what the + [tracks board](../2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign) + lists to reshape or retire, `/history`'s baseline and accepted checkpoint; +- the feed's identity check (the era), `available_from` and the + rebuilt range, and whether the sequencer refuses a client from an older era + whose count lies past the rebuilt history end, which today only the client's + own adopt check catches; +- `/nonce` after a recovery, which reads only the live rows; +- the L2 tx count's older name, `executed_input_count`, which the HTTP fields + and the `X-Executed-Input-Count` header still carry (the storage columns say + `l2_tx_count`); +- the WS headers. + +## Later + +- **A metrics endpoint.** The sequencer exports no metrics: an operator reads + logs and `/healthz`. Design one on the internal listener, beside the other + internal routes, starting from the two numbers `/healthz` reports + ([what the sequencer does](../../recovery/README.md#what-the-sequencer-does)). +- **A batch proxy, to rotate the submitter's key.** The canonical machine + compiles in the address that posts batches, so the key behind that address + can never change. A proxy contract would stand at it: the proxy forwards a + batch to the InputBox from one configured hot wallet, the sequencer's, and a + cold multisig can replace that wallet. The machine then pins the proxy, + which never changes. A rotation may still need a cockroach recovery; that is + to work out. + +## Machine sourcing + +A separate task. Setup only takes a machine; finding a trusted one is the +operator's job. + +- **From a validator node at a settled epoch**, so its root can be checked on + chain. Review our sling node: does it keep the machines at the end of past + epochs, or only the newest? +- **A playbook** for finding a machine at or below a given L2 tx count, and for + checking it. +- **A backup procedure for machines.** Storing directory trees for a long time + is a solved problem; pick a mature tool rather than building one. + +## libdex + +[libdex as its author describes it](../../review/2026-11-12-libdex-answers.md) +records the answers to the first round of questions. They add two items to +[build](#build), the state's range read and closing a batch on its real size. +They also open two canonical rules in the +[register](../../review/register.md#canonical-rules-open-for-the-c-port): what +the scheduler does with an over-long deposit, and where the nonce lives. + +## Questions for Bart + +What an op pays is the application's choice, with charging what the sender +has as the recommendation, and admission's refusal by policy answers a +redundant price push; neither is a question. + +- When a deleted session wallet is registered again, where does its nonce + start? If it can start below where it stopped, what keeps the wallet's + earlier ops, which are public on L1, from running again? +- What does `validate_user_op` check besides the nonce? Each of those checks + becomes part of execution, which always runs and always counts. +- When a user op arrives as an L1 input, who is its sender: the L1 address + that sent the input, or the signer of a signature inside it? If a + signature, what keeps that op from running twice? +- Does a client ever send several ops from one wallet without waiting for + each answer? Does anything need a wallet's ops to run with no gap between + them? +- Does the native engine lay out the state range byte for byte as the guest + does? On which hosts has that been run? +- When there are numbers: the machine cycles of an idle tick and of the worst + one. + +## Measure + +The [recovery measurements](../../review/2026-11-10-recovery-measurements.md) +give setup's rebuild rate, the cost of reading a base machine, the wallet's +largest drain and the drills' times. What they leave out: + +- **The L1 sync** through a deployment's RPC provider, the one part of + setup's time the provider bounds. +- **libdex's numbers**: its execution rate in setup, its cost per drained + direct, and its `state` size, since every guest number there is the + wallet's ([questions](#questions-for-bart)). +- **Production-like hardware.** One Apple machine on macOS ran every measurement; + re-measure the rebuild on the recovery host's Linux storage. diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index add97ae6..502bb19f 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -2,8 +2,9 @@ An app plugs into the sequencer by implementing [`Application`](../../sequencer-core/src/application/mod.rs). It owns its state, -including execution progress; the protocol defines how that progress advances. -The shared execution boundary verifies each successful transition. Application +including each sender's nonce and its L2 tx count; the protocol +defines how they advance, and the shared executor decides which transactions +run and verifies each successful transition. Application code remains self-trusted: Rust capabilities cannot establish the correctness of a native engine, its FFI, or its canonical counterpart ([threat model](../threat-model/README.md), "Self-trust"). @@ -19,49 +20,54 @@ a reference wallet integration. | Method | Contract | |---|---| -| `validate_user_op(sender, op, current_fee) -> Result` | Pure predicate: `Accept`, `Reject(InvalidReason)`, or fatal engine failure. No state change. | -| `apply_valid_user_op(valid, safe_block) -> Result` | Execute and advance embedded progress exactly once on success, using the frame clock. | -| `apply_direct_input(input) -> Result` | Execute and advance embedded progress exactly once on success, using the input's L1 block. | -| `progress() -> ApplicationProgress` | Return the count/clock pair from the engine by value. | - -Execution callers use `validate_and_execute_user_op`, `execute_valid_user_op`, -and `execute_direct_input`. The first enforces `max_fee >= current_fee` before -app validation and rejects an accepted op at the exhausted nonce `u32::MAX` -after it. App validation checks the nonce and fee balance; a rejection is not -persisted. Trusted replay -uses the stored `ValidUserOp`, whose fee and sender were established at -inclusion, without validating a second time. - -Before an apply hook, the boundary computes the checked expected successor. -After `Ok`, it asserts that the engine reports exactly that successor and -returns the input's pre-execution count as its history offset. An engine may -use `ApplicationProgress::advance` or implement the same transition natively. -Adapters report the engine-owned progress rather than maintaining a separate -count or clock mirror. -Overflow fails before the hook runs. An error defines no successor: callers -terminate the execution path and discard the instance, without attempting to -roll back or inspect partially updated state. - -`Application::max_method_payload_bytes()` returns the engine's stable method -payload bound without constructing an instance. Zero permits only empty method -payloads. HTTP rejects oversized method payloads; the lane uses the declared -bound plus signed-op metadata to compute batch capacity. It is not a canonical -scheduler rejection rule. +| `user_nonce(sender) -> Result` | Report the sender's next nonce, zero for an address that has sent nothing, without changing state. | +| `apply_user_op(op) -> Result` | Execute a user op found at its sender's next nonce, advancing that nonce and the count by exactly one each. | +| `apply_direct(direct) -> Result` | Execute a direct, advancing the count by exactly one and moving no nonce. | +| `l2_tx_count() -> L2TxCount` | Report the count ([§4](#4-the-l2-tx-count-l2_tx_count)). | + +A transaction reaches the application only through the shared executor, +[`execute`](../../sequencer-core/src/application/mod.rs), in the canonical +machine, the lane and every replay alike. It runs a direct always. For a user +op it reads `user_nonce` and skips the op, changing nothing, when the op's +nonce is not the sender's next or that next nonce is `u32::MAX` +([user nonces](#user-nonces)); otherwise it calls `apply_user_op`. A user op +reaches the application with its sender, signed nonce, the frame's fee +exponent and the frame's safe block as its block +([`UserOpTx`](../../sequencer-core/src/l2_tx.rs)); a direct, with its sender, +inclusion block, input index, timestamp and payload +([`Direct`](../../sequencer-core/src/l2_tx.rs)). The application keeps no clock; +the [scheduler clock](scheduler-semantics.md#the-scheduler-clock) is the +scheduler's. + +`apply_user_op` never refuses: whatever the op does or pays, even a fee the +sender cannot cover, it executes. Charging what the sender has, a saturating +subtraction, is the recommendation; the wallet charges the smaller of the fee +and the balance and credits the sequencer with exactly that amount. + +Before an apply hook, the executor computes the checked successor of the +count. After `Ok`, it asserts that the engine reports exactly that successor +and, for a user op, that the sender's nonce advanced by exactly one, and it +returns the pre-execution count as the transaction's index in the history. An +engine may use `L2TxCount::advance` or implement the same transition +natively. Adapters report the engine-owned count rather than maintaining a +separate mirror. Overflow fails before the hook runs. An error defines no +successor: callers terminate the execution path and discard the instance, +without attempting to roll back or inspect partially updated state. ## Cross-cutting contracts ### 1. Determinism & purity -Execution must be deterministic over `(input, current state)`, including output -bytes and their order. Do not depend on wall time, randomness, pointer values, -nondeterministic iteration, or host-specific arithmetic in consensus paths. -Validation is read-only even though it may run on a different schedule during -live execution, canonical execution, and replay. State changes happen through -apply hooks; dump creation may change backing resources but preserves logical -state. +Execution must be deterministic over `(transaction, current state)`, including +output bytes and their order. Do not depend on wall time, randomness, pointer +values, nondeterministic iteration, or host-specific arithmetic in consensus +paths. `user_nonce` and admission are read-only, and they run on a different +schedule during live execution, canonical execution, and replay. State changes +happen through apply hooks; dump creation may change backing resources but +preserves logical state. -Wrapping an existing engine must preserve its transaction encoding, -rejection/inclusion semantics, and canonical state bytes. +Wrapping an existing engine must preserve its transaction encoding, inclusion +semantics, and canonical state bytes. `Application: Send + Sized` permits moving the engine to the lane's blocking worker. It requires neither `Sync` nor `Clone`: the lane owns one mutable @@ -74,91 +80,119 @@ rules. These outcomes have different protocol meanings: -- A validation rejection changes no state, consumes no nonce or fee, and is - not persisted. `InvalidReason` covers nonce mismatch, an exhausted nonce, - insufficient max fee, and insufficient fee balance. -- A successfully applied input is included even if the business operation - fails or is ignored. It advances progress, and an included user op consumes - its sender's nonce ([below](#user-nonces)). The wallet also charges the fee - for a malformed method or failed transfer after validation; malformed direct - inputs are included no-ops. These semantics must agree with the canonical - application. -- `AppError`, whether `Internal` or `Io`, is fatal in validation and execution. - It must not be disguised as a client rejection or an included no-op. - Fatal here means discarding the engine: the host still distinguishes a - terminal `Internal` fault from an operational I/O failure that may clear - after restart. - -Every input executed successfully live must execute successfully against the -same prior state on replay. The sequencer persists included inputs and replays -them on restart. It does not recover an instance after a failed hook. +- A **skipped** user op, one the executor finds at the wrong nonce, changes + no state and consumes no nonce or fee. A **refused** op, one the lane turns + away at `POST /tx` ([§3](#3-admission--policy-never-consensus)), is never + included. Neither is persisted. +- An **executed** transaction is included even if the business operation + fails, is ignored, or cannot pay its whole fee. It advances the count, and an + executed user op consumes its sender's nonce ([below](#user-nonces)). The + wallet also charges the fee for a malformed method or failed transfer; + malformed direct inputs are included no-ops. These semantics must agree with + the canonical application. +- `AppError`, whether `Internal` or `Io`, is fatal in every method. It must not + be disguised as a refusal or an included no-op. Fatal here means discarding + the engine: the host still distinguishes a terminal `Internal` fault from an + operational I/O failure that may clear after restart. + +Every transaction executed live must execute against the same prior state on +replay. The sequencer persists included transactions and replays them on +restart through the same executor, without admission. A stored user op the +executor skips on replay, or a live op it skips after the lane's pre-check +passed, is an impossible state: the lane stops with a terminal error. It does +not recover an instance after a failed hook. #### User nonces -Every application follows one nonce rule. The sequencer relies on it: -`GET /nonce` derives a sender's next nonce from the user ops it persisted, -without asking the engine. +Every application follows one nonce rule, and the executor applies it the same +way for all of them. The sequencer relies on it: `GET /nonce` derives a +sender's next nonce from the user ops it persisted, without asking the engine. - A genesis state starts every account at nonce 0. -- Validation accepts an op only when its nonce equals the sender's expected - nonce. -- Each included user op, business failures included, advances its sender's - expected nonce by exactly one. Nothing else changes a nonce: not direct - inputs, not other senders' ops. +- The executor runs an op only when its nonce equals the sender's next nonce, + as `user_nonce` reports it. +- Each executed user op, business failures and unpaid fees included, advances + its sender's next nonce by exactly one, which the executor asserts. Nothing + else changes a nonce: not direct inputs, not other senders' ops. The executor + cannot check a direct's side, since it cannot enumerate the senders a direct + might touch. - Nonces are `u32`. `u32::MAX` has no successor, so an account that reaches - it is exhausted: when app validation accepts an op at `u32::MAX`, the shared - execution boundary rejects it (`NonceExhausted`). Any other op carrying - `u32::MAX` fails app validation with the expected nonce. - -Under this rule a sender's expected nonce is one past its latest included op. -The exception is a sender with no surviving op since the era baseline. A -genesis baseline puts it at 0, but a [rebuilt baseline](../recovery/cockroach.md) -carries nonces the rebuilt database has no ops for, so `GET /nonce` reads 0 -until that sender's first op, and again if a later recovery invalidates all of -its post-rebuild ops. An engine outside the rule (gapped or keyed -nonces, a deposit that resets one) makes `GET /nonce` quote nonces the engine -rejects. + it is exhausted: the executor skips its op at `u32::MAX` + (`NonceExhausted`). Any other op carrying `u32::MAX` is skipped as a wrong + nonce. + +Under this rule a sender's next nonce is one past its latest included op. The +exception is a sender with no live op. `GET /nonce` counts only the +era's live rows, and every setup, genesis base included, rebuilds the history +up to its stop block first ([era baseline](application-history.md#era-baseline)), +so a sender whose last op lies in the rebuilt history or below the base count +reads 0 until its first live op. The lane's refusal names the expected nonce, which corrects +the client. An engine outside the rule (gapped or keyed nonces, a +deposit that resets one) makes `GET /nonce` quote nonces the lane refuses. The rule keeps every account's nonce in state forever. A batch-scoped alternative makes replay state prunable and lets the fee move to the batch; it is a [deferred design](../plans/2026-10-batch-scoped-replay-protection.md). -### 3. The safe-block clock — `last_executed_safe_block` - -The clock is the maximum block carried by an executed input: frame `safe_block` -for user ops, L1 inclusion block for directs, or zero at genesis. On success, -the engine advances it with `max(old_clock, input_clock)`. Input clocks are -not monotone: [gate d](scheduler-semantics.md#processing-an-input) orders -frame clocks only within a batch, so a later batch's user ops may run at a -`safe_block` below an earlier input's clock (a direct never runs below the -stored clock); the `max` keeps the stored clock monotone. Count zero implies clock zero. Both -fields survive checkpoint round-trips. - -The [frame-clock policy](scheduler-semantics.md#sequencer-frame-clock-policy) -may deliver a delayed head jump as one step. Newly covered directs execute -first with their exact inclusion blocks; all user ops in the frame receive -the same frame clock. An empty clock-only frame does not execute an input or -advance the app clock. - -Recovery reads this clock as `A`, the safe block reflected in the checkpoint; -a wrong clock mis-defines the reconstructed `(A, B]` fridge range -([cockroach recovery](../recovery/cockroach.md)). - -### 4. Canonical history cursor: `executed_input_count` - -The count is the next application-history offset, never a SQLite cursor. -An input executed at count `X` returns offset `X` and advances to `X + 1`. -Included no-ops count; rejections, merely queued inputs, envelopes, and empty -batches do not. Arithmetic is checked, never wrapping or saturating. - -The count belongs to the checkpoint's logical state. Standard recovery may -roll it back to a retained prefix and then advance over replacement inputs; -cockroach recovery supplies an absolute starting count from the recovered -engine. SQLite stores an independent expected snapshot count and per-input +### 3. Admission — policy, never consensus + +Admission answers whether the operator wants an op. Only the sequencer's lane +asks: the canonical machine never does, a guest build need not link it, and +its answer never changes what an included op does +([a user op's fate](scheduler-semantics.md#a-user-ops-fate)). + +| Method | Contract | +|---|---| +| `admit_user_op(sender, op, fee) -> Result` | Say, without changing state and without checking the nonce, whether the operator wants the op at this fee exponent. | +| `max_method_payload_bytes() -> usize` | Return the engine's stable method payload bound without constructing an instance. | +| `scheduler_limits() -> SchedulerLimits` | Return the staleness window, the fridge bound and the direct payload cap the application's template compiles in, without constructing an instance. | + +The staleness window, `max_wait_blocks`, is at least one block. It bounds how +long a sequencer can hold a direct back, and it is all the time operators have +to fix an outage before the era ends +([the window](../recovery/README.md#the-window), +[the budget](../recovery/README.md#the-budget)). `run` refuses a window too +short for its batches: the batch-open deadline in blocks plus a 100-block +submission margin must fit inside it. Changing the window is a new template +and a new setup. + +Setup pins the window and the direct payload cap with the deployment identity. +Storage selects the directs to execute by the pinned cap, and `run` refuses a +binary whose window or cap differs from the pinned one. The pin ties a binary +to its data directory, not to the deployed template: a machine's scheduler +export carries no constants, so building the sequencer and the template from +the same application is the operator's to keep. + +The lane checks a live op with +[`check_live_user_op`](../../sequencer-core/src/application/mod.rs), in the +order whose texts clients see: the fee cap against the frame's price, then the +executor's own nonce rule, then `admit_user_op`. A refusal is the `POST /tx` +422, with the refusal's text as its body. `Underfunded` covers a sender who +cannot pay the frame fee; `Other` carries any other policy, its text verbatim. +The wallet refuses only underfunded ops. + +A zero method payload bound permits only empty method payloads. HTTP rejects +oversized method payloads; the lane uses the declared bound plus signed-op +metadata to compute batch capacity. It is not a canonical scheduler rule. The +scheduler limits are canonical: the [fridge](scheduler-semantics.md#the-fridge) +admits by them, and the lane drops the same oversized directs. + +### 4. The L2 tx count: `l2_tx_count` + +The count is the next application-history offset, never a SQLite cursor. It +counts executed L2 transactions, not L1 inputs. A transaction executed at +count `X` returns index `X` and advances it to `X + 1`. Included no-ops count; +skipped ops, refusals, queued or dropped directs, envelopes, and empty batches +do not ([what counts](scheduler-semantics.md#what-counts)). Arithmetic is +checked, never wrapping or saturating. + +The count belongs to the checkpoint's logical state. Within an era it only +advances; a recovery from a canonical machine starts at that machine's count +([§7](#7-canonical-recovery-integration)). SQLite stores an independent expected snapshot count and per-input execution offsets, checked during catch-up. -HTTP snapshot metadata and mandatory WS claims carry the history version and -this count. Both restart and subscriber replay read the same current application +HTTP snapshot metadata and mandatory WS claims carry the era id and this +count. Both restart and subscriber replay read the same current application sequence. The [history guide](application-history.md) owns coordinates and replica bootstrap; the [API contract](../../README.md) owns wire shapes. @@ -216,9 +250,9 @@ checkpoint and must not depend on the process: which may become a file or directory. On `Ok`, all files and directory entries referencing them, including the parent of `prefix`, must be durable against an immediate kernel crash. SQLite references the checkpoint only - after this returns. The live logical state and progress stay unchanged; + after this returns. The live logical state and its count stay unchanged; subsequent execution cannot mutate the checkpoint. -- `from_dump(prefix)` restores equivalent logical state, including progress, +- `from_dump(prefix)` restores equivalent logical state, including the count, into an independent mutable engine. Execution cannot change the source checkpoint or another restored instance. The restored engine must remain usable after the source checkpoint is garbage-collected. @@ -243,37 +277,41 @@ a broken referenced checkpoint versus a retryable storage failure. Genesis construction stays on the concrete engine because its inputs vary by implementation. `CanonicalState::canonical_snapshot_bytes` is a separate -inspection trait required by the shared Rust scheduler's inspection method and -canonical harness, not by the native sequencer. Human-readable debugging state +inspection trait required by `CanonicalRunner::inspect` and the canonical +harness, not by the native sequencer. Human-readable debugging state also stays on the concrete application. ### 7. Canonical recovery integration -Every supported production application must provide a reproducible mapping from -a trusted canonical machine checkpoint and pinned deployment configuration to -its native recovery artifact. All logical state needed for future application -execution, including progress, must be recoverable this way. Native caches and -backing resources may be reconstructed; indispensable mutable application state -cannot exist only in the sequencer's local storage. - -For the native DEX integration, the native application state is the designated -canonical machine drive/memory region. The integration must pin its location, -layout, and extraction procedure for each supported image. Other applications -may require a different mapping. The recovery bundle also needs the exact L1 -boundary and next scheduler nonce, obtained from trusted canonical execution; -these are separate from merely extracting application bytes. -The exporter must also verify the canonical pending-direct queue satisfies -[checkpoint eligibility](../recovery/cockroach.md#checkpoint-eligibility). -An accurate application clock and `A < B` do not prove that condition after -faulty sequencing. - -Each integration supplies a versioned export command and operator procedure, -and demonstrates recovery from a non-genesis canonical checkpoint before -production use. The [recovery readiness requirements](../recovery/cockroach.md#recovery-readiness-before-deployment) -own the procedure and drill. This is an integration/release requirement, not a -claim that the private engine or current tooling has passed it. - -Canonical extraction belongs to application-specific recovery tooling. The -runtime `Application` trait stays independent of machine image layouts and host -export tooling; requiring a method to compile would establish its availability, -not the correctness or operational readiness of the recovery path. +Every supported production application must rebuild its native instance from +a trusted canonical machine. `from_canonical_state(state_file, count)` builds +an instance from the machine's comparison bytes, which setup reads from the +machine's `state` inspect report, at the machine's L2 tx count, which +the machine's +`scheduler` report carries with the rest of the scheduler's state +([state export](scheduler-semantics.md#state-export)). A dump of the result +holds the same comparison bytes. The count is passed in because an +application's bytes need not hold it; the wallet's do, and it refuses a file +whose count differs. + +So all logical state needed for future execution lives in the comparison +bytes. Native caches and backing resources may be rebuilt, but indispensable +mutable application state cannot exist only in the sequencer's local storage. +For the native DEX integration, the comparison bytes are the designated +canonical machine drive or memory region; the integration pins its location +and layout for each supported image. + +`setup --base ` is the sequencer's recovery from a canonical +machine ([what setup does](../recovery/cockroach.md#what-setup-does)): it loads +the stored machine, answers its `scheduler` and `state` inspect queries, and +builds the instance with `from_canonical_state`. The `state` report must fit +one 2 MiB report; a read through the labeled range, as the watchdog does, is +not built yet ([register](../review/register.md#sequencer-recovery-follow-ups)). +Each integration's conformance tests check the projection; setup's own tests +hand it a machine's export and state bytes directly, and the devnet round +trips from a stored machine (`recovery_from_stored_machine_round_trip_test` +and its C-host twin) build the reference wallet from a real machine's `state` +report ([drills](../watchdog/incident-runbook.md#drills)). +The runtime `Application` trait stays independent of machine image layouts and +host tooling: reading the state source and the `scheduler` report is the +host's work. diff --git a/docs/protocol/application-history.md b/docs/protocol/application-history.md index f249d67e..9523b875 100644 --- a/docs/protocol/application-history.md +++ b/docs/protocol/application-history.md @@ -1,13 +1,13 @@ # Application history and replay -Application history is the sequencer's current execution order: included user +Application history is the sequencer's current execution order: executed user operations and external direct inputs. It contains an optimistic suffix that -recovery may replace. A stable offset identifies an input only together with -its history version; receiving it does not establish L1 acceptance. +a new era may revoke. A stable offset identifies an input only together with +its era; receiving it does not establish L1 acceptance. This document owns history coordinates, recovery boundaries, and replica bootstrap. The [Application contract](application-contract.md) owns execution -and progress; the [snapshot lifecycle](../snapshots/lifecycle.md) owns artifact +and the count; the [snapshot lifecycle](../snapshots/lifecycle.md) owns artifact publication, selection, and retention; the [README API](../../README.md#api) owns routes, wire fields, and refusal codes. @@ -17,175 +17,218 @@ These facts answer different questions: | Fact | Meaning | |---|---| -| `ExecutedInputCount` | Number of application inputs already executed. At count `N`, entry `N` executes next. | -| `ApplicationProgress.last_executed_safe_block` | Maximum clock carried by an executed input: frame safe block for user ops, inclusion block for directs. | -| Latest surviving frame's `safe_block` | Complete L1 interval accounted for by local ordering, bounded below by the era baseline. | -| `safe_accepted_batches` | Safe L1 landings accepted by the scheduler rules and matched to local sealed bytes. | +| `L2TxCount` | Number of L2 transactions already executed; at count `N`, entry `N` executes next. | +| Latest frame's `safe_block` | Complete L1 interval accounted for by local ordering, bounded below by the era baseline. | +| `batches.accepted_input` | The safe L1 landing of a local sealed batch, accepted by the scheduler rules and matched to its bytes. | A frame can account for an empty interval or only batch envelopes, advancing L1 accounting without executing an application input. An accepted batch can confirm existing execution without adding an input. Empty batches can share -an application count. Neither the count nor the app clock substitutes for the -L1 accounting boundary or acceptance facts. +an application count. The count substitutes for neither the L1 accounting +boundary nor the acceptance facts. `safe_inputs` retains all InputBox observations, including batch envelopes. -`application_inputs` contains only the current ordered application sequence; -each row has a mandatory pre-execution offset, an owning batch/frame, and -exactly one reference to a user op or external direct input. Payloads remain -in those source tables. Included business failures and malformed-direct no-ops -advance the count; validation rejections and envelopes do not. - -Restart and egress read this same sequence. Replay executes stored valid -user ops with their recorded fee and frame clock, without revalidation; -directs use their original inclusion block. Timestamps and transaction hashes -are provenance, not extra application-transition inputs. The +An era's history lives in two tables, joined at its +[rebuilt history end](#era-baseline). `rebuilt_history` holds what setup +rebuilt from L1, from the base count up to that end; `application_inputs` +holds the live rows after it. Each live row has a mandatory pre-execution +offset, an owning batch/frame, and exactly one reference to a user op or +external direct input. Payloads remain in those source tables. Included +business failures, underfunded ops and +malformed-direct no-ops advance the count; refusals and envelopes do not. A +direct whose payload is over the application's cap never executes, as in the +[canonical fridge](scheduler-semantics.md#the-fridge), so it gets no row and no +offset: one storage predicate filters it from every query that enumerates +directs to execute. + +Restart and egress read this same sequence. Replay runs each stored user op +through the shared executor with its recorded fee and frame safe block, without +admission; the executor's nonce check still runs, and a stored op it skips is +an impossible state that stops the lane. A direct replays with its inclusion +block, input index and timestamp, which are application inputs. Transaction +hashes are provenance, not transition inputs. The [execution contract](application-contract.md#the-execution-methods) defines the -shared execution boundary. +shared executor. ## Identity and available history -A `HistoryClaim` combines: +A history's identity is its era: a UUIDv4 created by setup, identifying one +local history. A `HistoryClaim` is an era id and a next input, the +application's actual L2 tx count. -- **Era**: a UUIDv4 created by setup/rebuild, identifying one local history. -- **Recovery generation**: a revision within that era, advanced atomically - whenever automatic recovery invalidates at least one valid batch. -- **Next input**: the application's actual executed-input count. - -If the era baseline count is `K` and the current head is `H`, available entries +If the era's base count is `K` and the current head is `H`, available entries occupy `[K, H)`. A claim at `H` waits for future entries; a claim below `K` or -above `H` is refused. Identity is checked before position. Equal counts cannot -authorize resuming a different era or generation, even if the consumer believes -its state precedes the replaced suffix. The compatibility query below can -authorize rebinding a surviving checkpoint to the current version; WS itself -continues to require exact identity. - -Automatic recovery invalidates a batch suffix, removes its current application -rows, advances the generation, and opens the replacement Tip in one transaction. -Replacement inputs reuse suffix offsets under the new generation. Original L1, -batch, frame, and user-op source records remain; the invalidated flattened -sequence is not separately retained. A repair that invalidates nothing leaves -the generation unchanged. The [recovery guide](../recovery/README.md) owns -repair selection and guards. - -### Checkpoint compatibility after standard recovery - -Each generation transition records the surviving application count after the -invalidated rows are removed and before the replacement Tip adds any directs. -This cut commits atomically with invalidation, the generation advance, and -reopening. An invalidated empty batch still creates a transition with the old -head as its cut; a repair that invalidates nothing creates neither. The cuts are -immutable and retained for the era's lifetime. - -For a checkpoint saved in generation `g`, `/history` returns the current version -at generation `G`, head `H`, and preserved count: +above `H` is refused. Identity is checked before position, and equal counts do +not by themselves authorize resuming a different era. + +Inside an era the history only appends: a batch is never replaced, and a +restart changes nothing. A state saved at any count from `K` to `H` therefore +resumes under the same era id. The one event a client handles is a new era, +which a cockroach recovery creates after the old one ends, for example on a +history the scheduler can no longer accept +([the void history](../recovery/README.md#the-void-history)). The +[adopt rule](#adopting-a-new-era) rebinds a saved state to the new era, and +the [client's playbook](../recovery/cockroach.md#the-clients-playbook) says +what a client does around it. WS itself requires the exact era. The client +owns checkpoint consistency: its application state, projection, and claim +describe the same executed prefix. + +### Era baseline + +Every setup starts from a base, genesis or a stored canonical machine, and +rebuilds the history from L1 through a stop block it pins after flushing the +submitter's wallet +([what setup does](../recovery/cockroach.md#what-setup-does)). +Once the baseline's artifact is durable, setup publishes the era in one +transaction: one immutable row in the `era` table, whose existence is what +makes the database set up, and the baseline snapshot. It publishes: + +| Field | Value | +|---|---| +| Base count `K` | The base's L2 tx count, where the era's history starts; `/history` reports it as `available_from` | +| Stop block `C` | The safe block setup pinned after the flush; `/history` reports it as `l1_stop_block` | +| Rebuilt history end `E` | The L2 tx count at the end of block `C`, where the live rows start; `/history` reports it as `rebuilt_history_end` | +| Anchor `N'` | The scheduler's next batch nonce at the end of block `C`, which the era's first batch carries; `/history` reports it as `next_batch_nonce` | +| First queued input | The InputBox index of the oldest direct still queued at the end of block `C`, or the input count through `C` when none is; not on the wire | +| Baseline snapshot | The application's state at count `E` | + +A genesis base has `K = 0` and still runs the rebuild, so a deployment whose +L1 history executed transactions before its stop block starts with `E > 0`. The baseline is +the application's state at the end of block `C`, which the canonical machine +reaches too, so until a batch of the era is accepted the sequencer serves it +as the comparison checkpoint: inclusion block `C`, count `E`, next nonce `N'`. + +Setup opens no batch. `run`'s first frame drains the directs still queued at +`C`, from the first queued input on, as the canonical scheduler's next frame +does. Accepted-batch scanning begins after `C`. The era row survives baseline +artifact GC. + +### The rebuilt history + +`rebuilt_history` holds one row per transaction that executed while setup fed +the scheduler the inputs from the base's position through the last input at +or below `C`, keyed by its count +([what setup does](../recovery/cockroach.md#what-setup-does)). +A direct row keeps its input index, and the feed takes its sender, payload, +block, timestamp and transaction hash from `safe_inputs`. A user op row keeps +its sender, nonce, fee, data and safe block. Each row keeps the scheduler's +next batch nonce when it handed the row out: the batch whose frame holds or +drained it, or the next batch to come. A user op the executor skipped has no +row. + +Setup is the table's only writer, and only before it publishes the era: the +schema refuses an insert after that, any update, any deletion, a count that +does not follow the last, and a direct out of input order. The era's `K` and +`E` are exactly the span of the rows. + +### Adopting a new era + +A cockroach recovery replaces the era, so a client's claim from the old one +is refused with `ERA_CHANGED`. A client holding state at count `X` adopts the +new era at that count when ```text -P = min(H, cut[g+1], ..., cut[G]) +K <= X <= E ``` -For `g = G`, `P = H`. A checkpoint at count `X` is eligible to resume under the -returned version exactly when `K <= X <= P`. Check each saved checkpoint using -its own era and generation, then choose the newest eligible one. For cuts -`0 -> 1: 3` and `1 -> 2: 5`, a generation-0 checkpoint at 4 is invalid, while a -generation-1 checkpoint at 4 is eligible. Looking only at the latest cut would -incorrectly reuse the former. Cuts and current history are read together; a -missing intervening transition is an invariant failure, not permission to take -the minimum over an incomplete ledger. - -The client owns checkpoint consistency: application state, projection, and claim -must describe the same executed prefix. Once compatibility is established, -persist the new version with the restored checkpoint before continuing. If -another recovery wins the race with subscription, query again using that saved -version. A stale response cannot weaken WS admission. The query proves prefix -preservation under standard recovery's trusted local bookkeeping; it does not -inspect client state or establish trust after a software bug. A new era requires -the [manual projection recovery procedure](projection-replay.md#client-checkpoints). +and then subscribes with the current era id and `next_input = X`. +[`HistoryInfo::adopt_claim`](../../sequencer-core/src/history_api.rs) is the +check, and the [Rust SDK](../../sdk/rust-client/src/history.rs)'s +`adopt_current_era` reads `/history` and applies it. The server adds no check +of its own: the adopted claim carries the current era id, which the feed +admits like any other. -### Era baseline +| Refusal | Meaning | Remedy | +|---|---|---| +| `BelowBase` (`X < K`) | The era's base is newer than the client's state | The operator recovers again from an older base, or the client starts over from a current snapshot | +| `PastRebuiltEnd` (`X > E`) | The client's state holds rows the old instance served that never landed by `C` | The client restores an older state | + +Rows past `E` are the new era's live rows, served after the stop block, so an +old-era state there is not canonical even when the head has passed `X`. + +### What the count checks miss -Setup publishes a complete baseline only after its artifact is durable: -application count `K`, accounted L1 stop block `C`, starting batch nonce, and -history identity. Ordinary direct ordering and accepted-batch scanning begin -after `C`. The baseline's metadata survives root invalidation and artifact GC; -no padding inputs or separate mutable processing cursor represent its prefix. +The adopt rule compares counts, not rows. Two faults pass it: -Cockroach recovery drains queued directs through `C`, including young directs -that the canonical scheduler has not executed yet. Its output is a stable -restart baseline; its bytes need not equal canonical state at block `C`. -A later accepted batch snapshot supplies the first comparison checkpoint in that era. -Genesis supplies the trusted block-zero comparison state. The -[rebuild guide](../recovery/cockroach.md) owns checkpoint requirements and the -fixed stopping boundary. +- **A base machine that is not canonical.** Setup trusts its base. The + watchdog's first comparison of the era's baseline exposes it. +- **A client state that is wrong below its count**, such as one built on rows + the old instance served below `E` that the rebuild replaced. Only the + client's own check exposes it: subscribe from `K` and compare each row up to + `X` with the one it stored + ([the client's playbook](../recovery/cockroach.md#the-clients-playbook)). ## Consumers of checkpoints | Consumer | Starting point and continuation | |---|---| -| Native restart | Load the newest surviving batch snapshot, or baseline, check the engine's count against its row, then replay current application inputs. | +| Native restart | Load the newest surviving batch snapshot, or the era baseline at `E`, check the engine's count against its row, then replay the live rows after it. | | Sequencer replica | Download `/latest_snapshot`, restore its application state, and subscribe using the matching history claim. This follows optimistic execution. | | Watchdog | Start from independently trusted canonical machine state and replay L1. Compare at the sequencer's accepted checkpoint; the replica feed does not establish independent trust. | -| Application projection | Reconstruct additional transfer/order history using the era's historical L1 prefix, then join the application feed at the immutable baseline. The [projection replay contract](projection-replay.md) owns its checkpoint preparation and handoff. | +| Application projection | Restore its own checkpoint at a count the era serves, then follow the feed ([the client's playbook](../recovery/cockroach.md#the-clients-playbook)). The feed starts at the base count; the rows below it exist only on L1. | -A batch-close snapshot is identified by its local batch identity, not just its -count or nonce. Recovery can reuse a nonce and empty batches can repeat a count. +A batch-close snapshot is identified by its batch's nonce, not just its +count. Empty batches can repeat a count. Acceptance derives the comparison point without modifying the artifact. Selection must first choose the required accepted batch and then require its snapshot; a missing one cannot justify falling back to an older comparison. The [snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison) -explains block-boundary comparison and rollback retention. The +explains block-boundary comparison and which snapshots are retained. The [watchdog guide](../watchdog/README.md) explains independent verification. ## Replica bootstrap and resume 1. Download `GET /latest_snapshot`. The tar archive contains `info.toml` and the complete opaque application `state` file or directory. -2. Retain its `X-History-Era`, `X-Recovery-Generation`, and - `X-Executed-Input-Count` headers with those bytes. Restore the application and +2. Retain its `X-History-Era` and `X-Executed-Input-Count` headers with those + bytes. Restore the application and verify its count against that header. The archive alone does not carry the complete subscription claim. -3. Subscribe with the matching era, generation, and next-input count. Snapshot - selection, headers, and lease share one transaction; recovery during the - download can still invalidate the claim before subscription. Rebootstrap - if the server refuses that old era; within the same era, a compatible saved - checkpoint can instead be selected through `/history`. +3. Subscribe with the matching era and next-input count. Snapshot selection + and headers share one read, taken under the lease. If the server refuses + the era, because a new one started after the download, [adopt](#adopting-a-new-era) + the new one at the restored count, or rebootstrap. 4. Require each entry's offset to equal the application's current count, then execute it through the shared execution boundary. Successful application advances the count by one. Persist the history identity with the replica's state so that a later resume cannot combine different histories. 5. After an ordinary disconnect, reconnect with that saved identity and the - actual count. A generation mismatch permits the compatibility procedure - above. An unavailable prefix or lack of a compatible checkpoint requires a - current snapshot. A count ahead of the server's head is an invalid claim to correct. + actual count. An era mismatch permits the adopt rule. An unavailable prefix + requires a current snapshot. A count ahead of the server's head is an + invalid claim to correct. The [Rust SDK](../../sdk/rust-client/src/lib.rs) returns a `HistoryClaim` with its snapshot response and requires an explicit claim for subscriptions. The consumer owns restore, persistence, and reconnect. Fetching fresh identity -metadata alone cannot authorize old application state; an explicit compatibility -result can authorize a saved prefix within the same era. +metadata alone cannot authorize old application state; `adopt_current_era` +authorizes one from an older era. One durable page reader handles both backlog and live delivery, so there is no -separate cursor to switch at the live boundary. Each page reads identity, -bounds, and rows in one SQLite transaction. Interior gaps and invalid source -context fail loudly. Page size and send queues bound memory; available backlog +separate cursor to switch at the live boundary. It serves the rebuilt rows in +count order up to `E`, then the live rows, and a page may span that +seam. Each page reads identity, bounds, and rows in one SQLite transaction. +Interior gaps, a missing rebuilt row below `E`, and invalid source context +fail loudly. Page size and send queues bound memory; available backlog has no total replay cap. The [API contract](../../README.md#api) owns exact resource limits and handshake errors. -History replacement happens across a process boundary, after existing -subscriptions end. A session's mandatory claim therefore binds all its events; -there is no per-event generation or guaranteed farewell message. Revisit this -assumption if history can change within an admitted process or multiple local -writers become supported. +The history never changes inside an era, and a new era lives in a new data +directory served by a new process, after existing subscriptions end. A +session's mandatory claim therefore binds all its events; there is no +per-event identity or guaranteed farewell message. Revisit this assumption if +multiple local writers become supported. ## Code and validation map | Boundary | Code and tests | |---|---| | Coordinates and claim ordering | [`sequencer-core/src/history.rs`](../../sequencer-core/src/history.rs) — identity before position, inclusive head, checked counts. | -| Coherent application pages | [`storage/history/canonical.rs`](../../sequencer/src/storage/history/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | +| Coherent application pages | [`storage/history/canonical.rs`](../../sequencer/src/storage/history/canonical.rs) and its tests — source context, the rebuilt/live seam, nonzero baselines, gaps and SQL limits. | +| Rebuilt history | [`storage/rebuilt_history.rs`](../../sequencer/src/storage/rebuilt_history.rs) and its tests — contiguous counts, directs in input order, immutability, sealing at publication; [`commands/setup/rebuild.rs`](../../sequencer/src/commands/setup/rebuild.rs) — the export check, and the rebuild executing as it goes. | +| Era adoption | [`history_api.rs`](../../sequencer-core/src/history_api.rs) — `adopt_claim` and its range; the [SDK](../../sdk/rust-client/src/history.rs)'s `adopt_current_era`. | | Replay followed by live delivery | [`l2_tx_feed`](../../sequencer/src/egress/l2_tx_feed/) — bounded deep replay, identity refusals, shutdown and persistent faults; [`catch_up.rs`](../../sequencer/src/ingress/inclusion_lane/catch_up.rs) for native replay. | | Artifact and claim association | [`snapshot_endpoints.rs`](../../sequencer/src/integration_tests/snapshot_endpoints.rs) — headers, restore, archive contents, and lease lifetime. | -| Checkpoint compatibility | [`storage/history.rs`](../../sequencer/src/storage/history.rs) and recovery tests — immutable cuts, complete lineage, and transaction rollback; [`recovery_compatibility.rs`](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) — saved projections across missed recoveries and HTTP lookup/WS admission races. | The [integration validation record](../review/2026-09-16-track3-validation.md) records wallet replica and canonical-machine evidence. Remaining consumer and -deployment gates belong to the [Track 3 plan](../plans/2026-07-track3-feed-replay-design.md). +deployment gates are in the [register](../review/register.md#integration-work-owned-elsewhere). diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index 49b03f1c..2e9c686f 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -11,10 +11,16 @@ linking, genesis, host commands, and the reference wallet under `examples/`. ## Contract ownership - The [Application contract](application-contract.md) defines execution, - rejection, progress, deterministic outputs, and checkpoint obligations. + nonces, admission, the count, deterministic outputs, and checkpoint + obligations. - [Scheduler semantics](scheduler-semantics.md) defines canonical ordering and the acceptance boundary. The native engine does not acquire an ordering role - through this ABI. + through this ABI. A C or C++ guest reaches the reference scheduler through + [`scheduler.h`](../../bindings/c-scheduler/include/scheduler.h), a proposed + header in [`bindings/c-scheduler`](../../bindings/c-scheduler/README.md) + whose transaction records are this header's. A port of that header is held + to the [conformance vectors](scheduler-semantics.md#conformance) through the + crate's `tests/conformance.rs`. - The [C header](../../bindings/c-app-engine/include/application-engine.h) defines record layout, statuses, pointer ownership, and call lifetimes. An engine archive and its generated Rust bindings must agree on that header. @@ -25,16 +31,45 @@ linking, genesis, host commands, and the reference wallet under `examples/`. ## Execution and ownership Each opaque engine handle owns its mutable state. It may move between threads, -and calls on that handle are exclusive. Progress is returned together as -`ApplicationEngineProgress`; the shared execution boundary checks it against the -successful transition. The payload-bound getter takes no instance and returns a -stable value for the linked engine. Zero is a valid bound. +and calls on that handle are exclusive. `EngineApp` first checks +`application_engine_abi_version()` against the header's +`APPLICATION_ENGINE_ABI_VERSION`, before any other call, so an archive built +against another header is refused rather than run under changed rules. -Expected validation rejection leaves state unchanged. An execution, validation, -or output-drain failure discards the engine; the host owns process termination -and distinguishes terminal faults from retryable operational failures. -Exceptions must not cross the ABI. The header defines the lifetimes of borrowed -output and diagnostic buffers; the adapter copies them before reuse. +The calls map onto the [execution methods](application-contract.md#the-execution-methods) +and [admission](application-contract.md#3-admission--policy-never-consensus): + +- `application_engine_user_nonce` reports a sender's next nonce; the shared + executor, not the engine, compares it with each op and checks that an + execution advanced it by exactly one. +- `application_engine_execute_user_op` and + `application_engine_execute_direct_input` are total: every op handed to the + engine executes, a fee its sender cannot cover included, and every execution + advances the count by one. A direct arrives with its input index and + timestamp. +- `application_engine_l2_tx_count` reports the engine's L2 tx count, + which the executor checks against each successful transition. +- `application_engine_admit_user_op` is lane-only policy. It answers `INVALID` + with `INSUFFICIENT_FEE_BALANCE`, or with `REFUSED_BY_POLICY` and its text in + the last error message, which reaches the client verbatim. Retired reasons + keep their numbers, and the adapter treats them, or an unknown one, as fatal. +- `application_engine_scheduler_limits` and + `application_engine_max_method_payload_bytes` take no instance and return + stable values for the linked engine. The limits are the staleness window, + which is at least one block, the fridge bound and the direct payload cap. + Zero is a valid payload bound. +- `application_engine_from_canonical_state` opens an engine over a canonical + machine's comparison bytes at that machine's count + ([canonical recovery](application-contract.md#7-canonical-recovery-integration)). + `setup --base` calls it with the bytes the stored machine answers to the + `state` inspect query and the count from its `scheduler` report + ([state export](scheduler-semantics.md#state-export)). + +An admission, nonce-read, execution, or output-drain failure discards the +engine; the host owns process termination and distinguishes terminal faults +from retryable operational failures. Exceptions must not cross the ABI. The +header defines the lifetimes of borrowed output and diagnostic buffers; the +adapter copies them before reuse. Fee fields carry base-129/128 exponents. The shared max-fee comparison operates in log space; an application checking balances or charging fees uses a linear @@ -43,14 +78,14 @@ amount. The exact conversion contract lives on `fee_to_linear` in the table, intermediate flooring, and ascending bit order, not just the nominal exponential formula. [`build.rs`](../../sequencer-core/build.rs) generates the table and exponent bound. Native and canonical execution must agree on the -conversion, since different amounts can change rejection decisions and balances. +conversion, since different amounts can change balances and admission decisions. The same implementation may be compiled for native execution and the canonical machine. This does not establish equivalent behavior across targets: the application must preserve deterministic state and output bytes, including the checkpoint's canonical comparison file. Reference wallet ABI tests exercise the adapter contract; the `c_host_` process scenarios exercise snapshot replication, -restart, both recovery paths, and canonical-machine comparison as described in +restart, setup's rebuild, and canonical-machine comparison as described in the [build guide](../../bindings/c-app-engine/README.md#reference-wallet). These tests establish reference-wallet agreement for their workloads, not private DEX conformance. diff --git a/docs/protocol/projection-replay.md b/docs/protocol/projection-replay.md deleted file mode 100644 index 1667c59b..00000000 --- a/docs/protocol/projection-replay.md +++ /dev/null @@ -1,129 +0,0 @@ -# Application projections from historical L1 inputs - -A reader may maintain transfers, orders, or other application-specific history -absent from the latest application state. It owns that projection and its -checkpoints. The sequencer supplies raw scheduler inputs for the immutable era -baseline, then application inputs for the optimistic suffix. The canonical -scheduler remains the ordering authority. The [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only) -owns routes, fields, limits, and refusals; the -[history contract](application-history.md) owns application coordinates. - -The planned shared recovery tool supersedes this workflow, so it gets bug fixes -only; new client work should not build on it -([Track 3](../plans/2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign)). - -## Bootstrap and handoff - -1. Read `/history`. Pin the deployment identity and era, baseline application - count `K`, L1 stop block `C`, exclusive raw end `R`, and next scheduler nonce. - Use the deployment's application/genesis configuration and scheduler release, - including its batch codec, wait bound, and signing-domain name/version. -2. From trusted genesis, request historical records from raw index 0 and process - every record through the scheduler. Preserve InputBox order, including within - blocks. Malformed/rejected batch payloads still reach the scheduler: its - overdue-direct backstop runs before batch decoding. -3. Continue using the returned raw cursor until it equals `R`; page length and - block changes are not EOF. Drain the remaining directs through `C`, reproducing - the recovery terminal drain. Verify resulting count `K` and next nonce. -4. Refresh `/history?era_id=` for the current generation, confirm - the baseline association, and subscribe at application count `K`. The feed - contains entries `[K,head)` and then future entries. It does not redeliver the - inputs already included in the baseline. - -Raw records carry original inner payloads and authenticated sender/block context, -not the complete machine transport envelope. The signing domain uses the pinned -release plus deployment chain id and app address. Timestamps and transaction -hashes are provenance, not additional scheduler transition inputs. - -An ordinary generation change leaves `C/R/K/nonce` and historical pages intact. -If it occurs before subscription, refresh the same era's metadata and retry the -claim. An era change requires establishing correspondence with the new baseline; -never silently splice its pages into an earlier replay. Count/nonce equality is -a consistency check, not independent proof that a client computed correct state. - -The terminal-drained baseline need not equal canonical state at block `C`: -young directs may execute preemptively. A later accepted batch provides the -canonical comparison boundary. Keep that distinction when validating recovery. - -## Client checkpoints - -Save core application state, projection, and their actual `HistoryClaim` -consistently. An exact era-baseline checkpoint can be rebound to the current -generation after confirming the same immutable era/baseline. For suffix -checkpoints, query `/history?era_id=&from_generation=` -and require `K <= saved_count <= compatibility.preserved_input_count`. Query each -candidate using its own generation and choose the newest eligible backup. -Restore the complete application/projection checkpoint, persist the returned -version with it, and resume at its saved count. Repeat compatibility lookup if -another recovery causes WS to refuse that version. The -[history contract](application-history.md#checkpoint-compatibility-after-standard-recovery) -owns the calculation and its standard-recovery trust assumptions. - -For efficient manual recovery, prepare checkpoints at supported accepted L1 -block boundaries. Read the coherent `accepted_checkpoint` and `history.version` -from `/history`; restore an earlier compatible client backup and replay the -application feed exactly to the accepted count. Save the complete result with -that receipt. If the live reader is already ahead, use an independent restore; -the client owns the copy/replay cost. Bind the receipt to the matching history -and count. Count alone does not identify its block/nonce: empty batches can -share a count, and generations can reuse replaced offsets. - -Such a backup contains core state and projection at count `X`, inclusion block -`B`, next scheduler nonce `N`, and the application's own clock `A`. The -[manual recovery contract](../recovery/cockroach.md#checkpoint-eligibility) -requires no pending canonical direct at or below `A`, as well as `A < B` -(except known empty genesis) and `B <= C` for the target rebuild. Establish -queue eligibility against the canonical checkpoint; application-state equality -and the scalar bounds alone cannot prove it after faulty sequencing: - -1. Independently establish trust in the backup, projection implementation, and - checkpoint boundary under the [incident playbook](../recovery/cockroach.md#application-specific-reader-state). -2. Fetch raw records **after `A`**, not after `B`. Enqueue external directs from - `(A,B]` without executing them; skip batch envelopes in this seed range. -3. Process all records in `(B,C]` through the scheduler at nonce `N`, then perform - the same terminal drain and handoff as genesis bootstrap. - -Seeds and replay may share one paginated traversal. Page boundaries can split -block `B`; they must not cause premature replay or draining. An arbitrary -mid-batch optimistic checkpoint lacks this scheduler continuation. If an eligible -backup cannot be trusted, use an earlier eligible backup or trusted genesis. -Watchdog agreement on current application bytes does not certify the reader's -additional transfer/order history. - -If checkpointing during raw replay, persist the scheduler queue/nonce and raw -cursor alongside app/projection state, or resume from the prepared backup. -Persisting only an application count cannot resume an interrupted scheduler. - -## Worked recovery boundary - -With the 1200-block wait bound, consider valid direct inputs `D*` and accepted -user operations `U*` in this raw order: - -| Raw index | Block | Input | Application execution | -|---:|---:|---|---| -| 0 | 5 | `D0` | Queued | -| 1 | 12 | `D1` | Queued | -| 2 | 20 | Batch 0, safe block 10, `U0` | `D0`, `U0` | -| 3 | 20 | `D2`, after the batch | Queued | -| 4 | 24 | `D3` | Queued | -| 5 | 1230 | Malformed batch | Backstop executes `D1,D2,D3`; decoding rejects | -| 6 | 1232 | Batch 1, safe block 1228, `U1` | `U1` | -| 7 | 1235 | `D4` | Queued | - -The backup at `B=20` has count 2, clock `A=10`, and next nonce 1. A count-1 -backup sits partway through batch 0; it must execute `U0` before attaching this -receipt. Seed reconstruction starts at raw index 1, preserving both `D1` and the -same-block `D2` without executing batch 0 twice. - -For a rebuilt stop `C=1240`, the malformed input executes three overdue directs -without consuming a batch nonce. Batch 1 executes `U1`; terminal drain executes -`D4`. The resulting baseline is `K=7`, next nonce 2, raw end `R=8`, and app clock -1235. Subscribe at application offset 7; the next raw index is independently 8. - -The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) -restores a wallet checkpoint with separately saved notice history, deletes the -source backup, fetches one-record HTTP pages through the SDK, executes this -trace, and joins the real WS feed at nonzero `K`. It checks full wallet state and -explicit projection order against uninterrupted execution. It uses controlled -storage fixtures and a trusted checkpoint receipt; it does not establish Bart's -private database backup procedure or canonical-machine export conformance. diff --git a/docs/protocol/scheduler-semantics.md b/docs/protocol/scheduler-semantics.md index 131837f0..34f6f1f0 100644 --- a/docs/protocol/scheduler-semantics.md +++ b/docs/protocol/scheduler-semantics.md @@ -1,37 +1,110 @@ # Scheduler acceptance semantics -The canonical scheduler folds an application's L1 inputs, in L1 order, into -application state. This page specifies that fold for anyone who reimplements -it. The reference is [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs), -run in the canonical machine by the [canonical-app harness](../../examples/canonical-app/src/scheduler/mod.rs). -This text is the contract and the reference can have bugs, so report any -disagreement. The [application contract](application-contract.md) owns what -the application does with each input. Rules marked **pending** describe the -reference today; changing one changes the canonical machine image or its wire -format, so they are settled with the C++ port's author -([register](../review/register.md#canonical-rules-open-for-the-c-port)). +The canonical scheduler turns an application's L1 inputs, in InputBox order, +into one sequence of L2 transactions, and the application executes them. Every +validator runs it inside the canonical machine; the sequencer predicts it while +it sequences. This page specifies it for anyone who reimplements it. The +reference is the concrete [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs), +which decides the sequence, with [`application::execute`](../../sequencer-core/src/application/mod.rs) +running each transaction it hands out. +[`CanonicalRunner`](../../sequencer-core/src/scheduler/driver.rs) joins the two, +and the [canonical-app harness](../../examples/canonical-app/src/scheduler/mod.rs) +runs it in the machine. This text is the contract and the reference can have +bugs, so report any disagreement. The [application contract](application-contract.md) +owns what the application does with each transaction. Rules marked **Open** +are still to settle with the C++ port's author +([register](../review/register.md#canonical-rules-open-for-the-c-port)): +changing one changes the canonical machine image or its wire format. ## The one stream, the one classification +An **L2 transaction** is a direct input or a user op. The scheduler builds the +sequence one L1 input at a time: + +1. **Every input first runs the backstop.** Directs queued for at least + `max_wait_blocks` execute, oldest first, even when the input is a batch the + gates then reject. +2. **A direct input** joins the *fridge*, the queue of directs not yet + executed, unless its payload is over the cap. A full fridge first drains + its oldest blocks to make room ([the fridge](#the-fridge)). +3. **A batch** is accepted or changes nothing ([batch validity](#batch-validity)). +4. **Each frame of an accepted batch** first executes every queued direct up + to its safe block, oldest first, then hands out its user ops in order. +5. **A user op executes or is skipped** ([a user op's fate](#a-user-ops-fate)). + Each input is a Cartesi Rollups `EvmAdvance` from the InputBox. The scheduler -reads its `msgSender`, its `blockNumber` (the *inclusion block*), its `chainId` -and `appContract` (the signature domain), and its `payload`. It classifies by -sender only, never by payload content: +reads its `msgSender`, its `index`, its `blockNumber` (the *inclusion block*), +its `timestamp`, its `chainId` and `appContract` (the signature domain), and its +`payload`. It classifies by sender only, never by payload content: - `msgSender == sequencer_address`: the payload is a **batch**. -- Any other sender: a **direct input**, appended unchanged to the *fridge*, - the FIFO of directs not yet executed. +- Any other sender: a **direct input**. `sequencer_address` is deployment data compiled into the canonical machine. It -must equal the batch-submitter address that `setup` pins; nothing checks this -at runtime. If they differ, the machine treats every batch as a direct input -while the sequencer still reports its batches accepted, so only a watchdog -comparison reveals it. +must equal the batch-submitter address that `setup` pins, and nothing checks +this at runtime (a setup check is **Open**). If they differ, the machine treats +every batch as a direct input while the sequencer still reports its batches +accepted, so only a watchdog comparison reveals it. + +### What counts + +The application's **L2 tx count** is how many L2 transactions it has +executed. It names a state and the transaction that state executes next, so +the feed numbers its rows with it. It counts transactions, not L1 inputs. -The scheduler's state is `next_batch_nonce` (`u64`, 0 at genesis), the fridge, -and the application, which holds its executed-input count and clock. Nothing -else carries over between inputs: no earlier batch's `safe_block` is kept. The -fridge has no size bound (**pending**). +| Outcome of a user op | What happens | Counts | +|---|---|---| +| **Skipped** | Nothing: no nonce, no fee, no effect. | No | +| **Executed, succeeded** | The sender's nonce advances, the fee is charged and the effect applies. | Yes | +| **Executed, failed** | The sender's nonce advances and the fee is charged, but the effect fails, such as a transfer above the balance. | Yes | + +An op whose sender cannot pay the whole fee executes too. Every executed direct +counts, whatever the application does with it. Nothing else counts: not a +queued or dropped direct, not a skipped op, not a batch envelope, not anything +in a rejected or stale batch. A rejected or stale batch can still advance the +count through the backstop. + +### The scheduler's state + +| Field | Meaning | +|---|---| +| `l1_input_count` | How many InputBox inputs the scheduler has consumed: its position. | +| `next_batch_nonce` | The nonce the next accepted batch must carry; `u64`, 0 at genesis. | +| `clock` | The latest block the scheduler has executed at; 0 at genesis ([scheduler clock](#the-scheduler-clock)). | +| fridge | The admitted directs not yet executed, oldest first, each with its input index. | + +Nothing else carries over between inputs. **The scheduler's state never reads +the application's.** The nonce, the clock and the fridge depend only on the +inputs; the application affects them only by halting the machine, and whether +it executes a user op changes none of them. The scheduler runs no application +code either: the fridge admits by sender and payload length, and the nonce +check belongs to the executor. A port must keep this property, which lets the +scheduler run apart from the application. + +**The fridge is a queue.** Directs join at the back in input order and leave +only from the front, whichever rule drains them. So it always holds exactly the +admitted directs from one input index, the *first queued input*, up to the L1 +input count. + +The two counts differ, because a batch envelope or a queued direct is an L1 +input but not an L2 transaction. The application knows its L2 tx count, and a +client knows only that one; the machine knows both +([state export](#state-export)). + +### The invariant + +The [scheduler clock](#the-scheduler-clock) gives two properties after every +input: + +- **Blocks never go backwards** along the L2 sequence. Gate f gives this. +- **The clock splits the directs.** Every admitted direct at or below the clock + has executed, and every one above it is still queued. Strict frames, the + backstop's raise and whole-block overflow drains give this. + +The split gives the clock a meaning an application can use: at clock `c`, it +has applied every direct up to block `c` and none after. +[`Scheduler::resume`](../../sequencer-core/src/scheduler/mod.rs) refuses an +exported state whose queued directs break it. ## Wire format @@ -52,7 +125,7 @@ decrease and stay in bounds, so only the canonical encoding of a `Batch` decodes, and re-encoding returns the same bytes. `max_fee` and `fee_price` are fee exponents ([constants](#constants)). A `signature` of any length decodes, but only 65 bytes can recover a sender; a -fixed 65-byte field is **pending**. +fixed 65-byte field is **Open**. ## User-op signatures @@ -63,7 +136,7 @@ name `CartesiAppSequencer`, version `1`, the input's `chainId`, its `appContract` as `verifyingContract`, and no salt ([`build_input_domain`](../../sequencer-core/src/lib.rs)). The sender is not signed; it is recovered from `r ‖ s ‖ v` (alloy's `Signature::from_raw` and -`recover_address_from_prehash`), and any failure drops the op: +`recover_address_from_prehash`), and any failure skips the op: 1. The signature must be 65 bytes, and `v` must be 0, 1, 27, 28 or at least 35. The y-parity is `v` for 0 and 1; for `v ≥ 27` it is 1 when `v` is even, @@ -75,103 +148,359 @@ signed; it is recovered from `r ‖ s ‖ v` (alloy's `Signature::from_raw` and and the EIP-712 signing hash. This acceptance set is the library's, pinned on both the batch path and -`POST /tx` by the signature vectors except for `s ≥ n` and `v` from 29 to 34; -it is **pending**. Batches the sequencer +`POST /tx` by the signature vectors, and for `s = n` and `v` from 29 to 34 on +the batch path by the guest scenarios ([conformance](#conformance)). +Narrowing it to `v ∈ {27, 28}` and low `s` is **Open**. Batches the sequencer builds carry `v ∈ {27, 28}`, but keep a user's `s` as signed. ## Processing an input -Every input first runs the **censorship backstop**: pop and execute directs -from the front of the fridge while -`input.inclusion_block − front.inclusion_block ≥ MAX_WAIT_BLOCKS`. This bounds how -long a sequencer that stops submitting, or freezes `safe_block`, can hold a -direct back. Then a direct joins the fridge, and a batch runs these gates in -order; the first that applies decides the outcome: +1. **The backstop.** Pop and execute directs from the front of the fridge + while `input.inclusion_block − front.inclusion_block ≥ max_wait_blocks`, + raising the clock to each one's block. This bounds how long a sequencer + that stops submitting, or freezes `safe_block`, can hold a direct back. +2. **A direct** joins the fridge, unless its payload is over the cap, and once + there is room ([the fridge](#the-fridge)). The cap comes first: a direct + that never joins causes no overflow drain, though the backstop has run. +3. **A batch** runs the gates below. + +### Batch validity + +The first gate that applies decides the outcome: | # | Gate | Outcome | `next_batch_nonce` | |---|---|---|---| -| a | payload does not SSZ-decode as `Batch` | rejected | kept | -| b | `batch.nonce != next_batch_nonce` | rejected | kept | -| c | `frames` is empty | executed, as a no-op | +1 | -| d | frame structure is invalid (below) | rejected | kept | -| e | `inclusion_block − frames[0].safe_block ≥ MAX_WAIT_BLOCKS` | skipped as stale | kept | -| f | otherwise: execute every frame (below) | executed | +1 | - -Ages saturate at zero. An empty batch is valid at any age, since it has no -first frame (**pending**). Gate d rejects a batch whose frames include a -`safe_block` above `inclusion_block`, or a `safe_block` lower than the previous -frame's, or a `fee_price > MAX_EXPONENT`. Gate d checks all frames before -any frame drain or user-op execution; the overdue-direct backstop still runs -before the gates. Structural validation looks only inside the batch: a later -batch may start below an earlier one's last `safe_block`, and its user ops then -see the lower value. A -stale batch keeps the nonce, so every later batch fails gate b until one -reuses that nonce, as a batch rebuilt after sequencer recovery does. A -rejected and a skipped batch have the same effect: no state change, nonce -kept. The reject reason the reference reports is diagnostic. - -### Frame execution (gate f) - -For each frame, in order: +| a | The payload does not SSZ-decode as `Batch`. | Rejected | Kept | +| b | `batch.nonce != next_batch_nonce` | Rejected | Kept | +| c | `frames` is empty. | Accepted; executes nothing | +1 | +| d | The frame structure is invalid (below). | Rejected | Kept | +| e | `inclusion_block − frames[0].safe_block ≥ max_wait_blocks` | Skipped as stale | Kept | +| f | `frames[0].safe_block < clock` | Rejected | Kept | +| g | Otherwise: execute every frame. | Accepted | +1 | + +Gate d requires, for every frame: + +- its `safe_block` strictly below `inclusion_block`; +- its `safe_block` at or above the previous frame's; +- its `fee_price` at most `MAX_EXPONENT`. + +Gate d checks every frame before any frame executes; the backstop has already +run. Ages saturate at zero. An empty batch is accepted at any age, since it has +no first frame; rejecting empty batches is **Open**. A rejected batch and a +stale one have the same effect: no state change, nonce kept. So every later +batch fails gate b until one reuses that nonce, as only a new era's batches +do. The reject reason the reference reports is diagnostic. + +### Frame execution + +For each frame of an accepted batch, in order: 1. **Drain covered directs.** Pop and execute directs from the front of the fridge while their `inclusion_block ≤ frame.safe_block`. The `safe_block` is the sequencer's claim that it accounted for every direct up to that block; - the scheduler enforces the drain whatever the sequencer did. -2. **Execute the user ops** in order. Skip an op, with no effect, if (checked - in this order) its sender cannot be recovered, `max_fee < frame.fee_price`, - the application rejects it at fee `frame.fee_price`, or its `nonce` is - `u32::MAX`, which has no successor. Otherwise apply it with fee - `frame.fee_price` and clock `frame.safe_block` - ([`validate_and_execute_user_op`](../../sequencer-core/src/application/mod.rs)). - -## Outputs and the executed-input count + the scheduler enforces the drain whatever the sequencer did. Then raise the + clock to `frame.safe_block`. +2. **Hand out the user ops** in order. The scheduler skips an op whose + `max_fee` is below `frame.fee_price` or whose sender cannot be recovered + ([signatures](#user-op-signatures)); both leave no trace, so their order is + unobservable. The executor then reads the sender's next nonce from the + application and skips the op, checked in this order, when its nonce is not + that one, or when that one is `u32::MAX`, which has no successor. + + Otherwise the op executes with fee `frame.fee_price` and block + `frame.safe_block`: the application advances the sender's nonce and the + L2 tx count by one each, and the executor checks both. + +The application stores each sender's next nonce and reports it; shared code +compares it with the op's, so no application code decides which ops execute +([application contract](application-contract.md#user-nonces)). + +### A user op's fate + +Two parties decide, and they ask different questions: + +- **The canonical scheduler asks whether the sender authorized the op.** A + check that protects the user is consensus: the signature, the nonce and the + fee cap. An op that fails one is **skipped**, with no trace. +- **The sequencer's lane asks whether the operator wants the op.** A check that + protects the operator is admission: can the sender pay, does the payload fit + the batch, does the operator's policy allow it. An op that fails one is + **refused** at `POST /tx`. The canonical scheduler never asks: the fee is + owed to the sequencer, and only the sequencer includes ops. + +| An op that… | Checked by | Canonical scheduler | Sequencer's lane | +|---|---|---|---| +| has no recoverable sender | The scheduler | Skipped | Refused | +| caps its fee below the frame's price | The scheduler | Skipped | Refused | +| carries a nonce that is not its sender's next | The executor, from the application's stored nonce | Skipped | Refused | +| comes from a sender whose next nonce is `u32::MAX` | The executor | Skipped | Refused | +| comes from a sender who cannot pay the fee | The application's admission | Executed | Refused | +| exceeds the application's declared payload bound | The lane | Executed | Refused | +| breaks an operator policy | The application's admission, or the lane | Executed | Refused | +| fails in the application, such as a transfer above the balance | The application's execution | Executed | Admitted | +| is sponsored: the application charges it nothing | The application's execution | Executed | Admitted | +| is none of the above | | Executed | Admitted | + +- **Skipped** means nothing changes: no nonce, no count, no fee. +- **Executed** means the sender's nonce and the L2 tx count each + advance by one. What the op pays and whether its effect applies are the + application's choice ([application contract](application-contract.md#the-execution-methods)). + The application must not fail, since an application error halts the machine. +- **The lane refuses everything the scheduler skips**, by running the same + checks first ([`check_live_user_op`](../../sequencer-core/src/application/mod.rs)). + No op is admitted and then skipped. + +**An underfunded op executes.** Whether a sender can pay protects only the +sequencer from its own inclusion, so the lane checks it and consensus does not. +Which transactions execute then depends on signatures, fee caps and nonces +alone, never on a balance: a fault in the application's logic can change what +a transaction does, but not which transactions execute or what their counts +are. + +**A fee cap below the price skips the op.** The cap is part of what the sender +signed: execute this op only at a price up to the cap. Charging the price would +take more than the sender authorized, and executing at the cap would make a +frame's price mean something else for one op. The sequencer loses nothing: it +sets each frame's price, so it can put the op in a frame it fits. A skipped op +stays valid until its sender's nonce moves past it. + +**A sponsored op needs no rule of its own.** Whether an executed op pays is the +application's choice, and whether an unpaid op is wanted is admission's. An +operator can run its own price feed for nothing by funding the feed's account +where fees are credited to the operator, as the wallet does; by exempting the +role in the application; or by putting its ops in a frame priced at exponent 0. +A sponsored sender still signs a cap at or above the frame's price, and an +exemption must hang on something the signature proves, the sender's role, or +anyone can use it. + +## The scheduler clock + +The clock is the latest block the scheduler has executed at: an accepted +frame's safe block, or an executed direct's inclusion block. Every frame of an +accepted batch raises it, and so does every executed direct, whether a frame, +the backstop or an overflow drained it. It only rises: a raise takes the later +of the clock and the new block. Gate f rejects a batch whose first frame lies +below it, and gate d orders the rest. + +**Gate f alone keeps blocks from going backwards.** Every frame lies at or +above the clock, so a frame's directs come from the clock or above and at or +below its safe block, and its user ops see the safe block. The backstop needs +no help: a batch whose first frame lies below a block the backstop drained is +already stale by gate e. + +Three more rules keep the clock's [split of the directs](#the-invariant): + +- **Strict frames** (gate d) let a frame drain every direct of its own safe + block, because all of them landed before its batch. With equality allowed, a + direct from that block landing after the batch would stay queued at the + clock. Equality is also a claim no sequencer can make honestly: the block + was not finished when the batch was sealed. +- **The backstop raises the clock**, as every drain does, so no direct it + executed sits above the clock. The raise changes no outcome: gate e already + skips any batch it would reject. +- **Overflow drains whole blocks**, so no direct at or below the clock stays + queued ([the fridge](#the-fridge)). + +Unless an overflow overtakes it, a correct sequencer never trips gate f or the +strict rule: it never opens a frame below an earlier one, and it opens frames +only at safe heads it has already seen, so its batch lands in a later block. +Its own frame ordering ([I3](../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine)) +is consensus-critical: a bug there gets batches rejected. + +## The fridge + +- **Admission.** A direct whose payload is longer than the cap + (`max_direct_payload_bytes`) never joins: it never executes and never counts. + Every other direct joins, whatever it holds, and the application ignores what + it cannot read. The sequencer's lane drops the same directs by the same + constant, or the two diverge. +- **Why a constant.** A count bounds memory only if each entry's size is + bounded too: portals accept zero-value deposits with extra data up to the + InputBox's 64 KiB cap. A constant bounds the size without the scheduler + running application code. A filter written by the application would also + drop small garbage, but a zero-value portal deposit takes a slot for about + the same L1 gas, so it would buy little. +- **Entries.** Each entry keeps the direct as delivered, with the + [direct context](#direct-context) the application receives. +- **The bound.** The fridge is full when it holds `fridge_bound` entries or + more. When a direct arrives at a full fridge, the scheduler executes whole + blocks from the front, oldest first, until there is room, raising the clock + to each drained block. It never drains the arriving direct's own block: if + only that block's directs remain, the direct joins over the bound. The bound + is then soft by at most one block's directs, and memory stays bounded by what + one L1 block can carry. +- **An overflow can reject sealed batches.** The sequencer cannot predict an + overflow, and a batch it sealed earlier then lands in one of two ways. If its + first frame is at or above the last drained block, it is accepted, and the L2 + sequence is the one the sequencer predicted: the drained directs are the + first ones that frame would have drained. If it is below, gate f rejects the + batch, and every later one fails the nonce gate. The sequencer's acceptance + check does not see the clock, so it counts those batches as accepted and + diverges; the watchdog's divergence latch reports it, and a manual recovery + answers it. An overflow also moves which input executes the drained directs, + which the watchdog's comparison at that block can see. + +Both constants belong to the application's template, with the window +([`Application::scheduler_limits`](../../sequencer-core/src/application/mod.rs)). +Their values for the wallet are provisional, and the values and the largest +drain the guest can afford are **Open** +([register](../review/register.md#bound-the-canonical-direct-input-fridge)). + +## Direct context + +A direct executes with its inclusion block as its block, its input index and +its timestamp ([`Direct`](../../sequencer-core/src/l2_tx.rs)). The sequencer's +input table and the feed carry both. prevRandao is not passed: it would need a +column and a feed field, and the block proposer can bias it, so it comes only +on a stated need. A user op never sees its batch's context; it sees its frame's +safe block and fee. + +## Outputs and the L2 tx count An input's outputs are the application's notices and vouchers in execution -order: overdue directs, then each frame's drained directs and applied ops. -Reports are diagnostics outside the outputs tree: the reference reports a -rejected batch with one fixed string and a stale skip not at all, and the -[golden guest tests](../../examples/canonical-test/src/golden.rs) ignore them. - -The executed-input count is canonical state too. It advances by one for each -executed direct and each applied user op, including an app-level no-op or -business failure, and for nothing else: not for queueing a direct, a skipped -op, a batch envelope or an empty batch. A rejected or stale batch can still -advance it through the backstop. The [application contract](application-contract.md#4-canonical-history-cursor-executed_input_count) +order: the backstop's directs first, then, for a direct, any overflow drain, +or, for a batch, each frame's drained directs and executed ops. Reports are +diagnostics outside the outputs tree: the reference reports a rejected batch +with one fixed string and a stale skip not at all. + +The L2 tx count advances by one for each executed direct and each +executed user op, and for nothing else ([what counts](#what-counts)). The +[application contract](application-contract.md#4-the-l2-tx-count-l2_tx_count) owns the count; the [history contract](application-history.md) owns how -catch-up, recovery and the API use it. +catch-up, setup's rebuild and the API use it. ## Fatal conditions -The reference panics, halting the machine, when the application returns an -error; when the batch nonce or the executed-input count has no `u64` -successor; and when `blockNumber` or `chainId` does not fit `u64`. A halt is a -permanent fixed point, not a reject ([rollup host semantics](../cartesi-machine.md#rollup-host-semantics)), -so a port must halt on the same inputs. No canonical successor state exists. +The guest never answers an input with `RX_REJECTED`. A batch a gate rejects is +still accepted at the machine level. A revert would undo the whole input and +roll the scheduler's state back with the application's, so the scheduler's +state would depend on the application. It would also drop the backstop's +executions, which the reference keeps. + +The reference halts the machine when the application returns an error; when +the batch nonce, the L2 tx count or the L1 input count has no `u64` +successor; and when an input's `index`, `blockNumber`, `timestamp` or `chainId` +does not fit `u64`. A halt is a permanent fixed point, not a reject +([rollup host semantics](../cartesi-machine.md#rollup-host-semantics)), so a +port must halt on the same inputs. No canonical successor state exists. + +## State export + +The machine answers two inspect queries, so a reader can take its whole state: + +| Query | Report | +|---|---| +| `state`, or an empty payload | The application's comparison bytes, whole ([application contract](application-contract.md#6-checkpoint-lifecycle)). | +| `scheduler` | The scheduler's state and the L2 tx count, 48 bytes. | + +The `scheduler` report is +[`SchedulerExport`](../../sequencer-core/src/scheduler/mod.rs): six +little-endian `u64`s, in this order: + +1. `l1_input_count`, the machine's position; +2. `l2_tx_count`, the application's L2 tx count; +3. `next_batch_nonce`; +4. `clock`; +5. `first_queued_input`, the oldest queued direct's index, or the L1 input + count when none is queued; +6. `fridge_len`, how many directs are queued. + +The queued directs themselves are not exported. [The fridge is a +queue](#the-schedulers-state), so the first queued input and the L1 input count +name them, and L1 holds their bytes. +[`Scheduler::resume`](../../sequencer-core/src/scheduler/mod.rs) rebuilds a +scheduler from the report and those inputs, and refuses a range that does not +match it. The harness answers the `scheduler` query for every application. Any +other query, or an application failure while encoding its state, gets a report +saying so, never a halt. + +`setup --base` reads a stored machine's export through these two queries +([`sequencer::machine`](../../sequencer/src/machine.rs)), checks it against +its input table, and resumes the scheduler from it +([what setup does](../recovery/cockroach.md#what-setup-does)). +A C or C++ guest reaches the reference scheduler through +[`scheduler.h`](../../bindings/c-scheduler/include/scheduler.h), whose +`scheduler_state` returns the same fields but the L2 tx count, which the +application reports. ## Constants | Constant | Value | Home | |---|---|---| -| `MAX_WAIT_BLOCKS` | 1200 L1 blocks, about 4 h | [`lib.rs`](../../sequencer-core/src/lib.rs) | +| `max_wait_blocks` | 7200 L1 blocks, about 24 h, for the wallet | the application's template | | `MAX_EXPONENT` | 17101 | [`fee.rs`](../../sequencer-core/src/fee.rs) (`build.rs` generates it) | +| `fridge_bound` | 16384 entries for the wallet, provisional | the application's template | +| `max_direct_payload_bytes` | 1024 bytes for the wallet, provisional | the application's template | | `sequencer_address` | per deployment | compiled into the canonical machine | +[`SchedulerConfig`](../../sequencer-core/src/scheduler/mod.rs) carries all but +`MAX_EXPONENT`, built for an application by `SchedulerConfig::for_application` +from the submitter and the application's +[`scheduler_limits`](application-contract.md#3-admission--policy-never-consensus). +The window `max_wait_blocks` is at least one block, which the reference +asserts: with zero, the backstop could drain the arriving input's own block and +leave a direct from that block queued at the clock. + A fee exponent `n` stands for `(129/128)^n` smallest token units. The application converts it; [`fee_to_linear`](../../sequencer-core/src/fee.rs) owns the bit-exact conversion. A checked-in lookup table in its place is -**pending** ([Track 5](../plans/2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut)). +**Open** ([Track 5](../plans/2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut)). ## Conformance -[`consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json) -pins EIP-712 hashes, keccak256, the signature cases above (every accepted `v` -form, `v = 2`, high `s`, 64 bytes, `r = 0`, `s = 0`, `r = n`) with the sender -each path recovers, SSZ bytes, fee conversions, `EvmAdvance` calldata, and -guest scenarios with their exact outputs. It is never regenerated: a mismatch -is a consensus change. Gate edge cases are pinned by the reference's unit tests -and the [guest tests](../../examples/canonical-test/src/main.rs); vectors for -them are **pending**. +[`consensus_vectors_v2.json`](../../tests/fixtures/consensus_vectors_v2.json) +is this template's vector set. It is written once per template and never +regenerated: a mismatch is a consensus change to explain, not a reason to +update the file, and the next template ships a new file. It holds three kinds +of case: + +- **Byte-level cases**, carried unchanged from the earlier template's set: + EIP-712 hashes, keccak256, the signature cases above (every accepted `v` + form, `v = 2`, high `s`, 64 bytes, `r = 0`, `s = 0`, `r = n`) with the sender + each path recovers, SSZ bytes, the wallet's output payloads, fee conversions + and `EvmAdvance` calldata. +- **Guest scenarios**, each replayed from genesis on the devnet wallet + template. A step is one input (sender, block, timestamp, payload) with its + outcome, the transactions it handed out in order and whether each user op + executed, its exact outputs, and the `scheduler` report after it; a scenario + ends with the `state` report. They restate the earlier template's scenarios + under strict frames, and cover every gate (gate d on a later frame as well as + the first), the clock, the backstop (before a dropped direct too), the + payload cap, each fate of a user op, and `s = n` and `v` from 29 to 34 on the + batch path. The `scheduler` report after every step makes every scenario + assert that [the clock splits the directs](#the-invariant). +- **Scheduler-only scenarios**, under their own configuration with a fridge + bound of 2, which the wallet's bound of 16384 keeps the guest scenarios from + reaching. They pin the overflow drain: whole blocks, oldest first, never the + arriving block, and the clock it raises. Resume cases pin which exported + states and re-supplied ranges resume, and how a resumed scheduler continues. + +The skip for a sender whose next nonce is `u32::MAX` has no scenario, since +reaching it from genesis takes 2^32 ops; the executor's unit tests in +[`application/mod.rs`](../../sequencer-core/src/application/mod.rs) hold it, +and a C engine's caller must apply it too. + +The expected values came once from the reference, `CanonicalRunner` over the +wallet for the guest scenarios and the `Scheduler` alone for the rest, and four +checks hold them: + +| Check | Where | What it compares | +|---|---|---| +| Native, guest | `the_reference_reproduces_every_guest_scenario` in [`wallet.rs`](../../examples/app-core/src/application/wallet.rs) | Every field of every step and the final state, and the clock's split from the pinned data alone. | +| Native, scheduler only | `the_reference_reproduces_every_scheduler_only_scenario` in [`scheduler/mod.rs`](../../sequencer-core/src/scheduler/mod.rs) | Every field of every step, and each resume's result and refusal reason. | +| In the machine | The golden tests in [`golden.rs`](../../examples/canonical-test/src/golden.rs), on the built image | The guest scenarios' outputs, whether a reject report appears, and both inspect reports. | +| Through the header | [`conformance.rs`](../../bindings/c-scheduler/tests/conformance.rs) in `c-scheduler` | For both kinds of scenario, each input's outcome and reject reason, every handed-out transaction, the state after it, and whether each resume is refused. | + +The last check is the one a C or C++ port of +[`scheduler.h`](../../bindings/c-scheduler/include/scheduler.h) is held to. It +calls only the header's functions and reads only the file, but it links this +crate's reference export, so today a port replays the file through the header +the same way in its own harness; packaging the check to link another +implementation is in the +[todo](../plans/2026-11-sequencer-recovery/todo.md#build). It compares the +reject reason because the header reports it, though no output depends on it. +The header has one status for every refused resume, so only the native check +compares the refusal's reason. ## Sequencer side @@ -181,15 +510,15 @@ The rest of this page is for sequencer developers. Gate d constrains frame clocks only within a batch: non-decreasing and no later than its inclusion block. The scheduler does not require a frame for every L1 -block. During an admitted live run, the sequencer advances logical frame time +block. During a live run, the sequencer advances logical frame time when the latest persisted safe head `H` is at least five blocks beyond the open frame clock `S`. It then opens exactly one frame at `H`, drains the complete newly-covered direct range, and uses `H` for subsequent user ops. An observation jump from 100 to 132 therefore creates one frame at 132, not synthetic frames at each missed five-block boundary. Intermediate empty frames would execute -nothing, so their omission is scheduler-equivalent. Bootstrap and recovery are -anchoring transitions rather than live clock ticks: they may open a fresh Tip -at their proven checkpoint/current safe head without applying the five-block +nothing, so their omission is scheduler-equivalent. Opening a batch at +`run`'s start is an anchoring transition rather than a live clock tick: it +opens the batch at the current safe head without applying the five-block delta. Direct-input presence is not another rotation condition: the five-block tick @@ -207,11 +536,18 @@ second timer. ### The three implementations and why they agree -The sequencer predicts this fold in two more places: the off-chain predicate -`scheduler_accepts`, which builds the gold frontier, and the inclusion lane's -live prediction. The recovery fold drives `Scheduler` itself. +The sequencer predicts this scheduler in two more places: the off-chain +predicate `scheduler_accepts`, which builds the gold frontier, and the +inclusion lane's live prediction. [I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) -maps all three and their one deliberate difference: the predicate skips gate d. +maps all three and their deliberate differences: the predicate skips gates d +and f, a self-trust whose reason is recorded at +[`scheduler_accepts`](../../sequencer-core/src/protocol.rs). The lane shares +the reference's code where it can: it runs every transaction, live and on +replay, through `application::execute`; it refuses at `POST /tx`, through +`check_live_user_op`, every op the executor would skip; and it admits directs +by the application's payload cap, through one storage predicate that every +query enumerating directs to execute uses. [I2](../invariants.md#i2-drain-attribution-accumulated-directs-land-in-the-clock-advanced-frame) and [I3](../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) are the lane's mirrors of the drain order and of frame monotonicity, and diff --git a/docs/recovery/README.md b/docs/recovery/README.md index f12e6a47..c130e03b 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -1,369 +1,267 @@ -# Automatic Recovery - -Automatic recovery repairs the sequencer's optimistic history after a liveness -failure. It stops issuing soft confirmations, settles outstanding submissions -when necessary, and replaces the affected suffix before resuming. It uses the -existing SQLite database and assumes the sequencer's own code and accepted -local history are correct. - -For lost or unusable local state, including after a sequencer bug, use -[cockroach recovery](cockroach.md): fix the bug, choose a trusted canonical -application checkpoint, and rebuild from L1 in a fresh data directory. Automatic -recovery does not establish trust in a corrupted application state. - -This page owns the automatic procedure and its rationale. Start with the -lifecycle and dispatch below; read the safety arguments and model boundaries -when changing recovery. The [scheduler contract](../protocol/scheduler-semantics.md) -owns canonical acceptance rules; the [invariant register](../invariants.md) owns +# Batch landing and outages + +The sequencer serves an optimistic history: it confirms a user op at once and +posts it to L1 later, in a batch. That promise holds only while its batches +land inside the application's staleness window. This page owns what happens +when they do not: the rule that ends an era, what the sequencer does about it, +and what an operator does. + +The sequencer repairs nothing by itself. Its one recovery is +[cockroach recovery](cockroach.md): an operator rebuilds it in a new data +directory from a trusted canonical machine and L1, which starts a new era. +[Why there is no automatic repair](#why-there-is-no-automatic-repair) gives +the reasons. + +The [scheduler contract](../protocol/scheduler-semantics.md) owns the canonical +acceptance rules, and the [invariant register](../invariants.md) owns cross-module enforcement. -## The state being repaired +## Accepted and unaccepted batches -The local batch tree has one valid path: an **accepted prefix**, followed by an -**optimistic suffix** ending at the open **Tip**. Recovery invalidates a suffix -and opens a new Tip from the surviving path. Invalidated batch, frame, and -user-op source facts remain available for audit. +The sequencer's batches form one list in creation order: an **accepted +prefix**, then the batches L1 has not accepted yet, ending at the one **open +batch** that still takes user ops. -“Accepted” (also called **Gold** in code and models) means the safe-input -projection applied the scheduler's acceptance rules and matched the landed -bytes to a valid local closed batch. It does not mean an independent canonical -machine was observed executing it. A foreign or different accepted payload -records canonical divergence and forbids automatic repair. +**Accepted** means the sequencer read a safe L1 input, applied the scheduler's +acceptance rules to it, and matched the landed bytes to its own closed batch at +that nonce. The match is recorded on the batch as `accepted_input` +([I9](../invariants.md#i9-acceptance-identity-accepted-nonce-n-means-our-batch-n)). +It does not mean a canonical machine was seen executing it; the +[watchdog](../watchdog/README.md) checks that. A foreign or different payload +accepted at one of the sequencer's nonces records a **divergence**, which ends +the era as well +([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). -Keep three identities separate: +Everything past the accepted prefix is a prediction. A user op in an unaccepted +batch has been confirmed to its sender and served on the feed, but L1 has not +agreed to it yet. -| Identity | Meaning during recovery | -|---|---| -| Local `batch_index` | Unique creation identity; never reused. Invalidation targets a local suffix. | -| Scheduler batch nonce | Ordering identity; derived when storage creates the batch. A replacement branch reuses the invalidated suffix's nonces. | -| L1 wallet nonce | Transaction slot; a batch transaction and a flush no-op may compete for it. Covered slots must be consumed at safe depth before post-flush repair. | - -A parentless root uses the deployment's immutable anchor nonce: zero at genesis, -or the scheduler's next nonce after cockroach recovery. Production needs no -accepted ancestor or submitted sentinel to repair a fully invalidated branch -([I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). - -## Lifecycle and startup dispatch - -The recovery cycle crosses a process boundary: - -1. **Detect and stop.** `DangerDetector` polls local `Storage::check_danger`; - any non-`Safe` result stops normal operation. It neither writes the database - nor calls L1. Expected-recovery and retryable exits close intake and drain - workers; diagnosed terminal runtime faults abort immediately. -2. **Respawn.** The orchestrator restarts expected recovery (`10`) and retryable - exits (`20`). Terminal exit `30` or `SIGABRT` requires investigation before a - deliberate restart. Process - ownership and shutdown belong to the [authority-boundary ADR](../plans/2026-08-authority-boundary-adr.md). -3. **Recover under the process lock, with no workers.** Check local terminal - facts before any provider call, attempt initial L1 sync, select at most one - repair, then inspect again after a repair. -4. **Prepare, admit, launch.** Prepare resources without starting tasks. A final - current inspection must still authorize serving. It creates a single-use - `RuntimeAdmission` witness, consumed synchronously by worker launch. - -Startup first refuses a persisted canonical divergence or a missing rollback-safe -checkpoint. After initial sync, one consistent `RecoveryInspection` selects: - -| Local fact | Action | -|---|---| -| `Safe` + open Tip | Ready for runtime preparation. | -| `Safe` + no Tip | `EnsureOpenTip`: create it under a transaction guard. | -| `TipInDanger(N)` | `RecoverTip { N }`: invalidate that Tip and reopen, without flushing. | -| `ClosedBatchInDanger(N)` | Flush → sync → guarded post-flush cascade. | -| `L1ViewStale` | Retry; the persisted view or clock cannot authorize serving. | -| `EstimatedBatchInDanger(N)` | Retry; an estimate alone cannot authorize invalidation. | -| `CanonicalDivergence(N)` or missing recovery checkpoint | Refuse; automatic recovery cannot repair this trust failure. | - -Only an initial-sync **provider failure** may fall back to a still-usable -persisted view. Other failures keep their typed retry/refuse classification. -Post-flush sync has no such fallback: it must establish the view required by -the cascade. - -Every repair must commit an open Tip. A fresh inspection afterward must report -`Safe` with a Tip and no terminal facts; otherwise the boot exits. Startup does -not attempt a second repair in that invocation. Final admission repeats the -same policy after preparation, because preparation can outlive the freshness -of the L1 view. A new repair requirement also exits rather than launching. - -Preparation validates the rollback checkpoint's metadata and performs snapshot -hygiene, but does **not** restore the application. After launch, the inclusion -lane loads a surviving snapshot and completes catch-up before processing new user ops. -Admission authorizes worker launch; application restoration can still fail. - -Startup logs `danger_status`, `danger_batch_index`, and `recovery_decision`, -then any invalidated indexes. Errors retain their retry/refuse classification -and diagnostic cause; the orchestrator owns restart policy and alert routing. - -## Detection and timing - -Canonical staleness and local danger use different reference blocks and thresholds: +## The window -```text -scheduler rejects a batch with at least one frame when: - inclusion_block - first_frame.safe_block >= MAX_WAIT_BLOCKS +`max_wait_blocks` is a constant of the application's template +([scheduler limits](../protocol/application-contract.md#3-admission--policy-never-consensus)). +The scheduler uses it twice +([processing an input](../protocol/scheduler-semantics.md#processing-an-input)): -sequencer observes danger when: - current_safe_block - first_frame.safe_block >= danger_threshold - danger_threshold = MAX_WAIT_BLOCKS - preemptive_margin_blocks -``` +- **The backstop.** Before anything else, every input executes the directs + that have been queued for `max_wait_blocks` or more. +- **Staleness.** A batch that lands `max_wait_blocks` or more after its first + frame's safe block is skipped, and the batch nonce stays. Every later batch + then fails the nonce gate. -An old batch can already have landed while fresh. Its inclusion block decides -acceptance; its age at the current safe head does not undo that acceptance. -The detector therefore examines the first unaccepted closed batch and the Tip, -not accepted history. A wire batch with zero frames is never stale and consumes -its nonce; a normal local batch with zero user ops still has a first frame and -can age. - -### Danger threshold - -The threshold means “stop and recover,” not “this batch cannot land.” The Tip -may still be canonically fresh when invalidated, and closed batches can become -accepted while the flush is running. - -The margin provides headroom before canonical expiry; it is not a grace period -after detection. Startup can repair immediately. Defaults and validation live -in [`TimingArgs`](../../sequencer/src/commands/config.rs): with `MAX_WAIT_BLOCKS` -1200 and margin 300, observed danger starts at age 900 blocks. Neither that -margin nor the fee policy bounds the time required to finish recovery. - -A Tip ages from its first frame, so `run` also refuses a -`max_batch_open_seconds` that, rounded up to blocks at `seconds_per_block`, -exceeds the danger threshold minus `BATCH_SUBMISSION_MARGIN_BLOCKS` -([`RunConfig`](../../sequencer/src/commands/config.rs)); otherwise a low-volume -Tip reaches danger before it can be sealed, submitted and safe-accepted. The -check assumes L1 blocks are no faster than `seconds_per_block` and does not -bound inclusion time. - -### When safe-head progress stops - -A responsive RPC endpoint can keep returning an old view. The detector uses -both the safe block's timestamp and local time since the last recorded safe-head -advance. [`check_danger_in`](../../sequencer/src/storage/recovery.rs) checks in -this order: - -1. Canonical divergence. -2. Missing or old safe-block timestamp → `L1ViewStale`. -3. Observed closed-batch danger, then observed Tip danger. -4. Clock regression of at least one block-time against either persisted time - baseline → `L1ViewStale`; sub-block skew is tolerated. -5. Estimated missed blocks (`elapsed / seconds_per_block`) reduce the danger - threshold. An unresolved batch crossing it gives `EstimatedBatchInDanger`. -6. Otherwise `Safe`. - -An old view blocks repair selection before observed-age checks. A regressed -clock does not suppress danger already established by observed block numbers; -a remaining clock fault still prevents admission after repair. Estimates stop -new soft confirmations but never decide which work to invalidate. - -## Repairs and their guards - -### Closed batches: flush, sync, cascade - -**Flush the covered wallet slots.** The durable wallet-nonce watermark `W` is an -upper bound on every slot this deployment may have broadcast. Every broadcaster -raises it durably **before** sending at a new nonce. A crash between those steps -may cover a slot that was never used; it must not leave a sent slot uncovered -([I14](../invariants.md#i14-watermark--wallet-nonce-of-every-tx-ever-broadcast)). - -The flusher submits zero-value self-transfers at unresolved slots from the -account's Latest wallet nonce through `max(Pending, W + 1) - 1`. It completes only when: +The sequencer's lane predicts neither. It assumes its batches land fresh, and +until one lands stale the prediction holds: the backstop executes the directs +the next frame would have drained, in the same order. + +## The void history + +Let `H` be the safe head, with every input through `H` read and judged. Take +the oldest batch L1 has not accepted, closed or open, and let `s` be its first +frame's safe block. Any landing still to come is in block `H + 1` or later. So +once ```text -Pending <= Safe && Safe >= W + 1 +H + 1 - s >= max_wait_blocks ``` -Here Latest, Pending, and Safe are account transaction counts, not block -numbers; absent `W` contributes a lower bound of zero. An original batch or a -no-op may win each slot. Completion means every covered slot is consumed at -safe depth, even if the local node forgot an original transaction. It does not -require erasing that transaction's bytes from every mempool. - -The flusher returns the **safe block number** at which it observed completion. -Startup keeps it only in the current call; a crash or retry loses that -observation and the next attempt flushes again. Flush changes no local recovery -facts except the wallet watermark. - -**Sync through that observation.** Sync ingests safe InputBox events and updates -the local scheduler-acceptance projection. It does not query a canonical -application machine. A provider failure here retries the boot; there is no -fallback to the pre-flush view. - -**Cascade under an immediate SQLite transaction.** Its guard requires no -canonical divergence, a rollback-safe checkpoint, and a persisted safe head at -least as high as the flush observation. Then choose the pivot: - -- First valid closed batch beyond the accepted frontier, regardless of age. -- If none remains, the Tip only if it has reached `danger_threshold`. -- Otherwise invalidate nothing, retaining a fresh Tip or opening a missing one. - -The cascade deliberately runs even if refreshed danger is now `Safe`. The -closed-suffix policy follows from having completed flush and sync; it does not -repeat the trigger test. There is no extra inspection between flush and sync: -the process lock and absence of workers exclude competing local writers, and -sync is the step that can discover new divergence. Revisit this sequence if -startup gains concurrent writers. - -Flush completion depends on L1 progress. Replacement attempts can be rejected -or remain uncompetitive, and provider failures can interrupt the attempt. -Retries preserve safety but establish no recovery deadline. Pricing and its -accepted liveness limits belong to the [L1 fee policy](../l1-fee-policy.md). - -### Open Tip: repair without flushing - -An open Tip has never been submitted, so invalidating it creates no L1-slot -race. `RecoverTip { N }` rechecks divergence, checkpoint availability, and -**exactly** `TipInDanger(N)` inside its transaction, then invalidates that Tip -and opens a fresh one. It does not invalidate closed batches or fall back to -repairing a changed decision. - -The threshold is a policy choice: retaining an aging Tip would make the -runtime detector stop service again. Waiting for `MAX_WAIT_BLOCKS` would keep -the same suspected prediction alive without solving that cycle. - -`EnsureOpenTip` is a separate action. Its transaction requires `Safe`, no Tip, -no divergence, and a rollback-safe checkpoint. It opens the Tip without -invalidating history. Guarded writes matter even without concurrent writers: -wall-clock aging alone can change the decision after inspection, which retries. -An open Tip found by the guard refuses instead: aging cannot create one, so -only another writer or an inspection bug can. - -### Atomic history change and replay - -Invalidation, history rewind, generation change, and Tip creation share one -transaction. Invalidation removes the suffix's `application_inputs` projection; -raw source facts remain. `RecoveryGeneration` increments once iff at least one -valid batch was invalidated. Each advance appends an immutable generation cut: -the surviving count after suffix deletion, before reopening can insert any -replacement directs. Failed reopening rolls all of this back. The cuts let -readers determine whether a saved checkpoint survived several recoveries; -the [history contract](../protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) -owns that query. Invalidating an empty batch records the old head; a no-op repair -records no transition. - -The new Tip follows the latest surviving batch, or uses the immutable root -anchor if none survives. It attributes direct inputs after the surviving -frame's drain boundary, with the era baseline block as a floor. Storage records -those application entries; the launched lane executes them during catch-up. -This prevents a restored prefix's directs from being executed twice. - -Recovery requires the latest accepted batch snapshot, or the era baseline -before any local batch is accepted. A newer surviving optimistic snapshot can -reduce replay, but cannot replace that rollback guarantee. Snapshot publication, -artifact validation, leases, and GC are owned by the -[snapshot lifecycle](../snapshots/lifecycle.md); history coordinates and -subscription behavior by the [API contract](../../README.md). - -## Why the closed-suffix policy is safe - -**Settling slots removes the zombie race.** Before the flush, an old batch may -still win an L1 slot after local invalidation. Reusing its scheduler nonce too -early can let later old batches execute against the replacement branch. The -[historical counterexample](history/README.md) demonstrates this failure. -Detecting danger before settlement is necessary; mutating the closed suffix -before settlement is the unsafe step. - -After completion, the original transactions cannot newly win those consumed -slots on descendants of the observed safe chain. Sync through the observation -accounts for the originals that did win. Replacements use later wallet slots -and reuse only the scheduler nonces beyond the accepted prefix. This relies on -the trusted, consistent L1 view and dedicated submitter key in the -[threat model](../threat-model/README.md), and on every broadcaster preserving -the watermark. - -**A skipped batch does not advance the scheduler nonce.** When nonce `N` arrives -stale, its frames are skipped and later `N+1`, `N+2`, … envelopes encounter a -nonce mismatch. The overdue-direct backstop still runs before envelope -classification, so “skipped batch” does not mean the whole input has no state -effect. A missing batch whose slot was consumed by a no-op also leaves the -expected nonce unchanged. Later input cannot retroactively make already -rejected envelopes execute. - -**Discarding the entire remaining closed suffix is a convergence policy.** It -can include rejected landings, no-op-replaced transactions, and batches never -submitted at all. Some of that work could theoretically be resubmitted fresh; -“everything past Gold is doomed” is not a general impossibility proof. Recovery -sacrifices it to avoid preserving a partly submitted suffix and restarting into -the same danger/flush cycle. The cost is invalidated soft confirmations. - -If every closed batch became accepted, an aging Tip can still need repair: -its first frame may share the preceding batch's safe block, while its age is -measured at the later post-flush head. A fresh Tip survives without a generation -change. Thus flush alone does not imply invalidation. - -**Content identity is a prerequisite.** The input reader checks at/above-anchor -accepted wire bytes against the local valid closed batch. A foreign or different -payload persists divergence and freezes the acceptance frontier. Startup checks -that marker before L1 access, after sync, inside repair transactions, and before -admission. Repairing the tree's shape cannot recover missing canonical effects; -investigate the fault and use [cockroach recovery](cockroach.md). Check scope -and enforcement are owned by [I9 and I15](../invariants.md). - -## Formal Verification - -Two bounded TLA+ models check complementary safety obligations. They are not -a refinement proof of the Rust implementation, a proof of their composition, -or a liveness guarantee. Read both before changing recovery code. - -### `preemptive.tla`: batches and wallet slots - -[`preemptive.tla`](preemptive.tla) models safe-block advancement, wallet-slot -competition between batches and no-ops, scheduler acceptance, and branch -invalidation. Its `Inv` checks `ZombieSafety` at every reachable state: -`schedulerExpected = CountGold(spine)`. It also checks batch-nonce contiguity, -invalid-branch ancestry, wallet-slot uniqueness, and L1/scheduler cursor bounds. - -Several details must not be read as literal production behavior: - -| Model | Production mapping or limit | -|---|---| -| A Gold genesis sentinel at nonce zero | Production opens a parentless root at its stored anchor, without a submitted sentinel. Root/anchor cases are tested in Rust. | -| `SubmitBatch` assigns the pending suffix with `max(walletNonce, nextL1Slot)` | The poster derives the suffix and Latest account nonce, and raises the durable watermark before sending. The model expression is not a Rust nonce-allocation recipe. | -| Tip advancement and submission can interleave with recovery; dead batches can race after model invalidation | Production stops workers and settles covered slots before the closed cascade. These additional modeled interleavings do not establish coverage of different production actions. | -| `Resolve` handles a stale Silver frontier or a Tip at `MAX_WAIT_BLOCKS` | Production also invalidates a killed/unsubmitted closed suffix after flush and repairs a Tip at the earlier danger threshold. Those actions need the arguments above and Rust tests. | - -The model's `Gold`, `Silver`, `Bronze`, `Pending`, and `Tip` colors describe -stages of inclusion and acceptance. `Gold* Silver* Bronze* Pending* Tip` is -**not** an invariant: flushing can leave a killed Pending before a surviving -Silver. Do not build implementation assumptions on that ordering. - -The configured finite bounds are in [`preemptive.cfg`](preemptive.cfg). -The model has neither crash/restart nor the wall-clock freshness policy. - -### `admission.tla`: startup and permission to launch - -[`admission.tla`](admission.tla) models the local terminal gate, initial-sync -fallback, at most one repair, flush observation and mandatory sync, guarded -cascade, post-repair inspection, task-free preparation, and final admission. -Retry, refusal, or owner loss starts a fresh attempt over surviving durable -facts; the flush observation and admission witness do not survive. - -Its invariants cover terminal dominance, repair preconditions, a caught-up -post-flush view, and admission soundness. It abstracts successful repair as -producing a Tip; concrete transactions and rollback are checked by Rust tests. -Neither model covers external era/generation metadata, the application-input -projection, or snapshot artifact/lease/GC durability. Those obligations remain -in storage constraints, tests, and the [snapshot lifecycle](../snapshots/lifecycle.md). - -Run the configured checks with: - -```bash -tlc -workers auto -deadlock docs/recovery/admission.tla -tlc -workers auto -deadlock docs/recovery/preemptive.tla -just -f docs/recovery/justfile check-all +that batch is stale wherever it lands, its nonce is never consumed, and no +later batch can be accepted. Everything served past the accepted prefix is +**void**, and the era is over. + +- **It is a proof from observed L1.** It uses no clock, no estimate and no + margin, so it cannot fire early. +- **It never fires on a batch that could still land.** The sequencer's + acceptance check is looser than the scheduler's + ([I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)), + so a batch it counts as unaccepted is unaccepted on chain. +- **It stays true.** `H` only rises, `s` is fixed when the batch opens, and + nothing repairs. So it is not stored: every start derives it again from the + database, before L1 is reachable. +- **The open batch counts.** Its first frame is fixed when it opens, so it + ages whether or not it holds a user op. + +The trigger is not the backstop's drain. A drain alone does not contradict the +lane, and a history can be void with no direct queued at all. + +```rust +/// The oldest batch L1 has not accepted, closed or open, at the safe head. +pub struct LandingLag { + pub batch_nonce: u64, + pub first_frame_safe_block: u64, + pub safe_block: u64, +} + +impl LandingLag { + /// The batch's age at the earliest block it can still land in: `H + 1 - s`. + pub fn blocks(&self) -> u64; + /// No block after the safe head can accept this batch. + pub fn is_void(&self, max_wait_blocks: u64) -> bool; +} ``` -## Implementation and test map +## What the sequencer does + +A recorded divergence and a void history are the two **terminal facts**. While +either holds, `run` refuses, and neither ever stops holding. -| Concern | Owner and useful tests | +**At start**, `run`: + +1. checks the terminal facts on the database as it is, before any L1 call; +2. syncs L1 to the safe head; +3. checks the terminal facts again; +4. opens a batch if none is open, which is how the first `run` of an era gets + its first batch; +5. launches its workers. + +It repairs, flushes and replaces nothing. When the first sync fails because the +provider is down, `run` starts on its stored view, as a running sequencer +carries on through the same outage. The first `run` of an era is the +exception: it has no open batch yet, and it exits 20 rather than open one at a +safe head that may be hours old, since a batch's first frame is fixed when it +opens. Any other sync failure exits 20 or 30, by its class. + +**While running**, one worker reads the terminal facts every two seconds and +stops the process on either. It does not write the database and does not call +L1. The lane does not check for a void history, and it sees a divergence only +at its periodic frontier read, so a user op confirmed in the seconds between +the safe head showing a fact and the stop is lost with the rest. + +**How it stops.** A start that finds a terminal fact exits 30 and names it. A +running process that finds one aborts (SIGABRT, 134), as it does for any +diagnosed terminal fault; a clean shutdown that finds one exits 30 instead of +0. A supervisor treats all three as terminal +([restart policy](../watchdog/operator-deployment.md#sequencer-restart-policy)). + +**It reports two numbers**, with no threshold of its own: + +| Number | Meaning | +|---|---| +| `landing_lag_blocks` | `H + 1 - s` for the oldest unaccepted batch: its age at the next block. The history is void when it reaches `max_wait_blocks` | +| `safe_head_age_seconds` | The wall clock less the safe block's L1 timestamp. The lag freezes while the provider is down; this number does not | + +Both are in [`GET /healthz`](../../README.md#health-probes-internal-only), +which answers `null` for them until the worker's first reading, and in a +`landing lag` log line, written at start and every five minutes. + +**What it does not do:** + +- **It does not stop intake.** An operator who wants no new confirmations + during an outage closes `POST /tx` at the reverse proxy. +- **It does not estimate.** A stale L1 view refuses nothing, at start or while + running. +- **It does not stop posting.** The poster retries as always; a void batch + wastes gas only until the next safe head shows it. + +## The budget + +From the moment batches stop landing, the operators have the window, less the +age of the oldest unaccepted batch at that moment, less the safe head's lag of +about 64 blocks. With the wallet's 24-hour window and the default two-hour +batch, that is between about 21.5 and 23.5 hours. A stopped sequencer spends +the same budget: its open batch ages while it is down. + +A running sequencer with no traffic stays inside it. The lane seals the open +batch after `max_batch_open_seconds`, whatever it holds, and the poster posts +it. `run` refuses a `max_batch_open_seconds` that leaves no room to land: the +window less a submission margin of 100 blocks +([`RunConfig`](../../sequencer/src/commands/config.rs)). The check assumes L1 +blocks are no faster than `seconds_per_block` and does not bound inclusion +time. + +## Scenarios + +| Situation | The sequencer | Operators | Clients | +|---|---|---|---| +| Batches stop landing; fixed inside the window | Keeps serving and sealing; posts the backlog in order once it can | Paged by the landing lag; fix the cause | Nothing | +| A restart during that outage | Starts as always | Nothing more | Reconnect | +| Not fixed: the oldest unaccepted batch is void | Stops; every `run` refuses | Cockroach recovery | A new era | +| Blind past the window: the provider is down | Cannot see it; stops at the first sync that shows it | The same | The same | +| Stopped past the window, even with nothing confirmed | Refuses at the next start | The same | The same | +| A foreign batch at one of its nonces | Records the divergence and stops; every `run` refuses | Cockroach recovery | A new era | + +Each row is a devnet drill +([`test_cases.rs`](../../tests/e2e/src/test_cases.rs)), run in CI by +`just test-rollups-e2e`: + +| Situation | Drill | +|---|---| +| Fixed inside the window | `batches_stop_landing_inside_the_window_then_land_test`, and at a restart `dropped_submission_is_posted_again_after_restart_test` | +| A restart during the outage | `restart_after_a_short_outage_keeps_history_test`, `restart_inside_the_window_keeps_history_then_lands_test`, `warm_restart_with_l1_down_test` | +| Void while running | `void_history_while_running_stops_for_good_test`, through the rebuild to the new era | +| Blind past the window | `blind_past_the_window_stops_at_the_first_sync_that_shows_it_test`, and at start `blind_start_boots_on_the_stored_view_then_stops_at_the_first_sync_test` | +| Stopped past the window | `stopped_past_the_window_refuses_every_start_test`, and with nothing confirmed `stopped_past_the_window_with_nothing_confirmed_refuses_test` | +| A foreign batch at one of its nonces | `foreign_batch_at_our_nonce_stops_for_good_test` | +| The first `run` of an era with the provider down | `first_run_of_an_era_with_l1_down_retries_test` | + +## When batches are not landing + +1. **Watch the landing lag and the safe head's age.** Alert at a fraction of + the window that leaves time to act. +2. **Find the cause**: the provider, the submitter's balance, fees + ([fee policy](../l1-fee-policy.md)), L1 itself, or a crash loop. +3. **Keep `run` up.** It posts the backlog the moment it can. A stopped + sequencer posts nothing, and its open batch keeps aging. +4. **To stop adding confirmations that may be lost**, close `POST /tx` at the + proxy. The feed and the poster carry on. +5. **Do not flush the wallet.** `flush-mempool` runs only with `run` stopped, + and it replaces the batch transactions waiting in the mempool with no-ops. + `run` must then post those batches again, which spends the window. + +## When the sequencer has stopped for good + +The running process aborts, and every later start exits 30, naming the batch, +its first frame's safe block, the safe head and the window +([how it stops](#what-the-sequencer-does)). Restarting does not help, and +nothing in the old data directory needs repair: follow +[the operator's playbook](cockroach.md#the-operators-playbook) for a cockroach +recovery. A void history is not a disagreement with the canonical machine, so +the watchdog has not latched and that playbook's last step has no latch to +clear. + +What is lost is what any cockroach recovery loses: everything served that had +not landed by the stop block. Those user ops are revoked, not cancelled; anyone +holding one can submit it again while its sender's nonce is unused. + +Clients handle one event, the new era, by +[the client's playbook](cockroach.md#the-clients-playbook). + +## Why there is no automatic repair + +A repair in place would stop the sequencer short of the window, settle the +submitter's wallet, cut the history back to the accepted prefix and resume in +the same database. The sequencer does not do it, for these reasons: + +- **It would save no confirmation.** It discards everything past the accepted + prefix, which is exactly what a void history loses. +- **It could not finish before the people do.** To replace closed batches it + must first land transactions, so with a dead provider or an empty wallet it + waits for the same fix. +- **Nobody is asleep.** It would fire hours into an incident that has already + paged everyone. +- **Every client would have to mirror it**: notice the cut, roll back, + subscribe again. A client builds the cockroach playbook anyway. +- **It would be the most complex part of the sequencer**: replacing a batch + reuses its nonce, so a transaction of the old batch that lands late executes + against the new history unless every wallet slot is settled first. The + [archive](history/README.md) keeps the models of that race and its + counterexample. + +What it would buy is unattended resumption in the same database, after an +outage or a stop of any length. The price of doing without it is that a +sequencer stopped or blind for longer than the window needs a cockroach +recovery, even when it had confirmed nothing. + +Inside an era no batch nonce is reused, so that race cannot occur: a late +transaction of the sequencer's carries the bytes of a batch it still holds, +and it is accepted as that batch or skipped. + +## Code map + +| Concern | Owner | |---|---| -| Startup dispatch, error classification, final admission | [`recovery/mod.rs`](../../sequencer/src/recovery/mod.rs); procedure tests substitute only L1 sync/flush, keeping real SQLite inspections and repairs. | -| Detection, mutation guards, pivot and atomic cascade | [`storage/recovery.rs`](../../sequencer/src/storage/recovery.rs), [`recovery/tests.rs`](../../sequencer/src/storage/recovery/tests.rs); exact Tip guard, safe-view floor, unconditional post-flush policy, generation rollback, root nonce, and direct replay. | -| Observed danger versus estimates, accepted frontier and content identity | [`recovery/tests.rs`](../../sequencer/src/storage/recovery/tests.rs) (`check_danger`): stale-view precedence, clock faults; [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) tests: accepted frontier, content identity, divergence freeze; [`l1_submission.rs`](../../sequencer/src/storage/l1_submission.rs) tests: reused nonces. | -| Slot settlement and broadcast coverage | [`recovery/flusher.rs`](../../sequencer/src/recovery/flusher.rs), [`l1/watermark.rs`](../../sequencer/src/l1/watermark.rs), [`submitter/poster.rs`](../../sequencer/src/l1/submitter/poster.rs). | -| Task-free preparation, launch, restore and catch-up | [`commands/run/`](../../sequencer/src/commands/run/), [`inclusion_lane/mod.rs`](../../sequencer/src/ingress/inclusion_lane/mod.rs). | - -The accepted-frontier cache stores acceptances, not scan progress. Rejected -inputs after the frontier may be rescanned on later syncs; a separate persistent -cursor would need nonce-reuse reasoning and tests. This is a storage performance -tradeoff, not an extra recovery phase or a modeled TLA+ invariant. +| The landing lag and the terminal facts | [`storage/landing.rs`](../../sequencer/src/storage/landing.rs) | +| Startup: check, sync, check, open a batch | [`commands/run/`](../../sequencer/src/commands/run/) | +| The watching worker and the reading `/healthz` serves | [`landing.rs`](../../sequencer/src/landing.rs), [`egress/api/health.rs`](../../sequencer/src/egress/api/health.rs) | +| Acceptance, content identity and the divergence marker | [`storage/acceptance.rs`](../../sequencer/src/storage/acceptance.rs) | +| The staleness formula and the off-chain acceptance check | [`protocol.rs`](../../sequencer-core/src/protocol.rs) | +| The batch-open check | [`commands/config.rs`](../../sequencer/src/commands/config.rs) | diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 2a33cb4d..8996b1af 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -1,303 +1,381 @@ -# Cockroach recovery: rebuild from L1 - -Cockroach recovery (`setup --recovery`) creates a fresh sequencer starting state -from a trusted application checkpoint and historical L1 inputs. Use it when the -local database is lost or cannot be trusted, including after a sequencer bug has -corrupted its state or produced malformed batches. **Find and fix the bug before -rebuilding.** The operator initiates recovery; the command automates the rebuild. - -The procedure is **flush → fold → fill**: - -1. **Flush** outstanding submitter transactions and choose a fixed safe L1 - stopping block at or after the trusted checkpoint's inclusion block. -2. **Fold** the input history through the canonical scheduler, starting from the - trusted checkpoint. Every input receives its normal scheduler treatment: - accepted batches execute, malformed or rejected batches are skipped, and - direct inputs are queued and drained. Finally, drain every remaining direct - input through the stopping block. -3. **Fill** a fresh database with the recovered application state and next batch - nonce, ready for `run`. - -The result has accounted for the whole input prefix and carries no inherited -speculative user-operation suffix. It does not need to reach the moving tip: -normal operation handles inputs after the stopping block. - -This is a **resume baseline**. The final drain can execute young direct inputs -that the canonical scheduler still has queued at that block. The resumed frame -covers them before new user operations; the baseline itself is not a canonical -comparison checkpoint at that block. - -[Standard recovery](README.md) instead uses the existing database to repair an -optimistic suffix automatically. It assumes the local bookkeeping is trustworthy. +# Cockroach recovery -## Recovery readiness before deployment +Cockroach recovery starts a new sequencer when the old one is gone or cannot be +trusted: it had a bug, it served transactions that never landed, its SQLite is +corrupt, or its disk failed. The watchdog reports most of these as a divergence +([incident runbook](../watchdog/incident-runbook.md)). It is the sequencer's +only recovery: a void history and a recorded divergence both stop the +sequencer for good and send the operator here +([batch landing and outages](README.md)). + +The operator starts a recovery and one command does the work: `setup`, the +same command that starts a new deployment. Every setup starts from a **base** +and rebuilds the history from L1; a recovery is a setup on a chain that +already has history, usually from a stored canonical machine. + +**Find and fix the bug before rebuilding.** The canonical image keeps its own +bugs: a fix to code the image shares makes the sequencer diverge from it, and +nothing but the watchdog's next comparison would notice. -Canonical-to-native recovery is a required -[application integration capability](../protocol/application-contract.md#7-canonical-recovery-integration). -Each production deployment must have a release-matched export command, a -completed application-specific runbook, and a successful recovery rehearsal. -An incident is an execution of that prepared procedure, not the first attempt -to determine a state layout or assemble recovery metadata. - -The application runbook must name: - -- The canonical image, native engine, state layout, exporter, and tool versions. -- How to select and preserve a trusted CM checkpoint and its exact L1 boundary. -- The command extracting the application state, count/clock, and next scheduler - nonce into the bundle accepted by `setup --recovery`, including supported - checkpoint boundaries and verification of the - [pending-direct eligibility condition](#checkpoint-eligibility) below. -- Artifact locations, access/backup procedures, validation commands, and the - commands to rebuild, restart, compare, and resume affected readers. -- A rehearsed fallback to an earlier trusted checkpoint or genesis if the - preferred artifact is unavailable, with measured replay time and disk needs. - -The qualifying drill starts with a **non-genesis canonical machine checkpoint**, -pinned deployment data, and L1 access, while the old native database and dumps -are unavailable. Run the actual exporter and restore its output; check the -native application bytes/progress and scheduler nonce against the canonical -source. Exercise directs pending at the checkpoint and inputs arriving after it. -The exporter must refuse a checkpoint with a pending direct at or below its -application clock, even when `A < B`; rehearse the earlier-checkpoint/genesis -fallback. Also cover an eligible nonempty queue whose directs are all above `A`. -Run `setup --recovery` in a fresh directory, resume sequencing, and compare -against independent canonical execution after a new batch is accepted. The -terminal-drained baseline itself need not equal canonical state at `C`. - -Automate this integration check where the artifacts are available, and require -passing evidence for the supported release before production deployment. Record -artifact versions, commands, checks, and timings; repeat when the state mapping, -checkpoint/recovery behavior, or relevant release artifacts change. A native -dump round-trip or a test using shared scheduler fixtures alone does not exercise -the canonical-machine export boundary. The -[Track 6 integration plan](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) -tracks the remaining tooling and validation work; this requirement is not an -implemented deployment gate. - -## Incident playbook - -The objective is a usable, independently trusted starting state. A newer sound -checkpoint reduces replay work; finding the exact first bad execution or the -latest possible sound checkpoint is not a prerequisite for recovery. - -1. **Stop and preserve.** Stop the sequencer and prevent automatic restarts. - Preserve its data directory, logs, and available archives before rebuilding. - Copy the watchdog state directory's `config.json`, its head under - `checkpoints/`, and its `incident/` evidence once `status` no longer shows - the evidence collection `running`; a latched watchdog does no other work, - so this needs no pause. Preserve affected client databases and their - checkpoint metadata separately. -2. **Establish the cause and reference.** Check deployment identity, canonical - machine image, bootstrap boundary, and the reported comparison boundary. - A configuration mismatch is different from faulty execution. Fix the cause - before running the replacement sequencer; retain an independently trusted - canonical machine or earlier checkpoint as the reference. -3. **Select a sound application checkpoint.** The watchdog's durable head is - its last successful comparison checkpoint, or its operator-trusted initial - checkpoint if no comparison succeeded. A failed comparison does not replace - it; initialization and idle ticks are not successful comparisons. A latched - `state_mismatch` usually also keeps `incident/canonical` (its evidence - lists it), the canonical machine at the latched block, derived - independently of the sequencer, and - `sequencer-watchdog replay` re-derives one at any block from a trusted - machine ([incident runbook](../watchdog/incident-runbook.md)). Use that - canonical state, or independently validate a retained candidate at the same - exact L1 boundary. Current-state equality can establish a usable application - state without establishing that all earlier executions were correct. -4. **Prepare a restorable native bundle.** Validate the candidate's restored - application state, embedded count/clock, and next batch nonce against the - canonical reference at block `B`. Verify its pending-direct queue meets - [checkpoint eligibility](#checkpoint-eligibility); otherwise choose an - eligible earlier checkpoint or genesis. Keep the artifact and its boundary - metadata together and record how its trust was established. The receipt alone - is not evidence that a faulty sequencer executed correctly. -5. **Rebuild in a fresh directory.** Use the invocation below. Recovery chooses - its post-flush stopping block `C`, replays from the trusted checkpoint, and - publishes a new era. Preserve that baseline artifact and its metadata for - client alignment before resumed operation can collect it. Resume independent - watchdog comparison when a new accepted comparison checkpoint is available. - -### Obtaining the recovery artifact - -The watchdog stores a whole CM, including scheduler state; it does not save a -native `/finalized_snapshot` archive. Use the application's rehearsed canonical -export command to obtain the native bundle, or a retained native archive whose -state and resume metadata can be validated against the canonical reference. -The mapping is required even when it is a direct extraction of the labeled -state range. The generic watchdog does not implement that application-specific command. -A comparison file alone need not contain everything an engine requires to restore. - -Retain verified native archives outside sequencer GC if they are the intended -recovery source. The watchdog normally prunes its previous CM checkpoint, and -sequencer GC may remove the native artifact from the last passing comparison -after a newer batch is accepted. Downloading `/finalized_snapshot` after an -alarm can return the faulty newer state. The -[backup guide](../watchdog/operator-deployment.md#checkpoint-disk-usage-and-backups) -describes retention; the application runbook supplies the tested conversion. -Use its rehearsed earlier-checkpoint/genesis fallback when the preferred source -is unavailable. - -### Application-specific reader state - -A reader such as Bart's indexer owns more state than the sequencer application. -Choose its latest checkpoint whose execution provenance and indexing behavior -remain trustworthy after diagnosing the incident. Its boundary may differ from -the sequencer's chosen checkpoint. Compare against an independent reconstruction -of the required projection when needed; matching current balances or positions -does not validate historical transfers, deals, or portfolio records. - -Restore that complete client checkpoint and its scheduler continuation metadata, -then replay canonical L1 inputs through the replacement era's `C` and perform -the terminal drain. Establish agreement with the replacement application -baseline, including count `K`, before binding the reader to the new history -claim. If no client checkpoint can be trusted, rebuild its projection from a -trusted origin. A current application snapshot cannot supply omitted history. - -The [projection replay contract](../protocol/projection-replay.md) describes the -historical-input API, checkpoint preparation, and handoff metadata. Its reference -test exercises a trusted client checkpoint; the application-specific backup and -incident validation remain integration work. This manual procedure does not -require automated cross-era checkpoint matching. - -## Run a rebuild - -Select and prepare the trusted checkpoint using the incident playbook above. - -The loader requires an application artifact, `info.toml`, and `checkpoint.toml`. -The [recovery export workflow](../snapshots/lifecycle.md#http-and-recovery-exports) -describes this bundle. An export receipt checks metadata agreement; it does not -prove the checkpoint correct. An ordinary optimistic snapshot, a subscriber's -dump, or a bare local `dumps//` directory is insufficient. - -With the deployment's [setup configuration](../../README.md#running) and -batch-submitter signing key configured, use a fresh data directory: - -```sh -cargo run -p wallet-sequencer -- setup --recovery \ - --data-dir \ - --checkpoint-block \ - --checkpoint-dump-dir +## One setup, from any base + +```text +wallet-sequencer setup [--base ] [--dry-run] ``` -Recovery signs L1 transactions, so the key must match the configured submitter. -After success, start `run` with that same data directory. A completed rebuild -refuses another `setup --recovery`; failures before completion publish no partial -baseline. If the RPC node has not reached the checkpoint, recovery exits with -retryable code 20. Synchronize that node and retry with the same checkpoint and -incomplete data directory. +Every application binary takes these arguments; `wallet-sequencer` is the +reference. A base is a stored machine of the deployment's template, stopped +between inputs. Without `--base`, the base is genesis: L2 tx count zero, the +scheduler's genesis state, and the binary's genesis application. Setup rebuilds +the history from the base through a **stop block**, a safe L1 block it pins +after flushing the submitter's wallet, and publishes a new **era**: a history +that starts at the base's L2 tx count, holds the rebuilt history through the +stop block, then the rows `run` sequences. + +**Recovery is needed exactly when L1 shows the old submitter's activity past +the base**: a submitter input after the base's last input, or unsettled +transactions in the submitter's wallet. Without that activity the rebuild only +moves directs. With it, the rebuild re-judges the old instance's batches by the +canonical scheduler's rules. Setup does the same either way, and its summary +says which case it found. Which base to use is the operator's call: any base +works, genesis included, though a rebuild from genesis takes longer as the +deployment ages. + +What the old instance served and never landed by the stop block is gone. Its +user ops are revoked, not cancelled: they carry no expiry, so anyone holding +one can resubmit it while its sender's nonce is unused. + +### Inputs + +| Input | Notes | +|---|---| +| A base machine | `--base `, optional; setup trusts it, and the operator checks it ([before anything](#before-anything)). | +| The submitter key | `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE`, as for `run`; it must sign for the submitter address, and `--dry-run` takes none. | +| An empty data directory | Absent, empty, or holding only `sequencer.lock`; setup refuses anything else. | +| The setup configuration | The RPC endpoint, chain id, application address, submitter address and fee source ([Running](../../README.md#running)). | + +Setup takes no client count; each client checks its own +([the client's playbook](#the-clients-playbook)). -## Implementation contract +### Outputs -Read this section when changing checkpoint loading, replay, or baseline -publication. The [scheduler contract](../protocol/scheduler-semantics.md) owns -input interpretation; recovery uses that same scheduler implementation. +Setup and its dry run print the same summary: -### Data dictionary +```text +base: L1 inputs, L2 tx count +submitter inputs past the base: +unsettled submitter transactions: +inputs the rebuild replays: (through block ) + +``` + +Setup's block is the stop block, and it adds the stop block and the L2 tx count +at its end. The dry run's block is the safe head it synced to. + +**The dry run** reads the base, syncs L1 into a scratch directory under the +system's temporary directory, and runs the export check (step 4) there. It +takes no key, flushes nothing and never touches the data directory, and it +reports the escape facts rather than refusing on them. It syncs every input +from the deployment block, so it costs as much time and temporary disk as +setup's own sync. -The replay boundaries are: +**Setup** publishes the new era: -| Value | Meaning | +| Field | Value | |---|---| -| `S`, `B`, `N` | Trusted checkpoint state, inclusion block, and next batch nonce. | -| `A` | Last executed application safe block reported by `S`. | -| `C` | Fixed post-flush safe stopping block. | -| `S'`, `N'` | Recovered state and next batch nonce. | -| `K` | Application count in `S'`; the first later application input has offset `K`. | - -### Checkpoint eligibility - -At the exact end-of-block boundary `B`, the canonical scheduler must have **no -pending direct with inclusion block `<= A`**. The checkpoint therefore already -accounts for every direct through `A`; all remaining directs are reconstructed -from `(A, B]`. The exporter must inspect the canonical queue to establish this -condition, including when validating a retained native artifact against the CM. -If it fails, refuse the export and select an eligible earlier checkpoint or -genesis. - -Honest live sequencing establishes this condition: a frame's safe block precedes -its L1 inclusion, so all directs it covers have already arrived. The overdue-direct -backstop preserves it too: by the time a block's directs are overdue, that whole -block has been observed. The FIFO drain executes all equally aged directs from -that block and all older ones, so advancing `A` leaves none of them pending. - -The canonical scheduler also accepts equality. After faulty sequencing, a batch at -block 10 can execute at clock 10 before another direct in that block arrives. -An empty batch at block 11 advances the nonce without draining that direct. -The truthful checkpoint has `A=10 < B=11`, yet seeding `(A,B]` would omit it. -Application-state equality and `A < B` alone cannot establish eligibility. - -Loading checks the receipt's block against configured `B` and its nonce against -`info.toml`. It requires `A < B`, except for known empty genesis (`B`, nonce, and -application count all zero). At non-genesis `A = B`, a direct arriving after the -accepted batch in block `B` could still be pending but disappear from the seed -range. The bundle contains no scheduler queue evidence, so the loader cannot -verify the pending-direct condition. Eligibility, checkpoint state, and nonce -remain operator-trusted; the later content-identity check does not verify this -prefix. Exporter enforcement belongs to the application's required recovery -integration, not the generic loader. - -The complete ordering is `A < B <= C`, with `A = B = 0` allowed for empty -genesis. Before sourcing or executing the fold, recovery requires `B <= C`. -Otherwise publishing the checkpoint state at an earlier baseline block could -make normal reconciliation execute already-accounted direct inputs again. - -### Flush and stopping block - -Recovery reads the wallet-nonce watermark from the fresh data directory, so the -previous database's watermark is not used, even when that database survives. -Flushing therefore depends on the provider's pool view. A transaction dropped there but -alive elsewhere may escape; a later accepted foreign or mismatched landing -freezes the rebuilt instance and requires another rebuild. The provider is -trusted fail-stop, as specified in the [threat model](../threat-model/README.md). - -After flushing, raw L1 ingestion must reach at least `C`. It may advance farther, -but the fold stops at `C`. Accepted-batch projection is deferred until the new -baseline and batch tree exist. - -A trusted checkpoint can be ahead of an honest replacement node that is still -synchronizing. A successful flush only settles the wallet slots known to that -node; it does not establish `C >= B`. If `C < B`, recovery refuses with retryable -exit 20, even when the later re-sync head has reached `B`: that newer observation -does not replace the fixed stopping block. It publishes no baseline and must be -retried after the node catches up. - -### Replay boundaries - -Seed the scheduler's pending-direct queue from `(A, B]`, excluding inputs sent by -the batch submitter. Then replay **all raw inputs** in `(B, C]` in L1 order with -expected nonce `N`. Drain the remaining directs through `C` to obtain `(S', N')`. -For an [eligible checkpoint](#checkpoint-eligibility), the disjoint ranges -preserve pending directs without executing its accepted batches again. - -On the first `run` sync, acceptance starts at nonce `N'` and scans only blocks -**strictly after `C`**. Nonce filtering alone would let a previously rejected -future-nonce batch in the old prefix be reinterpreted as accepted. Raw inputs -ingested beyond `C` remain for normal reconciliation. Newly executed application -inputs are recorded beginning at `K`; raw L1 indices and application offsets are -separate coordinates. - -### Publish the baseline - -Write and durably sync the immutable application dump first. One SQLite -transaction then registers a fresh UUIDv4 era at generation zero, count `K`, -boundary `C`, anchor nonce `N'`, a parentless root frame at `C`, the snapshot, and -`setup_complete`. The collapsed prefix creates no application-input rows. - -Artifact failure leaves setup incomplete; transaction failure leaves at most an -orphan artifact. Identity pinning and raw L1 ingestion may survive an incomplete -attempt, but the lock and setup admission prevent serving a partial baseline. - -`C` remains the fallback reconciliation boundary if standard recovery invalidates -the root. The [history contract](../protocol/application-history.md#era-baseline) -owns these immutable coordinates; [snapshot lifecycle](../snapshots/lifecycle.md) -owns restore selection, rollback-safe retention, and eventual baseline disposal. +| Base count | The base's L2 tx count, where the era's history starts. | +| Stop block | The safe block pinned after the flush, stored as `stop_block`. | +| Rebuilt history end | The L2 tx count at the end of the stop block, where the live rows start. | +| First queued input | The InputBox index of the oldest direct still queued at the end of the stop block, or the input count through it when none is. | +| Anchor batch nonce | The scheduler's next batch nonce at the end of the stop block. | +| Baseline | A snapshot of the application at the end of the stop block, registered at the rebuilt history end. | + +`/history` reports the base count as `history.available_from`, and the stop +block, its input count, the anchor and the rebuilt history end under +`baseline` ([history metadata](../../README.md#history-metadata-internal-only)). +The first queued input stays off the wire. + +### What setup does + +**Before writing anything**, setup runs every check that can fail and be +retried: the RPC's chain id, a safe head sampled before discovery, and the +application's deployment block at or below that safe head. It also prepares +the fee source (a fixed price is held to the fee floor here), checks that the +key signs for the submitter address, and reads the base (step 1), whose state +bytes go to a scratch directory under the system's temporary directory. A +failure here, or a kill while a large machine loads, leaves the directory +empty but for the lock file, so retrying in the same directory is safe. + +1. **Reads the base.** For a machine, setup loads it privately and answers two + inspect queries ([reading a stored machine's export](../cartesi-machine.md#reading-a-stored-machines-export)): + `scheduler`, the 48-byte scheduler export, and `state`, the application's + canonical state bytes ([state export](../protocol/scheduler-semantics.md#state-export)). + It builds the application with `Application::from_canonical_state` at the + export's L2 tx count and refuses one that reports another count. For + genesis, it builds the binary's genesis application, which must start at + count zero. +2. **Flushes and pins the stop block.** From here setup writes: it pins the + deployment identity (the application's staleness window and direct-payload + cap with it) and the + first fee quote, then replaces whatever the provider's pool holds for the + submitter account and waits until that is safe + ([the flush and the stop block](#the-flush-and-the-stop-block)). The stop + block is a safe block observed after it. +3. **Fills the input table and checks for an escape.** It syncs every L1 input + from the deployment block to a safe head at or past the stop block. Then it + refuses, with exit 40, if the wallet is still unsettled or a submitter + input lies past the stop block. Either means a transaction escaped the + flush or the old instance still runs. This is setup's one check on the old + instance. +4. **Checks the base's export against L1.** Among the inputs the base + consumed, the admitted directs above its clock (not from the submitter, + payload within the application's cap) must number exactly the export's + queue length and start at its first queued input + ([the invariant](../protocol/scheduler-semantics.md#the-invariant)). The + base must also lie at or below the stop block, and the scheduler must + resume from the export over its queued inputs, with its clock at or below + the stop block. A mismatch means the machine is not canonical, or its + scheduler drained wrongly. +5. **Rebuilds the history with the scheduler.** It resumes the scheduler from + the export, re-supplying the inputs from the first queued input up to the + base's position, then feeds it every later input through the stop block, + never the sync's overshoot. The scheduler never reads the application: + what it hands out depends only on its state and the inputs. After the last + input its clock must lie at or below the stop block. +6. **Executes each transaction as it is handed out.** The application runs it + at once through `sequencer_core::application::execute`, the boundary the + lane and the guest share. A direct always executes; a user op executes or + is skipped ([a user op's fate](../protocol/scheduler-semantics.md#a-user-ops-fate)). + An executed transaction is stored in the `rebuilt_history` table with its + L2 tx count; a skipped user op is not stored. An application error stops + setup naming the input, the transaction's position among those the input + released, and the transaction. At the end, the directs executed must number + the admitted directs the scheduler drained, counted in the input table. +7. **Publishes the new era.** It writes the baseline dump, then one transaction + inserts the era row, which carries the batches' anchor nonce and whose + existence marks setup complete, and registers the baseline. Setup opens no + root frame and no Tip. + +The rebuild reads inputs and writes rows in pages of 1024, so memory stays +bounded however long the rebuilt range is. Nothing is published until step 7. +A setup that fails after writes begin leaves a directory to discard, whatever +its exit code: another setup there exits 30. + +### The flush and the stop block + +Setup's database is new, so it has no wallet-nonce watermark from the old +instance, even when the old directory survives: a lost database takes its +watermark with it. The flush is therefore best effort. It sends no-ops for the +wallet slots the provider's pool knows about, which normally holds the +transactions in flight when the old instance stopped; on a settled wallet it +returns at once. A transaction dropped from that pool but alive elsewhere can +escape it. If it lands before setup's sync, the escape check refuses; if it +lands later, the new era's content-identity check sees a foreign landing, +freezes the accepted frontier, and the deployment needs another recovery. The +provider is trusted fail-stop, as the +[threat model](../threat-model/README.md#rpc-endpoint) specifies. + +The stop block is fixed by the flush. The sync may run past it, but the rebuild +stops there, and a sync that ends below it refuses. In the new era, acceptance +scans only blocks after the stop block, starting at the anchor nonce, so no +landing in the rebuilt prefix is reinterpreted. Inputs past the stop block wait +in the input table for `run`. + +## After setup + +- **`run` opens the era's first batch** at its own safe head: its start opens + a batch when none is open ([what the sequencer does](README.md#what-the-sequencer-does)). + That batch's root frame first drains the directs still queued at the stop + block, from the first queued input on, as the canonical scheduler's first + frame does; submitter and over-cap inputs never execute. +- **The watchdog compares the baseline.** The application's state at the end + of the stop block is the canonical machine's state there, so until a batch of + the era is accepted the sequencer serves the baseline as its comparison + checkpoint at the stop block, for a genesis base too + ([snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison)). + The watchdog compares it on its next tick, once no latch holds it; a mismatch + there means the base machine was not canonical. +- **The feed serves the new era from the base count**: the rebuilt rows up + to the rebuilt history end, then the live rows, in one read + transaction. A direct's row takes its sender, payload, block, timestamp and + transaction hash from the input table. A claim from an older era is refused + with `ERA_CHANGED`; its client adopts the new era + ([the client's playbook](#the-clients-playbook)). +- **`GET /nonce` counts only the live rows.** For a sender whose last op lies + in the rebuilt history it reads low until that sender's next op is sequenced. + A wallet that signs at the nonce its own state holds is unaffected, and a + `422` bad-nonce rejection names the expected nonce. +- **The rebuilt history is permanent.** Its rows come from safe blocks, so they + never change: nothing removes rows inside an era, and storage triggers + refuse an update or a deletion. + +## How long a recovery takes + +Setup's time has three parts, and the base decides the largest one. The +[recovery measurements](../review/2026-11-10-recovery-measurements.md) hold +the numbers for the reference wallet. + +- **The sync** reads every input from the deployment block, so the RPC + provider bounds it; its database writes are a small part. The dry run syncs + the same inputs, so it takes as long. A sync through a deployment's provider + is not measured yet. +- **The flush** returns at once on a settled wallet, and otherwise waits until + its no-ops are safe. +- **The rebuild** recovers each user op's signature, one op after another on + one core, and that is almost all of its time; executing the transactions, + writing the rows and publishing take the rest. Its time therefore follows the user ops + past the base, not the length of the history: a recent base keeps a + recovery short, and genesis costs the whole history. The dry run prints how + many inputs the rebuild replays; the measured rate per user op turns the ops + among them into a time. Execution is the application's own cost, so measure + a native engine's with it. + +Reading the base machine is never the slow step. Memory stayed bounded in the +measurements, as the paging in [what setup does](#what-setup-does) intends, +but one reading is unexplained +([register](../review/register.md#sequencer-recovery-follow-ups)). On disk, +the rebuilt history about doubles the database. When the new era's first +batch lands, the canonical machine drains every direct still queued in that +one input; for the wallet, a fridge full to its bound drains well inside the +guest's budget for one input. + +## Before anything + +- **The old instance stays down.** The new data directory is pinned to the + same submitter address, and the process lock guards one directory, not two. + The flush settles transactions, not processes, so no command that signs with + the key may run against the old directory again + ([restart policy](../watchdog/operator-deployment.md#sequencer-restart-policy)). + Setup's escape check catches an old instance that is mid-flight when it + looks, not one started afterwards. +- **The base machine must be canonical.** Setup trusts it. The export check + catches a machine whose queued directs disagree with L1, but nothing in setup + checks the application state; the watchdog's first comparison of the new era + does. The operator checks where the machine came from: a watchdog checkpoint, + `incident/canonical`, or `sequencer-watchdog replay --out` were derived from + L1 independently of the sequencer, and a machine from a validator node at a + settled epoch can be checked against the root on chain. +- **The base must load.** It must be a machine of the deployment's template, + stored by the Cartesi Machine version the sequencer links + ([versions](../cartesi-machine.md#versions-are-all-or-nothing)), and stopped + between inputs, waiting at an `RX_ACCEPTED` yield. Not every machine the + watchdog stores does + ([which ones](../watchdog/README.md#machines-a-recovery-can-start-from)). + Setup refuses a halted machine, so a canonical machine that halted cannot be + recovered: the application is dead on chain. +- **Recovery repairs only the sequencer.** What the canonical machine executed + stays executed, a faulty sequencer's accepted batches included. +- **Preserve the evidence first.** Keep the old data directory, its logs, and + the watchdog's `incident/` before anything is rebuilt; affected client + databases are preserved separately. + +## Recovery readiness before deployment + +A recovery runs once in a deployment's life, so it must be ready before it is +needed: + +- **The application rebuilds from a machine.** It implements + `from_canonical_state`, and its template answers the `scheduler` and `state` + queries ([canonical recovery integration](../protocol/application-contract.md#7-canonical-recovery-integration)). + The `state` answer must fit one 2 MiB report; a larger state needs another + state source ([the loader's limits](../cartesi-machine.md#reading-a-stored-machines-export)). + The [measurements](../review/2026-11-10-recovery-measurements.md#b-reading-a-base-machine) + give how many wallet accounts fit; measure the application's own `state` + size against it, since until setup has a range read a machine whose state + has outgrown it cannot be a base. +- **The operator can find a trusted machine** at or below a given L2 tx count, + and keeps machines backed up. The watchdog's checkpoints are one source + ([backups](../watchdog/operator-deployment.md#checkpoint-disk-usage-and-backups)); + sourcing from a validator node is + [open work](../plans/2026-11-sequencer-recovery/todo.md#machine-sourcing). +- **Each client keeps backups it can restore at a count it trusts** + ([client backups](../plans/2026-11-sequencer-recovery/client-backups.md)). +- **The operators have run the drill.** The + [incident runbook's drills](../watchdog/incident-runbook.md#drills) say which + parts exist as tests. + +## The operator's playbook + +1. **Stop the old instance**, keep it down, and preserve its evidence + ([before anything](#before-anything)). +2. **Fix the cause** when a bug caused the incident. +3. **Ask each client for a state it trusts**, as an L2 tx count. +4. **Pick a base** at or below every client's count: a machine, or genesis. + `setup --base --dry-run` prints its L2 tx count and the old + instance's activity past it. +5. **Run `setup --base `** with the submitter key, in an empty data + directory. +6. **Run `run`** on that directory. +7. **Tell each client to realign** at its own count. +8. **Clear the watchdog's latch, then watch its next comparison.** If a + divergence latched it, clear it once `run` has started + ([incident runbook](../watchdog/incident-runbook.md#3-resolve)). The next + tick compares the new era's baseline at the stop block. + +## The client's playbook + +A client of the old era holds state at some L2 tx count `X`. It realigns on the +feed it already uses, with no file and no separate tool: + +1. **Restore a trusted state** at its count `X`. +2. **Read `/history`** for the new era's identity, its base count + (`history.available_from`) and its rebuilt history end + (`baseline.rebuilt_history_end`). +3. **Check `X` against that range** and take the claim to resume with: + `HistoryInfo::adopt_claim(X)` in `sequencer-core`, or + `SequencerClient::adopt_current_era(X)` in the Rust SDK. Below the base + count (`BelowBase`), the base is newer than the client's state, so the + operator must recover from an older machine. Past the rebuilt history end + (`PastRebuiltEnd`), the state holds rows the old instance served that never + landed by the stop block: restore an older state. +4. **Optionally check its rows.** Subscribe from the base count and compare + each row up to `X` with the row it stored. Any difference means its state is + not canonical. +5. **Subscribe with the adopted claim** and follow the feed as always. The + claim carries the new era's id, and the server checks it like any + other; it does not check the rebuilt history end itself, so step 3 is the + client's to do. + +A reader that owns more state than the application, such as an indexer of +transfers or orders, restores a checkpoint of that state too, at the same +count; matching current balances does not validate historical records. A +reader with no checkpoint it trusts can rebuild from the feed only when the +era's base is genesis: the feed starts at the base count, and the rows below +it exist only on L1. + +## What can go wrong + +| Problem | Seen where | Remedy | +|---|---|---| +| The data directory holds anything | Setup refuses it untouched, exit 30 | Run setup in an empty directory. | +| L1 is unreachable, or the deployment block is not yet safe | Setup's checks before writing, exit 20 | Retry in the same directory. | +| The key does not sign for the submitter address | Setup's identity check before writing, exit 30 | Configure the pinned submitter's key. | +| The base machine cannot be loaded, is not between inputs, or does not answer | Setup's step 1, exit 30 | Pick another machine ([which watchdog machines qualify](../watchdog/README.md#machines-a-recovery-can-start-from)). | +| The base's `state` report is over 2 MiB | Setup's step 1, exit 30: the machine did not answer the `state` query, because it halted | Recover from an older machine whose state fits, or from genesis. | +| The application cannot be built from the machine's state | Setup's step 1, exit 30 | A guest that fails to encode its state still answers `state`, with its error text (`inspect failed: ...`) as the report, so a decode error here may mean the guest's encoder failed: query the machine's `state` and read the report. | +| A transaction escaped the flush, or the old instance runs | Setup's escape check, exit 40 | Stop the old instance, discard the directory, run setup again. | +| The RPC node lags the base, so the base lies past the stop block | Setup's export check, exit 30 | Let the node catch up, discard the directory, run setup again. | +| The base's export disagrees with L1 | Setup's export check, exit 30 | Recover from a trusted machine; if this one is canonical, its scheduler drained wrongly. | +| A failure after writes began, retryable or not | Setup's exit code, then exit 30 on any retry in place | Discard the directory and run setup again. | +| The application fails on a rebuilt transaction | Setup stops naming the input index, the position and the transaction, exit 30 | Replay a trusted machine to that input: if it halts there too, the application is dead; if not, fix the native engine or its projection. | +| The base machine is not canonical | The new era's first watchdog comparison | Recover from a trusted machine. | +| The base is newer than a client's state | The client's `BelowBase`, or `HISTORY_UNAVAILABLE` on subscribe | Recover again from an older machine. | +| A client's state holds rows that never landed | The client's `PastRebuiltEnd` | The client restores an older state. | +| A client's state is wrong below its count | Only the client's own checks, such as the row comparison | The client restores an older state. | ## Code map -| Responsibility | Code | +| Concern | Code and tests | |---|---| -| Load, flush, sync, fold | [`commands/setup/mod.rs`](../../sequencer/src/commands/setup/mod.rs) | -| Durable baseline artifact | [`commands/setup/fill.rs`](../../sequencer/src/commands/setup/fill.rs) | -| Atomic baseline completion | [`storage/lifecycle.rs`](../../sequencer/src/storage/lifecycle.rs) | -| Scheduler fold | [`scheduler/fold.rs`](../../sequencer-core/src/scheduler/fold.rs) | -| Accepted-prefix boundary | [`storage/safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) | +| Arguments and the key | [`commands/config.rs`](../../sequencer/src/commands/config.rs) `SetupConfig`; its `setup_*` tests. | +| The procedure, the empty-directory rule, the dry run and the escape check | [`commands/setup/mod.rs`](../../sequencer/src/commands/setup/mod.rs); [`chain_tests.rs`](../../sequencer/src/commands/setup/chain_tests.rs) runs setup against Anvil on a chain with history and compares it with `CanonicalRunner`. | +| The base, and the seam tests use instead of an emulator | [`commands/setup/base.rs`](../../sequencer/src/commands/setup/base.rs) `ReadMachineExport`; [`machine.rs`](../../sequencer/src/machine.rs) `read_machine_export`, whose unit tests drive every failure arm through a scripted emulator, and `canonical-test`'s `export` cases run on the built image. | +| Export check, and the rebuild executing as it goes | [`commands/setup/rebuild.rs`](../../sequencer/src/commands/setup/rebuild.rs); `Scheduler::resume` in [`sequencer-core`](../../sequencer-core/src/scheduler/mod.rs); `admitted_directs_in` in [`storage/l1_inputs.rs`](../../sequencer/src/storage/l1_inputs.rs) for the closing direct count. | +| Publication | [`commands/setup/fill.rs`](../../sequencer/src/commands/setup/fill.rs); `publish_era` in [`storage/lifecycle.rs`](../../sequencer/src/storage/lifecycle.rs). | +| The rebuilt history table and the `era` row | [`storage/rebuilt_history.rs`](../../sequencer/src/storage/rebuilt_history.rs), [`storage/history.rs`](../../sequencer/src/storage/history.rs) (`Era`, `NewEra`, `era_in`), and their triggers in [`0001_schema.sql`](../../sequencer/src/storage/migrations/0001_schema.sql). | +| The root frame's drain from the first queued input | `next_undrained_safe_input_index_in` in [`storage/ingress.rs`](../../sequencer/src/storage/ingress.rs). | +| The baseline as the first comparison checkpoint | `finalized_dump_in` in [`storage/snapshots.rs`](../../sequencer/src/storage/snapshots.rs). | +| The feed across the rebuilt and live rows | [`storage/history/canonical.rs`](../../sequencer/src/storage/history/canonical.rs). | +| Adoption | `HistoryInfo::adopt_claim` in [`history_api.rs`](../../sequencer-core/src/history_api.rs); `adopt_current_era` in [`sdk/rust-client`](../../sdk/rust-client/src/history.rs). | +| The flush | [`l1/flusher.rs`](../../sequencer/src/l1/flusher.rs) `flush_to_safe`. | +| End to end on the devnet | [`tests/e2e/src/recovery_round_trip.rs`](../../tests/e2e/src/recovery_round_trip.rs): the round trips from genesis and from a stored machine, and setup's refusal of a set-up directory, each for the Rust and C hosts ([drills](../watchdog/incident-runbook.md#drills)). | diff --git a/docs/recovery/history/2026-10-shared-recovery-tool.md b/docs/recovery/history/2026-10-shared-recovery-tool.md new file mode 100644 index 00000000..fd17e85e --- /dev/null +++ b/docs/recovery/history/2026-10-shared-recovery-tool.md @@ -0,0 +1,757 @@ +# Shared recovery tool + +**Status:** archived on 2026-11-05, never implemented. It is superseded by +[sequencer recovery](../../plans/2026-11-sequencer-recovery/README.md), which +reads the whole scheduler state from the machine instead of rebuilding the +fridge from L1. +Kept as the record of the 2026-10-28 to 2026-11-02 design sessions; the facts +that still hold live in the new plan and the [register](../../review/register.md). +Its claims were checked by reading the code at `3e92563`, +[PR #56](https://github.com/cartesi/sequencer/pull/56) at `ee5c72c`, Dave at +`dbcca7d4` and the emulator at v0.22.0-test3 (`7adfd492`, whose header and CLI +match v0.22.0), and by adversarial review; none was executed. + +Cockroach recovery rebuilds a sequencer that is ready for new user ops after an +instance loses its database or its behavior cannot be trusted. The operator +triggers it by hand and keeps that instance down. The design takes the worst +case: the submitter key is ours, but the instance's database, snapshots and +receipts prove nothing. The canonical scheduler is the authority, bug for bug. +The rebuilt sequencer is assumed correct; if it is not, fix it and rebuild +again. Nothing has to be replayed from genesis: the tool starts from a +canonical machine and reads only the L1 blocks after that machine's clock. + +What makes the tool *shared* is a client: Bart's DEX backend keeps a richer +state than the sequencer and must realign after a rebuild, from its own backup +and with its own code. Bart operates both; only their development is split. +Recovery runs rarely, so each step must be simple enough to explain, practice +and trust. + +## The idea + +Every executed L2 transaction carries a block: a direct its inclusion block, a +user op its frame's `safe_block`. Let `T` be the canonical sequence of executed +L2 transactions, in execution order. Each application state carries its +progress: its **effective count** `X`, how many transactions it has executed, +and its **effective block** `E`, the block of the latest one. + +> A state on the canonical history with effective count `X` has executed +> exactly `T[0..X)`. Nothing but executed transactions changes an application, +> so the state is missing exactly `T[X..K)`, where `K` is the count at the stop +> block `C`. + +Recovery hands each consumer its missing part, starting from its **base**: the +canonical state it starts from. The rebuilt sequencer's base is a canonical +machine at position `n_m` and count `X_m`, and it needs `T[X_m..K)`. A client's +base is a count `X_c` at which its state is canonical, reached from its own +backup, and it needs `T[X_c..K)`. + +The hard part is computing `T[X_m..K)` without scanning from genesis. The +canonical machine makes it possible. Its application state is the sequencer's +starting state, and three numbers that its scheduler reports under the next +template let the tool replay the canonical scheduler from where the machine +stopped: + +- its **position** `n_m`: how many InputBox inputs it has consumed; +- its **batch nonce** `N_m`: the nonce the next batch must carry; +- its **scheduler clock** `L_m`: the block its fridge has been drained through, + the latest of every accepted frame's `safe_block` and every drained direct's + block. + +The tool reads the inputs of L1 blocks `(L_m, C]` once and splits them at the +machine's position: + +1. **Below `n_m`** the machine has already consumed the inputs. The tool drops + their batches without judging them and keeps their well-formed directs. + Those are exactly the machine's *fridge*: the directs it admitted but has + not executed yet. +2. **From `n_m` on**, it runs the canonical scheduler from the machine's own + state: batch nonce `N_m`, that fridge and clock `L_m`. + +The tool does not know which user ops the application will include, so it hands +the sequencer *candidates*. The sequencer executes them; the ones that execute +form `T[X_m..K)`, and each client executes its own part of that list. + +## Terms + +- **Template**: the initial machine whose root hash the on-chain application + pins as `getTemplateHash()`. A deployed application keeps its template for + life ([Cartesi Machine facts](../../cartesi-machine.md#versions-are-all-or-nothing)), + so a rule that changes the guest ships with a new template, which is to say + a new deployment. A **machine** is any stored state of the canonical + machine, such as a validator's at the end of an epoch or a watchdog + checkpoint. +- **Effective count `X` and effective block `E`**: an application's progress, + `executed_input_count` and `last_executed_safe_block` in code + ([application contract](../../protocol/application-contract.md#3-the-safe-block-clock--last_executed_safe_block)). + Both follow executed transactions only: a skipped op, a rejected batch or a + queued direct moves neither. The count numbers the feed's rows. +- **Position `n`**: how many InputBox inputs a state has consumed, which is + also the index of the next input. It differs from the effective count `X`, + because a batch envelope or a queued direct is an input but not a + transaction. The watchdog's checkpoint names, `-`, hold + positions. +- **Scheduler state**: the batch nonce `N`, the fridge and, under the next + template, the scheduler clock `L`. `L` and `E` stay separate on purpose. A + frame raises `L` to its `safe_block` even when it runs nothing, while `E` + moves only with executed transactions. The scheduler reads `L`, so it never + depends on which ops the application included. Merged, `E` would move when + nothing executed: two states at one count could then differ, and a consumer + that sees only executed rows could not follow `E`. +- **`C`** is the stop block, a safe head fixed after the flush; **`N'`** and + **`L'`** are the batch nonce and the scheduler clock at its end. +- **`P` and `B`**, as in the watchdog's latch record: `P` is its head, the last + block the sequencer agreed with, or the uncompared bootstrap block before + the first agreement, and `B` the block of the comparison that failed. +- **Conformance vectors**: + [`consensus_vectors_v1.json`](https://github.com/cartesi/sequencer/blob/3e92563a3947673b3931a6b43eecc16ec4e0ba48/tests/fixtures/consensus_vectors_v1.json), + frozen cases (EIP-712 hashes, signatures, SSZ bytes, fee conversions, guest + scenarios with their exact outputs) that every port of the scheduler must + reproduce ([conformance](../../protocol/scheduler-semantics.md#conformance)). + +## The tool as a function + +In ML notation: + +```ocaml +(* One InputBox input. Its chain id and application address, the signature + domain, are deployment constants. *) +type raw_input = { index : index; block : block; timestamp : time; + prev_randao : word256; sender : address; payload : bytes } + +(* What the scheduler hands the application, in execution order. A direct is + an input from any sender but the sequencer, passed on whole. *) +type direct = raw_input +type tx = + | User_op of { sender : address; nonce : nonce; max_fee : fee; fee : fee; + payload : bytes; safe_block : block } + | Direct of direct + +(* The scheduler's state. It reads nothing from the application. *) +type sched = { nonce : nonce; (* N: the next batch's nonce *) + fridge : direct queue; (* admitted directs not yet executed *) + clock : block } (* L *) + +val step : sched -> raw_input -> sched * tx list + +(* What the base machine reports through its scheduler query. *) +type base = { position : index; nonce : nonce; clock : block } (* n, N, L *) + +(* Candidates from the base's position through the end of stop, and the + scheduler state there. *) +val tool : gateway -> base -> stop:block -> tx list * sched +``` + +In code: + +```ocaml +let tool gateway m ~stop = + let inputs = read gateway ~after:m.clock ~through:stop in + let before, after = List.partition (fun i -> i.index < m.position) inputs in + (* Phase 1: the fridge. Batches the machine consumed are not judged again. *) + let fridge = List.filter (fun i -> is_direct i && well_formed i) before in + (* Phase 2: the canonical scheduler, from the machine's own state. *) + run_steps { nonce = m.nonce; fridge; clock = m.clock } after +``` + +**Why the split is exact.** Under the next template every scheduler state keeps +one invariant: *every admitted direct at or below `L` has executed, and every +one above `L` is still in the fridge.* Every drain empties the fridge's front +through some block, oldest first: + +- a frame drains every direct up to its `safe_block`, which lies strictly below + its batch's block, so all of them have arrived, and it raises `L` to that + `safe_block` even when it drains nothing; +- the backstop drains every direct at least `MAX_WAIT_BLOCKS` old; +- an overflow drains whole blocks older than the current input's; + +and the backstop and an overflow raise `L` to the last block they took. So `L` +stays below the current input's block. A newly admitted direct therefore lands +above `L`, and so does every input from `n_m` on: the scan `(L_m, C]` holds all +of phase 2. And the machine's fridge is exactly the well-formed directs above +`L_m` with index below `n_m`. Three template rules make this hold +([rules](#rules-for-the-next-template)). Without them a block can be split +between executed and queued directs, the scan could miss the rest of the +machine's last block, and the tool would need the fridge's first index as a +fourth value. + +**Why batches before `n_m` are not judged again.** The machine already judged +them, and a second judgment can differ. A faulty submitter lands batch nonce 10 +at block 200, with a frame at 190; the canonical scheduler expects 9 and +rejects it. Batch 9 lands at block 201, with its one frame at 185, and runs. So +a machine at the end of block 201 has `N_m = 10` and `L_m = 185`, and its scan +`(185, C]` includes block 200. A replay that judged the block-200 batch again +would find the expected nonce and a first frame at or above `L_m`, accept it, +run ops the canonical machine never ran, and then reject the real batch 10. + +**Why each value is needed.** + +- **The position.** The batch nonce cannot replace it, as the example shows, + and neither can a block. A machine can stop inside a block: Dave's last + finalized machine does when an input lands after the sealing transaction in + the same block. Even at a block's end, the machine cannot know whether its + last block held more inputs. Each input carries its index on L1, so the tool + still queries by block range and splits by index. +- **The batch nonce**, for the nonce gate. +- **The clock**, for the floor gate. A batch whose frame at 100 runs ops and + whose frame at 140 runs nothing leaves `L = 140` and `E = 100`. A later batch + starting at 120 is rejected canonically; a replay floored at `E` would accept + it. + +**What the tool reads besides L1.** `step` reads no application state. It does +read what the template fixes: the sequencer address, `MAX_WAIT_BLOCKS`, the +fridge bound and the application's stateless rule for well-formed directs, so +for libdex the scheduler library embeds a port of Bart's rule. Signature +recovery is pure, so `step` does it. The user ops it emits are *candidates*: +the application that executes them decides which ones execute, from its +senders' nonces (and, without the +[authorization rule](#rules-for-the-next-template), their fee balances). The +canonical machine is `step` followed by the guest's application. That holds +while the guest never answers an input with `RX_REJECTED`, as the +[fatal conditions](../../protocol/scheduler-semantics.md#fatal-conditions) require: +the host's revert would roll the scheduler state back with the application, a +[question for Bart](#questions-for-bart). + +**Cost.** The scan reads blocks `(L_m, C]`. While well-formed directs keep +arriving, the backstop keeps `L` within `MAX_WAIT_BLOCKS` (about 4 hours) of the +latest input. After a quiet stretch `L_m` can trail further, but a quiet +stretch carries few inputs. For the default machine the range also spans +Dave's settlement lag, [still to be measured](#measure-before-choosing). + +**Completeness.** A provider that drops a log would drop a direct from the +fridge or a batch from the replay. The tool reads the InputBox's input count at +blocks `L_m` and `C`, checks that the indices it reads run without gaps from +the first count to the second, and refuses a base whose position lies outside +those two counts. The count at `L_m` needs an RPC with state at that block, an +archive node for an old base, as the watchdog's `init` does +([commands](../../watchdog/README.md#commands)). + +## The consumers + +```ocaml +(* Executes each tx through the shared execution boundary, as the canonical + application does: a user op its state does not authorize is skipped. + Returns the new state and the transactions it executed, with their counts. *) +val execute : state -> tx list -> state * (count * tx) list + +let sequencer m ~stop = + let candidates, sched = tool gateway (scheduler_query m) ~stop in + let state, executed = execute (project m) candidates in (* T[X_m..K) *) + (import state sched, executed) + +(* s: a client state restored to its base, a count between X_m and K. *) +let client s executed = fst (execute s (from_count (count s) executed)) +``` + +Only the sequencer executes candidates, because only its state sits at the +machine's position. It holds the nonces (and, without the authorization rule, +the balances) that decided inclusion there. This yields `T[X_m..K)` with each +transaction's count, and the sequencer writes that list for the clients. A +client executes the part from its own base `X_c`, which needs +`X_m <= X_c <= K`: the machine must be no newer than any client's base. + +Only the sequencer receives `sched`. It imports `N'` and `L'`, and its root +frame drains the queued directs ([below](#queued-directs-at-the-stop-block)). A +client receives those directs as the new era's first feed rows, like any other +row, and never holds scheduler state. + +### Why a list and not a plugin + +The alternative is a `RichClientApp` plugin that the tool runs to produce each +client's end state. The list is better for three reasons: + +- **Data crosses the boundary, not code.** The tool depends only on the + scheduler, which a template fixes; a client's projection, schema and storage + change on their own schedule. +- **One execution boundary.** A client already executes L2 transactions + through the shared execution boundary, and recovery reuses it. An entry + carries its count (the feed's offset) and what the canonical application + reads; it carries no batch nonce (a direct the backstop drains belongs to no + batch) and no transaction hash. A plugin would be a second path that runs + once in a deployment's life and rots. +- **History, not only state.** Bart's client keeps trades and transfers, and + executing its part of the list replays that history, in canonical order. + +### What the count is for + +The tool positions the machine by its input position and never reads a count. +The consumers use the count three ways: + +- **Positioning a client.** A client has no input position, only its count, so + its part of the list is cut by count. +- **Joining the new era.** The era serves rows from `K`, which `/history` + reports as `available_from`. After its part of the list, a client compares + its count with `K` and subscribes there. The comparison must be explicit: the + feed refuses a count below `K` but serves one above it. +- **Matching the machine.** The wallet's comparison bytes embed the count, so + the sequencer's `K` must equal the machine's, and the projection from a + machine must carry it. + +## States at a base + +Each consumer's state must sit exactly at its base: a state one transaction +short or past it would drop or repeat one. + +### The sequencer's base + +The sequencer's state is projected from a machine. The projection must carry +the effective count and block that the consumers' counts start from: the +wallet's `state` inspect report holds both, and libdex keeps its state as a +flat buffer in one of the machine's drives, which may or may not hold them +([application contract](../../protocol/application-contract.md#7-canonical-recovery-integration)). +The three values come from the machine's `scheduler` inspect query, a rule for +the next template. Any canonical machine the operator trusts will do if it is +no newer than any client's base; the tool enforces no provenance. After a +divergence every client's base lies at or before the watchdog's head `P`, +which rules out a machine past it. + +| Machine | Position | Trust | +|---|---|---| +| Dave's last finalized machine, the default | the current sealed epoch's `inputIndexLowerBound`, inside a block when an input lands after the sealing transaction in the same block; the scheduler query reports it too | its root is on chain | +| The watchdog's head | the input count in its name | the operator's watchdog replayed it from L1; before its first agreement, the operator trusted it at `init` | +| A `replay --out` machine | the input count at the end of the block it was replayed to | the operator's watchdog replayed it from L1 | +| `incident/canonical`, only for a deployment without clients | the input count at the end of the latched block `B`, past every client's base | the operator's watchdog replayed it from L1 | +| The template | zero, with nonce and clock zero; the tool then reads every input since the deployment block | its hash is on chain | + +A client whose base is older than the default machine needs an older machine: +`replay --out` derives one from an archived checkpoint or the template. + +A matching root authenticates a machine only from a source we trust, such as +our own Dave node: a stored machine's root comes from its stored hash tree, not +from its contents ([hashing](../../cartesi-machine.md#hashing)). An upcoming +emulator release brings a mechanism to verify an untrusted load; until then, +`verify_hash_tree` is not that mechanism and is +[on the watchdog's to-do list](../../plans/2026-09-watchdog-v2.md#remaining-work). + +### The client's base + +A client's base is a count `X_c` at which its state is canonical. A watchdog +agreement vouches for one, since the sequencer reports its count with each +comparison. It vouches for the state, not for the client's rows before it +([after an incident](#after-an-incident)). An accepted checkpoint alone does +not: accepted means only that our batch landed and passed a partial gate check, +which skips gate d +([I1](../../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)), +so `/history` keeps advancing past a divergence that only the watchdog sees. +For the wallet the comparison pins the count, since its bytes embed it; for +libdex that depends on Bart's answer. + +The sequencer's digest response already carries its count, but the watchdog +parses only the block and the digest, and deletes each checkpoint once a newer +one agrees. So it must append each agreement to a log kept outside its state +directory, which a re-`init` wipes: the agreement's block, the sequencer's +count, and the era and generation that count was read under, which the digest +response must add. + +A client reads its base from that log, not from a sequencer, which may be gone +or wrong. It takes the newest agreement on its own history, normally the head. +An agreement is on the client's history when its count is at or below the +client's and either: + +- its era is the client's and its generation is the client's or an earlier + one, since standard recovery never cuts an accepted row; or +- its era is one the client left at a rebuild, and its count is at or below + the base the client crossed each later rebuild from. + +So a client that just realigned keeps the base it crossed from until its new +era logs an agreement, which is also the comparison that checks the rebuild. +Before a deployment's first agreement the log is empty, and a client's base is +count zero, its initial state. + +A client that follows the optimistic feed is past its base by the time the +watchdog compares, and its database can hold terabytes. Three rules make any +count on its history restorable, with any backup: + +1. **Every commit is a boundary and keeps its rows.** Apply each row, from the + feed or from the executed list, or each run of rows, in one SQLite + transaction that also writes the count, era and generation and stores the + rows themselves. Any consistent copy of the database is then at a + transaction boundary, says which, and holds every row before it. +2. **A cut discards later snapshots.** A standard recovery cuts the history at + its preserved count + ([checkpoint compatibility](../../protocol/application-history.md#checkpoint-compatibility-after-standard-recovery)); + a rebuild cuts it at the client's base. The new history replaces the rows + past the cut and reuses their counts. So once realigned, the client deletes + every snapshot past the cut and stops snapshotting its old live database. +3. **Snapshot anywhere.** Take snapshots when convenient; none needs to line up + with a checkpoint. + +To reach `X_c`, restore the newest snapshot at or below `X_c`. Then replay the +rows from that snapshot up to `X_c` through the client's normal apply, and +check the count. The rows can come from the live database or from any newer +snapshot, since each holds every row before it. Never restore a snapshot past +`X_c`: its state includes rows past the base, which may not be canonical. +Replay is exact only if applying a row reads nothing but the database and the +row. A rebuilt sequencer serves only its own era, so the client keeps its old +live database until it has realigned. +[Notes on snapshotting](#notes-on-snapshotting-a-large-sqlite-database) cover +the storage side. + +### After an incident + +The sequencer may start from any canonical machine no newer than every +client's base. After a sequencer incident a client starts from the newest +logged agreement on its history, normally the watchdog's head. An agreement +vouches for the sequencer's state, not for the order of the rows before it: +rows that commute reach the same state in either order, so a matching state +does not validate the client's history +([reader state](../cockroach.md#application-specific-reader-state)). +If diagnosis finds the fault began before that agreement, the client starts +from an older logged agreement that predates it, and the sequencer then needs +a machine no newer than that, which `replay --out` derives. + +- **The watchdog latched a `state_mismatch`.** Its head stays at `P`, which + need not be the block before `B`. A tick compares only the newest accepted + checkpoint, so the accepted checkpoints in `(P, B)` were never compared, and + the compared state can first differ anywhere in `(P, B]` + ([incident runbook](../../watchdog/incident-runbook.md)). The client followed + the faulty sequencer through all of them. The other latches call for no + rebuild: a dead canonical machine ends the application + ([preconditions](#preconditions-the-design-must-state)), and a regression is + an operator error + ([runbook](../../watchdog/incident-runbook.md#inclusion_block_regressed)). +- **The sequencer lost its database, or its content-identity check tripped** + ([I15](../../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). + A tripped check answers the finalized routes with 503 and does not latch the + watchdog, so its head stays at the last comparison before the trip. +- **The client has a bug.** This is the client's own repair, not a rebuild: it + restores a snapshot older than the bug's first effect and replays its rows + with the fixed code, across any rebuild, since the executed list's entries + are rows in its database. It then deletes every snapshot newer than the one + it restored, since they carry the bug's state. + +## Queued directs at the stop block + +At the end of `C` the canonical scheduler can still hold queued directs. By the +invariant above, they are exactly the well-formed directs in blocks `(L', C]`. + +The rebuilt sequencer starts from the scheduler state at the end of `C`. It +keeps no fridge: its input table, which setup fills from the deployment block, +already holds every direct. Today the lane drains directs from the first input +after the later of a floor block and its latest frame's `safe_block`, and a +rebuild puts that floor at `C`. The change is that the floor becomes `L'`, in an +era field of its own: `base_safe_block` stays `C`, since it also serves as +`/history`'s `l1_stop_block` and the end of historical inputs. So the root +frame, at `C`, first executes the queued directs, as the first canonical frame +drains them, before any user op, and clients receive them as ordinary feed +rows. A cascade that invalidates every frame +([I16](../../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)) +opens a new root, which drains above `L'` again. Acceptance still scans only +blocks after `C` ([replay boundaries](../cockroach.md#replay-boundaries)). + +Today's procedure instead executes the queued directs during recovery (the +terminal drain) and publishes the result, and each client repeats that drain +after restoring the scheduler's queue and nonce +(projection replay, a contract since deleted). That state is a +prediction: the canonical machine reaches it only inside the post-`C` input +that drains the last of the queued directs, often executing more there, so no +comparison can check it, a client that applied it must roll it back if a +second rebuild comes, and a rebuild that started from it would execute those +directs twice. Track 3 kept the terminal drain because replacing it would carry +pending work across the baseline +(the Track 3 plan's scope boundaries, since deleted); +that work crosses as one block, `L'`. + +## The procedure + +1. **Choose the bases.** Each client restores its state to the newest logged + agreement on its history that precedes any fault the diagnosis found + ([the client's base](#the-clients-base)). The sequencer then takes a + canonical machine no newer than any of them: by default Dave's last + finalized one, else the watchdog's head or a `replay --out` machine + ([the sequencer's base](#the-sequencers-base)). +2. **Flush, then fix `C`.** Settle the old instance's transactions, then take + a safe head observed after they settle as `C`. The flush is best effort: it + replaces what the provider's pool holds for the submitter account and waits + until that is safe. It cannot rely on the old watermark, which does not + survive every event that calls for a rebuild + ([flush boundary](../cockroach.md#flush-and-stopping-block)). As a + standalone subcommand, it is the only step that needs the key. Refuse if a + submitter input above `C` is already ingested. +3. **Run the tool.** Read the machine's position, batch nonce and clock with + its scheduler query. The tool refuses a position below the InputBox count at + `L_m` or past the end of `C`, reads blocks `(L_m, C]`, and writes the + candidates and the scheduler state at `C`, the latter last as the commit + point. +4. **Rebuild the sequencer.** Execute the candidates from the state projected + from the machine, write the executed list for the clients, and import the + result with `N'` and `L'` in one transaction. Then start the sequencer with + `run` promptly. A late start is safe: once the root frame ages past the + danger threshold, startup's + [Tip repair](../README.md#open-tip-repair-without-flushing) reopens + it at the current safe head, still draining above `L'`. +5. **Realign clients.** Each client: + 1. reads `/history` for the new era and generation; + 2. executes its part of the executed list, committing its entries as rows + under that era and generation (rule 1); + 3. checks that its count equals `available_from`, and records with it the + base it crossed the rebuild from; + 4. deletes every snapshot past its base (rule 2), then takes a snapshot; + 5. subscribes at its count with that era and generation. + +The new era's first watchdog comparison checks the result a batch later: a +wrong list, `N'` or `L'`, or a guest compiled for another sequencer address, +shows up there as a divergence. Replaying the watchdog's head to `C` with +`replay`, which starts only at the end of a block, checks it before `run`, at +an emulated cost. The tool cannot see a halt, which reads application state; a +consumer that reaches the halting transaction fails there, as the canonical +machine did. + +## Rules for the next template + +The owner settled these on 2026-10-29 and 2026-11-02. Each changes the guest, +so each needs Bart's agreement and ships with the next template, ours and his +port's ([register](../../review/register.md#canonical-rules-open-for-the-c-port)). +The tool relies on three of them: whole-block overflow, strict frame clocks and +the scheduler clock. They keep `L` below the current input's block, which makes +phase 1 exact and puts every input from the position on inside the scan. Phase +2 then replays whatever the template applies. The tool runs `step` without +application state; the only application code inside `step` is the fridge's +well-formedness rule, so that rule may read only the direct and fixed +configuration. + +- **Fridge bound.** Only well-formed directs enter the fridge, under an + application-defined rule that reads only the direct and fixed configuration, + and a count bounds it + ([register](../../review/register.md#bound-the-canonical-direct-input-fridge)). + A direct that is not well-formed is dropped without executing. An overflow + drains whole blocks from the front, oldest first, until there is room. The + bound must hold every direct one block can carry, about 1,500 to 2,000 at + today's 60M gas limit, for the deployment's life. Then an overflow never + needs the current input's block, and it drains at most one block more than + it must. The sequencer cannot mirror an overflow, so the batches it already + sealed diverge, and a manual cockroach recovery answers that. Making + standard recovery repair one is deferred on complexity, since standard + recovery is the hardest part of this software. +- **Strict frame clocks.** A frame's `safe_block` must lie strictly below its + batch's inclusion block; gate d accepts equality today. A correct sequencer + never posts equality, since its frames take the L1 safe head and its batch + lands later. Equality lets a frame drain only part of its own block. The + frozen conformance vectors post such a frame in 18 of their 19 guest + scenarios, so this rule ships with v2 vectors. +- **The scheduler clock and floor gate.** The scheduler keeps `L`: the latest + of every accepted frame's `safe_block` and every drained direct's block. A + new gate, placed after gate e, rejects a batch whose first frame lies below + `L`; empty batches stop at gate c and never reach it. It reads `L`, never `E` + ([terms](#terms)). Applications may then read blocks as time: a direct's + inclusion block and a user op's frame `safe_block` never go backwards along + `T`. Raising `L` on a backstop drain changes no gate outcome, not even a + report: the backstop drains only directs at least `MAX_WAIT_BLOCKS` old, so + any batch whose first frame lies below one is stale and stops at gate e. + Recovery still needs that raise: without it, the directs the backstop + executed would sit above `L_m`, and the tool would queue and execute them + again. On the sequencer side, the gate makes + [I3](../../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) + consensus-critical: every transition that opens a Tip needs an assert that it + starts at or above the latest valid frame. `scheduler_accepts` leaves the + floor out, as it leaves out gate d + ([I1](../../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)): + those asserts keep the sequencer's own batches at or above it, and an + overflow's raise, which rejects batches the lane already sealed, is the + divergence a manual rebuild answers. +- **Authorization decides inclusion.** A user op is included exactly when its + sender authorized it here: its signature recovers the sender, its nonce is + the sender's next and has a successor, and its `max_fee` covers the frame + fee. An op that fails any of these has no effect on anyone. An included op + always advances its sender's nonce and pays the fee, capped at the balance + (a saturating subtraction); anything else that stops it is a business + failure, which still advances the nonce and charges the fee. The wallet + changes too: its apply returns an error on an unpayable fee, which would + halt the machine under this rule. The sequencer keeps checking balances at + admission, so its batches are unchanged. +- **Direct inputs carry their context.** An application may read a direct's + input index, block timestamp and prevRandao, never a batch's. The fridge and + the `Direct` the scheduler applies carry them. The sequencer's input table + and the feed's direct rows already carry the index and timestamp, and both + must add prevRandao, or a client could not apply such a direct. +- **A `scheduler` inspect query** returning the machine's position, batch + nonce and clock: the tool's input. The position is the last input's index + plus one, batches included, or zero for the template. + +Shared scheduler code serves every deployment; if each sequencer release +supports one template, which the proposed template tie below would enforce, +no rule needs a switch per deployment. + +## Preconditions the design must state + +- **The old instance stays down.** Recovery deliberately creates a second data + directory pinned to the same submitter address, the old one can still run + with the same key, and + [the flush fences transactions, not processes](../cockroach.md#incident-playbook). + No command that signs with the key may run against the old directory again + ([restart policy](../../watchdog/operator-deployment.md#sequencer-restart-policy)). + A sequencer address that is an L1 contract forwarding batches from a hot key + that a cold multisig rotates (a separate design, out of scope) would fence + at L1 instead. +- **Key loss or compromise means a new deployment** while the submitter is an + account compiled into the template ([README](../../../README.md#running)). +- **A dead canonical machine is unrecoverable.** Cockroach recovery repairs + only the sequencer's view. PR #56 makes gate d reject an out-of-range + `fee_price`, so on templates built from it that sequencer bug becomes a + divergence recovery can repair; any other + [fatal condition](../../protocol/scheduler-semantics.md#fatal-conditions) still + ends the application. +- **Consumers execute bit for bit.** The sequencer executes the candidates, and + every client its part of the executed list, through the canonical validation + and execution; for Bart's client that means embedding libdex exactly as the + guest runs it. +- **The template tie.** A fix to code the template shares diverges the + sequencer from it, which today only the watchdog's next comparison notices + ([cockroach recovery](../cockroach.md)). Setup would pin the + application's `getTemplateHash()`, and `run` would refuse a template other + than the one its release declares. Tying the watchdog to the validators is + [a later to-do](../../plans/2026-09-watchdog-v2.md#remaining-work). +- **Detection.** The watchdog's divergence latch tells the operator that the + sequencer disagrees with the canonical machine, after a bug or a fridge + overflow alike. Whether its `range` state source works with Bart's template + is unverified ([state sources](../../watchdog/README.md#state-sources)). +- **Users.** Revoked user ops are not cancelled + ([cockroach recovery](../cockroach.md)); this is accepted, and + expiry is future work. +- **The canonical prefix stays.** Batches the faulty sequencer got accepted + stay canonical, and recovery does not undo them. +- **Practice.** Both of today's rebuild drills restore an honest instance's + own archive through `setup --recovery` + ([incident runbook](../../watchdog/incident-runbook.md#drills)): a disaster, + not a bug. Rebuilding from a machine, and realigning a client, are + unpracticed, and a path run once in a deployment's life rots. + +## Decisions + +The owner settled these on 2026-10-29 and 2026-11-02: + +| Decision | Choice | +|---|---| +| Shape | From L1, the scheduler library and three values the base machine reports (position, batch nonce, clock): candidates from the machine's position and the scheduler state at `C`. No fold from genesis to reach the machine | +| Phases | Blocks `(L_m, C]` read once: below the machine's position, directs only, and batches never judged again; from it, the canonical scheduler | +| Clients | Each executes its part of the sequencer's executed list, from its own count, instead of a plugin per client; the machine is no newer than any client's base | +| Sequencer's state | Projected from a machine the operator trusts, by default Dave's last finalized one; no enforced provenance | +| Clocks | `E` (the application's, from executed transactions) and `L` (the scheduler's, from accepted frames and drains) stay separate | +| Queued directs at `C` | A direct floor at the block `L'`, replacing the terminal drain. It applies the tool's phase-1 rule (every queued direct lies above the clock), the lane's floor is already a block, and a database today's procedure rebuilt keeps its floor at its `C`. Settled on 2026-11-02, in place of the index `F` chosen on 2026-10-29 | +| Flush | Best effort, trusting the provider's pool view | +| Fencing | An operator requirement, in the threat model's [operator mistakes](../../threat-model/README.md#operator-mistakes) | +| Next template | The fridge bound with whole-block overflow, strict frame clocks, the scheduler clock and floor gate, authorization deciding inclusion, direct context, and the `scheduler` query | +| Fridge overflow | A manual cockroach recovery; standard recovery does not repair it | + +These stay open: + +| Decision | Recommendation | +|---|---| +| Client's base | The newest agreement on its history, from a log in which the watchdog records each agreement with the sequencer's count, era and generation. The alternative, a hash chained over executed transactions that the guest keeps in its state, would check a client's whole history, order included, and drop the log and the cut rule, at the cost of a template change | +| Client backups | Every commit writes the count and keeps its rows, the executed list's included; a cut discards later snapshots; snapshots taken anywhere and replayed to the base | +| Executed list's delivery | A file the rebuild writes when it executes the candidates (step 4), before `run` starts. Serving it as the new era's first feed rows instead needs an immutable store for rows before `K` and a batch nonce for directs the backstop drained | +| Template tie | One template per release, enforced at setup and `run` | +| Shared gate function | Defer: it changes the gold frontier, the heart of standard recovery | +| `trait Scheduler` | Defer: for libdex deployments the scheduler library is ours, with libdex's direct rule ported, held to Bart's guest by the versioned gate vectors of the next template | +| Threat-model statement | Add that key loss or compromise means a new deployment until the forwarding contract | + +## Questions for Bart + +- Does libdex's drive hold only its flat buffer, or a padding tail too? Does + that buffer hold the effective count and block, so that a state projected + from a machine starts at the canonical count and a comparison pins it? +- Can your guest answer the `scheduler` query: its position, batch nonce and + clock? +- Which of a direct's index, timestamp and prevRandao does libdex read? +- Is your guest's compiled sequencer address the pinned submitter? Can its + harness or scheduler answer an input with `RX_REJECTED`? +- Does your scheduler reproduce the conformance vectors, and reject an + out-of-range `fee_price` at gate d, as ours does since PR #56? Will it adopt + strict frame clocks with the v2 vectors? +- Which directs are well-formed for libdex, and how large can one be? The + fridge's count bound bounds memory only if each entry's size is bounded, and + it must hold every direct one block can carry. +- Does libdex's validation refuse a user op for any reason besides its + signature, nonce and fee cap? Under the authorization rule, every other + reason becomes a business failure that still advances the nonce and charges + the fee. +- Can your client follow [the backup rules](#the-clients-base): the count in + every commit with its rows, and snapshots discarded past a cut? How large is + its database, which SQLite version does it link, and does applying a row + read anything besides the database and what a list entry carries, such as + the row's batch nonce or a direct's transaction hash? +- What are libdex's per-op cycles and state size? Who fixes libdex during an + incident? + +## Measure before choosing + +- The scan of blocks `(L_m, C]`, and signature recovery and execution over it, + at the production deployment's Dave lag, which sets how long the default + machine's list is. +- The guest's cost to drain a full fridge in one input, which the root frame + after a rebuild and the first frame after an outage both do, and one + overflow's drain of a full block. +- A client's replay, from its newest snapshot at or below its base to that + base. +- The time to recover, end to end, in a drill. + +## Notes on snapshotting a large SQLite database + +From the SQLite, Litestream, OpenZFS and AWS documentation as of 2026-10-29: + +- Volume or filesystem snapshots (ZFS, LVM thin, EBS) are the cheap choice at + terabyte scale: atomic and incremental. A ZFS snapshot holds every completed + write, so SQLite recovers it to its last committed transaction. An EBS + snapshot holds only what reached the volume, as a power cut would: with + `synchronous` NORMAL or FULL that only moves its count back, but with OFF it + can be corrupt. The database and its `-wal` file must be in the same + snapshot. Restore by cloning a snapshot, never by rolling back the live + volume, whose rows the replay reads. +- Litestream suits gigabytes rather than terabytes: by default it uploads a + full snapshot every 24 hours and keeps 24 hours. It restores only to the + end of one of its own files (`-txid`, `-timestamp`), which can batch many + transactions, so pick one at or below the base and let the rows in the live + database cover the rest. It deletes only by age, so it cannot discard the + restore points past a cut (rule 2): at a cut, replicate to a new path, and + restore from the old path only at or below the cut. +- The backup API and `VACUUM INTO` copy every page on every run. On a busy + database a copy either restarts after each write from another connection or + holds one read transaction, which stops WAL checkpoints until it ends. +- Every SQLite engine that opens a WAL database needs 3.51.3 or later (or the + 3.44.6 or 3.50.7 backport) once two connections, in threads or processes, + can write or checkpoint at once: earlier versions can corrupt it. Litestream + checkpoints with its own embedded engine, fixed from v0.5.17. WAL mode fixes + the page size, so choose it at creation. +- Retention is by position, not age: always keep the newest snapshot at or + below the client's base and every newer one on the current history. The + watchdog's head stays put while it is latched, idle or stopped, so an age + window, such as Litestream's default of 24 hours, can delete the only usable + base. A bug in the client, or a sequencer fault diagnosed before the head, + needs a snapshot older than it, which only a longer retention covers. + +## Remaining work + +- Settle the open decisions; the template rules wait on Bart's answers. +- Tool: the two-phase replay from the three values, from a `step` split out of + `Scheduler`, with the scan's completeness check; and the flush + subcommand. Today's scheduler recovers every signature, hands an op to the + application in two calls (`validate_user_op`, then `apply_valid_user_op`) + and keeps no input index in the fridge, so an application that records what + it is handed cannot produce the candidates. +- Guest, for the next template: the rules above, the `scheduler` query, and + v2 conformance vectors. +- Sequencer: project a machine's state, execute the candidates, write the + executed list, import `N'` and `L'`, and drain directs above `L'`. A database + that today's procedure already rebuilt has executed its queued directs in + the terminal drain, so its floor stays at its `C`. Draining above `L'` + replaces the terminal drain, so rewrite what pins that drain today: the + projection replay steps, the README's nonce note, the history guide's era + baseline, snapshot lifecycle, invariants I6, I8, I11 and I20, and the + rebuild drill's replica check. I1's documented omissions add the floor gate. +- Clients: the backup rules, and a driver that executes a part of the executed + list through the shared execution boundary and commits its entries as rows. +- Watchdog: parse the count, era and generation from the digest response, + which must add the latter two, and append each agreement to a log that + survives a re-`init`; and `export-state`, which writes a machine's + state-source bytes for the sequencer's projection, and the position, batch + nonce and clock its `scheduler` query reports for the tool, and nothing + application-specific. +- Dave's machine: read the last finalized root and the sealed epoch's + `inputIndexLowerBound` from the consensus contract, which nothing in this + repository reads yet, and check the index against the scheduler query's + position. +- Drills that rebuild from a machine and realign a client, in the + [incident runbook](../../watchdog/incident-runbook.md#drills); until the tool + lands, `setup --recovery` keeps its archive drill. +- Then the deletions the + [tracks board](../../plans/2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign) + lists for the tool's landing, and `setup --recovery`, which the tool replaces. diff --git a/docs/recovery/history/2026-11-recovery-from-a-machine.md b/docs/recovery/history/2026-11-recovery-from-a-machine.md new file mode 100644 index 00000000..9182ab2d --- /dev/null +++ b/docs/recovery/history/2026-11-recovery-from-a-machine.md @@ -0,0 +1,605 @@ +# Recovery from a canonical machine + +**Status:** archived on 2026-11-05, never implemented. A candidate of +2026-11-03 that simplified the [shared recovery tool](2026-10-shared-recovery-tool.md); +the owner's review of it led to +[sequencer recovery](../../plans/2026-11-sequencer-recovery/README.md), which +supersedes both. Its claims were checked by reading the code at `4e7b34b` +(main with PR #56), Dave at `dbcca7d4` and the emulator at v0.22.0-test3, and by +adversarial review; none was executed. + +Cockroach recovery rebuilds a sequencer after an instance loses its database +or its behavior cannot be trusted. The operator triggers it by hand and keeps +that instance down. The design takes the worst case: the submitter key is +ours, but the instance's database, snapshots and receipts prove nothing. The +canonical scheduler is the authority, bug for bug. A client, Bart's DEX +backend, keeps a richer state than the sequencer and must realign after a +rebuild, from its own backup and with its own code. Bart operates both; only +their development is split. Recovery runs rarely, so each step must be simple +enough to explain, practice and trust. + +## The idea + +Let `T` be the canonical sequence of executed L2 transactions, in execution +order. Each application state carries its **count** `X`: how many of them it +has executed. + +> A state on the canonical history with count `X` has executed exactly +> `T[0..X)`. Nothing but executed transactions changes an application, so the +> state is missing exactly `T[X..K)`, where `K` is the count at the stop block +> `C`. + +Recovery picks one **base** for every consumer: a canonical machine, at +position `n_m` and count `X_m`. The rebuilt sequencer projects its state from +that machine. Each client restores its own state to `X_m` from its backup and +checks it against the machine. Then every consumer needs the same thing, +`T[X_m..K)`. + +The rebuild starts from the machine, not from genesis. Under the next template +the machine's scheduler reports four numbers: + +- its **position** `n_m`: how many InputBox inputs it has consumed; +- its **batch nonce** `N_m`: the nonce the next batch must carry; +- its **fridge start** `F_m`: the index of the oldest direct still in its + fridge, or `n_m` when the fridge is empty; +- its **scheduler clock** `L_m`: the floor its next batch must not start + below. + +The rebuild reads its input table from index `F_m` and splits it at `n_m`: + +1. **Phase 1**, inputs from `F_m` below `n_m`: the machine already consumed + them. Keep the well-formed directs, which are exactly the machine's fridge, + and drop the batches without judging them. +2. **Phase 2**, inputs from `n_m` through `C`: run the scheduler and the + application from the machine's state, with batch nonce `N_m`, that fridge + and clock `L_m`. + +Phase 2 records every transaction that executes. That record, the **executed +list** `T[X_m..K)`, is all the sequencer hands the clients. What is shared is a +list of transactions, not a tool. + +## Terms + +- **Template**: the initial machine whose root hash the on-chain application + pins as `getTemplateHash()`. A deployed application keeps its template for + life ([Cartesi Machine facts](../../cartesi-machine.md#versions-are-all-or-nothing)), + so a rule that changes the guest ships with a new template, which is to say + a new deployment. A **machine** is any stored state of the canonical + machine. +- **Count `X`**: how many transactions a state has executed: + `executed_input_count` in code, though it counts transactions, not inputs. + A skipped op, a rejected batch, a dropped direct or a queued direct does not + move it. It numbers the feed's rows. +- **Position `n`**: how many InputBox inputs a state has consumed, which is + also the index of the next input. It differs from the count, because a batch + envelope or a queued direct is an input but not a transaction. +- **Scheduler state**: the batch nonce `N`; the fridge, the admitted directs + not yet executed, oldest first, each with its input index; and, under the + next template, the scheduler clock `L`: the last accepted frame's + `safe_block`, raised by an overflow to the youngest drained direct's block. + Under the next template the application keeps no clock of its own + ([progress is the count alone](#rules-for-the-next-template)). A + transaction's block reaches it as an argument: a direct's inclusion block, a + user op's frame `safe_block`. +- **Comparison digest**: the SHA-256 of an application's comparison bytes, the + state the watchdog already compares + ([application contract §6](../../protocol/application-contract.md#6-checkpoint-lifecycle)). +- **`C`** is the stop block, a safe head fixed after the flush. **`N'`**, + **`F'`** and **`L'`** are the batch nonce, fridge start and scheduler clock at + its end; `F'` is the first input after `C` when nothing is queued. +- **`P` and `B`**, as in the watchdog's latch record: `P` is its head, the last + block the sequencer agreed with, or the trusted bootstrap block before the + first agreement, and `B` the block of the comparison that failed. + +## The rebuild + +```ocaml +(* What the base machine reports through its scheduler query. *) +type base = { position : index; nonce : nonce; + fridge_start : index; clock : block } (* n, N, F, L *) + +(* One row of the executed list: a transaction and its count. *) +type row = { count : count; tx : tx } + +(* The states at the end of stop, and the executed list T[X_m..K). *) +val rebuild : machine -> stop:block -> state * sched * row list + +let rebuild m ~stop = + let b = scheduler_query m and app = project m in + (* Phase 1: the fridge. Batches the machine consumed are not judged again. *) + let fridge = input_table ~from:b.fridge_start ~below:b.position + |> List.filter (fun i -> is_direct i && App.well_formed app i) in + (* Phase 2: the scheduler and the application, from the machine's state. *) + let s = Scheduler.resume app ~nonce:b.nonce ~fridge ~clock:b.clock in + Scheduler.run_recording s (input_table ~from:b.position ~through:stop) +``` + +**Why phase 1 is exactly the fridge.** Only well-formed directs enter the +fridge, and every drain takes its front: a frame, the backstop and an overflow +all execute the oldest queued directs first. So the queued directs are always +every admitted direct from the oldest queued one on, and the machine's fridge +is exactly the well-formed directs with index from `F_m` below `n_m`. + +**Why batches below `n_m` are not judged again.** The machine already judged +them, and a second judgment can differ. A faulty submitter lands batch nonce 10 +at block 200; the canonical scheduler expects 9 and rejects it. Batch 9 lands +at block 201 and runs, so a machine at the end of block 201 has `N_m = 10`. A +rebuild that judged the block-200 batch again would find the expected nonce, +accept it, run ops the canonical machine never ran, and then reject the real +batch 10. + +**Why each value is needed.** + +- **The position.** The batch nonce cannot replace it, as the example shows, + and neither can a block. A machine can stop inside a block: Dave's last + finalized machine does when an input lands after the sealing transaction in + the same block. Even at a block's end, the machine cannot know whether its + last block held more inputs. +- **The batch nonce**, for the nonce gate. +- **The fridge start.** Among the directs below the position, some have + executed and some still wait, and nothing else tells them apart: an + overflow, for one, drains part of a block. +- **The clock**, for the floor gate. A batch whose frame at 100 runs ops and + whose frame at 140 runs nothing leaves `L = 140`. A later batch starting at + 120 is rejected canonically; a rebuild that floored at the block of the + latest executed transaction, 100, would accept it. + +**Where the inputs come from.** The rebuild runs in the fresh data directory. +Setup fills its input table from the deployment block to a safe head at or +past `C`, and the L1 reader proves the table complete +([I6](../../invariants.md#i6-the-frame-clock-accounts-for-a-complete-l1-interval)): +indices contiguous from zero, and the InputBox count checked at the safe head +it commits. So the rebuild reads nothing from L1 itself and needs no archive +node, and the sequencer's lane later drains the queued directs from the same +table. The rebuild refuses a base that does not fit the table: `F_m` must not +exceed `n_m`, `n_m` must not exceed the inputs through `C`, and input `F_m`, +when it is below `n_m`, must be a well-formed direct. + +**What a row holds.** A row is a feed row without `batch_nonce`: a direct the +backstop drained belongs to no batch, and every row is already settled. A +direct's row carries its input index, timestamp and transaction hash. Today a +direct enters the scheduler and its fridge with only its sender, block and +payload, so each direct carries its input index through the fridge, and the +rebuild reads the timestamp and hash from the input table by that index. + +**What it costs.** Phase 1 reads the inputs from `F_m` below `n_m`, batches +included; the backstop keeps them within `MAX_WAIT_BLOCKS` of the machine's +last input. It only checks well-formedness, so it recovers no signatures and +executes nothing. Phase 2 recovers signatures and executes from the position +to `C`, so the cost is the base's age at `C`. + +**How close today's code is.** [`fold_replay`](../../../sequencer-core/src/scheduler/fold.rs) +already seeds a fridge from a range with batches dropped unjudged, then +replays the rest from a batch nonce. The rebuild changes its seed range to +`F_m` up to `n_m`, adds the clock to `Scheduler::resume_at`, carries each +direct's index through the fridge, drops the terminal drain, and records each +executed transaction with its count, where the scheduler already sees an +included op and every drained direct. + +**Our scheduler and the guest's.** The canonical machine is the guest's +scheduler followed by its application. The rebuild runs our scheduler with the +same application. It reproduces the machine while our scheduler agrees with +the guest's on every gate and drain, which the template's conformance vectors +test, and while the guest never answers an input with `RX_REJECTED`, as the +[fatal conditions](../../protocol/scheduler-semantics.md#fatal-conditions) +require. A rebuild that reaches a halting transaction fails there, as the +canonical machine did. + +## One base for every consumer + +The base is one canonical machine. It must predate any fault the diagnosis +found. Its count `X_m` must be at or below every client's count, because a +client holds rows only up to its own count and the old instance's feed is gone. + +| Machine | Trust | +|---|---| +| The watchdog's head | the operator's watchdog replayed it from L1 and agreed with the sequencer there; before the first agreement, it is the bootstrap machine the operator trusted at `init` | +| Dave's last finalized machine | its root is on chain | +| The template | its hash is on chain; its scheduler state is zero, so the rebuild replays every input since the deployment block | + +For a base at any other block, the watchdog's `replay --out` derives a machine +at the end of that block, starting from the template or from an archived +checkpoint whose block is trusted. Every machine reports its position, batch +nonce, fridge start and clock through its `scheduler` query. Which machine is +the default is [open](#decisions-for-the-owner). + +A matching root authenticates a machine only from a source we trust, such as +our own Dave node: a stored machine's root comes from its stored hash tree, not +from its contents ([hashing](../../cartesi-machine.md#hashing)). An upcoming +emulator release brings a mechanism to verify an untrusted load. + +**The sequencer** projects its state from the machine. The projection must +carry the count; for libdex, whose state is a flat buffer in one of the +machine's drives, that is a question for Bart. + +**Each client** restores its state to `X_m`. It takes its newest snapshot at +or below `X_m`, replays its own rows up to `X_m` through its normal apply, and +checks the count. Then it computes its comparison digest and compares it with +the machine's. + +- **A match** shows that the client's application state is canonical at + `X_m`, whatever path brought it there. It needs no watchdog agreement. +- **A mismatch** has one of two causes. Either the client is wrong (a bug, a + wrong restore or engine skew), and it fixes that itself and checks again; an + older machine cannot fix it. Or its history left the canonical one before + `X_m`; then the operator takes an older machine, and every consumer starts + there. + +A match says nothing about the client's own records before `X_m`: which rows +it holds, their order, or what it derived from them. Different rows can reach +the same state +([reader state](../cockroach.md#application-specific-reader-state)). +Those records are canonical only if the feed the client followed was. That is +why the base must predate any fault the diagnosis found. After a +`state_mismatch` latch that means at or before `P`, unless diagnosis places the +fault later: a tick compares only the newest accepted checkpoint, so the +compared state can first differ anywhere in `(P, B]` +([incident runbook](../../watchdog/incident-runbook.md)). + +The check needs the client to produce the comparison bytes exactly as the +canonical machine holds them. For libdex that is its state `M` as the guest's +drive holds it, zero tail included: a question for Bart. Because the check +tests the client's own execution, it also catches a wrong restore, engine skew +and any client bug that changes the application state. It cannot see a bug +confined to the client's own records. + +## Clients apply the list + +A client applies the executed list as rows, through the feed's apply: + +```ocaml +(* Every row executed canonically, so a refusal means a wrong base, list or + engine: realignment stops at that row. *) +let client s rows = + List.fold_left (fun s r -> + assert (r.count = count s); + apply_or_stop s r.tx) (* a user op must carry its sender's next nonce *) + s rows +``` + +The rebuild writes the list as a file, durably, before it publishes the era. +The file opens with a header: the new era, `X_m`, the machine's comparison +digest, `K` and the baseline's comparison digest. The operator keeps it until +every client has realigned. After the list, the client's count must equal the +new era's `available_from`, `K`, and its digest the baseline's. It subscribes +there and receives the directs still queued at `C` as the era's first feed +rows, like any other row. A client never holds scheduler state. + +## Client backups + +The plan asks two rules of Bart's client, and leaves the rest of its storage to +it: + +1. **Every commit is a boundary and keeps its rows.** Apply each row, from the + feed or from the executed list, or each run of rows, in one SQLite + transaction that also writes the count, era and generation and stores the + rows themselves. Any consistent copy of the database, taken at any time, is + then at a transaction boundary, says which, and holds every row before it. +2. **A cut discards later snapshots.** A standard recovery cuts the history at + its preserved count + ([checkpoint compatibility](../../protocol/application-history.md#checkpoint-compatibility-after-standard-recovery)); + a rebuild cuts it at `X_m`. The new history replaces the rows past the cut + and reuses their counts. So once realigned, the client deletes every + snapshot past the cut and stops snapshotting its old live database. + +To reach `X_m`, restore the newest snapshot at or below it, replay the rows +after it from the live database or a newer snapshot, and check the count and +the digest. Never restore a snapshot past `X_m`: its state includes rows past +the base. Replay is exact only if applying a row reads nothing but the database +and the row. + +Retention sets recovery time, not correctness. A pruned snapshot costs a longer +replay from an older one, never a wrong base, because the digest check catches +a wrong state. Keep recent snapshots and a thinning tail that outlasts the +oldest base an operator would pick: Dave's settlement lag plus the time to +respond to an incident. Once an incident opens, stop pruning until every client +has realigned. + +## The procedure + +1. **Choose the base and check the clients.** Pick a canonical machine that + predates any fault the diagnosis found and whose count is at or below every + client's. Read its scheduler values, count and comparison digest. Each + client restores a copy of its state to `X_m` and checks the digest. A client + that mismatches first checks its restore and its code; pick an older + machine only if its history left the canonical one before `X_m`. Nothing is + committed yet. +2. **Rebuild** with one new command, which replaces `setup --recovery`. It + needs the submitter key. In order, it: + 1. flushes the old instance's transactions and fixes `C` as a safe head + observed after they settle. The flush is best effort: it replaces what + the provider's pool holds for the submitter account and waits until that + is safe ([flush boundary](../cockroach.md#flush-and-stopping-block)); + 2. syncs the input table to a safe head at or past `C`; + 3. applies genesis setup's detection gate at `C`. Before publishing + anything, it refuses if the submitter's wallet nonce is unsettled or a + submitter input lies above `C`. Either means a transaction escaped the + flush or the old instance still runs. Stop that instance and rerun; the + rerun flushes again and fixes a later `C`; + 4. checks that the base fits the table, replays, and writes the baseline + dump and the executed list durably; + 5. publishes the era in one transaction: count `K`, stop block `C`, batch + nonce `N'`, direct floor `F'`, the baseline and setup completion. It + prints the baseline's digest, `N'`, `F'` and `L'` for the check. +3. **Check**, always in drills, and in an incident unless the replay would + outlast the outage budget. Replay the newest trusted machine that ends a + block, normally the watchdog's head, to `C` with `replay`, then read it with + `export-state`. Its digest, batch nonce, fridge start and clock must equal + the values the rebuild printed. On a mismatch, fix the code and rebuild + before any client realigns. +4. **Run.** Start the sequencer with `run`. Its startup opens the root frame at + its own safe head, which drains every queued direct from `F'` before any + user op, as the first canonical frame drains them. Clear the watchdog's + latch; its next tick compares the baseline at `C`. +5. **Realign clients.** Each client applies the executed list from `X_m`, + committing its rows under the new era and generation. It checks that its + count equals `available_from` and its digest the baseline's, deletes every + snapshot past `X_m`, takes a snapshot, and subscribes. + +## Queued directs at the stop block + +At the end of `C` the canonical scheduler can still hold queued directs: the +well-formed directs from index `F'` through the last input at or below `C`. + +The rebuild publishes the era the way genesis setup does. It opens no root +frame; `run` opens one at its own safe head. + +The era gets a new field, the **direct floor**, set to `F'`. The lane drains +queued directs from the later of the floor and the first input after its +latest frame's `safe_block`. Today the lane floors at `base_safe_block`, a +block, and a rebuild sets it to `C`. The floor is optional, and a migration +writes none. An era without one, whether from genesis setup or from today's +procedure, drains from the first input after its `base_safe_block`, as today. + +`base_safe_block` stays `C`. The baseline has judged every batch through `C`, +so acceptance scans only blocks after it +([replay boundaries](../cockroach.md#replay-boundaries)), and no batch +at or below `C` is judged again. The directs from `F'` are not yet executed; +the floor covers them. + +A baseline this rebuild built is the canonical state at the end of `C`: the +machine's state plus `T[X_m..K)`, with the queued directs still queued. So the +sequencer serves it on `/finalized_state` as the era's first comparison +checkpoint, at `C`, as it serves genesis at block 0, and the watchdog compares +it on its next tick. A baseline from today's procedure holds the terminal +drain and stays a restore artifact, as today; an era with no floor recorded +marks it. + +Today's procedure instead executes the queued directs during recovery (the +terminal drain) and publishes the result, and each client repeats that drain +(projection replay, a contract since deleted). That state is a +prediction: the canonical machine reaches it only inside the post-`C` input +that drains the last of the queued directs, so no comparison can check it, and +a rebuild that started from it would execute those directs twice. + +## Rules for the next template + +Each changes the guest, so each needs Bart's agreement and ships with the next +template, ours and his port's +([register](../../review/register.md#canonical-rules-open-for-the-c-port)). The +rebuild relies on the `scheduler` query and on a fridge that only pops its +front, as today's does. The floor gate serves applications, not recovery. + +- **Fridge bound.** Only well-formed directs enter the fridge. The rest are + dropped: they never execute and never count, where today every direct + counts. The application decides well-formedness with a function on its + trait that reads only the direct and the application's fixed configuration, + such as the wallet's portal and token addresses, never state that + transactions change; Bart's guest calls libdex's own. The sequencer's lane + drops the same directs with the same function, or it diverges. A count + bounds the fridge, and an overflow drains just enough of its front to make + room, raising `L` to the youngest drained direct's block. The sequencer + cannot mirror an overflow, so the batches it already sealed diverge, and a + manual cockroach recovery answers that. +- **The floor gate.** The scheduler keeps `L`. A new gate, after gate e, + rejects a batch whose first frame lies below `L`; empty batches stop at gate + c and never reach it. So a transaction's block never goes backwards along + `T`, and applications may read blocks as time. The gate makes + [I3](../../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) + consensus-critical: every transition that opens a Tip, the sequencer's open + batch, must assert that it starts at or above the latest valid frame. + `scheduler_accepts` leaves the floor out, as it leaves out gate d + ([I1](../../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)). +- **A `scheduler` inspect query** returning the machine's position, batch + nonce, fridge start and clock. The position is the last input's index plus + one, batches included, or zero for the template. The fridge start is the + oldest queued direct's index, or the position when the fridge is empty, so + the guest's fridge keeps each direct's index. +- **Progress is the count alone.** `last_executed_safe_block` leaves the + shared progress contract, the C engine ABI's progress and the wallet's + comparison bytes. Nothing reads it in this design: the count positions + states, and `L` is the scheduler's clock. Today two paths read it, as a seed + bound: `setup --recovery` and a client's projection replay. Both leave with + the rebuild. +- **Direct inputs carry what libdex reads.** A user op never sees its batch's + context. Whether a direct's input index, timestamp and prevRandao reach the + application waits on Bart's answer; the template passes exactly the fields + libdex reads. The index and timestamp cost nothing, since the sequencer's + input table and the feed's direct rows carry them. prevRandao needs a column + and a feed field, and the block proposer can bias it, so it comes only with a + stated need. + +Inclusion stays as today: the application's validation decides it, fee +balance included, through the shared boundary that the canonical scheduler and +live inclusion both use. The rebuild executes with the machine's full state, +so it includes exactly what the canonical machine included. + +## Preconditions + +- **The old instance stays down.** Recovery deliberately creates a second data + directory pinned to the same submitter address, the old one can still run + with the same key, and + [the flush fences transactions, not processes](../cockroach.md#incident-playbook). + No command that signs with the key may run against the old directory again + ([restart policy](../../watchdog/operator-deployment.md#sequencer-restart-policy)). +- **The submitter key survives.** The rebuild signs the flush and `run` submits, + so a lost or leaked key means a new deployment while the submitter is an + account compiled into the template. +- **Recovery repairs only the sequencer's view.** Whatever the canonical + machine executed stays, a faulty sequencer's accepted batches included. A dead + canonical machine is unrecoverable; PR #56 makes gate d reject an + out-of-range `fee_price`, so on templates built from it that sequencer bug + becomes a divergence recovery can repair. +- **Consumers execute bit for bit.** Our scheduler reproduces the guest's, + which the conformance vectors test. The sequencer and every client run the + canonical validation and execution; Bart's client embeds libdex exactly as + the guest runs it and can produce its comparison bytes. +- **The template tie.** The sequencer runs the same application code as the + template, and the template never changes, so a sequencer release that fixes + that code would disagree with the canonical machine. Setup would pin the + application's `getTemplateHash()`, and `run` would refuse a template other + than the one its release declares. +- **Detection.** The watchdog's divergence latch tells the operator that the + sequencer disagrees with the canonical machine, after a bug or a fridge + overflow alike. + +## What changes from the current plan + +Rows marked *settled* reopen a decision the owner settled on 2026-10-29 or +2026-11-02. + +| Current plan | This candidate | Why | +|---|---|---| +| An application-free tool emits candidates; the sequencer executes them (*settled*) | The rebuild runs the scheduler with the application and records what executes | Only the sequencer executes the tool's output, and it holds the application anyway | +| The tool reads L1 by block range through its own gateway, with a completeness check that needs an archive node for an old base | The rebuild reads the fresh data directory's input table, which the L1 reader proves complete | Setup already syncs that table; the second reader duplicated it | +| The fridge's start found by block: strict frame clocks, whole-block overflow, the clock raised by every drain, the bound holding a block (*settled*) | The machine's `scheduler` query also reports its fridge start, an index, so the guest's fridge keeps each direct's index; `L` is raised only by an overflow | Reading by index needs none of those rules: the fridge only pops its front | +| An overflow drains whole blocks (*settled*) | It drains just enough, as settled on 2026-10-29 | Whole blocks served only the block-based fridge start | +| The direct floor at the block `L'` (*settled*) | The index `F'` | The same reason; it is the floor chosen on 2026-10-29 | +| Separate steps joined by files, with a commit point | One command, one transaction, as `setup --recovery` is today | Removes a file format, a crash window and a step to practice | +| A base per consumer (*settled*); clients vouched by a log of watchdog agreements with era and generation, a rule for which agreement is on a client's history, a crossed-base record and count zero before the first agreement | One base for every consumer, at or below every client's count; each client checks its digest against the machine | Each client is checked directly against one machine, trusted by its source, so no agreement log is needed; the check also tests the client's own execution | +| Dave's last finalized machine is the default base (*settled*) | Open | With one base, Dave's lag becomes a stretch every client rewrites from the list | +| `incident/canonical` as a base for a deployment without clients | Dropped | One rule for the base: it predates the fault | +| A list entry carries its count and what the canonical application reads, but no batch nonce and no transaction hash | A row is a feed row without `batch_nonce`: a direct's row carries its input index, timestamp and transaction hash. The file opens with a header: era, `X_m`, the machine's digest, `K` and the baseline's digest | A client applies and stores list rows exactly as it does feed rows, and checks its count and digests against the header | +| A client executes its part, skipping an op its state refuses | A client applies rows and stops at the first refused row | Every row executed canonically, so a refusal means a wrong base, list or engine | +| Replaying the watchdog's head to `C` is optional; the first comparison checks the rebuild a batch later | A check step before `run`, always in drills | Catches a wrong list or scheduler state before any client realigns | +| Authorization decides inclusion (*settled*) | Dropped; inclusion stays as today | Neither design needs it, since the application decides inclusion in both. It adds a halt path (the wallet's apply errors on an unpayable fee), and because it caps the fee at the balance, a faulty sequencer could include ops from unfunded senders and make libdex store their nonces at no cost | +| The effective block kept apart from `L` in the progress contract (*settled*) | Dropped from the contract | Nothing reads it; the count positions states and `L` is the scheduler's clock | +| The well-formedness rule ported into our scheduler library | A function on the application's trait, reading only the direct and fixed configuration | The rebuild has the application at hand; no port to keep in sync | +| Dropped directs: what the lane does is not stated | The lane drops them with the same function | Otherwise every dropped direct diverges the count | +| Direct context with prevRandao (*settled*) | The fields libdex reads; prevRandao only on need | A template's libdex is fixed, so a field it does not read is pure cost; prevRandao needs a column and a feed field, and the proposer can bias it | +| The import opens a root at `C`; a late start needs Tip repair | Publishing the era opens no root; `run` opens it at its own safe head | Publishing becomes genesis setup with other numbers | +| The rebuilt baseline is only a restore artifact | The era's first comparison checkpoint, at `C`, when this rebuild built it | Without the terminal drain it is the canonical state there | +| "Refuse if a submitter input above `C` is already ingested", in the flush step | Genesis setup's detection gate, at `C` | One existing gate instead of a new rule | +| The flush as a standalone subcommand | A step of the rebuild command | The key is needed by `run` on the same directory anyway | +| Retention by position, anchored on the client's base | Retention sets recovery time; the digest check guards correctness | A pruned snapshot costs time, never a wrong base | +| `/history`'s baseline and accepted checkpoint reshaped or retired with the new sync | Deleted when the rebuild lands | Nothing reads them in this design | + +Kept as they are: the split at the machine's position, the floor gate, +fencing, the best-effort flush, backup rules 1 and 2, and the executed list +delivered as a file. + +## Decisions for the owner + +| Decision | Recommendation | +|---|---| +| Default machine | Open; this reopens the settled choice of Dave's machine. The watchdog's head is fresh and ends a block, so the list is short; it is trusted through the operator's watchdog. Dave's last finalized machine is anchored on chain but lags, so every client rewrites that lag from the list, losing the feed's batch nonces there | +| One base for every consumer | Adopt; this reopens the per-consumer bases settled on 2026-10-29, and settles the open choice of a client's base | +| Fridge start | The index the machine reports, in place of the block rules settled on 2026-11-02 | +| Authorization decides inclusion | Drop it, unless its semantics are wanted for their own sake: an included op always advances the nonce and pays what it can. Keeping it needs Bart to list every reason libdex refuses an op besides its nonce or an account binding, since each would become a charged business failure | +| Progress is the count alone | Adopt with the next template | +| Direct context | Wait for Bart's answer; prevRandao only on a stated need | +| Template tie | One template per release, enforced at setup and `run` | +| Shared gate function, `trait Scheduler` | Defer, as before | + +## Questions for Bart + +- Does libdex's drive hold only its flat buffer, or a padding tail too? Does + that buffer hold the count? +- Can your client produce its comparison bytes: libdex's state exactly as the + guest's drive holds it, zero tail included? +- Can your guest answer the `scheduler` query: its position, batch nonce, + fridge start and clock? +- Does libdex read a transaction's block as time, for example to expire + orders? That is the floor gate's reason. +- Which of a direct's index, timestamp and prevRandao does libdex read? +- Is your guest's compiled sequencer address the pinned submitter? Can its + harness or scheduler answer an input with `RX_REJECTED`? +- Does your scheduler reproduce the conformance vectors, and reject an + out-of-range `fee_price` at gate d, as ours does since PR #56? +- Which directs are well-formed for libdex, and how large can one be? +- Can your client follow the two backup rules? How large is its database, + which SQLite version does it link, and does applying a row read anything + besides the database and the row? +- What are libdex's per-op cycles and state size? Who fixes libdex during an + incident? + +## Measure before choosing + +- The rebuild's replay from the base's position to `C`, at the watchdog head's + age and at the production Dave lag. +- The guest's cost to drain a full fridge in one input, which the first frame + after an outage and the root frame after a rebuild both do. +- A client's restore to `X_m` and its digest over a large state. +- The time to recover, end to end, in a drill. + +## Notes on snapshotting a large SQLite database + +From the SQLite, Litestream, OpenZFS and AWS documentation as of 2026-10-29: + +- Volume or filesystem snapshots (ZFS, LVM thin, EBS) are the cheap choice at + terabyte scale: atomic and incremental. A ZFS snapshot holds every completed + write, so SQLite recovers it to its last committed transaction. An EBS + snapshot holds only what reached the volume, as a power cut would: with + `synchronous` NORMAL or FULL that only moves its count back, but with OFF it + can be corrupt. The database and its `-wal` file must be in the same + snapshot. Restore by cloning a snapshot, never by rolling back the live + volume, whose rows the replay reads. +- Litestream suits gigabytes rather than terabytes: by default it uploads a + full snapshot every 24 hours and keeps 24 hours. It restores only to the + end of one of its own files (`-txid`, `-timestamp`), which can batch many + transactions, so pick one at or below the base and let the rows in the live + database cover the rest. It deletes only by age, so it cannot discard the + restore points past a cut (rule 2): at a cut, replicate to a new path, and + restore from the old path only at or below the cut. +- The backup API and `VACUUM INTO` copy every page on every run. On a busy + database a copy either restarts after each write from another connection or + holds one read transaction, which stops WAL checkpoints until it ends. +- Every SQLite engine that opens a WAL database needs 3.51.3 or later (or the + 3.44.6 or 3.50.7 backport) once two connections, in threads or processes, + can write or checkpoint at once: earlier versions can corrupt it. Litestream + checkpoints with its own embedded engine, fixed from v0.5.17. WAL mode fixes + the page size, so choose it at creation. + +## Remaining work + +- The owner's decisions above; the template rules wait on Bart's answers. +- Rebuild: generalize `fold_replay` to the two phases, add the clock to + `Scheduler::resume_at`, carry each direct's index through the fridge, drop + the terminal drain, record each executed transaction with its count, refuse + a base that does not fit the table, and write the list. Fold the flush, the + sync, the detection gate and the publication into one command that prints + the baseline's digest and scheduler values. +- Sequencer: the lane drops directs the well-formedness function refuses, + before storage gives them a count; the era's direct floor at the index `F'`, + read where the lane finds the next undrained input; publishing the era + without opening a root; the rebuilt baseline served as the era's first + comparison checkpoint, only in an era that records a floor; and the floor + gate in I1's documented omissions, with I3's assert at every transition that + opens a Tip, which I3 lists under what enforces it. Draining from `F'` + replaces the terminal drain, so rewrite what pins that drain today: the + projection replay steps, the README's nonce note, the history guide's era + baseline, snapshot lifecycle, invariants I6, I8, I11 and I20, and the + rebuild drill's replica check. +- Guest, for the next template: the rules above, the `scheduler` query, the + well-formedness function on the application's trait and the C binding, + progress as the count alone (with I19, the application contract's §3, the + history guide's progress facts and the snapshot format), and new conformance + vectors. +- Clients: the two backup rules, the digest checks at `X_m` and at `K`, and a + list driver that stops at the first refused row. +- Watchdog: `export-state`, which writes a machine's state-source bytes, its + comparison digest and the values its `scheduler` query reports, and nothing + application-specific. +- Dave's machine, if it is chosen as the base: read its last finalized root + from the consensus contract, which nothing in this repository reads yet. +- Threat model: what losing or leaking the submitter key means, and that a + revoked user op is not cancelled: anyone holding it can submit it again while + its sender's nonce is unused, and an application that wants expiry checks a + deadline in the payload against the op's `safe_block`. +- Drills that rebuild from a machine with directs queued at `C` and realign a + toy-wallet client, in the [incident runbook](../../watchdog/incident-runbook.md#drills), + on the devnet in CI. A path run once in a deployment's life rots. +- Then the deletions the + [tracks board](../../plans/2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign) + lists for the tool's landing, plus `/history`'s baseline and accepted + checkpoint, which nothing reads in this design, and `setup --recovery`. diff --git a/docs/recovery/history/README.md b/docs/recovery/history/README.md index c4ea14ed..fe485b64 100644 --- a/docs/recovery/history/README.md +++ b/docs/recovery/history/README.md @@ -1,9 +1,22 @@ # Recovery Design History -This directory preserves the **historical optimistic recovery design** and its -counterexample. It is not the production recovery procedure. The -[current recovery guide](../README.md) owns automatic recovery; the -[cockroach guide](../cockroach.md) owns manual rebuilding. +This directory preserves recovery designs the sequencer does not implement: +the **optimistic recovery design** and its counterexample, the **preemptive +recovery** with its two models, and two superseded cockroach-recovery plans. +The sequencer's one recovery is [cockroach recovery](../cockroach.md); +[batch landing and outages](../README.md) owns what happens when batches stop +landing and [why there is no automatic repair](../README.md#why-there-is-no-automatic-repair). + +## Superseded cockroach-recovery plans + +Archived on 2026-11-05. Each opens with the commit it was checked against, +and its links point at the code and docs of that commit: some name headings +and files that later changes removed, so read them at that commit. + +| Plan | What it proposed | Why it was superseded | +|---|---|---| +| [Shared recovery tool](2026-10-shared-recovery-tool.md) | A tool that rebuilds the scheduler's history from L1, starting from three values a machine reports, shared by the sequencer and its clients | Reading the whole scheduler state from the machine needs no replay from L1 and no shared tool | +| [Recovery from a canonical machine](2026-11-recovery-from-a-machine.md) | The rebuild folded into setup, from four values a machine reports | The owner's review led to exporting the whole scheduler state, fridge included | ## The optimistic alternative @@ -16,7 +29,7 @@ from the invalidated suffix may remain in the network as zombies competing with new recovery batches. The recorded bounded check reported 194M states with no invariant violations -after the Silver-only fix. This is model evidence, not a proof of production +after the Silver-only fix. This is model evidence, not a proof of that recovery or its completion time. Bounds are in [`optimistic.cfg`](optimistic.cfg). ## The counterexample: invalidating before slot resolution @@ -52,26 +65,40 @@ argument depended on. The retained optimistic model's `Resolve` therefore requires a Silver frontier that is stale by inclusion: that original batch is already on safe L1 and cannot be displaced by a replacement. -## Why production uses preemptive recovery - -Production closes intake and performs recovery offline. For closed-batch -recovery, the flush consumes every covered wallet-nonce slot at safe depth, -**whether the original transaction or a no-op wins**, then re-syncs the accepted -prefix before cascading. It does not require the original frontier batch to -become Silver. An unsubmitted open Tip has no wallet slot to settle. - -This gives recovery a sequential procedure and stops new soft confirmations -while submission uncertainty is being resolved. The optimistic alternative -keeps serving through that interval and may add confirmations that a later -cascade revokes. - -The tradeoff is downtime. Flush completion requires L1 progress; fee headroom -does not guarantee replacement or establish a deadline. The -[current recovery guide](../README.md) owns the safety conditions, and the -[L1 fee policy](../../l1-fee-policy.md) owns the accepted liveness limits. - -## Running the historical model +## The preemptive recovery and its models + +The preemptive design repaired the history in place. A detector stopped the +process at a threshold before staleness; the next start flushed every wallet +slot the closed unaccepted batches had used, so that the original or a no-op +landed at safe depth in each, then re-synced the accepted prefix, invalidated +the suffix after it, and opened a fresh batch at the first unaccepted nonce. +It did not require the original frontier batch to be on safe L1, and it closed +intake while it ran. The sequencer does not implement it. + +Two bounded TLA+ models checked it. They were not a refinement proof of the +Rust code, a proof of their composition, or a liveness guarantee. + +- [`preemptive.tla`](preemptive.tla) ([bounds](preemptive.cfg)) models + safe-block advancement, wallet-slot competition between batches and no-ops, + scheduler acceptance and branch invalidation. Its `Inv` checks + `ZombieSafety` (`schedulerExpected = CountGold(spine)`) with batch-nonce + contiguity, invalid-branch ancestry, wallet-slot uniqueness and cursor + bounds. +- [`admission.tla`](admission.tla) ([bounds](admission.cfg)) models the + startup controller: the local terminal gate, the initial-sync fallback, at + most one repair, the flush observation and mandatory sync, the guarded + cascade, and admission before any worker exists. Its invariants cover + terminal dominance, repair preconditions, a caught-up post-flush view, and + admission soundness. + +Both models exist because replacing a batch reuses its batch nonce, so a late +transaction of the replaced batch can race the replacement. Inside an era no +batch nonce is reused: a late transaction of the sequencer's carries the bytes +of a batch it still holds, and it is accepted as that batch or skipped. The +race these models guard against cannot occur there. + +## Running the models ```bash -tlc -workers auto -deadlock docs/recovery/history/optimistic.tla +just -f docs/recovery/history/justfile check-all ``` diff --git a/docs/recovery/admission.cfg b/docs/recovery/history/admission.cfg similarity index 100% rename from docs/recovery/admission.cfg rename to docs/recovery/history/admission.cfg diff --git a/docs/recovery/admission.tla b/docs/recovery/history/admission.tla similarity index 100% rename from docs/recovery/admission.tla rename to docs/recovery/history/admission.tla diff --git a/docs/recovery/history/justfile b/docs/recovery/history/justfile new file mode 100644 index 00000000..8a334a14 --- /dev/null +++ b/docs/recovery/history/justfile @@ -0,0 +1,19 @@ +tlc := env("TLC", "tlc") + +# Archived models of recovery designs the sequencer does not implement. They +# are dev analysis tools for revisiting those designs, not checks of the code. + +# Check the startup repair and runtime admission model +check-admission: + {{tlc}} -workers auto -deadlock admission.tla + +# Check the preemptive recovery model (~90s) +check-preemptive: + {{tlc}} -workers auto -deadlock preemptive.tla + +# Check the optimistic recovery model (~3min) +check-optimistic: + {{tlc}} -workers auto -deadlock optimistic.tla + +# Check every archived model +check-all: check-admission check-preemptive check-optimistic diff --git a/docs/recovery/preemptive.cfg b/docs/recovery/history/preemptive.cfg similarity index 100% rename from docs/recovery/preemptive.cfg rename to docs/recovery/history/preemptive.cfg diff --git a/docs/recovery/preemptive.tla b/docs/recovery/history/preemptive.tla similarity index 100% rename from docs/recovery/preemptive.tla rename to docs/recovery/history/preemptive.tla diff --git a/docs/recovery/justfile b/docs/recovery/justfile deleted file mode 100644 index e3d078c6..00000000 --- a/docs/recovery/justfile +++ /dev/null @@ -1,17 +0,0 @@ -tlc := env("TLC", "tlc") - -# Check startup recovery procedure and runtime admission safety -check-admission: - {{tlc}} -workers auto -deadlock admission.tla - -# Check the preemptive recovery spec (~90s) -check-preemptive: - {{tlc}} -workers auto -deadlock preemptive.tla - -# Check the historical optimistic design (~3min). A dev analysis tool for -# revisiting that design, not a check of the current procedure. -check-optimistic: - {{tlc}} -workers auto -deadlock history/optimistic.tla - -# Check the specs of the current recovery procedure -check-all: check-admission check-preemptive diff --git a/docs/review/2026-09-16-track3-validation.md b/docs/review/2026-09-16-track3-validation.md index b1498882..ab9a8ca6 100644 --- a/docs/review/2026-09-16-track3-validation.md +++ b/docs/review/2026-09-16-track3-validation.md @@ -12,7 +12,7 @@ measurements must not be attributed to its rebased trees. The latency baseline already contains the initial versioned-history foundation; it isolates the subsequent application-history refactor, not the cost of the complete stack. -Retained for the [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md) +Retained for the [Track 3 integration gates](register.md#integration-work-owned-elsewhere) and the ADR's [performance posture](../plans/2026-08-authority-boundary-adr.md#performance-posture), which cites its ACK latency as regression evidence: this is the wallet baseline against which native-engine and deployment results can be assessed. Replace or diff --git a/docs/review/2026-10-13-whole-repo-review.md b/docs/review/2026-10-13-whole-repo-review.md index 31dd5795..0ffd565b 100644 --- a/docs/review/2026-10-13-whole-repo-review.md +++ b/docs/review/2026-10-13-whole-repo-review.md @@ -84,7 +84,7 @@ vectors pass unchanged (W3b). ### Step 0: decisions (settled 2026-10-13) 1. **Bart: done.** His client treats cockroach recoveries as absent, and it can - keep doing so once the tool exists. The [Track 3 plan](../plans/2026-07-track3-feed-replay-design.md#follow-up-sequence) + keep doing so once the tool exists. The Track 3 plan (`docs/plans/2026-07-track3-feed-replay-design.md`, since deleted) and the [tracks board](../plans/2026-07-coordination-tracks.md) still describe him integrating scheduler replay; correct that text in W1. 2. **Fees stay in log space.** The batch-level fee is deferred together with @@ -96,12 +96,9 @@ vectors pass unchanged (W3b). - The NUM note's batch-fee half stays deferred, recorded in [batch-scoped replay protection](../plans/2026-10-batch-scoped-replay-protection.md). 3. **N8 typed signing: deferred.** Out of the current scope. -4. **Recovery-tool shape: a later dedicated session.** The direction mapper's - sketch is under [Consumer surfaces](#consumer-surfaces). - - Bart's client ignores cockroach recoveries, so a v1 can be the extraction - of `setup --recovery` into a keyless tool that emits only the sequencer - bundle; the client output can wait. - - Whether checkpoints carry scheduler state stays coupled to the fridge bound. +4. **Recovery-tool shape: a later dedicated session**, held on 2026-10-28. The + [shared recovery tool plan](../recovery/history/2026-10-shared-recovery-tool.md) owns + the proposed design and its open decisions. ### W1: correctness and safety (about 18 small patches, no decisions) @@ -370,17 +367,6 @@ What survives: the ingress routes, `/finalized_state`, `/digest` and the canonical page reader with `HistoryClaim` (native restart uses it), the listener split, and in-era generation signalling. -The recovery-tool sketch (from the direction mapper): - -- **Inputs:** deployment identity, a checkpoint (genesis or scheduler state at B), - the stop block C from `flush`, and an input source over (B,C]. -- **Outputs:** written file-first, with `receipt.toml` last as the commit point - (tool-minted era, B, C, N', K, digest). One output is a sequencer bundle for - `setup --import`. The other is a versioned client trace of (B,C] with outcomes - and app outputs. -- **Client rule:** a client imports at an era boundary and never subscribes - across one. - ## Correctness findings | Id | Sev / status | Where | Scenario | Fix | diff --git a/docs/review/2026-10-14-guest-image-refresh.md b/docs/review/2026-10-14-guest-image-refresh.md index bfae761f..46616850 100644 --- a/docs/review/2026-10-14-guest-image-refresh.md +++ b/docs/review/2026-10-14-guest-image-refresh.md @@ -23,7 +23,7 @@ hash, and how a behavior-preserving move is checked, is in with `rust-src` in the cross toolchain (see `docs/cartesi-machine.md`). Two cold builds of the W3b final commit (`756a8ac`) agreed, and so did builds from three checkout paths. -- Behavior: [`tests/fixtures/consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json), +- Behavior: [`tests/fixtures/consensus_vectors_v1.json`](https://github.com/cartesi/sequencer/blob/3e92563a3947673b3931a6b43eecc16ec4e0ba48/tests/fixtures/consensus_vectors_v1.json), pinned at `6000e3a` before anything moved and byte-identical at the final commit. The workspace tests check it natively at every commit, and `just canonical-test-guest` (15 cases, 5 golden) checks it in the built image diff --git a/docs/review/2026-11-10-recovery-measurements.md b/docs/review/2026-11-10-recovery-measurements.md new file mode 100644 index 00000000..ea20e3a9 --- /dev/null +++ b/docs/review/2026-11-10-recovery-measurements.md @@ -0,0 +1,277 @@ +# Recovery measurements — 2026-11-10 + +Scope: what `setup` and the canonical guest cost on the recovery path, for the +[recovery plan's measure items](../plans/2026-11-sequencer-recovery/todo.md#measure). +Four costs: setup's rebuild and execution of the history, reading a base +machine's export, the guest draining a full fridge in one input, and the devnet +recovery drills end to end. + +Retained for the register's +[sequencer recovery follow-ups](register.md#sequencer-recovery-follow-ups) and +its [fridge bound](register.md#bound-the-canonical-direct-input-fridge) entry. +Measured at `36337b9`, with the devnet image +`examples/canonical-app/out/canonical-machine-image` built from it. Replace +the guest numbers when libdex's guest exists, since every one of them is the +reference wallet's; re-measure the rebuild if its signature recovery changes; +delete this record when no register entry cites it. + +## Environment + +- Apple M5 Max (18 cores in two performance levels, 6 and 12), 36 GiB, macOS + 26.6.2, APFS on the internal SSD. +- Rust 1.99.0 from the dev shell, release profile (the workspace sets no + `[profile.release]`, so opt-level 3, no LTO). SQLite bundled with + `libsqlite3-sys` 0.38.2, with the pragmas `Storage` sets in production. +- Signature recovery is alloy's `Signature::recover_address_from_prehash` on + `k256` 0.13.4, the pure-Rust backend, as the scheduler runs it. +- Cartesi Machine emulator 0.22.0 (libcartesi from the dev shell), through the + `cartesi-machine` Rust bindings the sequencer links. + +The harnesses were throwaway code, not committed; the methods below say enough +to rebuild them. + +## Summary + +| What | Result | For an operator | +|---|---|---| +| Rebuild and execute a history | 12,000 to 12,300 rows/s and 718 to 733 inputs/s end to end, from 10^4 to 10^6 inputs, from genesis or a base | Recovery time is the number of user ops past the base times about 85 µs | +| What dominates the rebuild | Signature recovery: 78 µs of each op's 83 µs of rebuild (95%) | Only parallel recovery would shorten setup much | +| Read a base machine's export | 32 ms at the largest state that fits | Never the slow step | +| Largest `state` report that fits | 40,328 accounts that hold only a balance; 27,593 when every account also has a nonce | This is the limit that binds first | +| Cycle headroom of the 2^36 inspect cap | at least 3,269 times the cycles of the largest `state` report that fits | Comfortable | +| Drain 16,384 queued directs in one input | 1.94 G cycles, 2.1 s in the emulator, 145,000 times under the per-input budget | Comfortable | +| Devnet recovery drills | 36 to 41 s per round trip | Mostly the scenario's own waits, not setup | + +## a. Setup's rebuild and execution + +### Method + +A temporary `#[ignore]` test in `sequencer::commands::setup`, run with +`cargo test -p sequencer --release --lib`, with the rebuild's test-only page +size raised to the production 1,024 rows for the run. + +- **The input table.** A fresh setup database with the test deployment + identity pinned. The inputs come in cycles of four: three ERC-20 deposits + from the portal at block `b`, then a batch from the submitter at block + `b + 1` with two frames at safe block `b`, each of 32 signed ops. The + deposits fund 1,024 senders in turn. Each op is a wallet transfer of 1 to + one of 100,000 recipients, at fee price 10 with `max_fee` 10. One op in 20 + repeats the op before it, so the executor skips it for its nonce. A cycle + makes 67 rows, 3 deposits and 64 ops, and 3.2 ops per cycle are skipped. +- **The sync.** Excluded, because its time is the RPC provider's. Its database + part was timed: the inputs were appended through + `append_ingested_safe_inputs_with_timestamp`, the input reader's write, in + chunks of 1,024 inputs. +- **Genesis.** The base is the genesis export and the genesis wallet. +- **A base in the middle.** The native `CanonicalRunner` runs the first + `4 × (cycles / 2) + 3` inputs and its export and wallet become the base, so + three deposits are queued in the base's fridge. Building that base is not + timed. +- **The timed steps** are those of `rebuild_and_publish`, each timed alone: + `check_export`, `count_submitter_inputs`, `rebuild_history`, + `execute_history` with the wallet, and `fill::publish`. "Total" is their + sum. The stop block is the last block. +- **The decomposition.** After setup, the same pages fed to a resumed + `Scheduler` with nothing written ("scheduler only"); then 25,600 distinct ops + hashed (`eip712_signing_hash`) and recovered one by one. + +### Results + +Each row is one run. A second 10^6 run, made to sample memory, gave +1,336.3 s of rebuild and 1,392.7 s in all (12,027 rows/s). Durations come from +Rust's `Instant`, which on macOS does not count system sleep; the host slept +during the long runs. + +| Inputs in the table | Base | Inputs replayed | Rows (skipped) | Rebuild | Execute | Publish | Total | Inputs/s | Rows/s | +|---|---|---|---|---|---|---|---|---|---| +| 1,000 | genesis | 1,000 | 16,750 (800) | 1.36 s | 0.04 s | 0.02 s | 1.42 s | 706 | 11,828 | +| 10,000 | genesis | 10,000 | 167,500 (8,000) | 13.2 s | 0.42 s | 0.02 s | 13.7 s | 731 | 12,242 | +| 10,000 | input 5,003 | 4,997 | 83,750 (4,000) | 6.6 s | 0.21 s | 0.02 s | 6.8 s | 733 | 12,284 | +| 100,000 | genesis | 100,000 | 1,675,000 (80,000) | 132.1 s | 4.3 s | 0.08 s | 136.6 s | 732 | 12,264 | +| 100,000 | input 50,003 | 49,997 | 837,500 (40,000) | 65.9 s | 2.3 s | 0.05 s | 68.3 s | 732 | 12,263 | +| 250,000 | genesis | 250,000 | 4,187,500 (200,000) | 332.5 s | 11.0 s | 0.16 s | 343.8 s | 727 | 12,180 | +| 1,000,000 | genesis | 1,000,000 | 16,750,000 (800,000) | 1,329.4 s | 52.8 s | 1.74 s | 1,384.6 s | 722 | 12,097 | + +`check_export` and `count_submitter_inputs` took at most 0.7 s together. The +10^6 runs and the 250,000 run, which sampled memory, skipped the +decomposition. + +What else the runs showed: + +- **Signature recovery dominates.** One recovery costs 78 µs (78.2 to + 80.7 µs across runs) and the EIP-712 hash 1.2 µs. At 10^5 inputs the 1.6 + million recoveries come to about 125 s of the 132 s rebuild. The scheduler + alone over the same pages took 129.3 s; writing the rows took the other + 2.9 s (2%), and reading the input pages 0.07 s. +- **Execution is the application's cost.** The wallet ran 320,000 to 450,000 + rows/s, 3 to 4% of the total. Another application's execution must be + measured with it. +- **A base in the middle halves the time** because it halves the inputs + replayed: the rate is the same, so setup's time follows the work past the + base, not the history's length. +- **Memory stays bounded.** Sampled every 2 to 5 s with `ps`, the test + process stayed between 26 and 55 MB resident at 250,000 and at 10^6 inputs; + it grew only during execution and publication, with the wallet. One reading + is unexplained: `/usr/bin/time -l` around the first 10^6 run's whole + `cargo test` command reported a 2.9 GB peak, against 141 MB at 10^4 and + 10^5; the sampling would miss a spike shorter than 5 s. +- **Disk.** The input table took 244 MB at 10^5 inputs and 2.4 GB at 10^6; + after a genesis setup the database was 510 MB and 5.1 GB: the rebuilt + history about doubles it. The sync's database writes took 0.47 s for 10^5 + inputs and 5.3 s for 10^6. +- **The wallet's state.** With 100,000 recipients the native wallet reached + 5.3 MB, more than a machine's `state` report can carry (b). + +## b. Reading a base machine + +### Method + +A temporary example binary in the `sequencer` crate drove the devnet image +through the emulator bindings as `testsi` does: one `EvmAdvance` per input, the +root hash sent as the revert hash, then a run to the next manual yield. To +grow the wallet it queued deposits, each from a new owner at one block, then +sent one batch whose single empty frame drains them, in rounds of at most +16,384. It stored the machine at each size. For each stored machine it then: + +- loaded it as `read_machine_export` does, sent the `scheduler` and then the + `state` query, and read `mcycle` before and after each run to the manual + yield, with the wall time of that run; +- called `sequencer::machine::read_machine_export` on it and timed the whole + call: the load, both queries and the state file's write. + +Only balances grow this way: a deposit's owner has no nonce entry. + +### Results + +The `scheduler` query took about 4,400 cycles and 0.1 ms at every size +(12,623 on the template). The `state` query: + +| Accounts | `state` report | Cycles | Wall | Under the 2^36 cap | `read_machine_export` | +|---|---|---|---|---|---| +| 0 (template) | 76 B | 4,851 | under 0.1 ms | 14 million times | 2 ms | +| 1,024 | 53,324 B | 574,395 | 1.6 ms | 119,638 times | 3 ms | +| 4,096 | 213,068 B | 2,176,863 | 4.3 ms | 31,568 times | 5 ms | +| 16,384 | 852,044 B | 8,531,035 | 14.2 ms | 8,055 times | 15 ms | +| 24,576 | 1,278,028 B | 12,809,394 | 20.9 ms | 5,365 times | 21 ms | +| 32,768 | 1,704,012 B | 17,070,474 | 27.5 ms | 4,026 times | 27 ms | +| 40,000 | 2,080,076 B | 20,913,146 | 33.0 ms | 3,286 times | 32 ms | +| 40,328 | 2,097,132 B | 21,020,105 | 32.4 ms | 3,269 times | 32 ms | +| 40,329 | does not fit | 29,843,541, then halts | 54.5 ms | | 56 ms, refused | +| 65,536 | does not fit | 47,672,415, then halts | 86.0 ms | | 88 ms, refused | + +Loading a stored machine took 3 ms. + +What the table means: + +- **The report limit is exactly 2 MiB of report bytes.** The wallet's report + is `76 + 52 × balances + 24 × nonces` bytes; 40,328 balances make 2,097,132 + bytes and fit, and 40,329 make 2,097,184 and do not. When the report does + not fit, the guest's `emit_report` fails, the guest panics and the machine + halts; `read_machine_export` then refuses the base with "the machine did not + answer the `state` query: the machine halted". Only the in-memory copy + halts, since the loader maps the stored files copy-on-write. +- **The account count where it stops fitting** depends on how many accounts + hold a nonce. Every account that has sent an op holds both entries, 76 bytes, + so 27,593 such accounts fit. +- **The cycle cap is not what binds.** The query costs about 520 cycles per + account, so the largest report that fits uses 1/3,269 of the 2^36 cap. The + halting queries used under 30 million cycles at 40,329 accounts and under 48 + million at 65,536, both far below it. + +## c. Draining a full fridge in one input + +### Method + +The same driver, from the template: N deposits from new owners queued at one +block, then one batch at the next block whose single frame, with no ops, names +that block as its safe block. That frame drains the whole fridge, as the first +frame after an outage and the era's root frame both do. The cycles and the wall +time are those of the batch's run to its manual yield. Each size refills from +the previous size's stored full fridge. Variants: 1,024-byte payloads (the cap) +instead of 72 bytes, and directs from a sender that is not the portal, which +the wallet counts but does not credit, so they emit no notice. + +All sizes fill in seconds, so the template's bound of 16,384 and four times it +were measured directly; nothing is extrapolated. + +### Results + +| Queued directs | Cycles of the drain | Per direct | Wall | Notices | +|---|---|---|---|---| +| 1,024 | 123,952,092 | 121,047 | 0.14 s | 1,024 | +| 4,096 | 483,428,709 | 118,025 | 0.52 s | 4,096 | +| 16,384 (the bound) | 1,936,703,678 | 118,207 | 2.07 s | 16,384 | +| 65,536 (four times the bound, one block) | 7,798,563,505 | 118,997 | 8.37 s | 65,536 | +| 16,384, 1,024-byte payloads | 1,982,798,459 | 121,020 | 2.15 s | 16,384 | +| 16,384, not from the portal | 16,696,478 | 1,019 | 0.02 s | 0 | + +- **The cost is linear,** about 118,000 cycles per drained deposit, and the + emulator ran about 930 million cycles per second. +- **The deposit's execution and its notice dominate.** Draining directs that + execute without crediting or emitting costs about 1,000 cycles each; payload + size barely matters. +- **Queuing is cheap:** about 4,400 cycles per queued 72-byte direct and 6,000 + per 1,024-byte one, under 0.05 ms of emulation each. +- **The per-input budget** is 2^48 − 1 cycles from each delivered advance + ([emulator facts](../cartesi-machine.md)). A drain at the bound uses + 1/145,000 of it, and four times the bound also completed: memory and the + accumulated outputs held too. + +## d. The devnet recovery drills + +From the drills' run on the same tree (the devnet agent's report): the local +Anvil devnet, debug builds as the e2e harness runs them, each recovery scenario +run three times, the six interleaved, with +`target/debug/rollups-e2e --exact`, and the divergence drill once, +every run passing. The times are each scenario's whole wall time. + +| Scenario | Wall time | +|---|---| +| `recovery_from_genesis_round_trip_test` | 36 to 37 s | +| `c_host_recovery_from_genesis_round_trip_test` | 36 to 37 s | +| `recovery_from_stored_machine_round_trip_test` | 40 to 41 s | +| `c_host_recovery_from_stored_machine_round_trip_test` | 37 to 41 s | +| `setup_refuses_a_set_up_data_directory_test` | 0.5 to 0.8 s | +| `c_host_setup_refuses_a_set_up_data_directory_test` | about 0.5 s | +| `watchdog_divergence_drill_test` | about 23 s, one run | + +The whole devnet suite, 53 scenarios, ran in 512 s. + +A round trip runs an old instance and pumps traffic, stops it, queues directs, +runs setup, starts the new era, lands its first batch and checks two fresh +watchdogs. The drill does not time setup alone. Its history is a few dozen +blocks of devnet traffic, which by (a) rebuilds in a small fraction of the +time; the rest is the scenario's steps and its waits, bounded as follows: mining until a block is safe (at most 64 mines, +100 ms apart), waiting for finality to advance (at most 40 tries, 1 s apart), +and watchdog commands (at most 120 s each). The stored-machine round trips +also run the watchdog's `init` and `replay` to make their base machine. + +## Conclusions + +- **The `state` report's 2 MiB is the limit that binds.** It holds 27,593 to + 40,328 wallet accounts, and the synthetic history in (a) outgrew it within + 10,000 inputs. Setup refuses a base machine whose state passes it, so such a + deployment can recover only from genesis or an older base; that is the + register's + [range-read follow-up](register.md#sequencer-recovery-follow-ups); this + record gives that entry its exact numbers. +- **The inspect cycle cap is comfortable** at more than 3,000 times the cost + of the largest report that fits. A range read would answer smaller queries, + so its headroom must be measured when it is built. +- **The fridge's largest drain is comfortable for the wallet:** 2 s and + 1/145,000 of the per-input budget at the bound. libdex's per-direct cost is + its own to measure, one of the questions for Bart. +- **Setup's time is about 85 µs per user op past the base, on one core.** + That is about 42 million ops per hour on this machine. A base whose age holds + 30 million ops, a week at a sustained 50 ops/s, takes about 43 minutes to + rebuild; genesis takes as long as the whole history. A recent base is what + keeps a recovery short. +- **One optimization would matter**, if rebuild time ever does. The + scheduler reads nothing from the application, so the ops of a page could be + recovered on all cores before they are handed out in order; that would + divide the recovery's 95% of the rebuild by up to the number of cores + (unmeasured). Nothing else in setup takes more than about 3% of its time. + +Limits of these numbers: one machine, macOS, an SSD; the synthetic mix is one +choice, so scale by rows and ops rather than inputs; the guest numbers are the +reference wallet's; the emulator ran in this process, not inside a node. diff --git a/docs/review/2026-11-12-libdex-answers.md b/docs/review/2026-11-12-libdex-answers.md new file mode 100644 index 00000000..1c98a670 --- /dev/null +++ b/docs/review/2026-11-12-libdex-answers.md @@ -0,0 +1,155 @@ +# libdex, as its author describes it — 2026-11-12 + +Scope: what libdex, the first application built on this sequencer, does today, +from its author's answers to fourteen questions on 2026-11-12. Each section +gives the answers, then what they mean for this tree, checked against +`2aaa4d7`. The second part is our reading, not the author's words. + +Retained for the [recovery plan's todo](../plans/2026-11-sequencer-recovery/todo.md#libdex) +and the register's [fridge](register.md#bound-the-canonical-direct-input-fridge) +and [canonical rules](register.md#canonical-rules-open-for-the-c-port) +entries. It describes libdex at one date and no number here is measured. +Replace it when libdex ports to this tree's headers. + +## The revision libdex builds against + +libdex builds against `8a7c914` (2026-10-06), on a branch that bumps the +contracts to `3.0.0-alpha.10`. Its author will move to the next release whose +sequencer, node, contracts, emulator and CLI work together. + +What it means: that revision has `application_engine_validate_user_op` and no +scheduler header. The port crosses to admission, total execution and +[`scheduler.h`](../../bindings/c-scheduler/include/scheduler.h) in one step, +so a change to the signed op or to either header costs libdex least before +that port. + +## Ops and their sizes + +- A price push's method is `86 + 27 × N` bytes for `N` markets, `N ≤ 255`: 167 + bytes at 3 markets, 518 at 16, 6,971 at 255. Fewer than 16 markets are + expected at launch; the number is a business choice. +- Every other op is 9 to 50 bytes. +- Pushing is open to anyone, but a push must carry the oracle's signature + beside its sender's. +- A late push is included and discarded today, which costs a fee and machine + time for nothing. libdex can tell a redundant push without advancing its + tick. Whether pushers are refunded is undecided on libdex's side. + +What it means: + +- Admission's `APPLICATION_ENGINE_REFUSED_BY_POLICY` answers the redundant + push, and it is cheap for libdex to compute. Nothing changes here. +- The lane closes a batch on `count × (71 + max_method_payload_bytes)` + against a target of about 12.6 KB. With the bound at 167 bytes a batch + closes at 53 ops, at 518 bytes at 22, and at 6,971 bytes at 2, where about + 110 small ops fit. Closing on real size + ([todo](../plans/2026-11-sequencer-recovery/todo.md#build)) has to land + before libdex ports. +- The fee assumes 126 bytes an op. A push takes 250 bytes on the wire at 3 + markets, 601 at 16 and 7,054 at 255: 2, 4.8 and 56 times the assumption, + against the fee's tenfold slack. A per-byte price can wait while the markets + are few, and cannot at 255. + +## The scheduler + +- The header looks right to its author, who will know for sure on integrating + it. Resuming from the five numbers is expected to work after the port. +- libdex's scheduler asserts on engine state, which can go, and calls the + engine's `validate_user_op` to decide whether a user op is skipped. +- The scheduler's state lives in process memory, outside the engine's state. +- `fee_to_linear` is ported and matches bit for bit, in 64 bits. An exponent of + 5,612 or more is unpayable, and its author proposed a reason code for it. + +What it means: + +- Here the scheduler never asks the engine. The executor compares the nonce + and execution is total + ([user nonces](../protocol/application-contract.md#user-nonces)), so + `validate_user_op` has no place in the port as the contract stands + ([canonical rules](register.md#canonical-rules-open-for-the-c-port)). +- The guest must answer the `scheduler` query itself + ([canonical machine](../plans/2026-11-sequencer-recovery/application.md#the-canonical-machine)). +- The engine header already covers an unpayable fee: admission reports + `APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE` with `required` all ones. + +## L1 inputs + +- Every user op can also be sent as an L1 input. +- The only other input is a portal's ERC-20 deposit. libdex ignores a + deposit's extra bytes, and its author asks that a long deposit be trimmed + and never dropped. + +What it means: + +- No filter on sender or shape exists: any sender may send any op. +- The scheduler drops a direct over `max_direct_payload_bytes` + ([the fridge](../protocol/scheduler-semantics.md#the-fridge)). A deposit with + long extra data is such a direct, and the portal has already taken its + tokens, so dropping it loses them. + +## Accounts and nonces + +- Only an ERC-20 deposit of at least `MIN_COLLATERAL` creates an account. A + user op never does. System accounts, the fee collector among them, exist + from genesis, and the fee collector may not send user ops. +- When the account table is full, a deposit is refunded at once by a + withdrawal voucher. +- A session wallet can be deleted and used again. After that the engine's + nonce and `GET /nonce` can disagree; libdex's clients sign with the engine's. + +What it means: libdex cannot follow the nonce rule. The rule gives every +address a nonce from 0 that each executed op advances and nothing else +changes; libdex has no place for the nonce of a signer without an account, +and a deleted wallet's nonce does not survive. If a reused wallet restarts +below where it stopped, its earlier ops, which are public on L1, can run +again; that is to confirm. + +## Clocks + +libdex has two. The oracle's clock, fed by price pushes, is the one traders +see. The block clock, fed by the safe blocks of frames and directs, detects a +stale oracle and halts trading. libdex reads no timestamp and stores none. + +What it means: after an outage the block clock is ahead of the oracle's, so +libdex halts trading until a fresh push. Recovery needs nothing more. + +## State + +The state is a 512 MiB address range, memory-mapped, in a layout Boost's +containers define, and mostly zeros. Its first 4 MiB hold the table an +emergency withdrawal reads. The L2 tx count is inside, named +`executed_input_count`. + +What it means: setup reads a base machine's state only through an inspect +report of at most 2 MiB, so it cannot read a libdex base until it reads the +range ([todo](../plans/2026-11-sequencer-recovery/todo.md#build)). The native +engine must lay out the same bytes as the guest, on every host a recovery +runs on; that is to confirm. + +## Machine cycles + +- Placing an order costs by how far it crosses the book: a large market + order matches hundreds of orders, a resting limit order may match none. +- A deposit is a few checks. +- A price push is the most expensive op, since it advances a tick: + liquidations, pegged orders and socialized losses. An idle tick is linear + in the accounts; the worst case is many billions of cycles. + +What it means: a push can arrive as a direct, so a drain of the fridge can +hold as many ticks as the fridge has entries. At ten billion cycles a tick, +an assumed figure, about 28,000 ticks fit one input's budget, under twice the +wallet's provisional bound of 16,384 entries. libdex's bound has to be priced +by its worst direct. + +## The client + +- The backend calls `/history` and `/ws/subscribe`, the frontend `/tx`, the + price pusher `/readyz` and `/tx`. +- The indexer cannot go back to an earlier count and has no backups yet. + After a recovery it reprocesses from genesis, which takes long. With backups + its author guesses minutes to reprocess a few hours. +- It links the SQLite 3.53.4 amalgamation statically. + +What it means: the list names no caller of `/nonce`, `/fee` or `/domain`. A +libdex recovery's time includes a full reindex until the indexer has +[backups](../plans/2026-11-sequencer-recovery/client-backups.md). diff --git a/docs/review/register.md b/docs/review/register.md index bb06b137..a41bc67f 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -18,39 +18,57 @@ are proposed work. ### Bound the canonical direct-input fridge -The shared [scheduler](../../sequencer-core/src/scheduler/mod.rs) retains raw -direct payloads without a byte budget. The age backstop does not bound memory, -and HTTP controls cannot limit canonical L1 traffic. QA's guest/node OOM -evidence supports investigating this independently of the toy wallet's state -growth; its workload-specific thresholds are not deployment capacity bounds. +The scheduler bounds its fridge by a count of entries, and a direct whose +payload is over a cap never joins it. Both are template constants the +application supplies through `Application::scheduler_limits`; the wallet's +values, 16384 entries and 1024 bytes +([`wallet.rs`](../../examples/app-core/src/application/wallet.rs)), are +provisional, and changing the cap means a new setup: `run` refuses a data +directory whose pinned cap differs. The staleness window comes from the same +method and is pinned the same way. The +[scheduler contract](../protocol/scheduler-semantics.md) owns the rules: an +overflow drains whole blocks of the oldest entries, which live prediction +cannot mirror, so an overflow is a deliberate divergence that the watchdog's +divergence latch reports and cockroach recovery answers. -Direction: use application-defined, deterministic format/sender rules to avoid -retaining irrelevant bytes, and a generous fixed capacity pinned per deployment. -Filtering or compacting must preserve valid deposits and agree across canonical -execution, live prediction, application-history progress, and recovery. Define -capacity in protocol bytes/slots, not allocator-dependent memory usage. +To settle with Bart: -Size against the cheapest accepted L1 inputs, including transaction batching, -actual batch-inclusion delays and the backstop window. The five-block local -frame tick is not a canonical drain interval. Include temporary allocations, -application execution and accumulated outputs in the guest budget. +- The payload cap's value. A count bounds memory only if each entry's size + is bounded too, and portals accept zero-value deposits with arbitrary extra + data, up to the 64 KiB input cap; the cap is the longest payload the + application accepts. libdex accepts every user op as a direct, so its cap + is at least its largest price push, 6,971 bytes + ([libdex's sizes](2026-11-12-libdex-answers.md#ops-and-their-sizes)). +- No filter on a direct's sender or shape. One would have to be a function of + the input alone, since the scheduler reads no application state, the lane + mirrors the fridge, and a resume re-admits from the raw inputs. libdex takes + any op from any sender as a direct, so the input shows nothing but its + length ([libdex's L1 inputs](2026-11-12-libdex-answers.md#l1-inputs)). +- The count. Above what L1 can include within the backstop window, roughly + 2 million zero-value deposits at mainnet's 60M gas limit today (30-40k gas + each when a contract batches them; measure it on the pinned contracts), + overflow cannot happen at all; the image fixes the bound for the + deployment's life, so leave headroom for higher gas limits. A smaller count + still makes overflow costly, and its consequence is a manual rebuild, not a + dead application. One block can carry about 1,500 to 2,000 directs at + today's gas limit; a block that fills the bound alone joins over it. +- The largest drain. The first frame after an outage and the root frame after + a rebuild both drain a full fridge in one input, so the bound must fit that + drain in the guest's budget, temporary allocations, execution and + accumulated outputs included. The five-block local frame tick is not a + canonical drain interval. The wallet drains its bound of 16,384 deposits in + one input in 1.94 G cycles, 1/145,000 of the per-input budget, and four + times the bound also completes + ([2026-11-10 measurement](2026-11-10-recovery-measurements.md#c-draining-a-full-fridge-in-one-input)); + libdex's drain is its own to measure, and its worst direct is a price push + that advances a tick + ([libdex's cycles](2026-11-12-libdex-answers.md#machine-cycles)). -Overflow must preserve valid deposits and have deterministic, non-catastrophic -behavior; uninterrupted service and soft confirmations need not survive it. -Early FIFO execution is a candidate, not a settled algorithm: it can reorder -still-fresh delayed batches. Decide whether incompatible batches are rejected -or overflow is detected as an exceptional ordering event that stops optimistic -service and requires cockroach recovery. Matching batch bytes/nonce alone does -not reveal this divergence. Silent loss of valid deposits is not a recovery -strategy. - -Next: settle that transition and measure headroom, then exercise delayed fresh -batches, same-block overflow, outages, and replay/rebuild agreement. Preserve -the [checkpoint eligibility rule](../recovery/cockroach.md#checkpoint-eligibility): -forcing one direct while leaving another from the same block queued can require -an earlier checkpoint or genesis. The [scheduler contract](../protocol/scheduler-semantics.md) -and [application contract](../protocol/application-contract.md) own the boundaries -the implementation must update together. +Next: settle these values and measure libdex's headroom, then exercise a +same-block overflow on devnet. An overflow changes the +application state after a block's last accepted batch, so the watchdog's +block-boundary comparison does not hold for that block +([snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison)). ### Batch-size accounting omits SSZ overhead @@ -79,39 +97,19 @@ counts, then correct the estimate and check the separately configured useful can happen meanwhile. Revisit on a partial failure where block polling stalls while the endpoint otherwise serves; the fix is an outer timeout around the watch. Evidence: [`submitter/poster.rs`](../../sequencer/src/l1/submitter/poster.rs). -- **Intermittent C-host recovery WebSocket reset.** At `dc4dd78` on - 2026-09-19, `c_host_recovery_after_stale_batches_test` failed an expected - message receive with `Connection reset without closing handshake` in the - [push run](https://github.com/cartesi/sequencer/actions/runs/35456024614/job/105931662252). - The [PR run](https://github.com/cartesi/sequencer/actions/runs/35456514750/job/105934071941) - passed all 49 scenarios on the same source tree. The failed run retained no - child-process log artifact, so the reset's cause is unclassified. CI now - uploads `tests/e2e/results/*.log` on failure. On recurrence, use those logs - to identify the server's last events and the failing receive before changing - timeouts, teardown, or retry behavior. Evidence: the - [recovery scenario](../../tests/e2e/src/test_cases.rs) and - [WS receive helper](../../tests/harness/src/ws.rs). - **Transient SQLite contention stops the submitter and the input reader** (input-reader half checked 2026-10-13 at `1a9615d`). Submitter read handles use a 50 ms busy timeout; the input reader opens writer handles (5 s busy timeout) for its cursor read and its append. In both, a storage/open failure escapes the worker loop. BUSY/LOCKED are - nonterminal but project to unclassified exit 1, causing respawn/recovery - under a restarting supervisor. Frequency and benefit of local retry are + nonterminal but project to unclassified exit 1, causing a respawn under a + restarting supervisor. Frequency and benefit of local retry are unmeasured. Check contention for both workers before choosing a bounded retry or timeout change; preserve other errors. Evidence: [`storage/open.rs`](../../sequencer/src/storage/open.rs), [`submitter/worker.rs`](../../sequencer/src/l1/submitter/worker.rs), [`l1/reader.rs`](../../sequencer/src/l1/reader.rs), [`commands/error.rs`](../../sequencer/src/commands/error.rs). -- **Overlapping admission policies.** The generic - [`lifecycle` preflight](../../sequencer/src/storage/lifecycle.rs) contains - setup/rebuild branches, but production calls it only for run/flush; setup - and rebuild reach the shared completion rule only inside - `complete_baseline_setup`. [`setup`](../../sequencer/src/commands/setup/mod.rs) - has its own admission path with an intentionally tested already-complete - no-op. Consolidate when the shared recovery tool reshapes setup; there is no - demonstrated conflicting live route. - **Fee-observation visibility.** [`log_gas_price_updated_at_ms`](../../sequencer/src/storage/fee_oracle.rs) is persisted, with no production reader. Decide whether operator SQL @@ -122,13 +120,6 @@ counts, then correct the estimate and check the separately configured oracle rewrites the stamp on every successful poll, unchanged price included, so a stale stamp means failing polls. That is one FULL-sync commit per successful poll, an estimated (unmeasured) cost under 0.1% of writer time. -- **Recovery across external effects.** Model/storage tests do not replace - process-level zombie re-injection or a restart between flush completion and - cascade commit. Before adding harness machinery, identify the missing - observation: safe nonce consumption must exclude a later original landing, - and a restarted recovery must rederive facts without reusing the previous - attempt's flush witness. Use the [recovery model](../recovery/README.md) to - bound a scenario and assess whether existing component tests suffice. - **OpenAPI drift.** [`openapi.yaml`](../../openapi.yaml) is hand-maintained beside the [API contract](../../README.md#api). CI lints its structure, but nothing checks it against the routes or serde wire types; only the @@ -137,6 +128,18 @@ counts, then correct the estimate and check the separately configured generated client or documentation site comes to depend on the file, add a test that compares its paths with the routers and validates serialized wire types against its schemas. +- **The `safe` tag's cadence depends on the L1 node** (web sources read + 2026-11-12, nothing run). The sequencer takes its safe head from the `safe` + tag ([`l1/reader.rs`](../../sequencer/src/l1/reader.rs)). By default that is + the justified block: it moves one epoch at a time, 32 blocks, and trails the + head by an epoch or more. A consensus client that turns on the fast + confirmation rule, which needs no fork, reports a block a few behind the + head instead, and falls back to the finalized block when the rule stalls, so + the tag can move back. Providers enable it per network and some only by + opt-in. Before a deployment reads a node with the rule on, check that the + reader and the lane's frame clock tolerate a safe head that goes backwards, + and revisit the five-block frame tick + ([frame-clock policy](../protocol/scheduler-semantics.md#sequencer-frame-clock-policy)). ## Known optimizations @@ -150,17 +153,14 @@ at `6b2a233`. [2026-09-25 measurement](2026-09-25-sender-index-commit-cost.md) it raised the mean 64-op commit from 0.27 to 1.0 ms and p99 from 1.1 to 9 ms. It stays: any sender-keyed durable structure pays about the same once the sender - population is large, and a derived index needs no recovery maintenance. If + population is large, and a derived index needs no maintenance. If lane persistence becomes the throughput limit, the alternatives are a lane-written sender-to-next-nonce table, cheaper only while the sender - population fits in a few pages and needing a rewind on invalidation, and + population fits in a few pages, and moving checkpoints off the lane (next entry). Such a table is also where rebuilt-baseline nonces could be seeded; see [Track 6](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign). - On the read side, the lookup walks the sender's invalidated entries above its - current nonce: about 0.11 µs each, 16 ms at 100,000, bounded by that sender's - own rolled-back volume. A covering `(sender, nonce, batch_index)` index would - skip the table fetch per entry. Revisit with a measurement on production-like + Revisit with a measurement on production-like Linux storage, or when lane throughput is the limit. Evidence: [`0001_schema.sql`](../../sequencer/src/storage/migrations/0001_schema.sql), [`storage/ingress.rs`](../../sequencer/src/storage/ingress.rs). @@ -177,19 +177,15 @@ at `6b2a233`. pays a file open, a schema parse on its first statement, and cold page and statement caches: about 0.2 ms per request against about 2 µs for the nonce query itself ([measurement](2026-09-25-sender-index-commit-cost.md#read-cost)). - A small pool of long-lived read connections would remove that cost. Separately, - `current_fee_quote` builds a full `WriteHead`, including two `COUNT(*)` scans - of the Tip's user ops, to return two numbers; that cost is bounded by batch - size and unmeasured. Evidence: - [`ingress/api.rs`](../../sequencer/src/ingress/api.rs), - [`storage/queries.rs`](../../sequencer/src/storage/queries.rs). + A small pool of long-lived read connections would remove that cost. Evidence: + [`ingress/api.rs`](../../sequencer/src/ingress/api.rs). - **Per-batch snapshots run on the lane.** Every batch close writes its snapshot before the sealing commit, so admission waits for `Application::create_dump` and the sequencer's fsyncs. Closes fire on the fee policy's byte target or the batch-open deadline, not on L1 posting, so under load several land per L1 block and the stall grows with throughput over target - size. Exact per-batch snapshots stay: restart, acceptance comparison, and - recovery select the exact batch's snapshot + size. Exact per-batch snapshots stay: restart and acceptance comparison + select the exact batch's snapshot ([lifecycle](../snapshots/lifecycle.md#creation-and-restart)). For a large-state native engine, the mitigation is a `create_dump` that clones files on a copy-on-write filesystem (XFS with `reflink=1`, btrfs, APFS); see the @@ -204,8 +200,9 @@ at `6b2a233`. `1a9615d`). A reader tick fetches `[cursor, safe]` whole, holds every event in memory (decoded event plus raw log, roughly twice the input bytes; unmeasured) and commits once; one failed single-block `get_logs` restarts the - span. In `run` the span stays under `l1_read_stale_after_blocks`. Setup, - startup syncs after downtime and `setup --recovery` have no bound: a fresh + span. In `run` the span covers the blocks since the last successful tick, + which a provider outage lengthens without bound, and setup and startup syncs + after downtime have no bound either: a fresh database scans every input since the app's deployment, our batches included (the InputBox caps an encoded input at 64 KiB). Current histories are megabytes, and rerunning on a larger host is the fallback. The known @@ -213,18 +210,27 @@ at `6b2a233`. indices are contiguous from the stored count and the last one sits at block `b`, the stored and received inputs in blocks before `b` are complete, because InputBox indices follow L1 order. Commit those at head `b - 1`, with - that block's header timestamp and without refreshing `synced_at_ms` (I12), - and hold the inputs at `b` for the next window; an empty window commits + that block's header timestamp (I12), and hold the inputs at `b` for the + next window; an empty window commits nothing; the window that reaches the sampled safe head keeps today's count check; reject logs outside the requested window; a later failure still returns an error. A window end is never committed on contiguity alone: the - lane would stamp frames past a dropped tail input. The recovery fold still - loads every input after its checkpoint. Act when `safe_inputs` payload bytes + lane would stamp frames past a dropped tail input. Act when `safe_inputs` payload bytes approach a quarter of the recovery host's memory, when a recovery sync fails on a transient mid-crawl, or when the reader becomes a crate shared with another node. Evidence: [`l1/reader.rs`](../../sequencer/src/l1/reader.rs), [`l1/partition.rs`](../../sequencer/src/l1/partition.rs), [`storage/l1_inputs.rs`](../../sequencer/src/storage/l1_inputs.rs). +- **Setup recovers signatures on one core** (measured 2026-11-10). Each user + op's signature recovery costs about 78 µs and takes 95% of the rebuild, so + setup's time is about 85 µs per user op past the base + ([2026-11-10 measurement](2026-11-10-recovery-measurements.md#a-setups-rebuild-and-execution)). + The scheduler reads nothing from the application, so a page's ops could be + recovered on all cores before the scheduler hands them out in order; the + speedup is unmeasured. Nothing else in setup takes more than about 3% of its + time. Act when a recovery's rebuild time matters to an incident's duration. + Evidence: [`commands/setup/rebuild.rs`](../../sequencer/src/commands/setup/rebuild.rs), + `Scheduler` in [`sequencer-core`](../../sequencer-core/src/scheduler/mod.rs). ## Verification gaps @@ -234,11 +240,8 @@ at the smallest useful boundary when working on that behavior. | Boundary | Existing evidence and remaining check | |---|---| -| Elapsed-time danger | Storage/procedure coverage exists. Add a process scenario isolating `EstimatedBatchInDanger` from stale-view refusal: retry at exit 20 without a speculative cascade. Start in [recovery](../../sequencer/src/recovery/mod.rs) and the [E2E scenarios](../../tests/e2e/src/test_cases.rs). | -| Canonical divergence | Storage freeze and startup refusal are covered separately. Compose accepted divergent input, runtime stop, and refusal after respawn in a process test; frontier must remain frozen. See [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). | -| Rebuilt anchor | Anchor unit mechanics and rebuild round-trip are covered. Exercise a later full-tear cascade after a nonzero-anchor rebuild and verify submission resumes at that anchor. See [I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce). | | Same-block directs | `multi_deposit_reconciliation_test` accumulates directs in separate blocks. Queue portal sends, mine once, assert equal receipt blocks, and verify canonical/WS order and attribution in the [E2E scenarios](../../tests/e2e/src/test_cases.rs). | -| Boot failure exits | Exit 20 now has a process assertion; exits 40/1 still need exact assertions in applicable setup/failure scenarios, so the supervisor receives the intended recovery/retry hint. Start in the [E2E scenarios](../../tests/e2e/src/test_cases.rs) and [exit contract](../../sequencer/src/commands/error.rs). | +| Boot failure exits | Exit 20 has a process assertion, and exit 30 for setup on a set-up directory. Exit 40 is setup's escape check, covered only by unit and chain tests (`escape_check_*`, `the_escape_check_refuses_activity_past_the_stop_block`); exits 40 and 1 still need exact assertions in process scenarios, so the supervisor receives the intended hint. Start in the [E2E scenarios](../../tests/e2e/src/test_cases.rs) and [exit contract](../../sequencer/src/commands/error.rs). | | SDK HTTPS | A test pins that `https://` endpoints open with a TLS ClientHello. Certificate rejection is untested; it relies on reqwest's rustls platform verifier, with no override in the SDK. Add a rejection test if the SDK gains custom roots. See the [client](../../sdk/rust-client/src/lib.rs). | | Uniswap-mode boot | Source-boundary tests pin setup validation, lazy runtime refresh, and transient quote retention. Existing sequencer E2Es use fixed mode. Decide whether a mock-pool boot scenario warrants its harness cost when changing oracle integration. | @@ -247,36 +250,166 @@ have lease/GC coverage. If restart cost becomes a requirement, add an assertion that distinguishes restoring a dump from a correct but expensive genesis replay; no new snapshot lifecycle is implied. +## Sequencer recovery follow-ups + +What `setup --base` leaves open, checked on 2026-11-10 against the working +tree that adds it. The [recovery plan's todo](../plans/2026-11-sequencer-recovery/todo.md) +holds the remaining build work; these are the accepted limits and the +unverified paths. + +- **`/nonce` after a recovery.** `next_user_nonce` reads only the era's live + ops, so a sender whose last op lies in the rebuilt history or below the base + count reads 0 until its first live op. It stays because the application is + the nonce authority: the lane's `422` names the expected nonce, which + corrects the client in one round trip, and an exact answer would need the + lane to ask the application per request or a sender table seeded from the + baseline (the [sender index](#known-optimizations) entry). The + [README](../../README.md#get-noncesenderaddress) documents it. Revisit if a + wallet cannot recover from the `422` without user action. Evidence: + [`storage/ingress.rs`](../../sequencer/src/storage/ingress.rs). +- **A base machine's state through the range source.** Setup reads the + application's state only through the `state` inspect query + ([`machine.rs`](../../sequencer/src/machine.rs)), whose report must fit the + guest's 2 MiB transmit buffer; the watchdog's labeled-range state source has + no setup counterpart. The reference wallet's `state` report fits 27,593 + accounts that hold a balance and a nonce, or 40,328 that hold only a + balance; one more and the guest's inspect halts, and setup refuses the base + with exit 30. The 2^36-cycle cap on each query (`QUERY_MCYCLE_BUDGET`) has at + least 3,269 times headroom over the largest report that fits, so the report + size binds first ([2026-11-10 measurement](2026-11-10-recovery-measurements.md#b-reading-a-base-machine)). + This is acceptable for the wallet. libdex's state is a 512 MiB range + ([libdex's state](2026-11-12-libdex-answers.md#state)), so a libdex + deployment could recover only from genesis: add a range read, and measure + its queries against the cap, before libdex deploys + ([state sources](../watchdog/README.md#state-sources)). +- **Not every machine the watchdog stores is a base.** Setup loads only a + machine waiting at an `RX_ACCEPTED` yield, and the + [watchdog README](../watchdog/README.md#machines-a-recovery-can-start-from) + says which of its machines do. It stays because setup refuses one that does + not wait, at its first step with exit 30, before writing anything. Evidence: + [`replay.lua`](../../watchdog/replay.lua), [`tick.lua`](../../watchdog/tick.lua), + and the three yield tests in [`e2e.lua`](../../watchdog/tests/e2e.lua). +- **The loader's failure arms against the real emulator.** Unit tests in + [`machine.rs`](../../sequencer/src/machine.rs) drive every arm of + `read_machine_export` through a scripted emulator; whether the real + emulator behaves as scripted (an undeliverable response leaving `iflags.Y` + set, for one) rests on the [emulator facts](../cartesi-machine.md#reading-a-stored-machines-export). + Against the built image, canonical-test's `export` cases cover only the + answered queries and a missing directory; the measurement above also saw a + `state` report over 2 MiB halt the guest and `read_machine_export` refuse + the base. +- **A foreign scheduler in the rebuild.** An application's author may bring + their own scheduler, behind [`scheduler.h`](../../bindings/c-scheduler/include/scheduler.h), + for setup to run in the rebuild. Setup and the sequencer call the Rust + `Scheduler` directly: no `Scheduler` trait and no adapter over the header + exist, so no foreign scheduler can be linked yet + ([build](../plans/2026-11-sequencer-recovery/todo.md#build)). +- **Unverified on GitHub Actions:** the release workflow's native arm64 + runner (`ubuntu-24.04-arm`) and the amd64 release link, since the + sequencer links the emulator. That workflow runs only on a tag or by hand, + so no pull request exercises it. Evidence: + [`release.yml`](../../.github/workflows/release.yml). +- **Release notes owe an upgrade path.** No deployment crosses this release + in place: + - **A new setup.** The baseline schema changed in place, so a data directory + from before needs a new `setup`, which starts a new era and realigns every + client. `run` does not refuse an older directory by name; it fails on the + first statement the old schema cannot serve. + - **No automatic recovery.** A sequencer whose batches do not land inside + the window stops for good ([batch landing and outages](../recovery/README.md)). + Exit code `10` is gone and stays unused. + - **Removed flags.** `--preemptive-margin-blocks` and + `--l1-read-stale-after-blocks` are gone. A command line that still passes + one fails argument parsing with exit `2`, which a supervisor recipe keyed + on `30` does not catch; their environment variables are ignored. + - **Client interfaces.** A history's identity is its era id: the recovery + generation, `STALE_GENERATION`, `from_generation`, `compatibility` and + `X-Recovery-Generation` are gone, and the subscribe query and `/history` + answer `400` to a field they do not know. + `HistoryBaseline.rebuilt_history_end` is a required field, and + `/historical-l1-inputs` is gone. An SDK build from before cannot talk to + this server. + - **Applications.** `SchedulerLimits` carries `max_wait_blocks`, and a C + engine is rebuilt against ABI version 3. + - **The runtime.** The released binaries link the emulator statically, so + they need the Ubuntu 24.04 or Debian 13 C++ runtime. + - **The template hashes.** Read them from the released images + (`just canonical-print-build-hashes`), not from an earlier build: any + source change in a guest crate moves them, a rename included + ([what moves the template hash](../cartesi-machine.md)). +- **One unexplained memory reading in the rebuild.** Setup streams its passes + in pages, and a test process rebuilding 10^6 inputs, sampled every 5 s, + stayed between 26 and 55 MB resident. `/usr/bin/time -l` around one such + run's whole `cargo test` command reported a 2.9 GB peak, which sampling would + miss if it lasted under 5 s + ([2026-11-10 measurement](2026-11-10-recovery-measurements.md#a-setups-rebuild-and-execution)). + Recheck with a peak-memory reading of the setup process alone before + sizing a recovery host's memory. + ## Canonical rules open for the C++ port -The [scheduler spec](../protocol/scheduler-semantics.md) marks these rules -**pending**. Each changes the canonical machine image or its wire format, so -they are agreed with the port's author. The wire-format bump ships first, on -its own, after draining sealed batches and before the first production deploy; -the rest ship in the port's image. Checked 2026-10-14 at `32bf8de`; -evidence: the spec's links and -[`consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json). +Each of these changes the canonical machine image or its wire format, so it is +agreed with the port's author. The wire-format bump ships first, on its own, +after draining sealed batches and before the first production deploy; the +rest ship in the port's image. The +[scheduler contract](../protocol/scheduler-semantics.md) owns the rules in +force. - **Signature acceptance set.** The guest and `POST /tx` accept every `v ≥ 35` and the high-`s` twin, as alloy does. Proposed: `v ∈ {27, 28}` and low `s` - only. + only. `POST /tx` is not pinned for `s ≥ n` or `v` from 29 to 34, which only + the batch path's guest scenarios cover, and neither path for `s > n`. - **Empty batches** are valid at any age and consume a nonce, though the sequencer never sends one. Reject them only if the port forces a new image anyway. - **Wire format.** A fixed 65-byte signature field, with the [SSZ overhead](#batch-size-accounting-omits-ssz-overhead) fix; drain sealed batches before the switch. +- **An over-long direct is dropped, deposits included.** A direct over + `max_direct_payload_bytes` never joins the fridge + ([the fridge](../protocol/scheduler-semantics.md#the-fridge)). A portal + deposit with long extra data is such a direct, and the portal has already + taken its tokens, so the drop loses them. Proposed: cut the payload to the + cap and let it join, handing the application the original length so it can + tell a cut input; libdex reads only a deposit's head + ([libdex's L1 inputs](2026-11-12-libdex-answers.md#l1-inputs)). The cut is + still a function of the input alone, so the lane and a resume agree. +- **The nonce rule does not fit libdex.** The rule gives every address a + nonce from 0 that each executed op advances and nothing else changes; the + executor asserts it and `GET /nonce` relies on it + ([user nonces](../protocol/application-contract.md#user-nonces)). libdex + creates an account only on a deposit, bounds its account table, and deletes + session wallets with their nonces + ([libdex's accounts](2026-11-12-libdex-answers.md#accounts-and-nonces)). If + a posted batch carries an op from a signer without an account, the op has + no nonce to advance and the executor's assertion stops the rebuild. To + decide before the port: keep the counter, let the application answer "no + account", and require that a nonce never goes down; or move replay + protection into the scheduler in a form whose state can be dropped + ([deferred plan](../plans/2026-10-batch-scoped-replay-protection.md)). - **Riding the port's image:** the [fee table](../plans/2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut), - the [fridge bound](#bound-the-canonical-direct-input-fridge), and a setup check - of the guest's compiled `sequencer_address`. With them come versioned gate - and signature vectors, adding the unpinned `s ≥ n` and `v` from 29 to 34. + the [fridge bound and cap values](#bound-the-canonical-direct-input-fridge), + and a setup check of the guest's compiled `sequencer_address`. The port's + image ships its own vector set, as + [`consensus_vectors_v2.json`](../../tests/fixtures/consensus_vectors_v2.json) + is this template's ([conformance](../protocol/scheduler-semantics.md#conformance)). ## Integration work owned elsewhere -- [Track 3 follow-ups](../plans/2026-07-track3-feed-replay-design.md): - private-engine snapshot-to-live replication and representative deployment latency, - including checkpoint creation and complete L1-reconciliation turns. +- [Track 3](../plans/2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign) + follow-ups, each needed before relying on it in production: + - Bart's private DEX indexer and its database backups, while his client + adopts the API. Bugs it finds are fixed; missing fields or an awkward + workflow feed the new client sync. + - Private-engine snapshot-to-live replication. + - Ingress latency, feed catch-up headroom and recovery time on the target + deployment's workload, including checkpoint creation and complete + L1-reconciliation turns. +- **Not planned without a consumer need:** resumable snapshot transfer (when + artifact size makes an interrupted download costly) and session fencing (if + several local writers become supported). Rotating the submitter's key has a + [note](../plans/2026-11-sequencer-recovery/todo.md#later). - [Track 6](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign): external engine/scheduler agreement, application-specific recovery export, independent fee-conversion vectors, and consumer-driven ABI/checkpoint decisions. diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index ddc4779b..580be453 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -21,8 +21,8 @@ and its canonical state coincide; one write per `create_dump`. Here `prefix` is the app-owned `state` directory inside the sequencer's dump directory. Thus the wallet file is `dumps//state/state`. The surrounding `info.toml` and exported `checkpoint.toml` use the sequencer's metadata format -version; that version does not identify the wallet's SSZ schema. Application -progress is embedded in the wallet bytes. History identity and acceptance +version; that version does not identify the wallet's SSZ schema. The L2 tx +count is embedded in the wallet bytes. History identity and acceptance metadata come from the sequencer; see [application history](../protocol/application-history.md). ## Toy Wallet Wire Format @@ -44,12 +44,13 @@ metadata come from the sequencer; see [application history](../protocol/applicat - `nonces` (`Vec`) - `address` (`[u8; 20]`) - `nonce` (`u32`) -- `executed_input_count` (`u64`) -- `last_executed_safe_block` (`u64`) — the app's safe-block clock - (reported by `Application::progress`): max block carried by any - executed input. Recovery reads it as `A`, the safe block this state - reflects, so it must live in the canonical state bytes (both the - bare-metal and canonical-machine sides advance it identically). +- `l2_tx_count` (`u64`) — the L2 tx count, reported by + `Application::l2_tx_count`. `WalletApp::from_canonical_state` + refuses bytes whose count differs from the machine's. + +The wallet holds no clock. The safe-block clock belongs to the scheduler and +travels in its export, not in the application's bytes +([the scheduler clock](../protocol/scheduler-semantics.md#the-scheduler-clock)). ### Determinism @@ -70,7 +71,6 @@ The decoder rejects: - Malformed SSZ bytes (any decode error from the SSZ library). - A snapshot containing two entries in `balances` with the same address. - A snapshot containing two entries in `nonces` with the same address. -- Zero `executed_input_count` with a nonzero `last_executed_safe_block`. Duplicate checks prevent an entry from silently overwriting another during restore. The decoder accepts unique entries in any order; re-encoding the @@ -91,6 +91,12 @@ If a future change ever needs to break the wire format against live dumps: Until then, do not reorder, repurpose, or reinterpret existing fields in place. +The one in-place break so far removed the trailing `last_executed_safe_block` +field (the fixed part went from 84 to 76 bytes). It was deliberate and +pre-launch: it ships with a new template hash, and no live dump had to +survive it. A dump in the old layout fails to decode (`snapshot decode +failed`) rather than being misread. + ## Trust Model The file shares the persistent data directory's diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 37119dab..cb7f0b59 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -4,10 +4,10 @@ Snapshots preserve application state at a declared execution boundary. Their contents and boundary are immutable. L1 acceptance is a separate durable fact: there is no promotion operation and no mutable finalized pointer. -Keep three boundaries separate: the engine embeds its executed-input count and -safe-block clock; SQLite associates an artifact with a local batch or era -baseline; safe L1 acceptance selects a comparison checkpoint. Counts and clocks -alone do not establish acceptance. The [history guide](../protocol/application-history.md) +Keep three boundaries separate: the engine embeds its L2 tx count; +SQLite associates an artifact with a batch or the era baseline; safe L1 +acceptance selects a comparison checkpoint. A count alone does not establish +acceptance. The [history guide](../protocol/application-history.md) explains these coordinates. ## Artifact and boundary @@ -26,18 +26,17 @@ source deletion. Its canonical comparison file may be only one part of that artifact. The [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle) owns durability and engine behavior; [format.md](format.md) describes the wallet. -SQLite separates artifact ownership from snapshot boundaries: +SQLite holds one row per artifact and the boundary it holds: | Fact | Meaning | |---|---| -| `dumps` | Artifact path and active reader lease count | -| `snapshots` | Artifact's immutable application count and local batch identity | -| Baseline snapshot (`batch_index IS NULL`) | State from which this era starts | -| `safe_accepted_batches` | Scheduler-accepted, content-matched local batch landings | +| `snapshots` | Artifact path, immutable application count, and the batch's nonce | +| Baseline snapshot (`batch_nonce IS NULL`) | The application at the end of the era's stop block, at its rebuilt history end | +| `batches.accepted_input` | The safe input whose scheduler-accepted, content-matched landing accepted the batch | The application count is the next canonical input position, never a physical -SQLite cursor. The batch identity disambiguates empty batches, which can share -an application count, and replacement branches, which can reuse a nonce. +SQLite cursor. The batch's nonce disambiguates empty batches, which can share +an application count. ## Creation and restart @@ -49,44 +48,56 @@ Creation runs on the lane, so user ops queued behind a close wait for it. Each close logs the creation time as `snapshot_ms`, at DEBUG, or at WARN once it reaches half the `POST /tx` acknowledgement budget. -Setup similarly makes the baseline artifact durable before publishing the -complete era baseline, recovery root when applicable, and setup-completion -facts atomically. - -Restart requires the newest valid closed batch's snapshot, or the baseline -when no valid closed batch exists. A missing required snapshot fails loud. +Setup similarly makes the baseline artifact durable before one transaction +inserts the era row (which marks setup complete) and registers the baseline +snapshot; +it opens no root frame, which `run`'s startup does +([cockroach recovery](../recovery/cockroach.md#what-setup-does)). The rows of +the era's rebuilt history, from its base count to the baseline, have no +snapshots: they live in `rebuilt_history`, which the feed serves and restart +never replays. + +Restart requires the newest closed batch's snapshot, or the baseline when no +batch is closed. A missing required snapshot fails loud. The selected artifact and stored application count come from one row. Catch-up checks the restored engine's count and replays -application inputs from that count. Invalidated branches are excluded by the -same valid-batch relation used elsewhere. +application inputs from that count. ## Acceptance and comparison The newest accepted batch determines the comparison checkpoint. Its snapshot must exist; storage refuses a missing required row instead of falling back to an -older snapshot. Acceptance already includes scheduler validation and local -content identity, so merely observing an own-sender L1 input is insufficient. +older snapshot. Before the era's first accepted batch, the baseline is the +comparison checkpoint: the application's state at the end of the stop block is +the canonical machine's state there, whatever the base, so it is served with +the stop block as its inclusion block, the rebuilt history end as its count, +and the anchor as its next batch nonce. After a cockroach recovery, this is the +watchdog's first comparison of the new era. Acceptance already includes +scheduler validation and local content identity, so merely observing an +own-sender L1 input is insufficient. A persisted canonical-divergence marker refuses accepted-checkpoint selection, including when a matching batch precedes a divergent acceptance in the same -block. Selection checks the marker in the same SQLite transaction as its read -and any download lease. Finalized endpoints return HTTP 503 before consulting -conditional cache headers; they do not fall back to an older artifact. +block. Selection checks the marker in the same read transaction, which a +download runs under the lease registry's lock. Finalized endpoints return HTTP +503 before consulting conditional cache headers; they do not fall back to an +older artifact. This selection is independent of the lane's L1 reconciliation cursor. A crash between reader ingestion and lane reconciliation cannot miss a promotion or repeat one: acceptance is already durable, and the query derives the result. -The genesis baseline is known canonical at block zero. A rebuilt baseline is -**only a restore artifact** until a new batch is accepted. The recovery fold can -pre-execute queued direct inputs through its stop block; that state need not -equal the canonical machine's state at that block. It is never advertised as an -accepted comparison checkpoint solely because recovery produced it. - The current watchdog compares at the selected accepted batch's **L1 block boundary**. Every batch has a snapshot, and reader ingestion accounts for a complete safe block before publishing its accepted prefix. Selecting the latest accepted batch therefore includes later accepted batches in the same block. +On the canonical side, no later input in that block changes the application +state, even after faulty sequencing. The backstop already ran at the batch's +own input and the fridge is ordered by block, so nothing becomes overdue later +in the block; directs only enqueue; and rejected or stale batches change +nothing. A fridge bound that executes directs on arrival breaks this for the +block where it overflows +([register](../review/register.md#bound-the-canonical-direct-input-fridge)). Sparse snapshots are a future policy change: an older artifact cannot be labelled as that block's final state when a later accepted batch in the block has no artifact. That change must settle comparison positioning and replay @@ -100,40 +111,66 @@ These endpoints are operator-only and require network isolation: opaque `state` artifact. It may describe optimistic state. - `/finalized_state` streams only the canonical comparison file for the latest accepted checkpoint. `/finalized_state/inclusion_block` provides its block and - executed-input count for the watchdog's idle poll, and + L2 tx count for the watchdog's idle poll, and `/finalized_state/digest` adds the file's SHA-256 under the same lease. - `/finalized_snapshot` streams a complete recovery tar archive containing `info.toml`, `state`, and a generated `checkpoint.toml` acceptance receipt. The receipt supplies the accepted inclusion block and next batch nonce. -Snapshot bodies carry `X-History-Era`, `X-Recovery-Generation`, and -`X-Executed-Input-Count`. Acceptance endpoints also carry `X-Inclusion-Block`. -The metadata, selected artifact, and lease are captured in one transaction. +Snapshot bodies carry `X-History-Era` and `X-Executed-Input-Count`. +Acceptance endpoints also carry `X-Inclusion-Block`. +The metadata and selected artifact come from one read, taken under the lease. Restoring `/latest_snapshot` and subscribing with its history claim gives the consumer a coherent snapshot-plus-suffix starting point. -**Operator backup workflow:** download `/finalized_snapshot` and extract the -archive. Supply that extracted directory to `setup --recovery`, with the -checkpoint block recorded in its receipt. The loader checks the receipt against -the immutable dump metadata and configured block. Copying a bare local -`dumps//` directory is insufficient: it is a restore artifact and has no -acceptance receipt. Export creates the receipt without modifying local files. +**Operator backup:** `/finalized_snapshot` exports an accepted checkpoint with +its receipt, without modifying local files. A bare local `dumps//` +directory is only a restore artifact and has no acceptance receipt. No command +consumes the archive: a recovery rebuilds from a canonical machine, never from +a sequencer artifact ([cockroach recovery](../recovery/cockroach.md)). ## Retention, leases, and crash safety Garbage collection retains: 1. The newest accepted checkpoint, or the baseline before first acceptance. -2. Every valid snapshot beyond the accepted frontier. An intermediate optimistic +2. Every snapshot beyond the accepted frontier. An intermediate optimistic batch may become the next accepted head before its successors do. 3. Every artifact with an active reader lease. -Older accepted artifacts and invalidated branch artifacts are collectible. -The baseline artifact can be retired after an accepted checkpoint replaces its -rollback role; the era's immutable baseline metadata remains in SQLite. +Older accepted artifacts are collectible. +The baseline artifact can be retired after an accepted checkpoint replaces it +as the comparison checkpoint; the era's immutable baseline metadata and its rebuilt history +remain in SQLite. + +Leases live in the serving process, in one registry of per-snapshot counters +that `run` shares between the egress handlers, the lane's GC and startup +hygiene. A lease means something only while this process serves a body, and +only `run` serves and collects, under the data directory's exclusive process +lock; nothing durable records a lease, so a restart starts with none. A second +process that served or collected would need a shared lease record: revisit this +if one ever does. The registry follows one protocol: + +- **Acquire** runs on a blocking thread over a read-only connection, which it + opens and makes load the schema first. It locks the registry, then selects in + one deferred read transaction (the finalized endpoints' divergence check + included), ends it, increments the snapshot's counter and unlocks. The + selection's transaction begins under the lock, so what it reads postdates any + GC commit. +- **GC** begins its IMMEDIATE write transaction first, then locks the registry, + selects its candidates, drops every candidate whose counter is above zero, + removes the zero entries from the registry, deletes the remaining rows and + commits. It unlocks only after the commit returns: before that, an acquire + would still read the rows being deleted and lease a snapshot whose artifact + is about to go, and a missing artifact is terminal. +- **Release** is the guard's drop: it decrements the counter, takes no lock + and does no I/O, so it is safe on an async worker thread. A zero entry stays + until the next GC removes it. + +GC takes SQLite's writer lock, then the registry; acquire takes the registry, +then only reads. WAL readers never take the writer lock, and any wait a reader +does meet is bounded by its 50 ms busy timeout, so the two cannot deadlock. -Selection, lease acquisition, and GC serialize through SQLite write -transactions. A lease release guard is armed only after its increment commits. An HTTP body retains the lease through completion or disconnect. Archive production also retains ownership until it stops reading the source, so dropping the network stream cannot race producer reads against filesystem deletion. @@ -142,9 +179,9 @@ GC selects and deletes eligible database rows in one transaction, then removes the enclosing dump directories recursively. This filesystem operation disposes of all checkpoint resources. Filesystem deletion failure leaves a harmless orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing -state and is forbidden. Startup clears leases left by the dead process, -checks the rollback checkpoint's `info.toml` and format version, collects obsolete -rows, and sweeps orphan directories before workers start. The sweep matches +state and is forbidden. Startup, before the listeners are bound, checks the +accepted checkpoint's `info.toml` and format version, collects obsolete rows, +and sweeps orphan directories. The sweep matches artifacts by filesystem identity (device and inode), not by path, because setup and run can spell the same directory differently; it resolves every retained artifact before deleting anything and stops if one cannot be resolved diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 2cd9aa53..afe8f6cb 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -2,14 +2,14 @@ The security posture this codebase defends against. Defines what is in scope for security review, what is out of scope, and the trust level assigned to each actor and interface. -See [`../recovery/README.md`](../recovery/README.md) for the recovery subsystem, which operationalizes parts of this threat model (adversarial mempool, fail-stop L1 provider). +See [batch landing and outages](../recovery/README.md) for what the sequencer does when its batches stop landing, and [cockroach recovery](../recovery/cockroach.md) for its one recovery; both operationalize parts of this threat model (adversarial mempool, fail-stop L1 provider). ## Assets What we are protecting: - **Rollup state integrity.** The canonical on-chain state must reflect a deterministic replay of user operations and direct inputs. Any divergence between the sequencer's off-chain view and the scheduler's on-chain execution is a state-integrity failure. -- **Soft-confirmation honesty.** Every soft confirmation issued by the sequencer must land on L1 as promised, or be explicitly revoked via recovery. +- **Soft-confirmation honesty.** Every soft confirmation issued by the sequencer must land on L1 as promised, or be explicitly revoked by a cockroach recovery's new era. - **User funds.** No user operation, replay, or protocol break can cause users to lose funds. - **Batch-submitter key.** Held in operator infra; not hijackable by network attackers. @@ -41,8 +41,8 @@ no reorg can move the floor. Do not model InputBox forgery. The code supports exactly one endpoint, shared by the reader, submitter, poster, flusher, and fee oracle. Behind a load-balanced fleet, lagging replicas -can silently truncate `get_logs` ranges and desynchronize the flush and re-sync -views; recovery refuses if the re-sync lags the flush view. +can silently truncate `get_logs` ranges and desynchronize the flush and sync +views; setup refuses a sync that ends below its stop block. The reader fails loud on an incomplete `InputAdded` set ([I6](../invariants.md#i6-the-frame-clock-accounts-for-a-complete-l1-interval)): @@ -69,7 +69,14 @@ processes on one directory (a kernel process lock) or a mistyped `--data-dir` copied-directory coordination, and distributed fencing are out of scope; the lock is a local foot-gun guard ([ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership)). -Judge mechanisms for this class against this boundary. +Judge mechanisms for this class against this boundary. Cockroach recovery +deliberately creates a second data directory pinned to the same submitter +address, and the lock cannot stop the old one from running with the same key, +so the operator must keep the old instance stopped +([before anything](../recovery/cockroach.md#before-anything)). Setup's escape +check (exit 40) catches an old instance whose transactions are unsettled, or +whose batch landed past the stop block, when setup looks; one started later +shows only when its batch lands, as a content-identity divergence. ### Fee oracle @@ -80,13 +87,13 @@ market movement. Residual risks are TWAP lag during real moves and a thin or wrong pool; multi-hop pricing is out of scope. Setup writes the first quote under the same hard L1 requirement as the rest of -setup. `run` gates neither recovery nor admission on another quote: it starts +setup. `run` does not gate its start on another quote: it starts from the persisted `log_gas_price`, builds the source from the pinned identity without RPC, and refreshes in the background, retrying transient failures indefinitely and keeping the last price. `log_gas_price_updated_at_ms` is telemetry, not an expiry gate. -A shared-endpoint outage or stale view is already caught by safe-head progress. +A shared-endpoint outage or stale view already shows as the safe head's age. A pool-specific failure with a healthy reader is an accepted, unbounded economic risk: a stale-low price subsidizes DA and weakens the spam barrier, and a stale-high price rejects users. Neither changes canonical execution, because @@ -106,14 +113,19 @@ cannot front-run the op it counts. ### Self-trust -The sequencer trusts its own code in a specific sense: **impossible states are never *handled*.** There are no graceful fallback paths, no re-validation of a neighbor module's answer, no code that keeps running past a violated internal contract. Normal preemptive recovery addresses liveness failures such as infrastructure outages, network partitions, and gateway failure. +The sequencer trusts its own code in a specific sense: **impossible states are never *handled*.** There are no graceful fallback paths, no re-validation of a neighbor module's answer, no code that keeps running past a violated internal contract. Liveness failures such as infrastructure outages, network partitions, and gateway failure are retried until the staleness window closes on the oldest unaccepted batch; then the era ends ([batch landing and outages](../recovery/README.md)). If a bug corrupts local state or causes the sequencer to emit malformed batches, frames, or user operations, the operator must diagnose and fix it. Then -[cockroach recovery](../recovery/cockroach.md) can rebuild from a trusted canonical -application checkpoint and L1. The checkpoint must be trusted independently of -the faulty local state. Historical inputs, including malformed batches, receive -the canonical scheduler's normal treatment during replay. +[cockroach recovery](../recovery/cockroach.md) (`setup --base`) rebuilds a +second data directory from a canonical machine and L1, reading nothing from +the old one. The base machine is trusted input: it must be trusted +independently of the faulty local state, because setup checks only its +scheduler export against L1, not its application state, and the watchdog's +comparison of the new era's baseline is the first independent check. The +rebuild runs every input after the base, malformed batches included, through +the canonical scheduler's normal treatment, up to the stop block: a safe block +pinned after the flush, so the rebuilt history never changes. This is **not** a prohibition on checking: internal invariants are asserted loudly wherever a check is near-free (types, SQL constraints and triggers, boundary assertions). The [check policy](../invariants.md#the-check-policy-fail-loud) owns the rule, its reasons, and the decision test. @@ -135,12 +147,12 @@ blocking production diagnostics would require revisiting that assumption - Process crashes at arbitrary points, including mid-transaction - **Restart after a terminal exit (accepted residual window).** There is no boot gate on a prior terminal verdict: a deliberate restart after - exit 30 or SIGABRT boots through fact-derived recovery. Every fault whose evidence + exit 30 or SIGABRT boots through the same checks as any start. Every fault whose evidence the boot path reads re-refuses before the first soft confirmation — - canonical divergence (persisted fact), misconfiguration (re-checked every - boot), boot-path storage corruption, incomplete setup — and the - batch/frame spine is re-inspected by the runtime danger detector within - seconds of launch. The accepted residual is narrow: corrupt payload bytes + canonical divergence (persisted fact), a void history (derived from the + database), misconfiguration (re-checked every boot), boot-path storage + corruption, incomplete setup — and the worker that watches the terminal + facts reads them again within seconds of launch. The accepted residual is narrow: corrupt payload bytes in rows at/below the lane's resume checkpoint re-trip only when the WS feed pages them or the submitter re-encodes a pending batch. The feed can read the whole available history from era base `K` in bounded pages, with no @@ -150,10 +162,10 @@ blocking production diagnostics would require revisiting that assumption entered by restarting after a terminal exit (including supervisors that restart regardless of exit status), and it is bounded by backstops that never depended on a boot gate: - rollbackable soft confirmations, the watchdog's digest comparison, and the I15 + revocable soft confirmations, the watchdog's digest comparison, and the I15 divergence freeze. - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders -- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after baseline block `C` against the valid closed batch we sealed at that nonce. The complete prefix through `C` is opaque and is never reinterpreted using the recovered nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. Cockroach recovery does not read the previous database's watermark, so that flush relies on the provider's pool view; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). +- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: no batch nonce is reused inside an era, so a zombie of ours carries the bytes of a batch the sequencer still holds and is accepted as that batch or skipped; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after the era's stop block against the closed batch we sealed at that nonce. Across eras, setup's flush before a rebuild is best effort ([the flush and the stop block](../recovery/cockroach.md#the-flush-and-the-stop-block)), and a submission that escapes it is one of these foreign landings. The rebuilt prefix through the stop block is never reinterpreted using the anchor nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. Cockroach recovery's setup has no watermark from the previous database, so its flush relies on the provider's pool view; its escape check and then the content-identity check keep the residual zombie detected rather than silent (see [the flush and the stop block](../recovery/cockroach.md#the-flush-and-the-stop-block)). - L1 reorgs up to safe depth - Malicious `POST /tx` callers: malformed signatures, spoofed sender, replay across chains or apps, nonce manipulation - Malicious direct-input senders: arbitrary payload, any intent; sender authenticity is guaranteed by InputBox @@ -175,22 +187,21 @@ These are preconditions the sequencer takes as given. They are neither "trust" n ### L1 block-time coupling -The wall-clock fallback estimates missed blocks as the time since the last safe-head advance divided by `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` ([automatic recovery](../recovery/README.md#when-safe-head-progress-stops)). - -This assumes a **known, bounded-variance relationship** between elapsed wall-clock time and mined L1 block count. The assumption has three parts: - -1. **Known average block time** — `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default 12s, Ethereum mainnet) accurately reflects the target chain's block cadence. -2. **Bounded variance** — over the danger-threshold window (~4h on mainnet), the delta between `elapsed_seconds / avg_block_time` and actual mined blocks is small. On Ethereum mainnet this holds: slot proposers occasionally skip, but >99% of slots produce a block. -3. **Wall clock is accurate enough for elapsed-time estimation.** A clock that falls a full block-time or more behind a persisted safety baseline is detected and refuses operation; sub-block skew is tolerated. Gradual or systematic drift remains an external assumption. - -**Where it matters.** The missed-block estimate is a fallback for a safe head that stops advancing, whether the RPC is unreachable or still answers with a stalled view. +The sequencer acts on L1 block numbers, never on elapsed time: the void +history, the landing lag and every acceptance decision read observed safe +blocks ([batch landing and outages](../recovery/README.md)). +`CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default 12s, Ethereum mainnet) converts +between the two in three places only: `run`'s batch-open check, which refuses +a `max_batch_open_seconds` that leaves no room to land +([the budget](../recovery/README.md#the-budget)), the poster's confirmation +timeout, and the flush's waits. **Violation modes.** -- **Chain with unstable block time.** A chain where average block time drifts substantially (e.g., PoW networks under major hashrate swings) would make the estimate less reliable. Mitigation: `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` should be tuned conservatively (overestimate block time → underestimate missed blocks → more cautious recovery triggers). +- **Blocks faster than `seconds_per_block`.** The batch-open check then + underestimates how many blocks an open batch spans, and a batch sealed by + time can open closer to the window than the check allowed. Tune the value + for the target chain's fastest sustained cadence. - **Operator misconfigures `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`.** Typo or copy-paste error pointing at the wrong chain's cadence. Operator-trust scope. -- **Significant host clock drift.** A sequencer host whose clock lags or leads the real-world by minutes per day could slowly desynchronize its danger estimates from reality. - -**Corollary for test design.** To deterministically exercise the wall-clock fallback, tests must maintain this coupling: when advancing the L1 block count, they should also advance (or simulate) the corresponding wall-clock interval. Our e2e harness does the reverse — it rewinds `l1_safe_head.synced_at_ms` to an older timestamp, which is semantically equivalent to advancing the wall clock. ### L1 chain semantics @@ -220,5 +231,5 @@ For each code path under review: ## Related documents -- [`../recovery/README.md`](../recovery/README.md) — recovery design, TLA+ formal verification +- [`../recovery/README.md`](../recovery/README.md) — batch landing, the void history, and outages - [`../../AGENTS.md`](../../AGENTS.md) — architecture map, conventions, and working rules diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index 40328edf..ae510f36 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -120,7 +120,7 @@ is decoded and fed to the machine before the next is fetched. | `tick` | One compare cycle; writes `last_tick.json` and `status.prom`. | 0 ok or idle, 1 warning, 2 divergence | | `status` | JSON on stdout: config, head, latched divergence, last tick. Read-only; fails on a missing state directory. | 0, or 1 | | `clear --block B --reason TEXT` | Archive the divergence latched at block B into `incidents/`. Refuses any other block, except for an unreadable marker, which has none. | 0, or 1 | -| `replay --from DIR --from-block A --to-block B --out DIR` | Re-derive canonical state from a stored machine into a new directory and print its digest. The machine must be a trusted start, like init's bootstrap; `--out` must not exist, and both paths are absolute. Reads the state directory's `config.json` and never writes it. | 0, or 1 | +| `replay --from DIR --from-block START --to-block B --out DIR` | Re-derive canonical state from a stored machine into a new directory and print its digest, or the input it stopped at. The machine must be a trusted start, like init's bootstrap; `--out` must not exist, and both paths are absolute. Reads the state directory's `config.json` and never writes it. | 0, a stop included, or 1 | The `sequencer-watchdog` wrapper takes a non-blocking `flock` on `$CARTESI_WATCHDOG_STATE_DIR/run.lock` for `init`, `tick`, and `clear`; @@ -135,7 +135,12 @@ block must be safe on the RPC; its input count comes from `getNumberOfInputs` at that block, which needs an RPC with state there (an archive node for old blocks); and when no input precedes the block, the machine must be the application's template (its root hash equals the on-chain -`getTemplateHash()`). +`getTemplateHash()`). `replay` takes its start count the same way, from +`getNumberOfInputs` at `--from-block`, and like `init` checks it against the +machine only when it is zero (the template check): `--from-block` must be the +block through which the machine consumed every input, as a checkpoint's name +states. A wrong block that gives a nonzero count silently re-feeds or skips +inputs. Exit 1 covers three failure classes, which the log line and `last_tick.json` name: `transient` (the L1 RPC or the sequencer's HTTP API; the next tick may @@ -195,11 +200,11 @@ state/ ``` A checkpoint's name carries its L1 block and the InputBox input count through -that block. Checkpoints are published with the emulator's `sync_stored` and -`rename_stored`, which fsync and rename atomically, so the head needs no -pointer file and a crash leaves either the old head or the new one. Older -checkpoints are removed after the new head is published; an interrupted -removal is finished by the next agreeing tick. JSON files are atomic but not +that block, each zero-padded to 20 digits. Checkpoints are published with the +emulator's `sync_stored` and `rename_stored`, which fsync and rename +atomically, so the head needs no pointer file and a crash leaves either the old +head or the new one. Older checkpoints are removed after the new head is +published; an interrupted removal is finished by the next agreeing tick. JSON files are atomic but not fsynced. An `incident/` whose `divergence.json` is missing or torn still latches (kind `unreadable_marker`); a lost `incident/` latches again on the next tick if the divergence persists; a lost tick record is rewritten. Of the @@ -211,6 +216,22 @@ running `init` again; a torn one (`config.json is not valid JSON`) means wiping the state directory first. The watchdog stores whole machines, never the sequencer's restore archives. +### Machines a recovery can start from + +The sequencer's `setup --base` ([recovery](../recovery/cockroach.md)) loads +only a machine that waits for an input, at an `RX_ACCEPTED` yield. Not every +machine the watchdog stores does: + +| Stored machine | Waits for an input | Why | +|---|---|---| +| `checkpoints/-/` | Always | `init` refuses a bootstrap machine that does not wait, and a tick publishes only a machine whose state it has just read at that yield. | +| `incident/canonical/` | Only for `state_mismatch` | A `canonical_machine_dead` incident keeps the machine at its fixed point. | +| `replay --out` | Only when the result carries `sha256` | A result with `stop` still exits 0 and leaves the stopped machine at `--out`. | + +A rejected last input leaves the machine waiting too, because the executor +restores the snapshot taken before it. `watchdog/tests/e2e.lua` reads each +stored machine's yield from its registers. + ## Divergence latch A divergence is latched first, and alone: the tick creates `incident/` and diff --git a/docs/watchdog/incident-runbook.md b/docs/watchdog/incident-runbook.md index 90e8c891..c10a18cc 100644 --- a/docs/watchdog/incident-runbook.md +++ b/docs/watchdog/incident-runbook.md @@ -3,8 +3,11 @@ What to do when the watchdog latches a divergence: `tick` exits 2 and `cartesi_watchdog_status{state="failed"}` is 1. Every step names the command that performs it, and each scenario has a drill that practices it before an -incident does. The [watchdog README](README.md) owns how the latch and its -evidence are produced. +incident does. Resolving **Sequencer wrong** is drilled on the devnet for the +rebuild and a toy-wallet client's realignment; clearing the latch around a +rebuild and realigning an application's own readers have no drill yet +([Drills](#drills)). The +[watchdog README](README.md) owns how the latch and its evidence are produced. ## What you have @@ -82,7 +85,8 @@ finish, so the stop can take as long as the download; a supervisor that kills the sequencer first leaves `sequencer_bytes_missing: transient: …`. Do not wait for the file: step 2's verdict comes from `replay`. To end the download now, kill the collecting watchdog tick. Preserve the sequencer's data -directory before any restart (cockroach recovery, step 1). +directory before anything is rebuilt +([before anything](../recovery/cockroach.md#before-anything)). ## 2. Find the faulty side @@ -92,17 +96,19 @@ Re-derive the canonical state at B from a trusted machine that predates the incident, independently of the watchdog's checkpoint chain: ```bash -sequencer-watchdog replay --from --from-block \ +sequencer-watchdog replay --from --from-block \ --to-block --out /tmp/replay- ``` Use the application's template machine with `--from-block 0` (`replay` checks it against the on-chain template hash), or an archived checkpoint whose block -you trust. `replay` needs an RPC with state at `A` (an archive node for old -blocks), absolute paths, and an `--out` that does not exist; it never writes -the state directory. Run it with a watchdog release on the deployment's -Cartesi Machine version: templates, checkpoints and `incident/canonical` load -only in the version that stored them +you trust. `` is the block its name states; `replay` checks it +against the machine only for the template +([commands](README.md#commands)). `replay` needs an RPC with state at +`` (an archive node for old blocks), absolute paths, and an +`--out` that does not exist; it never writes the state directory. Run it with a +watchdog release on the deployment's Cartesi Machine version: templates, +checkpoints and `incident/canonical` load only in the version that stored them ([upgrades](operator-deployment.md#upgrades)). | `replay`'s `sha256` equals | Conclusion | @@ -135,19 +141,29 @@ regression; the watchdog idles. ## 3. Resolve -**Sequencer wrong.** Fix the bug, then rebuild the sequencer with -[cockroach recovery](../recovery/cockroach.md) from a trusted canonical -checkpoint: `incident/canonical` at B (or step 2's `replay --out` directory -when `incident/canonical` is absent), or the head at P. Resume the sequencer, -then clear the latch: +**Sequencer wrong.** Make sure the old instance never runs again with the +submitter key ([restart policy](operator-deployment.md#sequencer-restart-policy)), +and fix the bug. Then rebuild the sequencer in a new data directory with +`setup --base`, following +[the operator's playbook](../recovery/cockroach.md#the-operators-playbook). +Its base is a trusted canonical machine at or below every client's trusted +count: `incident/canonical` at B (or step 2's `replay --out` directory when +`incident/canonical` is absent), the head at P, or an older checkpoint. Setup +loads only a machine that waits for an input, and not every stored machine +does ([which ones](README.md#machines-a-recovery-can-start-from)). Start +`run` on the new directory, then clear the latch: ```bash -sequencer-watchdog clear --block --reason "sequencer fixed: " +sequencer-watchdog clear --block --reason "sequencer rebuilt: " ``` The watchdog keeps its head at P: recovering the sequencer does not change -canonical history, so the next tick replays to the new accepted block and -compares. +canonical history. Until a batch of the new era is accepted, the sequencer +serves the era's baseline as its comparison checkpoint at the stop block, so +the next tick replays from P to the stop block and compares the baseline. A +mismatch there means the base machine was not canonical: rebuild again from a +trusted one. Realign affected clients at their own counts +([the client's playbook](../recovery/cockroach.md#the-clients-playbook)). **Watchdog wrong.** Copy `incident/` somewhere for the record, fix the cause (configuration, bootstrap machine, image), wipe the state directory, run @@ -170,19 +186,35 @@ it confirms that has not landed when `foreclose()` executes never will. On Kubernetes, scale the Deployment to zero, because Deployments restart regardless of exit code ([restart policy](operator-deployment.md#sequencer-restart-policy)). Never restart the sequencer for a foreclosed application: batch posts fail at -gas estimation, and every recovery cycle sends a few no-op flush transactions. +gas estimation on every attempt, and the open batch ages until the history is +void. ## Drills +Every drill with a test runs in CI: the watchdog's Lua unit tests +(`just test-watchdog` locally) in the Rust job, and +`just test-rollups-e2e` and `just test-watchdog-e2e` in the `rollups-e2e` job +([`ci.yml`](../../.github/workflows/ci.yml)). The devnet scenarios are in +[`test_cases.rs`](../../tests/e2e/src/test_cases.rs), the recovery ones in +[`recovery_round_trip.rs`](../../tests/e2e/src/recovery_round_trip.rs). + | Scenario | Drill | Practices | |---|---|---| -| Watchdog wrong (`state_mismatch`) | `watchdog_divergence_drill_test` in `just test-rollups-e2e` | init refusing a non-template image at genesis; a wrong non-genesis bootstrap latching; `status`; `clear` refusing another block; `replay` from the trusted image matching the sequencer; re-latching after a premature `clear`; re-`init` agreeing | +| Watchdog wrong (`state_mismatch`) | `watchdog_divergence_drill_test` in `just test-rollups-e2e` | init refusing a non-template image at genesis; a wrong image, advanced through every input up to the first input's block, bootstrapped there and latching; `status`; `clear` refusing another block; `replay` from the trusted image matching the sequencer; re-latching after a premature `clear`; re-`init` agreeing | | Canonical machine dead | `tick latches a dead canonical machine and replay reproduces the stop` in `just test-watchdog-e2e` | the latch at the stopping input, and `replay` reproducing the stop | | Sequencer wrong (`state_mismatch`) | `tick latches a mismatch with the canonical machine and a byte diff` in `just test-watchdog-e2e` | the evidence: canonical machine, both byte strings, and the first difference | +| Sequencer wrong: the rebuild from genesis | `recovery_from_genesis_round_trip_test` and `c_host_recovery_from_genesis_round_trip_test` in `just test-rollups-e2e` | the dry run; `setup` in a new directory with two directs queued at the stop block; `run`'s root frame draining them first, then a live op; a toy-wallet client refused with `ERA_CHANGED`, refused at a count past the rebuilt history end, adopting the new era at its count and checking its rows; a fresh watchdog agreeing with the era's baseline at the stop block, and another with the new era's first accepted batch | +| Sequencer wrong: the rebuild from a machine | `recovery_from_stored_machine_round_trip_test` and `c_host_recovery_from_stored_machine_round_trip_test` in `just test-rollups-e2e` | the same from a machine that `init` and `replay --out` stored at a block where an old-era direct is still queued, its `scheduler` report checked, and a client below that base refused with `BelowBase` | +| Sequencer wrong: setup refusing a set-up directory | `setup_refuses_a_set_up_data_directory_test` and `c_host_setup_refuses_a_set_up_data_directory_test` in `just test-rollups-e2e` | the dry run leaving the directory untouched, a second setup refused with exit 30, and `run` starting on the directory as it was | +| Sequencer wrong: the latch and an application's readers | none yet | clearing a latch around a rebuild, and realigning a reader other than the toy wallet | | Evidence lost to a full disk | `main tick keeps a divergence latched when its evidence cannot be kept` in `just test-watchdog` | the latch and exit 2 surviving a failed evidence item, which `evidence` names | | `inclusion_block_regressed` | `tick latches a regression below an agreed block` and `tick idles while the sequencer has not reached the bootstrap block` in `just test-watchdog` | the latch, and idling instead behind a never-agreed bootstrap | Before production, run the first drill once against the staging deployment -with the operators who will be paged: bootstrap a scratch watchdog state -directory from a wrong image at a block after the first input, and walk this -runbook to the re-`init`. +with the operators who will be paged: advance a wrong image through every +input up to a block past genesis (the drill uses the first input's block) with +`cartesi-machine --cmio-advance-state`, bootstrap a scratch watchdog state +directory from it at that block, and walk this runbook to the re-`init`. A +wrong image that skips those inputs latches `canonical_machine_dead`, not +`state_mismatch`: the guest halts on the first input it receives out of +InputBox order. diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index 156b6728..407c4b6d 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -37,9 +37,9 @@ curl -fsS "$CARTESI_WATCHDOG_SEQUENCER_URL/finalized_state/inclusion_block" curl -fsS "$CARTESI_WATCHDOG_SEQUENCER_URL/finalized_state/digest" ``` -Both answer JSON once the sequencer has a comparable checkpoint and 404 before -that: genesis is comparable at block zero, while a rebuilt baseline waits for -its next accepted batch +Both answer JSON once the sequencer has a comparable checkpoint, which a +completed setup always provides: the era's baseline at its stop block, until +an accepted batch replaces it ([snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison)). They answer 503 while the sequencer's own divergence marker is set. @@ -196,13 +196,11 @@ but checkpoints. A retained checkpoint is a trusted machine at a known block. It can bootstrap a new state directory ([initialize](#initialize)), serve as the trusted source for `replay` in the [runbook](incident-runbook.md#2-find-the-faulty-side), and -feed the canonical-to-native export of -[cockroach recovery](../recovery/cockroach.md#incident-playbook). It is not a -sequencer restore archive; that is `/finalized_snapshot` +be the base of `setup --base` in a +[cockroach recovery](../recovery/cockroach.md). Its name counts InputBox +inputs, not L2 transactions; `setup --base --dry-run` prints its L2 tx +count. It is not a sequencer restore archive; that is `/finalized_snapshot` ([recovery exports](../snapshots/lifecycle.md#http-and-recovery-exports)). -Complete the application's -[recovery readiness drill](../recovery/cockroach.md#recovery-readiness-before-deployment) -before production. ## Deployment facts @@ -373,14 +371,16 @@ checkpoint from a watchdog that already follows the deployment. ## Sequencer restart policy The sequencer's [exit codes](../../README.md#running) are its whole restart -contract: there is no database gate and no acknowledgement command. Standard -recovery is automatic on every boot, and a persistent terminal fault -re-detects fail-loud when the faulty state is next read, so the supervisor's +contract: there is no database gate and no acknowledgement command. A start +repairs nothing. A recorded divergence or a void history refuses every boot +([what the sequencer does](../recovery/README.md#what-the-sequencer-does)), +and any other persistent terminal fault re-detects fail-loud when the faulty +state is next read, so the supervisor's handling of 30 and SIGABRT (stop and page, do not auto-restart) is what bounds a crash loop. The final `sequencer exiting` line logs at ERROR for exits 30 and 1, and at -WARN for 10, 20 and 40, with `verdict` and `supervisor_action` fields; the +WARN for 20 and 40, with `verdict` and `supervisor_action` fields; the SIGABRT line says so. Alert on the exit status, not the log level. - **Supervisor recipes.** systemd can act on the code directly: @@ -390,28 +390,57 @@ SIGABRT line says so. Alert on the exit status, not the log level. re-detection windows, serving traffic in between. There, the crash-loop bound is your alerting, not the restart policy: page immediately on `lastState.terminated.exitCode == 30` (and on signal exits / 134). +- **Keeping a sequencer down.** Foreclosure, after which the InputBox refuses + the application's inputs, and cockroach recovery need an instance stopped + for good: disable its systemd unit, since `RestartPreventExitStatus` covers + only exit 30 and SIGABRT, or scale its Kubernetes Deployment to zero. Never + again run a command that signs with the submitter key, `run` or + `flush-mempool`, against its data directory. +- **Setup runs once per data directory, never on every boot.** It refuses a + directory that holds anything, a completed setup included, with exit 30, so + a setup unit or init container that runs before each `run` pages on the + second boot. Run it as a one-off Job or unit against a new directory. +- **Setup takes the submitter key**, from the same variables as `run`, because + it flushes the submitter's wallet; mount the key for the setup Job too. + `--dry-run` takes none. - **A fresh deployment needs a setup retry budget.** `setup` exits 20 until the L1 safe head reaches the application's deployment block, about one to two epochs on Ethereum (6.4–12.8 min); give a setup Job or unit a longer retry budget (a Kubernetes Job's default backoffLimit 6 allows about - 10.5 min). + 10.5 min). This and every other exit 20 from setup's checks before it + writes leave the directory empty, so a retry in place is safe. A setup that + fails after it begins writing leaves a directory to discard, whatever its + exit code, and a retry in place exits 30 + ([what setup does](../recovery/cockroach.md#what-setup-does)). +- **The sequencer binary links the Cartesi Machine emulator**, statically, to + read a base machine. The host needs no emulator install, but it needs the + C++ runtime the emulator was built against: Ubuntu 24.04, Debian 13 or newer, + with `libgomp1`, `libslirp0`, `libstdc++6` and `ca-certificates`. Each + release archive's `RUNNING.md` lists them. - **After an unclean death (OOM, node reboot, SIGKILL) no action is needed**: the next start re-derives everything from facts. For postmortems, the `terminal_faults` table records the cause of every command that returned through its bracket with a terminal verdict (best-effort, append-only, traveling with the data directory — - `SELECT * FROM terminal_faults ORDER BY fault_id DESC`; each `run` boot - logs the latest row at WARN unless a later completed setup resolved it). Any death that did not return + `SELECT * FROM terminal_faults ORDER BY fault_id DESC`, with a `sqlite3` + CLI of 3.37 or later, since the tables are STRICT; each `run` boot + logs the latest row at WARN, with its age). Any death that did not return through the bracket — SIGKILL, OOM, a node reboot, a terminal runtime abort, - a controller panic — leaves only the process logs. + a controller panic — leaves only the process logs. A recorded divergence + or a void history leaves no row either: the database holds the fact, and + every `run` on the directory refuses with it. - **Untrustworthy local state requires manual recovery.** The reader's content-identity divergence marker freezes the acceptance frontier and - blocks admission ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). + refuses every start ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). A watchdog state mismatch independently signals application-state disagreement ([incident runbook](incident-runbook.md)). Diagnose the cause - and follow the [incident playbook](../recovery/cockroach.md#incident-playbook) - for a fresh-directory rebuild when needed; standard recovery does not repair - bugs. + and follow the [incident runbook](incident-runbook.md#3-resolve), which + rebuilds in a new directory with `setup --base`; the sequencer repairs + nothing by itself. +- **Batches not landing is an outage to fix inside the window.** Keep `run` + up and follow [the outage playbook](../recovery/README.md#when-batches-are-not-landing); + once the history is void, every start refuses + ([when the sequencer has stopped for good](../recovery/README.md#when-the-sequencer-has-stopped-for-good)). ## Troubleshooting (live deployments) @@ -436,9 +465,9 @@ and failure class, as in | ` was initialized for another deployment or state source; wipe it to re-initialize` | The `init` environment names another chain, application, or state source than the directory holds | | `no input precedes block , so the machine at must be the application's template, but its root hash differs from the on-chain template hash` | The image is not this application's template: another chain's image, another build, or another application | | `L1 RPC: eth_call: …` during `init` | The RPC has no state at the bootstrap block; use an archive node | -| `sequencer /finalized_state/inclusion_block: HTTP 404` | No comparable checkpoint yet, or the URL reaches the wrong listener | +| `sequencer /finalized_state/inclusion_block: HTTP 404` | The URL reaches the wrong listener: a set-up sequencer always has a comparable checkpoint | | `sequencer /finalized_state/digest: HTTP 404` while `inclusion_block` answers | A proxy does not forward the digest route, or the sequencer comes from another release | -| `sequencer /finalized_state/inclusion_block: HTTP 503`, with `canonical divergence prevents accepted checkpoint selection` | The sequencer's own divergence marker is set: a sequencer incident for [cockroach recovery](../recovery/cockroach.md#incident-playbook) | +| `sequencer /finalized_state/inclusion_block: HTTP 503`, with `canonical divergence prevents accepted checkpoint selection` | The sequencer's own divergence marker is set: a sequencer incident for [cockroach recovery](../recovery/cockroach.md#the-operators-playbook) | | `L1 RPC safe head is behind target block ` | The RPC lags the sequencer's L1 view; the next tick retries | | `InputBox counts inputs at block but the scan reached index `, or `InputBox index where was expected: the L1 view is incomplete` | The provider returned incomplete logs, so switch providers; if it persists across them, a safe block below the head may have been replaced ([L1 chain semantics](../threat-model/README.md#l1-chain-semantics)), so confirm with `replay` from a trusted machine at a finalized block, then wipe the state directory and `init` from one | | `the L1 RPC does not serve chain ` | The RPC endpoint belongs to another chain | diff --git a/examples/app-core/src/application/mod.rs b/examples/app-core/src/application/mod.rs index 53f56e93..f7781620 100644 --- a/examples/app-core/src/application/mod.rs +++ b/examples/app-core/src/application/mod.rs @@ -7,4 +7,7 @@ mod wallet; pub use method::{MAX_METHOD_PAYLOAD_BYTES, Method, Transfer, Withdrawal}; pub use notice::{DepositNotice, TransferNotice}; -pub use wallet::{DEVNET_SEQUENCER_ADDRESS, SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; +pub use wallet::{ + DEVNET_SEQUENCER_ADDRESS, SEPOLIA_SEQUENCER_ADDRESS, WALLET_SCHEDULER_LIMITS, WalletApp, + WalletConfig, +}; diff --git a/examples/app-core/src/application/wallet.rs b/examples/app-core/src/application/wallet.rs index cb7a79a2..de9e0a15 100644 --- a/examples/app-core/src/application/wallet.rs +++ b/examples/app-core/src/application/wallet.rs @@ -15,13 +15,23 @@ use super::MAX_METHOD_PAYLOAD_BYTES as WALLET_MAX_METHOD_PAYLOAD_BYTES; use super::Method; use super::{DepositNotice, TransferNotice}; use sequencer_core::application::{ - AppError, AppOutput, AppOutputs, Application, ApplicationProgress, CanonicalState, - InvalidReason, ValidationOutcome, + Admission, AppError, AppOutput, AppOutputs, Application, CanonicalState, PolicyRefusal, + SchedulerLimits, }; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::ValidUserOp; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; +/// The wallet template's scheduler constants. Provisional: the window is +/// about 24 h of L1 blocks; a deposit's payload is 72 bytes plus its exec-layer +/// data, so 1 KiB leaves that data room, and a full fridge then holds at most +/// 16 MiB of payloads. +pub const WALLET_SCHEDULER_LIMITS: SchedulerLimits = SchedulerLimits { + max_wait_blocks: 7200, + fridge_bound: 16_384, + max_direct_payload_bytes: 1024, +}; + #[derive(Debug, Clone, Copy)] pub struct WalletConfig { pub erc20_portal_address: Address, @@ -61,7 +71,7 @@ pub struct WalletApp { // canonical entry order. balances: BTreeMap, nonces: BTreeMap, - execution_progress: ApplicationProgress, + l2_tx_count: L2TxCount, } /// Rollups-contracts v3.0.0-alpha.10 Erc20Portal. The contracts deploy at @@ -92,38 +102,23 @@ impl WalletApp { config, balances: BTreeMap::new(), nonces: BTreeMap::new(), - execution_progress: ApplicationProgress::default(), + l2_tx_count: L2TxCount::ZERO, } } /// Reconstruct from decoded snapshot parts. Used by `crate::wallet_snapshot::decode`. - /// - /// The progress pair comes from untrusted dump bytes, so an incoherent - /// pair is a typed decode error like every other corrupt-snapshot case — - /// not a panic escaping `from_dump`'s `Result`. pub(crate) fn from_snapshot_parts( config: WalletConfig, balances: BTreeMap, nonces: BTreeMap, - executed_input_count: u64, - last_executed_safe_block: u64, - ) -> Result { - let execution_progress = ApplicationProgress::try_new( - ExecutedInputCount::new(executed_input_count), - last_executed_safe_block, - ) - .ok_or_else(|| AppError::Internal { - reason: format!( - "snapshot progress is incoherent: zero executed inputs with \ - nonzero safe-block clock {last_executed_safe_block}" - ), - })?; - Ok(Self { + l2_tx_count: u64, + ) -> Self { + Self { config, balances, nonces, - execution_progress, - }) + l2_tx_count: L2TxCount::new(l2_tx_count), + } } // Accessors for the canonical snapshot encoder (`crate::wallet_snapshot`). @@ -150,21 +145,16 @@ impl WalletApp { } #[cfg(test)] - pub(crate) fn set_executed_input_count(&mut self, count: u64) { - self.execution_progress = ApplicationProgress::try_new( - ExecutedInputCount::new(count), - self.execution_progress.last_executed_safe_block(), - ) - .expect("coherent progress"); + pub(crate) fn set_l2_tx_count(&mut self, count: u64) { + self.l2_tx_count = L2TxCount::new(count); } fn balance_of(&self, addr: &Address) -> U256 { *self.balances.get(addr).unwrap_or(&U256::ZERO) } - // Wallet-specific read queries (not on the Application trait — the - // sequencer never asks: it derives nonces from persisted ops under the - // contract's nonce rule, and app-specific query surface belongs to the app). + // Wallet-specific read queries for tests and tooling. The sequencer reads + // the nonce through `Application::user_nonce`, which answers the same. pub fn current_user_nonce(&self, sender: Address) -> u32 { self.expected_nonce(&sender) } @@ -191,6 +181,8 @@ impl WalletApp { self.nonces.get(addr).copied().unwrap_or(0) } + /// The shared executor skips an op whose sender is at `u32::MAX` before it + /// reaches the wallet, so overflow here is an impossible state. fn bump_nonce(&mut self, addr: Address) { let next = self .expected_nonce(&addr) @@ -207,61 +199,25 @@ impl Default for WalletApp { } impl Application for WalletApp { - fn max_method_payload_bytes() -> usize { - WALLET_MAX_METHOD_PAYLOAD_BYTES + fn user_nonce(&self, sender: Address) -> Result { + Ok(self.expected_nonce(&sender)) } - fn validate_user_op( - &self, - sender: Address, - user_op: &UserOp, - current_fee: u16, - ) -> Result { - let expected_nonce = self.expected_nonce(&sender); - if user_op.nonce != expected_nonce { - return Ok(ValidationOutcome::Reject(InvalidReason::InvalidNonce { - expected: expected_nonce, - got: user_op.nonce, - })); - } - - // max_fee < current_fee is already checked by the free function - // validate_and_execute_user_op. No need to repeat here. - - let fee_cost = sequencer_core::fee::fee_to_linear(current_fee); + /// Charges what the sender has, up to the fee, and credits the sequencer + /// exactly that, so an underfunded op neither fails nor mints. The method + /// then runs on the remaining balance: one code path for every op. + fn apply_user_op(&mut self, op: &UserOpTx) -> Result { + let sender = op.sender; + let fee_cost = sequencer_core::fee::fee_to_linear(op.fee); let balance = self.balance_of(&sender); - if balance < fee_cost { - return Ok(ValidationOutcome::Reject( - InvalidReason::InsufficientFeeBalance { - required: fee_cost, - available: balance, - }, - )); - } - - Ok(ValidationOutcome::Accept) - } - - fn apply_valid_user_op( - &mut self, - user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - let sender = user_op.sender; - let fee_cost = sequencer_core::fee::fee_to_linear(user_op.fee); - let balance = self.balance_of(&sender); - if balance < fee_cost { - return Err(AppError::Internal { - reason: "validated user op cannot pay fee".to_string(), - }); - } + let charged = balance.min(fee_cost); self.bump_nonce(sender); - self.balances.insert(sender, balance - fee_cost); - self.credit(self.config.sequencer_address, fee_cost); + self.balances.insert(sender, balance - charged); + self.credit(self.config.sequencer_address, charged); let mut outputs = Vec::new(); - let method = Method::from_ssz_bytes(user_op.data.as_slice()).ok(); + let method = Method::from_ssz_bytes(op.data.as_slice()).ok(); match method.as_ref() { Some(Method::Transfer(transfer)) if self.debit_if_possible(sender, transfer.amount) => { self.credit(transfer.to, transfer.amount); @@ -290,17 +246,14 @@ impl Application for WalletApp { _ => {} } - self.execution_progress.advance(safe_block); + self.l2_tx_count.advance(); Ok(outputs) } - fn apply_direct_input( - &mut self, - input: &sequencer_core::l2_tx::DirectInput, - ) -> Result { + fn apply_direct(&mut self, direct: &Direct) -> Result { let mut outputs = Vec::new(); - let deposit = (input.sender == self.config.erc20_portal_address) - .then(|| Erc20Deposit::decode(&input.payload)) + let deposit = (direct.sender == self.config.erc20_portal_address) + .then(|| Erc20Deposit::decode(&direct.payload)) .transpose(); match deposit { Ok(Some(deposit)) => { @@ -316,10 +269,11 @@ impl Application for WalletApp { )); } else { warn!( - portal = %input.sender, + portal = %direct.sender, token = %deposit.token, sender = %deposit.sender, - block_number = input.block_number, + input_index = direct.input_index, + block_number = direct.block, "ignoring unsupported ERC-20 deposit token" ); } @@ -327,20 +281,48 @@ impl Application for WalletApp { Ok(None) => {} Err(reason) => { error!( - portal = %input.sender, - block_number = input.block_number, + portal = %direct.sender, + input_index = direct.input_index, + block_number = direct.block, error = %reason, "ignoring malformed trusted ERC-20 deposit payload" ); } } - self.execution_progress.advance(input.block_number); + self.l2_tx_count.advance(); Ok(outputs) } - fn progress(&self) -> ApplicationProgress { - self.execution_progress + fn l2_tx_count(&self) -> L2TxCount { + self.l2_tx_count + } + + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } + + fn scheduler_limits() -> SchedulerLimits { + WALLET_SCHEDULER_LIMITS + } + + /// Refuses an op the sender cannot pay for in full: it would pay the + /// operator less than the L1 data it costs. The nonce is not checked here. + fn admit_user_op( + &self, + sender: Address, + _op: &UserOp, + fee: u16, + ) -> Result { + let required = sequencer_core::fee::fee_to_linear(fee); + let available = self.balance_of(&sender); + if available < required { + return Ok(Admission::Refuse(PolicyRefusal::Underfunded { + required, + available, + })); + } + Ok(Admission::Admit) } fn from_dump(prefix: &Path) -> Result { @@ -379,6 +361,23 @@ impl Application for WalletApp { fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } + + /// The wallet's bytes hold its count, so a count that disagrees with the + /// machine's means a corrupt or misread export, never a state to install. + fn from_canonical_state(state_file: &Path, l2_tx_count: L2TxCount) -> Result { + let bytes = std::fs::read(state_file)?; + let app = crate::wallet_snapshot::decode(&bytes)?; + if app.l2_tx_count != l2_tx_count { + return Err(AppError::Internal { + reason: format!( + "canonical state holds L2 tx count {}, but the machine reports {}", + app.l2_tx_count.get(), + l2_tx_count.get() + ), + }); + } + Ok(app) + } } impl CanonicalState for WalletApp { @@ -396,23 +395,58 @@ mod tests { use rollups_types::alloy_sol_types::SolCall; use rollups_types::{Erc20Deposit, Erc20Transfer}; - use super::{ApplicationProgress, ExecutedInputCount, WalletApp, WalletConfig}; + use super::{L2TxCount, WalletApp, WalletConfig}; use crate::application::{DepositNotice, Transfer, TransferNotice, Withdrawal}; use sequencer_core::application::{ - AppError, AppOutput, Application, CanonicalState, InvalidReason, ValidationOutcome, + Admission, AppError, AppOutput, AppOutputs, Application, CanonicalState, Executed, + PolicyRefusal, Refusal, SkipReason, check_live_user_op, execute, }; - use sequencer_core::application::{ - ExecutionOutcome, execute_direct_input, execute_valid_user_op, validate_and_execute_user_op, - }; - use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; + use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use sequencer_core::user_op::UserOp; + fn user_tx(sender: Address, nonce: u32, fee: u16, data: Vec) -> L2Tx { + L2Tx::UserOp(UserOpTx { + sender, + nonce, + fee, + block: 123, + data, + }) + } + + fn direct_tx(sender: Address, payload: Vec) -> L2Tx { + L2Tx::Direct(Direct { + input_index: 7, + sender, + block: 123, + timestamp: 1_700_000_000, + payload, + }) + } + + /// Run `tx` through the shared executor and return its outputs. + fn executed(app: &mut WalletApp, tx: &L2Tx) -> AppOutputs { + match execute(app, tx).expect("execute") { + Executed::Done { outputs, .. } => outputs, + Executed::Skipped(reason) => panic!("expected execution, skipped: {reason}"), + } + } + + fn total_supply(app: &WalletApp) -> U256 { + app.balances.values().copied().sum() + } + + fn transfer_data(amount: u64, to: Address) -> Vec { + ssz::Encode::as_ssz_bytes(&super::Method::Transfer(Transfer { + amount: U256::from(amount), + to, + })) + } + #[test] fn sepolia_dedicated_submitter_is_the_batch_sender_and_fee_recipient() { use sequencer_core::batch::Batch; - use sequencer_core::scheduler::{ - ProcessOutcome, Scheduler, SchedulerConfig, SchedulerInput, - }; + use sequencer_core::scheduler::{InputOutcome, L1Input, Scheduler, SchedulerConfig}; use ssz::Encode; let submitter = address!("0x79b88d2f598d8D001832D69f5201332A3f1e14c8"); @@ -421,14 +455,16 @@ mod tests { assert_eq!(config.sequencer_address, submitter); assert_eq!(WalletConfig::default().sequencer_address, submitter); - let mut scheduler = Scheduler::new( - WalletApp::new(config), - SchedulerConfig::new(super::SEPOLIA_SEQUENCER_ADDRESS), - ); - let input = |sender| SchedulerInput { + let mut scheduler = Scheduler::genesis(SchedulerConfig::for_application::( + super::SEPOLIA_SEQUENCER_ADDRESS, + )); + let input = |index, sender| L1Input { + index, sender, - inclusion_block: 10, - domain: sequencer_core::build_input_domain(11155111, Address::ZERO), + block: 10, + timestamp: 0, + chain_id: 11155111, + app_contract: Address::ZERO, payload: Batch { nonce: 0, frames: Vec::new(), @@ -436,17 +472,17 @@ mod tests { .as_ssz_bytes(), }; - let previous = scheduler.process_input(input(previous_submitter)).unwrap(); - assert_eq!(previous.outcome, ProcessOutcome::DirectEnqueued); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); + let previous = scheduler.feed(input(0, previous_submitter)); + assert_eq!(previous, InputOutcome::DirectQueued); + assert_eq!(scheduler.state().next_batch_nonce, 0); - let current = scheduler.process_input(input(submitter)).unwrap(); - assert_eq!(current.outcome, ProcessOutcome::BatchExecuted); - assert_eq!(scheduler.next_expected_batch_nonce(), 1); + let current = scheduler.feed(input(1, submitter)); + assert_eq!(current, InputOutcome::BatchAccepted); + assert_eq!(scheduler.state().next_batch_nonce, 1); } #[test] - fn validation_is_repeatable_and_preserves_the_entire_state() { + fn admission_is_repeatable_and_preserves_state() { let mut app = WalletApp::default(); let sender = Address::from_slice(&[0x11; 20]); app.balances.insert(sender, U256::from(10_u64)); @@ -458,29 +494,93 @@ mod tests { }; for _ in 0..2 { - assert_eq!( - app.validate_user_op(sender, &user_op, 0).unwrap(), - ValidationOutcome::Accept - ); - user_op.nonce = 1; - assert_eq!( - app.validate_user_op(sender, &user_op, 0).unwrap(), - ValidationOutcome::Reject(InvalidReason::InvalidNonce { - expected: 0, - got: 1 - }) - ); - user_op.nonce = 0; + // Admission ignores the nonce: the executor's rule comes first. + for nonce in [0, 1] { + user_op.nonce = nonce; + assert_eq!( + app.admit_user_op(sender, &user_op, 0).unwrap(), + Admission::Admit + ); + } assert!(matches!( - app.validate_user_op(Address::ZERO, &user_op, 0).unwrap(), - ValidationOutcome::Reject(InvalidReason::InsufficientFeeBalance { .. }) + app.admit_user_op(Address::ZERO, &user_op, 0).unwrap(), + Admission::Refuse(PolicyRefusal::Underfunded { .. }) )); + assert_eq!(app.user_nonce(sender).unwrap(), 0); assert_eq!(app.canonical_snapshot_bytes().unwrap(), before); } } #[test] - fn execute_valid_user_op_charges_current_fee() { + fn admission_refuses_an_underfunded_op_with_the_served_text() { + let mut app = WalletApp::default(); + let sender = Address::from_slice(&[0x11; 20]); + let fee: u16 = 1000; + let required = sequencer_core::fee::fee_to_linear(fee); + let available = required - U256::from(1_u64); + app.balances.insert(sender, available); + let user_op = UserOp { + nonce: 0, + max_fee: fee, + data: Vec::new().into(), + }; + + let underfunded = PolicyRefusal::Underfunded { + required, + available, + }; + assert_eq!( + app.admit_user_op(sender, &user_op, fee).unwrap(), + Admission::Refuse(underfunded.clone()) + ); + let refusal = check_live_user_op(&app, sender, &user_op, fee).unwrap(); + assert_eq!(refusal, Some(Refusal::Policy(underfunded))); + assert_eq!( + refusal.unwrap().to_string(), + format!("insufficient balance for fee: required {required}, available {available}") + ); + + app.balances.insert(sender, required); + assert_eq!( + check_live_user_op(&app, sender, &user_op, fee).unwrap(), + None + ); + } + + #[test] + fn user_nonce_reports_next_nonce() { + let mut app = WalletApp::default(); + let sender = Address::from_slice(&[0x11; 20]); + let other = Address::from_slice(&[0x22; 20]); + assert_eq!(app.user_nonce(sender).unwrap(), 0); + + for expected in 1..=3 { + executed(&mut app, &user_tx(sender, expected - 1, 0, Vec::new())); + assert_eq!(app.user_nonce(sender).unwrap(), expected); + assert_eq!(app.current_user_nonce(sender), expected); + } + assert_eq!(app.user_nonce(other).unwrap(), 0); + } + + #[test] + fn wrong_nonce_is_skipped_without_state_change() { + let mut app = WalletApp::default(); + let sender = Address::from_slice(&[0x11; 20]); + app.balances.insert(sender, U256::from(10_000_u64)); + let before = app.canonical_snapshot_bytes().unwrap(); + + assert_eq!( + execute(&mut app, &user_tx(sender, 1, 100, Vec::new())).unwrap(), + Executed::Skipped(SkipReason::WrongNonce { + expected: 0, + got: 1 + }) + ); + assert_eq!(app.canonical_snapshot_bytes().unwrap(), before); + } + + #[test] + fn execute_user_op_charges_current_fee() { let mut app = WalletApp::new(WalletConfig::default()); let sender = Address::from_slice(&[0x22; 20]); let initial_balance = U256::from(1000_u64); @@ -488,21 +588,91 @@ mod tests { // fee exponent 100 → fee_to_linear(100) ≈ 2 ((129/128)^100 ≈ 2.17, truncated) let fee_exponent: u16 = 100; - let valid = ValidUserOp { - sender, - fee: fee_exponent, - data: Vec::new(), - }; let gas_cost = sequencer_core::fee::fee_to_linear(fee_exponent); - let outputs = execute_valid_user_op(&mut app, &valid, 123) - .expect("execute valid op") - .outputs; + let outputs = executed(&mut app, &user_tx(sender, 0, fee_exponent, Vec::new())); assert_eq!(app.current_user_nonce(sender), 1); assert_eq!(app.current_user_balance(sender), initial_balance - gas_cost); - assert_eq!(app.executed_input_count().get(), 1); - assert_eq!(app.last_executed_safe_block(), 123); + assert_eq!(app.l2_tx_count().get(), 1); + assert!(outputs.is_empty()); + } + + #[test] + fn underfunded_op_executes_charging_what_the_sender_has() { + let sequencer = address!("0x9999999999999999999999999999999999999999"); + let mut app = WalletApp::new(WalletConfig { + sequencer_address: sequencer, + ..WalletConfig::default() + }); + let sender = address!("0x1111111111111111111111111111111111111111"); + let recipient = address!("0x2222222222222222222222222222222222222222"); + let fee: u16 = 1000; + let fee_cost = sequencer_core::fee::fee_to_linear(fee); + let balance = fee_cost - U256::from(1_u64); + app.balances.insert(sender, balance); + app.balances.insert(sequencer, U256::from(5_u64)); + + let outputs = executed( + &mut app, + &user_tx(sender, 0, fee, transfer_data(1, recipient)), + ); + + assert!(outputs.is_empty(), "nothing is left to transfer"); + assert_eq!(app.current_user_balance(sender), U256::ZERO); + assert_eq!( + app.current_user_balance(sequencer), + U256::from(5_u64) + balance, + "the sequencer gains exactly what was charged" + ); + assert_eq!(app.current_user_balance(recipient), U256::ZERO); + assert_eq!(app.current_user_nonce(sender), 1); + assert_eq!(app.l2_tx_count().get(), 1); + } + + #[test] + fn underfunded_ops_conserve_total_supply() { + let sequencer = address!("0x9999999999999999999999999999999999999999"); + let mut app = WalletApp::new(WalletConfig { + sequencer_address: sequencer, + ..WalletConfig::default() + }); + let sender = address!("0x1111111111111111111111111111111111111111"); + let bystander = address!("0x3333333333333333333333333333333333333333"); + let fee: u16 = 1000; + app.balances.insert(sender, U256::from(7_u64)); + app.balances.insert(bystander, U256::from(11_u64)); + app.balances.insert(sequencer, U256::from(13_u64)); + let supply = total_supply(&app); + + // The sequencer's own underfunded op pays itself: no net change. The + // transfers exceed every balance, so only the fees move. + for (from, nonce) in [(sender, 0), (sender, 1), (sequencer, 0)] { + let data = transfer_data(1_000_000, bystander); + executed(&mut app, &user_tx(from, nonce, fee, data)); + assert_eq!(total_supply(&app), supply); + } + assert_eq!(app.current_user_balance(sender), U256::ZERO); + assert_eq!(app.current_user_balance(bystander), U256::from(11_u64)); + assert_eq!(app.current_user_balance(sequencer), U256::from(20_u64)); + assert_eq!(app.l2_tx_count().get(), 3); + } + + #[test] + fn zero_balance_op_charges_nothing_and_counts() { + let sequencer = address!("0x9999999999999999999999999999999999999999"); + let mut app = WalletApp::new(WalletConfig { + sequencer_address: sequencer, + ..WalletConfig::default() + }); + let sender = address!("0x1111111111111111111111111111111111111111"); + + let outputs = executed(&mut app, &user_tx(sender, 0, 1000, Vec::new())); + assert!(outputs.is_empty()); + assert_eq!(app.current_user_balance(sender), U256::ZERO); + assert_eq!(app.current_user_balance(sequencer), U256::ZERO); + assert_eq!(app.current_user_nonce(sender), 1); + assert_eq!(app.l2_tx_count().get(), 1); } fn encode_erc20_deposit_payload(token: Address, sender: Address, value: U256) -> Vec { @@ -521,26 +691,23 @@ mod tests { let nested_sender = address!("0x7777777777777777777777777777777777777777"); let before = app.current_user_balance(nested_sender); - let outputs = execute_direct_input( + let outputs = executed( &mut app, - &DirectInput { - sender: super::SEPOLIA_ERC20_PORTAL_ADDRESS, - block_number: 123, - payload: encode_erc20_deposit_payload( + &direct_tx( + super::SEPOLIA_ERC20_PORTAL_ADDRESS, + encode_erc20_deposit_payload( super::SEPOLIA_USDC_ADDRESS, nested_sender, U256::from(250_u64), ), - }, - ) - .expect("execute deposit direct input") - .outputs; + ), + ); assert_eq!( app.current_user_balance(nested_sender), before + U256::from(250_u64) ); - assert_eq!(app.executed_input_count().get(), 1); + assert_eq!(app.l2_tx_count().get(), 1); assert_eq!(outputs.len(), 1); match &outputs[0] { AppOutput::Notice(payload) => { @@ -559,23 +726,20 @@ mod tests { let nested_sender = address!("0x7777777777777777777777777777777777777777"); let before = app.current_user_balance(nested_sender); - let outputs = execute_direct_input( + let outputs = executed( &mut app, - &DirectInput { - sender: address!("0x3333333333333333333333333333333333333333"), - block_number: 123, - payload: encode_erc20_deposit_payload( + &direct_tx( + address!("0x3333333333333333333333333333333333333333"), + encode_erc20_deposit_payload( super::SEPOLIA_USDC_ADDRESS, nested_sender, U256::from(250_u64), ), - }, - ) - .expect("execute non-portal direct input") - .outputs; + ), + ); assert_eq!(app.current_user_balance(nested_sender), before); - assert_eq!(app.executed_input_count().get(), 1); + assert_eq!(app.l2_tx_count().get(), 1); assert!(outputs.is_empty()); } @@ -586,23 +750,16 @@ mod tests { let unsupported_token = address!("0x9999999999999999999999999999999999999999"); let before = app.current_user_balance(nested_sender); - let outputs = execute_direct_input( + let outputs = executed( &mut app, - &DirectInput { - sender: super::SEPOLIA_ERC20_PORTAL_ADDRESS, - block_number: 123, - payload: encode_erc20_deposit_payload( - unsupported_token, - nested_sender, - U256::from(250_u64), - ), - }, - ) - .expect("unsupported token should be ignored") - .outputs; + &direct_tx( + super::SEPOLIA_ERC20_PORTAL_ADDRESS, + encode_erc20_deposit_payload(unsupported_token, nested_sender, U256::from(250_u64)), + ), + ); assert_eq!(app.current_user_balance(nested_sender), before); - assert_eq!(app.executed_input_count().get(), 1); + assert_eq!(app.l2_tx_count().get(), 1); assert!(outputs.is_empty()); } @@ -610,19 +767,12 @@ mod tests { fn malformed_trusted_portal_deposit_is_a_no_op() { let mut app = WalletApp::new(WalletConfig::default()); - let outputs = execute_direct_input( + let outputs = executed( &mut app, - &DirectInput { - sender: super::SEPOLIA_ERC20_PORTAL_ADDRESS, - block_number: 123, - payload: vec![0xaa; 10], - }, - ) - .expect("malformed trusted portal payload should be ignored") - .outputs; - - assert_eq!(app.executed_input_count().get(), 1); - assert_eq!(app.last_executed_safe_block(), 123); + &direct_tx(super::SEPOLIA_ERC20_PORTAL_ADDRESS, vec![0xaa; 10]), + ); + + assert_eq!(app.l2_tx_count().get(), 1); assert!(outputs.is_empty()); } @@ -636,18 +786,10 @@ mod tests { // fee exponent 10 → fee_to_linear(10) = 1 ((129/128)^10 ≈ 1.08, truncated) let fee_exponent: u16 = 10; let gas_cost = sequencer_core::fee::fee_to_linear(fee_exponent); - let valid = ValidUserOp { - sender, - fee: fee_exponent, - data: ssz::Encode::as_ssz_bytes(&super::Method::Transfer(Transfer { - amount: U256::from(123_u64), - to: recipient, - })), - }; - - let outputs = execute_valid_user_op(&mut app, &valid, 0) - .expect("execute transfer") - .outputs; + let outputs = executed( + &mut app, + &user_tx(sender, 0, fee_exponent, transfer_data(123, recipient)), + ); assert_eq!( app.current_user_balance(sender), @@ -674,17 +816,11 @@ mod tests { let fee_exponent: u16 = 10; let gas_cost = sequencer_core::fee::fee_to_linear(fee_exponent); - let valid = ValidUserOp { - sender, - fee: fee_exponent, - data: ssz::Encode::as_ssz_bytes(&super::Method::Withdrawal(Withdrawal { - amount: U256::from(123_u64), - })), - }; + let data = ssz::Encode::as_ssz_bytes(&super::Method::Withdrawal(Withdrawal { + amount: U256::from(123_u64), + })); - let outputs = execute_valid_user_op(&mut app, &valid, 0) - .expect("execute withdrawal") - .outputs; + let outputs = executed(&mut app, &user_tx(sender, 0, fee_exponent, data)); assert_eq!( app.current_user_balance(sender), @@ -709,7 +845,7 @@ mod tests { } #[test] - fn business_failures_after_fee_validation_charge_fee_and_consume_nonce() { + fn business_failures_after_fee_admission_charge_fee_and_consume_nonce() { let sequencer = address!("0x9999999999999999999999999999999999999999"); let mut app = WalletApp::new(WalletConfig { sequencer_address: sequencer, @@ -721,22 +857,21 @@ mod tests { app.balances.insert(sender, funded); let fee: u16 = 10; let fee_cost = sequencer_core::fee::fee_to_linear(fee); - let too_much = U256::from(1_000_u64); + let too_much = 1_000_u64; // Selector 2 was the retired `Deposit { amount, to }` method, laid out // like `Transfer`; an op shaped like it must not mint to `to`. let mut retired_deposit = vec![2]; retired_deposit.extend(ssz::Encode::as_ssz_bytes(&Transfer { - amount: too_much, + amount: U256::from(too_much), to: recipient, })); let mut replay = app.clone(); for (i, data) in [ - ssz::Encode::as_ssz_bytes(&super::Method::Transfer(Transfer { - amount: too_much, - to: recipient, + transfer_data(too_much, recipient), + ssz::Encode::as_ssz_bytes(&super::Method::Withdrawal(Withdrawal { + amount: U256::from(too_much), })), - ssz::Encode::as_ssz_bytes(&super::Method::Withdrawal(Withdrawal { amount: too_much })), retired_deposit, ] .into_iter() @@ -749,25 +884,27 @@ mod tests { max_fee: fee, data: data.clone().into(), }; - let ExecutionOutcome::Included(live) = - validate_and_execute_user_op(&mut app, sender, &user_op, fee, safe_block).unwrap() - else { - panic!("a fee-funded op is included even when its method fails"); - }; - // Restart catch-up replays the persisted op without validation. - let replayed = - execute_valid_user_op(&mut replay, &ValidUserOp { sender, fee, data }, safe_block) - .unwrap(); + // The lane's sequence: the pre-check admits it, then it executes. + assert_eq!( + check_live_user_op(&app, sender, &user_op, fee).unwrap(), + None + ); + let tx = L2Tx::UserOp(UserOpTx::new(sender, &user_op, fee, safe_block)); + let live = execute(&mut app, &tx).unwrap(); + assert!( + matches!(&live, Executed::Done { outputs, .. } if outputs.is_empty()), + "a fee-funded op is included even when its method fails" + ); + // Restart catch-up replays the persisted op without admission. + let replayed = execute(&mut replay, &tx).unwrap(); assert_eq!(replayed, live); - assert!(live.outputs.is_empty()); let charged = fee_cost * U256::from(nonce + 1); assert_eq!(app.current_user_nonce(sender), nonce + 1); assert_eq!(app.current_user_balance(sender), funded - charged); assert_eq!(app.current_user_balance(sequencer), charged); assert_eq!(app.current_user_balance(recipient), U256::ZERO); - assert_eq!(app.executed_input_count().get(), u64::from(nonce) + 1); - assert_eq!(app.last_executed_safe_block(), safe_block); + assert_eq!(app.l2_tx_count().get(), u64::from(nonce) + 1); } assert_eq!( replay.canonical_snapshot_bytes().unwrap(), @@ -788,12 +925,7 @@ mod tests { let fee_exponent: u16 = 100; let gas_cost = sequencer_core::fee::fee_to_linear(fee_exponent); - let valid = ValidUserOp { - sender, - fee: fee_exponent, - data: Vec::new(), - }; - execute_valid_user_op(&mut app, &valid, 0).expect("execute op"); + executed(&mut app, &user_tx(sender, 0, fee_exponent, Vec::new())); assert_eq!( app.current_user_balance(sender), @@ -819,8 +951,7 @@ mod tests { app.balances.insert(bob, U256::from(5678_u64)); app.nonces.insert(alice, 4); app.nonces.insert(bob, 9); - app.execution_progress = ApplicationProgress::try_new(ExecutedInputCount::new(42), 777) - .expect("coherent progress"); + app.l2_tx_count = L2TxCount::new(42); let prefix = temp_dump_prefix(); app.create_dump(&prefix).expect("create dump"); @@ -844,7 +975,36 @@ mod tests { ); assert_eq!(restored.balances, app.balances); assert_eq!(restored.nonces, app.nonces); - assert_eq!(restored.execution_progress, app.execution_progress); + assert_eq!(restored.l2_tx_count, app.l2_tx_count); + } + + #[test] + fn from_canonical_state_round_trips_and_rejects_count_mismatch() { + let mut app = WalletApp::default(); + let sender = address!("0x1111111111111111111111111111111111111111"); + app.balances.insert(sender, U256::from(1234_u64)); + app.nonces.insert(sender, 4); + app.l2_tx_count = L2TxCount::new(42); + let bytes = app.canonical_snapshot_bytes().unwrap(); + + let prefix = temp_dump_prefix(); + app.create_dump(&prefix).expect("create dump"); + let state_file = WalletApp::state_file_in_dump(&prefix); + + let restored = WalletApp::from_canonical_state(&state_file, L2TxCount::new(42)) + .expect("matching count"); + let mismatch = WalletApp::from_canonical_state(&state_file, L2TxCount::new(41)); + std::fs::remove_dir_all(&prefix).expect("cleanup dump"); + + assert_eq!(restored.canonical_snapshot_bytes().unwrap(), bytes); + assert_eq!(restored.l2_tx_count().get(), 42); + match mismatch { + Err(AppError::Internal { reason }) => assert_eq!( + reason, + "canonical state holds L2 tx count 42, but the machine reports 41" + ), + other => panic!("expected a count mismatch, got {other:?}"), + } } #[test] @@ -863,15 +1023,10 @@ mod tests { let mut first = WalletApp::from_dump(&prefix).unwrap(); let second = WalletApp::from_dump(&prefix).unwrap(); - let user_op = ValidUserOp { - sender, - fee: 0, - data: Vec::new(), - }; - execute_valid_user_op(&mut first, &user_op, 11).unwrap(); + executed(&mut first, &user_tx(sender, 0, 0, Vec::new())); assert_ne!(first.canonical_snapshot_bytes().unwrap(), before); assert_eq!(live.canonical_snapshot_bytes().unwrap(), before); - execute_valid_user_op(&mut live, &user_op, 12).unwrap(); + executed(&mut live, &user_tx(sender, 0, 0, Vec::new())); assert_ne!(live.canonical_snapshot_bytes().unwrap(), before); assert_eq!(second.canonical_snapshot_bytes().unwrap(), before); assert_eq!( @@ -879,45 +1034,8 @@ mod tests { before ); std::fs::remove_dir_all(&prefix).unwrap(); - execute_valid_user_op(&mut first, &user_op, 13).unwrap(); - assert_eq!(first.executed_input_count().get(), 2); - } - - #[test] - fn safe_block_clock_advances_by_max_on_both_execution_paths() { - let mut app = WalletApp::new(WalletConfig::default()); - let sender = Address::from_slice(&[0x77; 20]); - app.balances.insert(sender, U256::from(10_000_u64)); - assert_eq!(app.last_executed_safe_block(), 0); - - // User op carries its covering frame's safe block. - let valid = ValidUserOp { - sender, - fee: 0, - data: Vec::new(), - }; - execute_valid_user_op(&mut app, &valid, 100).expect("execute op"); - assert_eq!(app.last_executed_safe_block(), 100); - - // A direct input advances the clock via its own inclusion block. - let direct = sequencer_core::l2_tx::DirectInput { - sender: Address::from_slice(&[0x88; 20]), - block_number: 150, - payload: Vec::new(), - }; - execute_direct_input(&mut app, &direct).expect("execute direct"); - assert_eq!(app.last_executed_safe_block(), 150); - - // max(): an older block must never regress the clock. A direct's - // inclusion block is <= its covering frame's safe block, so replays - // legitimately present blocks below the current clock. - let older_direct = sequencer_core::l2_tx::DirectInput { - sender: Address::from_slice(&[0x88; 20]), - block_number: 120, - payload: Vec::new(), - }; - execute_direct_input(&mut app, &older_direct).expect("execute older direct"); - assert_eq!(app.last_executed_safe_block(), 150); + executed(&mut first, &user_tx(sender, 1, 0, Vec::new())); + assert_eq!(first.l2_tx_count().get(), 2); } #[test] @@ -955,7 +1073,7 @@ mod tests { const CONSENSUS_VECTORS: &str = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../../tests/fixtures/consensus_vectors_v1.json" + "/../../tests/fixtures/consensus_vectors_v2.json" )); /// The guest emits these payloads as notices and voucher calldata, so their @@ -964,7 +1082,7 @@ mod tests { #[test] fn output_encodings_match_the_pinned_bytes() { let vectors: serde_json::Value = - serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v1.json"); + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json"); let pinned = &vectors["wallet_outputs"]; let text = |value: &serde_json::Value| value.as_str().expect("pinned string").to_owned(); let address = |key: &str| { @@ -1009,4 +1127,287 @@ mod tests { ); } } + + /// Replays every guest scenario of the vector file through the reference + /// scheduler and executor over the wallet, comparing every field of every + /// step and each scenario's final state. canonical-test holds the machine + /// to the same file, and c-scheduler's conformance test holds scheduler.h. + mod guest_vectors { + use std::collections::{BTreeMap, BTreeSet}; + + use alloy_primitives::{Address, U256, hex, keccak256}; + use serde_json::{Value, json}; + + use super::CONSENSUS_VECTORS; + use crate::application::{DEVNET_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; + use sequencer_core::application::{AppOutput, SkipReason}; + use sequencer_core::l2_tx::L2Tx; + use sequencer_core::scheduler::{ + Advanced, BatchRejectReason, CanonicalRunner, InputOutcome, L1Input, + SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, SchedulerConfig, SchedulerExport, TxFate, + }; + + fn text(value: &Value) -> &str { + value.as_str().expect("pinned string") + } + + fn number(value: &Value) -> u64 { + value.as_u64().expect("pinned number") + } + + fn bytes(value: &Value) -> Vec { + hex::decode(text(value)).expect("pinned hex bytes") + } + + fn address(value: &Value) -> Address { + text(value).parse().expect("pinned address") + } + + fn hex_of(bytes: &[u8]) -> String { + format!("0x{}", hex::encode(bytes)) + } + + fn pinned_output(value: &Value) -> AppOutput { + if let Some(notice) = value.get("notice") { + return AppOutput::Notice(bytes(notice)); + } + let voucher = &value["voucher"]; + AppOutput::Voucher { + destination: address(&voucher["destination"]), + value: text(&voucher["value"]) + .parse::() + .expect("pinned value"), + payload: bytes(&voucher["payload"]), + } + } + + /// The step fields that say what became of the input, in the file's form. + fn rendered(advanced: &Advanced) -> Value { + let reject = match advanced.outcome { + InputOutcome::BatchRejected(reason) => Some(match reason { + BatchRejectReason::DecodeFailed => json!({"reason": "decode_failed"}), + BatchRejectReason::WrongNonce { expected, got } => { + json!({"reason": "wrong_nonce", "expected": expected, "got": got}) + } + BatchRejectReason::SafeBlockNotBelowInclusionBlock => { + json!({"reason": "safe_block_not_below_inclusion_block"}) + } + BatchRejectReason::NonMonotonicSafeBlocks => { + json!({"reason": "non_monotonic_safe_blocks"}) + } + BatchRejectReason::FeePriceOutOfRange => { + json!({"reason": "fee_price_out_of_range"}) + } + BatchRejectReason::FirstFrameBelowClock { safe_block, clock } => json!({ + "reason": "first_frame_below_clock", + "safe_block": safe_block, + "clock": clock, + }), + }), + _ => None, + }; + let outcome = match advanced.outcome { + InputOutcome::DirectQueued => "direct_queued", + InputOutcome::DirectDropped => "direct_dropped", + InputOutcome::BatchAccepted => "batch_accepted", + InputOutcome::BatchStale => "batch_stale", + InputOutcome::BatchRejected(_) => "batch_rejected", + }; + let txs: Vec = advanced + .txs + .iter() + .map(|record| { + let batch_nonce = record.scheduled.batch_nonce; + match (&record.scheduled.tx, record.fate) { + (L2Tx::Direct(direct), TxFate::Done { .. }) => { + json!({"direct": direct.input_index, "batch_nonce": batch_nonce}) + } + (L2Tx::Direct(_), TxFate::Skipped(reason)) => { + panic!("a direct always executes, got {reason:?}") + } + (L2Tx::UserOp(op), fate) => json!({ + "user_op": { + "sender": op.sender.to_string(), + "nonce": op.nonce, + "fee": op.fee, + "block": op.block, + "data": hex_of(&op.data), + }, + "batch_nonce": batch_nonce, + "fate": match fate { + TxFate::Done { .. } => "executed", + TxFate::Skipped(SkipReason::WrongNonce { .. }) => "wrong_nonce", + TxFate::Skipped(SkipReason::NonceExhausted) => "nonce_exhausted", + }, + }), + } + }) + .collect(); + let mut fields = json!({"outcome": outcome, "txs": txs}); + if let Some(reject) = reject { + fields["reject"] = reject; + } + fields + } + + fn export_of(pinned: &Value) -> SchedulerExport { + SchedulerExport { + l1_input_count: number(&pinned["l1_input_count"]), + l2_tx_count: number(&pinned["l2_tx_count"]), + next_batch_nonce: number(&pinned["next_batch_nonce"]), + clock: number(&pinned["clock"]), + first_queued_input: number(&pinned["first_queued_input"]), + fridge_len: number(&pinned["fridge_len"]), + } + } + + /// What the pinned data alone says after each step: blocks never go + /// backwards along the handed-out sequence, and the clock splits the + /// admitted directs into executed (at or below it) and queued (above). + struct Invariant { + /// Admitted directs by input index, with their blocks. + admitted: BTreeMap, + executed: BTreeSet, + last_block: u64, + } + + impl Invariant { + fn check(&mut self, index: u64, step: &Value, scenario: &str) { + let block = number(&step["block"]); + if text(&step["outcome"]) == "direct_queued" { + self.admitted.insert(index, block); + } + for tx in step["txs"].as_array().expect("pinned txs") { + let tx_block = if let Some(direct) = tx.get("direct") { + let direct = number(direct); + assert!(self.executed.insert(direct), "{scenario}: {direct} twice"); + self.admitted[&direct] + } else { + number(&tx["user_op"]["block"]) + }; + assert!(tx_block >= self.last_block, "{scenario}: block went back"); + self.last_block = tx_block; + } + let export = export_of(&step["scheduler"]); + let queued: Vec = self + .admitted + .iter() + .filter(|(index, _)| !self.executed.contains(index)) + .map(|(index, _)| *index) + .collect(); + for (direct, direct_block) in &self.admitted { + assert_eq!( + self.executed.contains(direct), + *direct_block <= export.clock, + "{scenario}, step {index}: the clock must split the directs" + ); + } + assert_eq!( + export.fridge_len, + queued.len() as u64, + "{scenario}, step {index}" + ); + assert_eq!( + export.first_queued_input, + queued.first().copied().unwrap_or(export.l1_input_count), + "{scenario}, step {index}" + ); + } + } + + #[test] + fn the_reference_reproduces_every_guest_scenario() { + let vectors: Value = + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json"); + let guest = &vectors["guest"]; + let chain_id = number(&guest["chain_id"]); + let app_contract = address(&guest["app_contract"]); + let pinned_config = &guest["config"]; + let config = SchedulerConfig { + sequencer_address: address(&pinned_config["sequencer_address"]), + max_wait_blocks: number(&pinned_config["max_wait_blocks"]), + fridge_bound: number(&pinned_config["fridge_bound"]), + max_direct_payload_bytes: number(&pinned_config["max_direct_payload_bytes"]), + }; + // The vectors run on the devnet template exactly as the machine is built. + assert_eq!( + config, + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS) + ); + let senders: BTreeMap<&str, Address> = guest["senders"] + .as_object() + .expect("pinned senders") + .iter() + .map(|(name, value)| (name.as_str(), address(value))) + .collect(); + assert_eq!(senders["sequencer"], config.sequencer_address); + assert_eq!( + senders["portal"], + WalletConfig::devnet().erc20_portal_address + ); + let payloads = &guest["payloads"]; + let outputs = &guest["outputs"]; + + let scenarios = guest["scenarios"].as_array().expect("pinned scenarios"); + assert!(!scenarios.is_empty()); + for scenario in scenarios { + let name = text(&scenario["name"]); + let mut runner = + CanonicalRunner::genesis(WalletApp::new(WalletConfig::devnet()), config); + let mut invariant = Invariant { + admitted: BTreeMap::new(), + executed: BTreeSet::new(), + last_block: 0, + }; + let steps = scenario["steps"].as_array().expect("pinned steps"); + for (index, step) in (0_u64..).zip(steps) { + let input = L1Input { + index, + sender: senders[text(&step["from"])], + block: number(&step["block"]), + timestamp: number(&step["timestamp"]), + chain_id, + app_contract, + payload: bytes(&payloads[text(&step["payload"])]), + }; + let advanced = runner.process_input(input).expect("no application error"); + + let mut expected = json!({ + "outcome": step["outcome"].clone(), + "txs": step["txs"].clone(), + }); + if let Some(reject) = step.get("reject") { + expected["reject"] = reject.clone(); + } + assert_eq!(rendered(&advanced), expected, "{name}, step {index}"); + + let expected_outputs: Vec = step["outputs"] + .as_array() + .expect("pinned outputs") + .iter() + .map(|output| pinned_output(&outputs[text(output)])) + .collect(); + assert_eq!(advanced.outputs, expected_outputs, "{name}, step {index}"); + + let export = export_of(&step["scheduler"]); + assert_eq!(runner.export(), export, "{name}, step {index}"); + assert_eq!( + runner.inspect(SCHEDULER_INSPECT_QUERY), + Ok(export.encode().to_vec()), + "{name}, step {index}" + ); + invariant.check(index, step, name); + } + + let state = runner.inspect(STATE_INSPECT_QUERY).expect("state report"); + match (scenario.get("state"), scenario.get("state_keccak256")) { + (Some(pinned), None) => assert_eq!(hex_of(&state), text(pinned), "{name}"), + (None, Some(pinned)) => { + assert_eq!(keccak256(&state).to_string(), text(pinned), "{name}") + } + _ => panic!("{name} pins its state one way"), + } + } + } + } } diff --git a/examples/app-core/src/wallet_snapshot.rs b/examples/app-core/src/wallet_snapshot.rs index c0ffff5e..05404762 100644 --- a/examples/app-core/src/wallet_snapshot.rs +++ b/examples/app-core/src/wallet_snapshot.rs @@ -34,8 +34,7 @@ struct WalletSnapshot { pub sequencer_address: [u8; 20], pub balances: Vec, pub nonces: Vec, - pub executed_input_count: u64, - pub last_executed_safe_block: u64, + pub l2_tx_count: u64, } /// Deterministic SSZ bytes for `app`'s logical state; entries come out in @@ -63,8 +62,7 @@ pub fn encode(app: &WalletApp) -> Vec { sequencer_address: app.config().sequencer_address.into_array(), balances, nonces, - executed_input_count: app.executed_input_count().get(), - last_executed_safe_block: app.last_executed_safe_block(), + l2_tx_count: app.l2_tx_count().get(), } .as_ssz_bytes() } @@ -96,7 +94,7 @@ pub fn decode(bytes: &[u8]) -> Result { } } - WalletApp::from_snapshot_parts( + Ok(WalletApp::from_snapshot_parts( WalletConfig { erc20_portal_address: Address::from(decoded.erc20_portal_address), supported_erc20_token: Address::from(decoded.supported_erc20_token), @@ -104,9 +102,8 @@ pub fn decode(bytes: &[u8]) -> Result { }, balances, nonces, - decoded.executed_input_count, - decoded.last_executed_safe_block, - ) + decoded.l2_tx_count, + )) } #[cfg(test)] @@ -146,13 +143,66 @@ mod tests { ); app.nonces_mut() .insert(address!("0x1111111111111111111111111111111111111111"), 1); - app.set_executed_input_count(3); + app.set_l2_tx_count(3); let bytes = encode(&app); let restored = decode(&bytes).expect("decode snapshot"); assert_eq!(encode(&restored), bytes); } + /// A dump from before the clock's removal ends with + /// `last_executed_safe_block`. Read as the current layout it would lose + /// that field silently, so decoding must refuse it instead. + #[test] + fn decode_refuses_the_layout_with_the_safe_block_clock() { + #[derive(SszEncode)] + struct ClockedWalletSnapshot { + erc20_portal_address: [u8; 20], + supported_erc20_token: [u8; 20], + sequencer_address: [u8; 20], + balances: Vec, + nonces: Vec, + l2_tx_count: u64, + last_executed_safe_block: u64, + } + + let refused = |bytes: &[u8]| { + matches!( + decode(bytes), + Err(AppError::Internal { reason }) if reason.starts_with("snapshot decode failed") + ) + }; + // The empty-wallet golden of that layout, 84 fixed bytes. + let old_empty_golden = "3332de61a8bb9ac84893b2f552fe81c9a6dc54191c7d4b196cb0c7b0\ + 1d743fbc6116a902379c723879b88d2f598d8d001832d69f5201332a3f1e14c854000000540000\ + 0000000000000000000000000000000000"; + let old_empty_golden: Vec = (0..old_empty_golden.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&old_empty_golden[i..i + 2], 16).expect("hex nibble")) + .collect(); + assert_eq!(old_empty_golden.len(), 84); + assert!(refused(&old_empty_golden)); + + let account = address!("0x1111111111111111111111111111111111111111"); + let config = WalletConfig::default(); + let clocked = ClockedWalletSnapshot { + erc20_portal_address: config.erc20_portal_address.into_array(), + supported_erc20_token: config.supported_erc20_token.into_array(), + sequencer_address: config.sequencer_address.into_array(), + balances: vec![SnapshotBalance { + address: account.into_array(), + balance_be: U256::from(10_u64).to_be_bytes(), + }], + nonces: vec![SnapshotNonce { + address: account.into_array(), + nonce: 1, + }], + l2_tx_count: 3, + last_executed_safe_block: 1_000, + }; + assert!(refused(&clocked.as_ssz_bytes())); + } + #[test] fn encode_emits_entries_in_ascending_address_order() { // These rank differently under byte order, little-endian chunks and @@ -193,8 +243,7 @@ mod tests { balance(first_byte, 1), ], nonces: vec![nonce(last_byte, 8), nonce(first_byte, 7)], - executed_input_count: 0, - last_executed_safe_block: 0, + l2_tx_count: 0, }; assert_eq!(encode(&app), expected.as_ssz_bytes()); } diff --git a/examples/c-wallet-engine/src/lib.rs b/examples/c-wallet-engine/src/lib.rs index 80b51e75..963ea3f3 100644 --- a/examples/c-wallet-engine/src/lib.rs +++ b/examples/c-wallet-engine/src/lib.rs @@ -17,16 +17,38 @@ use alloy_primitives::Address; use app_core::application::{WalletApp, WalletConfig}; use c_app_engine::sys; use sequencer_core::application::{ - AppError, AppOutput, AppOutputs, Application, InvalidReason, ValidationOutcome, + Admission, AppError, AppOutput, AppOutputs, Application, PolicyRefusal, }; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; +#[unsafe(no_mangle)] +pub extern "C" fn application_engine_abi_version() -> u32 { + sys::APPLICATION_ENGINE_ABI_VERSION +} + #[unsafe(no_mangle)] pub extern "C" fn application_engine_max_method_payload_bytes() -> u64 { u64::try_from(WalletApp::max_method_payload_bytes()).expect("wallet payload bound exceeds u64") } +/// # Safety +/// `out_limits` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_scheduler_limits( + out_limits: *mut sys::ApplicationEngineSchedulerLimits, +) { + let limits = WalletApp::scheduler_limits(); + unsafe { + *out_limits = sys::ApplicationEngineSchedulerLimits { + max_wait_blocks: limits.max_wait_blocks, + fridge_bound: limits.fridge_bound, + max_direct_payload_bytes: limits.max_direct_payload_bytes, + } + }; +} + thread_local! { /// The last failure's message, and the buffer `state_file_in_dump` answers out of. /// @@ -48,7 +70,7 @@ fn set_error(message: impl AsRef) { }); } -/// Validation and execution: NOT_FOUND and INVALID_DUMP are lifecycle-only statuses. +/// Admission, nonce reads and execution: NOT_FOUND and INVALID_DUMP are lifecycle-only statuses. fn report(error: &AppError, what: &str) -> sys::ApplicationEngineStatus { match error { AppError::Io(err) => { @@ -62,10 +84,10 @@ fn report(error: &AppError, what: &str) -> sys::ApplicationEngineStatus { } } -/// Dump loading and creation: preserve the missing/corrupt classification across the ABI. These -/// are the kinds `referenced_artifact_io_is_terminal` and the host's AppBootstrap arm treat as -/// terminal; other I/O, PermissionDenied included, stays IO_ERROR, which the host classifies -/// operational. Diagnostics are not parsed. +/// Opening a dump or a canonical state file, and creating a dump: preserve the missing/corrupt +/// classification across the ABI. These are the kinds `referenced_artifact_io_is_terminal` and +/// the host's AppBootstrap arm treat as terminal; other I/O, PermissionDenied included, stays +/// IO_ERROR, which the host classifies operational. Diagnostics are not parsed. fn report_lifecycle(error: &AppError, what: &str) -> sys::ApplicationEngineStatus { let AppError::Io(err) = error else { return report(error, what); @@ -122,7 +144,8 @@ pub struct ApplicationEngine { /// Take a handle the C API was given. /// /// # Safety -/// `engine` must be a live handle from `application_engine_from_dump`, not yet destroyed. +/// `engine` must be a live handle from `application_engine_from_dump` or +/// `application_engine_from_canonical_state`, not yet destroyed. unsafe fn engine_ref<'a>(engine: *const ApplicationEngine) -> &'a ApplicationEngine { assert!(!engine.is_null(), "the C API forbids a null engine handle"); unsafe { &*engine } @@ -140,16 +163,16 @@ pub extern "C" fn application_engine_get_last_error_message() -> *const c_char { LAST_ERROR.with(|slot| slot.borrow().as_ptr()) } +/// Hand out a freshly opened wallet, or classify why it could not be opened. +/// /// # Safety -/// `prefix` is a NUL terminated path and `out_engine` is writable. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_from_dump( - prefix: *const c_char, +/// `out_engine` is writable. +unsafe fn open( + opened: Result, out_engine: *mut *mut ApplicationEngine, + what: &str, ) -> sys::ApplicationEngineStatus { - clear_error(); - let prefix = unsafe { path_from(prefix) }; - match WalletApp::from_dump(&prefix) { + match opened { Ok(app) => { let engine = Box::new(ApplicationEngine { app, @@ -159,15 +182,42 @@ pub unsafe extern "C" fn application_engine_from_dump( unsafe { *out_engine = Box::into_raw(engine) }; sys::APPLICATION_ENGINE_STATUS_OK } - // WalletApp::from_dump only reads and decodes; its Internal errors identify malformed bytes. + // The wallet's constructors only read and decode, and a count that disagrees with the + // bytes is a decode failure too, so their Internal errors identify malformed bytes. Err(AppError::Internal { reason }) => { - set_error(format!("from_dump failed: {reason}")); + set_error(format!("{what} failed: {reason}")); sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP } - Err(err) => report_lifecycle(&err, "from_dump"), + Err(err) => report_lifecycle(&err, what), } } +/// # Safety +/// `prefix` is a NUL terminated path and `out_engine` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_from_dump( + prefix: *const c_char, + out_engine: *mut *mut ApplicationEngine, +) -> sys::ApplicationEngineStatus { + clear_error(); + let prefix = unsafe { path_from(prefix) }; + unsafe { open(WalletApp::from_dump(&prefix), out_engine, "from_dump") } +} + +/// # Safety +/// `state_file` is a NUL terminated path and `out_engine` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_from_canonical_state( + state_file: *const c_char, + l2_tx_count: u64, + out_engine: *mut *mut ApplicationEngine, +) -> sys::ApplicationEngineStatus { + clear_error(); + let state_file = unsafe { path_from(state_file) }; + let opened = WalletApp::from_canonical_state(&state_file, L2TxCount::new(l2_tx_count)); + unsafe { open(opened, out_engine, "from_canonical_state") } +} + /// # Safety /// `engine` is a live handle, and it is not used again afterwards. #[unsafe(no_mangle)] @@ -181,11 +231,31 @@ pub unsafe extern "C" fn application_engine_destroy(engine: *mut ApplicationEngi /// # Safety /// Every pointer is non-null and its pointee outlives the call. #[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_validate_user_op( +pub unsafe extern "C" fn application_engine_user_nonce( + engine: *const ApplicationEngine, + sender: *const sys::ApplicationEngineEthereumAddress, + out_nonce: *mut u32, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_ref(engine) }; + let sender = Address::from(unsafe { (*sender).bytes }); + match engine.app.user_nonce(sender) { + Ok(nonce) => { + unsafe { *out_nonce = nonce }; + sys::APPLICATION_ENGINE_STATUS_OK + } + Err(err) => report(&err, "user_nonce"), + } +} + +/// # Safety +/// Every pointer is non-null and its pointee outlives the call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_admit_user_op( engine: *const ApplicationEngine, sender: *const sys::ApplicationEngineEthereumAddress, user_op: *const sys::ApplicationEngineUserOp, - current_fee: u16, + fee: u16, out_invalid: *mut sys::ApplicationEngineInvalid, ) -> sys::ApplicationEngineStatus { clear_error(); @@ -198,45 +268,45 @@ pub unsafe extern "C" fn application_engine_validate_user_op( data: unsafe { slice_from(&user_op.data) }.to_vec().into(), }; - match engine.app.validate_user_op(sender, &op, current_fee) { - Ok(ValidationOutcome::Accept) => sys::APPLICATION_ENGINE_STATUS_OK, - Err(err) => report(&err, "validate_user_op"), - Ok(ValidationOutcome::Reject(reason)) => { - let invalid = match reason { - InvalidReason::InvalidNonce { expected, got } => sys::ApplicationEngineInvalid { - reason: sys::APPLICATION_ENGINE_INVALID_NONCE, - values: sys::ApplicationEngineInvalidValues { - nonce: sys::ApplicationEngineInvalidNonce { expected, got }, + let refusal = match engine.app.admit_user_op(sender, &op, fee) { + Ok(Admission::Admit) => return sys::APPLICATION_ENGINE_STATUS_OK, + Ok(Admission::Refuse(refusal)) => refusal, + Err(err) => return report(&err, "admit_user_op"), + }; + let invalid = match refusal { + PolicyRefusal::Underfunded { + required, + available, + } => sys::ApplicationEngineInvalid { + reason: sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE, + values: sys::ApplicationEngineInvalidValues { + fee_balance: sys::ApplicationEngineInsufficientFeeBalance { + required: sys::ApplicationEngineUint256 { + bytes: required.to_be_bytes(), + }, + available: sys::ApplicationEngineUint256 { + bytes: available.to_be_bytes(), }, }, - InvalidReason::InsufficientFeeBalance { - required, - available, - } => sys::ApplicationEngineInvalid { - reason: sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE, - values: sys::ApplicationEngineInvalidValues { - fee_balance: sys::ApplicationEngineInsufficientFeeBalance { - required: sys::ApplicationEngineUint256 { - bytes: required.to_be_bytes(), - }, - available: sys::ApplicationEngineUint256 { - bytes: available.to_be_bytes(), - }, - }, + }, + }, + // The header gives this reason no union member: its text travels in the last error + // message, and the record carries the reason alone. + PolicyRefusal::Other(reason) => { + set_error(reason); + sys::ApplicationEngineInvalid { + reason: sys::APPLICATION_ENGINE_REFUSED_BY_POLICY, + values: sys::ApplicationEngineInvalidValues { + fee_balance: sys::ApplicationEngineInsufficientFeeBalance { + required: sys::ApplicationEngineUint256 { bytes: [0; 32] }, + available: sys::ApplicationEngineUint256 { bytes: [0; 32] }, }, }, - // The caller owns these guards and this entry point never checks them, so the app - // cannot produce these reasons. Reporting one would be a lie about which union - // member carries the diagnostics. - InvalidReason::InvalidMaxFee { .. } | InvalidReason::NonceExhausted => { - set_error("the app reported a caller-owned rejection"); - return sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR; - } - }; - unsafe { *out_invalid = invalid }; - sys::APPLICATION_ENGINE_STATUS_INVALID + } } - } + }; + unsafe { *out_invalid = invalid }; + sys::APPLICATION_ENGINE_STATUS_INVALID } /// Take an execution's outputs, refusing to run over ones still queued. @@ -266,29 +336,47 @@ unsafe fn execute( } } +/// The user op a record describes, its data copied. The wallet reads neither the nonce nor the +/// safe block, so only this crate's tests would see them dropped or swapped. +/// # Safety +/// The record's data span is readable. +unsafe fn user_op_from(user_op: &sys::ApplicationEngineUserOpTx) -> UserOpTx { + UserOpTx { + sender: Address::from(user_op.sender.bytes), + nonce: user_op.nonce, + fee: user_op.fee, + block: user_op.safe_block, + data: unsafe { slice_from(&user_op.data) }.to_vec(), + } +} + +/// The direct a record describes, its payload copied. The wallet reads neither the index, the +/// block nor the timestamp, so only this crate's tests would see them dropped or swapped. +/// # Safety +/// The record's payload span is readable. +unsafe fn direct_from(input: &sys::ApplicationEngineDirectInput) -> Direct { + Direct { + input_index: input.input_index, + sender: Address::from(input.sender.bytes), + block: input.block_number, + timestamp: input.timestamp, + payload: unsafe { slice_from(&input.payload) }.to_vec(), + } +} + /// # Safety /// Every pointer is non-null and its pointee outlives the call. #[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_execute_valid_user_op( +pub unsafe extern "C" fn application_engine_execute_user_op( engine: *mut ApplicationEngine, - user_op: *const sys::ApplicationEngineValidUserOp, - safe_block: u64, + user_op: *const sys::ApplicationEngineUserOpTx, out_output_count: *mut u64, ) -> sys::ApplicationEngineStatus { clear_error(); let engine = unsafe { engine_mut(engine) }; - let user_op = unsafe { &*user_op }; - let op = ValidUserOp { - sender: Address::from(user_op.sender.bytes), - fee: user_op.fee, - data: unsafe { slice_from(&user_op.data) }.to_vec(), - }; + let op = unsafe { user_op_from(&*user_op) }; // SAFETY: the caller guarantees `out_output_count` is writable. - unsafe { - execute(engine, out_output_count, |app| { - app.apply_valid_user_op(&op, safe_block) - }) - } + unsafe { execute(engine, out_output_count, |app| app.apply_user_op(&op)) } } /// # Safety @@ -301,18 +389,9 @@ pub unsafe extern "C" fn application_engine_execute_direct_input( ) -> sys::ApplicationEngineStatus { clear_error(); let engine = unsafe { engine_mut(engine) }; - let input = unsafe { &*input }; - let direct = DirectInput { - sender: Address::from(input.sender.bytes), - block_number: input.block_number, - payload: unsafe { slice_from(&input.payload) }.to_vec(), - }; + let direct = unsafe { direct_from(&*input) }; // SAFETY: the caller guarantees `out_output_count` is writable. - unsafe { - execute(engine, out_output_count, |app| { - app.apply_direct_input(&direct) - }) - } + unsafe { execute(engine, out_output_count, |app| app.apply_direct(&direct)) } } /// # Safety @@ -376,19 +455,10 @@ fn span_of(payload: &[u8]) -> sys::ApplicationEngineByteSpan { } /// # Safety -/// `engine` is a live handle and `out_progress` is writable. +/// `engine` is a live handle. #[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_progress( - engine: *const ApplicationEngine, - out_progress: *mut sys::ApplicationEngineProgress, -) { - let progress = unsafe { engine_ref(engine) }.app.progress(); - unsafe { - *out_progress = sys::ApplicationEngineProgress { - executed_input_count: progress.executed_input_count().get(), - last_executed_safe_block: progress.last_executed_safe_block(), - } - }; +pub unsafe extern "C" fn application_engine_l2_tx_count(engine: *const ApplicationEngine) -> u64 { + unsafe { engine_ref(engine) }.app.l2_tx_count().get() } /// # Safety @@ -435,3 +505,56 @@ pub unsafe extern "C" fn application_engine_state_file_in_dump( pub fn write_genesis(prefix: &Path, config: WalletConfig) -> Result<(), AppError> { WalletApp::new(config).create_dump(prefix) } + +#[cfg(test)] +mod tests { + use super::*; + + fn span(bytes: &[u8]) -> sys::ApplicationEngineByteSpan { + sys::ApplicationEngineByteSpan { + data: bytes.as_ptr(), + size: bytes.len() as u64, + } + } + + #[test] + fn records_unpack_every_field() { + let payload = [1_u8, 2, 3]; + let direct = sys::ApplicationEngineDirectInput { + sender: sys::ApplicationEngineEthereumAddress { bytes: [0x41; 20] }, + input_index: 3, + block_number: 7, + timestamp: 1_700_000_011, + payload: span(&payload), + }; + assert_eq!( + unsafe { direct_from(&direct) }, + Direct { + input_index: 3, + sender: Address::repeat_byte(0x41), + block: 7, + timestamp: 1_700_000_011, + payload: payload.to_vec(), + } + ); + + let data = [4_u8, 5]; + let user_op = sys::ApplicationEngineUserOpTx { + sender: sys::ApplicationEngineEthereumAddress { bytes: [0x42; 20] }, + nonce: 5, + fee: 300, + safe_block: 9, + data: span(&data), + }; + assert_eq!( + unsafe { user_op_from(&user_op) }, + UserOpTx { + sender: Address::repeat_byte(0x42), + nonce: 5, + fee: 300, + block: 9, + data: data.to_vec(), + } + ); + } +} diff --git a/examples/c-wallet-engine/tests/conformance.rs b/examples/c-wallet-engine/tests/conformance.rs index 33216bea..fa140481 100644 --- a/examples/c-wallet-engine/tests/conformance.rs +++ b/examples/c-wallet-engine/tests/conformance.rs @@ -2,17 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) use alloy_primitives::{Address, U256}; -use app_core::application::{Method, Transfer, WalletApp, WalletConfig, Withdrawal}; -use c_app_engine::{Application, EngineApp}; -use c_wallet_engine as _; +use app_core::application::{ + Method, Transfer, WALLET_SCHEDULER_LIMITS, WalletApp, WalletConfig, Withdrawal, +}; +use c_app_engine::{Application, EngineApp, sys}; use rollups_types::Erc20Deposit; use sequencer_core::application::{ - AppError, AppOutput, CanonicalState, ExecutionOutcome, execute_direct_input, - execute_valid_user_op, validate_and_execute_user_op, + Admission, AppError, AppOutput, CanonicalState, Executed, PolicyRefusal, SkipReason, + check_live_user_op, execute, }; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::fee::fee_to_linear; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use sequencer_core::user_op::UserOp; use ssz::Encode; +use std::os::unix::ffi::OsStringExt; use std::path::Path; fn fixture() -> (tempfile::TempDir, EngineApp, WalletApp, WalletConfig) { @@ -30,10 +34,12 @@ fn state_bytes(app: &mut EngineApp, path: &Path) -> Vec { std::fs::read(EngineApp::state_file_in_dump(path)).unwrap() } -fn deposit(config: WalletConfig, recipient: Address, amount: u64, block: u64) -> DirectInput { - DirectInput { +fn deposit(config: WalletConfig, recipient: Address, amount: u64, index: u64) -> L2Tx { + L2Tx::Direct(Direct { + input_index: index, sender: config.erc20_portal_address, - block_number: block, + block: 7 + index, + timestamp: 1_700_000_000 + index, payload: Erc20Deposit { token: config.supported_erc20_token, sender: recipient, @@ -41,40 +47,65 @@ fn deposit(config: WalletConfig, recipient: Address, amount: u64, block: u64) -> exec_layer_data: Vec::new(), } .encode(), + }) +} + +fn user_op(nonce: u32, max_fee: u16, data: Vec) -> UserOp { + UserOp { + nonce, + max_fee, + data: data.into(), } } +/// Run one transaction through the shared executor on both sides and demand the same fate. +fn execute_both(bridge: &mut EngineApp, native: &mut WalletApp, tx: &L2Tx) -> Executed { + let actual = execute(bridge, tx).unwrap(); + assert_eq!(actual, execute(native, tx).unwrap()); + assert_eq!(bridge.l2_tx_count(), native.l2_tx_count()); + actual +} + #[test] -fn abi_mixed_history_matches_native_outputs_progress_and_dump() { - let (dir, mut bridge, mut native, config) = fixture(); +fn abi_version_and_handle_free_getters_match_the_header_and_native() { + assert_eq!( + c_wallet_engine::application_engine_abi_version(), + sys::APPLICATION_ENGINE_ABI_VERSION + ); assert_eq!( EngineApp::max_method_payload_bytes(), WalletApp::max_method_payload_bytes() ); - assert_eq!(bridge.progress(), native.progress()); + assert_eq!(EngineApp::scheduler_limits(), WalletApp::scheduler_limits()); + assert_eq!(EngineApp::scheduler_limits(), WALLET_SCHEDULER_LIMITS); +} + +#[test] +fn abi_mixed_history_matches_native_outputs_count_and_dump() { + let (dir, mut bridge, mut native, config) = fixture(); + assert_eq!(bridge.l2_tx_count(), L2TxCount::new(0)); let sender = Address::repeat_byte(0x11); let recipient = Address::repeat_byte(0x22); for direct in [ - deposit(config, sender, 1000, 7), - DirectInput { + deposit(config, sender, 1000, 0), + L2Tx::Direct(Direct { + input_index: 1, sender: config.erc20_portal_address, - block_number: 4, + block: 4, + timestamp: 3, payload: vec![0xff], - }, - DirectInput { + }), + L2Tx::Direct(Direct { + input_index: 2, sender, - block_number: 9, + block: 9, + timestamp: 5, payload: vec![], - }, + }), ] { - assert_eq!( - execute_direct_input(&mut bridge, &direct).unwrap(), - execute_direct_input(&mut native, &direct).unwrap() - ); - assert_eq!(bridge.progress(), native.progress()); + execute_both(&mut bridge, &mut native, &direct); } - assert_eq!(bridge.progress().executed_input_count().get(), 3); - assert_eq!(bridge.progress().last_executed_safe_block(), 9); + assert_eq!(bridge.l2_tx_count().get(), 3); for (nonce, data, block) in [ ( @@ -96,69 +127,108 @@ fn abi_mixed_history_matches_native_outputs_progress_and_dump() { ), (2, vec![0xff], 13), ] { - let op = UserOp { - nonce, - max_fee: 10, - data: data.into(), - }; - let actual = validate_and_execute_user_op(&mut bridge, sender, &op, 10, block).unwrap(); - let expected = validate_and_execute_user_op(&mut native, sender, &op, 10, block).unwrap(); - assert_eq!(actual, expected); - let ExecutionOutcome::Included(receipt) = actual else { - panic!("funded operation was rejected") + let op = user_op(nonce, 10, data); + assert_eq!(bridge.user_nonce(sender).unwrap(), nonce); + assert_eq!( + bridge.user_nonce(sender).unwrap(), + native.user_nonce(sender).unwrap() + ); + assert_eq!( + bridge.admit_user_op(sender, &op, 10).unwrap(), + Admission::Admit + ); + assert_eq!(check_live_user_op(&bridge, sender, &op, 10).unwrap(), None); + assert_eq!(check_live_user_op(&native, sender, &op, 10).unwrap(), None); + + let tx = L2Tx::UserOp(UserOpTx::new(sender, &op, 10, block)); + let Executed::Done { outputs, .. } = execute_both(&mut bridge, &mut native, &tx) else { + panic!("an op at its sender's next nonce was skipped") }; match nonce { - 0 => assert!(matches!(receipt.outputs.as_slice(), [AppOutput::Notice(_)])), + 0 => assert!(matches!(outputs.as_slice(), [AppOutput::Notice(_)])), 1 => assert!( - matches!(receipt.outputs.as_slice(), [AppOutput::Voucher { value, .. }] if *value == U256::ZERO) - ), - 2 => assert!( - receipt.outputs.is_empty(), - "malformed method is an executed no-op" + matches!(outputs.as_slice(), [AppOutput::Voucher { value, .. }] if *value == U256::ZERO) ), + 2 => assert!(outputs.is_empty(), "malformed method is an executed no-op"), _ => unreachable!(), } - assert_eq!(bridge.progress(), native.progress()); } - assert_eq!(bridge.progress().executed_input_count().get(), 6); - assert_eq!(bridge.progress().last_executed_safe_block(), 13); + assert_eq!(bridge.l2_tx_count().get(), 6); + assert_eq!(bridge.user_nonce(sender).unwrap(), 3); + assert_eq!(bridge.user_nonce(recipient).unwrap(), 0); let checkpoint = dir.path().join("checkpoint"); assert_eq!( state_bytes(&mut bridge, &checkpoint), native.canonical_snapshot_bytes().unwrap() ); let restored = EngineApp::from_dump(&checkpoint).unwrap(); - assert_eq!(restored.progress(), bridge.progress()); + assert_eq!(restored.l2_tx_count(), bridge.l2_tx_count()); + assert_eq!(restored.user_nonce(sender).unwrap(), 3); } #[test] -fn abi_protocol_rejections_leave_state_and_progress_unchanged() { +fn abi_wrong_nonce_is_skipped_leaving_state_and_count_unchanged() { let (dir, mut bridge, mut native, config) = fixture(); let sender = Address::repeat_byte(0x11); - let direct = deposit(config, sender, 20, 7); - execute_direct_input(&mut bridge, &direct).unwrap(); - execute_direct_input(&mut native, &direct).unwrap(); - for (who, nonce, max_fee, current_fee) in [ - (sender, 9, 10, 10), - (sender, 0, 0, 10), - (Address::repeat_byte(0x55), 0, 10_000, 10_000), - ] { - let op = UserOp { - nonce, - max_fee, - data: vec![0xff].into(), - }; - let before = bridge.progress(); - let actual = validate_and_execute_user_op(&mut bridge, who, &op, current_fee, 99).unwrap(); + execute_both(&mut bridge, &mut native, &deposit(config, sender, 20, 0)); + let before = state_bytes(&mut bridge, &dir.path().join("before")); + + let op = user_op(9, 10, vec![0xff]); + let tx = L2Tx::UserOp(UserOpTx::new(sender, &op, 10, 99)); + assert_eq!( + execute_both(&mut bridge, &mut native, &tx), + Executed::Skipped(SkipReason::WrongNonce { + expected: 0, + got: 9 + }) + ); + assert_eq!(bridge.l2_tx_count().get(), 1); + assert_eq!(bridge.user_nonce(sender).unwrap(), 0); + let after = state_bytes(&mut bridge, &dir.path().join("after")); + assert_eq!(after, before); + assert_eq!(after, native.canonical_snapshot_bytes().unwrap()); +} + +#[test] +fn abi_unfunded_op_is_refused_by_admission_yet_executes_and_counts() { + let (dir, mut bridge, mut native, config) = fixture(); + let sequencer = config.sequencer_address; + let partial = Address::repeat_byte(0x11); + let unfunded = Address::repeat_byte(0x55); + execute_both(&mut bridge, &mut native, &deposit(config, partial, 20, 0)); + let fee = 10_000; + let required = fee_to_linear(fee); + assert!(required > U256::from(20)); + + for (offset, (who, available)) in [(partial, U256::from(20)), (unfunded, U256::ZERO)] + .into_iter() + .enumerate() + { + let op = user_op(0, fee, vec![0xff]); + let refusal = Admission::Refuse(PolicyRefusal::Underfunded { + required, + available, + }); + assert_eq!(bridge.admit_user_op(who, &op, fee).unwrap(), refusal); + assert_eq!(native.admit_user_op(who, &op, fee).unwrap(), refusal); + + // Admission is the operator's policy only. A batch carrying the op still runs it, + // charging what the sender has, and both the nonce and the count advance. + let tx = L2Tx::UserOp(UserOpTx::new(who, &op, fee, 99)); assert_eq!( - actual, - validate_and_execute_user_op(&mut native, who, &op, current_fee, 99).unwrap() + execute_both(&mut bridge, &mut native, &tx), + Executed::Done { + index: L2TxCount::new(1 + offset as u64), + outputs: vec![], + } ); - assert!(matches!(actual, ExecutionOutcome::Invalid(_))); - assert_eq!(bridge.progress(), before); + assert_eq!(bridge.user_nonce(who).unwrap(), 1); + assert_eq!(native.current_user_balance(who), U256::ZERO); } + assert_eq!(bridge.l2_tx_count().get(), 3); + assert_eq!(native.current_user_balance(sequencer), U256::from(20)); assert_eq!( - state_bytes(&mut bridge, &dir.path().join("rejected")), + state_bytes(&mut bridge, &dir.path().join("charged")), native.canonical_snapshot_bytes().unwrap() ); } @@ -169,29 +239,90 @@ fn abi_restored_instances_and_checkpoints_are_independent() { let source = dir.path().join("genesis"); let mut second = EngineApp::from_dump(&source).unwrap(); let original = std::fs::read(EngineApp::state_file_in_dump(&source)).unwrap(); - let input = deposit(config, Address::repeat_byte(0x11), 10, 7); - execute_direct_input(&mut first, &input).unwrap(); - assert_eq!(second.progress().executed_input_count().get(), 0); + let input = deposit(config, Address::repeat_byte(0x11), 10, 0); + let next = deposit(config, Address::repeat_byte(0x11), 10, 1); + execute(&mut first, &input).unwrap(); + assert_eq!(second.l2_tx_count().get(), 0); assert_eq!( std::fs::read(EngineApp::state_file_in_dump(&source)).unwrap(), original ); let frozen = dir.path().join("frozen"); let frozen_bytes = state_bytes(&mut first, &frozen); - execute_direct_input(&mut first, &input).unwrap(); + execute(&mut first, &next).unwrap(); assert_eq!( std::fs::read(EngineApp::state_file_in_dump(&frozen)).unwrap(), frozen_bytes ); std::fs::remove_dir_all(&source).unwrap(); - execute_direct_input(&mut second, &input).unwrap(); + execute(&mut second, &input).unwrap(); assert_eq!( state_bytes(&mut second, &dir.path().join("second")), frozen_bytes ); drop(first); let loaded = EngineApp::from_dump(&frozen).unwrap(); - assert_eq!(loaded.progress().executed_input_count().get(), 1); + assert_eq!(loaded.l2_tx_count().get(), 1); +} + +#[test] +fn abi_from_canonical_state_matches_native_and_classifies_bad_files() { + let (dir, _, mut native, config) = fixture(); + let sender = Address::repeat_byte(0x11); + execute(&mut native, &deposit(config, sender, 500, 0)).unwrap(); + let op = user_op( + 0, + 10, + Method::Withdrawal(Withdrawal { + amount: U256::from(5), + }) + .as_ssz_bytes(), + ); + execute( + &mut native, + &L2Tx::UserOp(UserOpTx::new(sender, &op, 10, 8)), + ) + .unwrap(); + let count = native.l2_tx_count(); + let machine = dir.path().join("machine"); + native.create_dump(&machine).unwrap(); + let state_file = WalletApp::state_file_in_dump(&machine); + + let mut bridge = EngineApp::from_canonical_state(&state_file, count).unwrap(); + assert_eq!(bridge.l2_tx_count(), count); + assert_eq!(bridge.user_nonce(sender).unwrap(), 1); + assert_eq!( + state_bytes(&mut bridge, &dir.path().join("projected")), + native.canonical_snapshot_bytes().unwrap() + ); + // The source stays the caller's: the instance outlives it and evolves as native does. + std::fs::remove_dir_all(&machine).unwrap(); + let next = deposit(config, sender, 1, 2); + assert_eq!( + execute(&mut bridge, &next).unwrap(), + execute(&mut native, &next).unwrap() + ); + assert_eq!( + state_bytes(&mut bridge, &dir.path().join("continued")), + native.canonical_snapshot_bytes().unwrap() + ); + + let rewritten = dir.path().join("rewritten"); + native.create_dump(&rewritten).unwrap(); + let state_file = WalletApp::state_file_in_dump(&rewritten); + let wrong_count = L2TxCount::new(native.l2_tx_count().get() + 1); + let garbage = dir.path().join("garbage"); + std::fs::write(&garbage, [0xff]).unwrap(); + for (file, count) in [(state_file.as_path(), wrong_count), (&garbage, count)] { + assert!(matches!( + EngineApp::from_canonical_state(file, count), + Err(AppError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData + )); + } + assert!(matches!( + EngineApp::from_canonical_state(&dir.path().join("missing"), count), + Err(AppError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound + )); } #[test] @@ -228,18 +359,59 @@ fn abi_malformed_dump_paths_remain_terminal_like_native_reads() { } } +/// An unpayable fee no longer fails an execution, so the fatal trigger is the engine's refusal +/// to run over undrained outputs. `EngineApp` always drains, so this drives the export directly; +/// the adapter's unit tests map INTERNAL_ERROR to `AppError::Internal`. #[test] -fn abi_execution_failure_returns_app_error_without_resuming_the_instance() { - let (_dir, mut bridge, _, _) = fixture(); - let op = ValidUserOp { +fn abi_execution_over_queued_outputs_is_refused_as_internal_error() { + let (dir, _, _, config) = fixture(); + let genesis = + std::ffi::CString::new(dir.path().join("genesis").into_os_string().into_vec()).unwrap(); + let mut engine = std::ptr::null_mut(); + assert_eq!( + unsafe { c_wallet_engine::application_engine_from_dump(genesis.as_ptr(), &mut engine) }, + sys::APPLICATION_ENGINE_STATUS_OK + ); + let payload = Erc20Deposit { + token: config.supported_erc20_token, sender: Address::repeat_byte(0x11), - fee: 10, - data: vec![], + value: U256::from(10), + exec_layer_data: Vec::new(), + } + .encode(); + let input = sys::ApplicationEngineDirectInput { + sender: sys::ApplicationEngineEthereumAddress { + bytes: config.erc20_portal_address.into_array(), + }, + input_index: 0, + block_number: 7, + timestamp: 1_700_000_000, + payload: sys::ApplicationEngineByteSpan { + data: payload.as_ptr(), + size: payload.len() as u64, + }, }; - // Calling the already-validated boundary on an unfunded op is an application invariant fault. - assert!(matches!( - execute_valid_user_op(&mut bridge, &op, 7), - Err(AppError::Internal { .. }) - )); - drop(bridge); + let mut outputs = 0; + let run = |engine, outputs: &mut u64| unsafe { + c_wallet_engine::application_engine_execute_direct_input(engine, &input, outputs) + }; + assert_eq!(run(engine, &mut outputs), sys::APPLICATION_ENGINE_STATUS_OK); + assert_eq!(outputs, 1, "a supported deposit emits its notice"); + + let mut untouched = u64::MAX; + assert_eq!( + run(engine, &mut untouched), + sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR + ); + assert_eq!(untouched, u64::MAX, "the count is written only on OK"); + let message = unsafe { + std::ffi::CStr::from_ptr(c_wallet_engine::application_engine_get_last_error_message()) + }; + assert!(message.to_str().unwrap().contains("still queued")); + assert_eq!( + unsafe { c_wallet_engine::application_engine_l2_tx_count(engine) }, + 1, + "the refused execution ran nothing" + ); + unsafe { c_wallet_engine::application_engine_destroy(engine) }; } diff --git a/examples/canonical-app/Cargo.toml b/examples/canonical-app/Cargo.toml index 85d66c5d..ffa83290 100644 --- a/examples/canonical-app/Cargo.toml +++ b/examples/canonical-app/Cargo.toml @@ -18,3 +18,7 @@ rollups-types = { workspace = true } [dev-dependencies] k256 = { workspace = true } +# The native tests sign user ops and build portal deposits and batches. +sequencer-core = { path = "../../sequencer-core", features = ["test-support"] } +rollups-types = { workspace = true, features = ["test-support"] } +ssz = { workspace = true } diff --git a/examples/canonical-app/src/bin/canonical-app-devnet.rs b/examples/canonical-app/src/bin/canonical-app-devnet.rs index 3480a213..b0c2507b 100644 --- a/examples/canonical-app/src/bin/canonical-app-devnet.rs +++ b/examples/canonical-app/src/bin/canonical-app-devnet.rs @@ -1,12 +1,14 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use app_core::application::{DEVNET_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; +use app_core::application::{WalletApp, WalletConfig}; use canonical_app::{SchedulerConfig, run_scheduler_forever}; use trolley::cmt::RollupCmt; fn main() { let rollup = RollupCmt::try_new().expect("failed to initialize rollup"); - let app = WalletApp::new(WalletConfig::devnet()); - run_scheduler_forever(rollup, app, SchedulerConfig::new(DEVNET_SEQUENCER_ADDRESS)); + // One address both submits the batches and collects the fees. + let wallet = WalletConfig::devnet(); + let config = SchedulerConfig::for_application::(wallet.sequencer_address); + run_scheduler_forever(rollup, WalletApp::new(wallet), config); } diff --git a/examples/canonical-app/src/bin/canonical-app-sepolia.rs b/examples/canonical-app/src/bin/canonical-app-sepolia.rs index f0139bdc..1148e7d7 100644 --- a/examples/canonical-app/src/bin/canonical-app-sepolia.rs +++ b/examples/canonical-app/src/bin/canonical-app-sepolia.rs @@ -1,12 +1,14 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use app_core::application::{SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; +use app_core::application::{WalletApp, WalletConfig}; use canonical_app::{SchedulerConfig, run_scheduler_forever}; use trolley::cmt::RollupCmt; fn main() { let rollup = RollupCmt::try_new().expect("failed to initialize rollup"); - let app = WalletApp::new(WalletConfig::sepolia()); - run_scheduler_forever(rollup, app, SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS)); + // One address both submits the batches and collects the fees. + let wallet = WalletConfig::sepolia(); + let config = SchedulerConfig::for_application::(wallet.sequencer_address); + run_scheduler_forever(rollup, WalletApp::new(wallet), config); } diff --git a/examples/canonical-app/src/lib.rs b/examples/canonical-app/src/lib.rs index 845c33e2..7f2a541c 100644 --- a/examples/canonical-app/src/lib.rs +++ b/examples/canonical-app/src/lib.rs @@ -3,4 +3,7 @@ pub mod scheduler; -pub use scheduler::{SchedulerConfig, run_scheduler_forever}; +pub use scheduler::{ + INVALID_BATCH_REPORT, SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, SchedulerConfig, + UNSUPPORTED_QUERY_REPORT, run_scheduler_forever, +}; diff --git a/examples/canonical-app/src/scheduler/mod.rs b/examples/canonical-app/src/scheduler/mod.rs index cbf5e4cb..0c11565f 100644 --- a/examples/canonical-app/src/scheduler/mod.rs +++ b/examples/canonical-app/src/scheduler/mod.rs @@ -1,64 +1,65 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! I/O shell that drives [`sequencer_core::scheduler`] inside the RISC-V guest: -//! reads `trolley` rollup inputs, narrows `U256` metadata to the scheduler's `u64` -//! domain, and emits the app's notices, vouchers and reports. +//! I/O shell that runs [`CanonicalRunner`] inside the RISC-V guest: reads +//! `trolley` rollup inputs, narrows their `U256` metadata to the scheduler's +//! `u64` domain, and emits the application's notices and vouchers, a report +//! for each rejected batch, and the answer to each inspect query. use alloy_primitives::U256; use rollups_types::{Notice, Voucher}; -use sequencer_core::application::AppOutput; -use sequencer_core::application::{Application, CanonicalState}; -use sequencer_core::scheduler::{InspectError, ProcessOutcome, Scheduler, SchedulerInput}; -use trolley::{Rollup, RollupRequest}; +use sequencer_core::application::{AppOutput, Application, CanonicalState}; +use sequencer_core::scheduler::{CanonicalRunner, InputOutcome, InspectError, L1Input}; +use trolley::{InputMetadata, Rollup, RollupRequest}; -pub use sequencer_core::scheduler::{STATE_INSPECT_QUERY, SchedulerConfig}; +pub use sequencer_core::scheduler::{ + SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, SchedulerConfig, +}; +/// The report a rejected batch leaves. Reports are diagnostics, not consensus, +/// so every reject reason shares this one text. +pub const INVALID_BATCH_REPORT: &[u8] = b"scheduler dropped invalid batch"; + +/// The report for an inspect query the guest does not answer. +pub const UNSUPPORTED_QUERY_REPORT: &[u8] = b"unsupported inspect query"; + +/// Runs the canonical machine from genesis. Every input is accepted at the +/// machine level; the guest halts (panics) only on the fatal conditions: an +/// application error, a counter with no `u64` successor, or metadata that does +/// not fit `u64`. pub fn run_scheduler_forever( mut rollup: R, app: A, scheduler_config: SchedulerConfig, ) -> ! { - let mut scheduler = Scheduler::new(app, scheduler_config); + let mut runner = CanonicalRunner::genesis(app, scheduler_config); loop { match rollup.next_input() { Ok(RollupRequest::Advance { metadata, payload }) => { - let inclusion_block = block_to_u64(metadata.block_number); - let domain = sequencer_core::build_input_domain( - chain_id_to_u64(metadata.chain_id), - metadata.app_contract, - ); - - let input = SchedulerInput { - sender: metadata.msg_sender, - inclusion_block, - domain, - payload, - }; - - let result = scheduler - .process_input(input) + // An application error leaves the input half done, with no + // canonical successor: halt before any of its outputs leave. + let advanced = runner + .process_input(l1_input(&metadata, payload)) .unwrap_or_else(|err| panic!("canonical application execution failed: {err}")); - for output in &result.outputs { + for output in &advanced.outputs { emit_app_output(&mut rollup, output) .unwrap_or_else(|err| panic!("scheduler failed to emit app output: {err}")); } - if matches!(result.outcome, ProcessOutcome::BatchRejected(_)) { + if matches!(advanced.outcome, InputOutcome::BatchRejected(_)) { rollup - .emit_report(b"scheduler dropped invalid batch") + .emit_report(INVALID_BATCH_REPORT) .unwrap_or_else(|err| { panic!("scheduler failed to emit invalid-batch report: {err}") }); } } Ok(RollupRequest::Inspect { payload }) => { - // Inspect is a public, read-only query endpoint: an unknown query - // (or a state-encode error) must not halt the guest. Emit a - // structured error report and keep serving, as it did before. - let report = match scheduler.inspect_state(&payload) { + // Inspect is a public, read-only endpoint: an unknown query or + // a state-encode error gets a report and must not halt. + let report = match runner.inspect(&payload) { Ok(bytes) => bytes, - Err(InspectError::UnsupportedQuery) => b"unsupported inspect query".to_vec(), + Err(InspectError::UnsupportedQuery) => UNSUPPORTED_QUERY_REPORT.to_vec(), Err(InspectError::Application(reason)) => { format!("inspect failed: {reason}").into_bytes() } @@ -72,6 +73,26 @@ pub fn run_scheduler_forever( } } +/// The scheduler's view of an `EvmAdvance`. The index and timestamp reach the +/// application through each direct; `prev_randao` is not part of the protocol. +fn l1_input(metadata: &InputMetadata, payload: Vec) -> L1Input { + L1Input { + index: narrow(metadata.index, "input index"), + sender: metadata.msg_sender, + block: narrow(metadata.block_number, "block number"), + timestamp: narrow(metadata.block_timestamp, "block timestamp"), + chain_id: narrow(metadata.chain_id, "chain id"), + app_contract: metadata.app_contract, + payload, + } +} + +/// Solidity hands these fields over as `uint256`. A value beyond `u64` has no +/// canonical successor state, so the guest halts on it, as a port must. +fn narrow(value: U256, field: &str) -> u64 { + u64::try_from(value).unwrap_or_else(|_| panic!("{field} does not fit u64")) +} + fn emit_app_output(rollup: &mut R, output: &AppOutput) -> trolley::RollupResult<()> { match output { AppOutput::Notice(payload) => { @@ -95,197 +116,746 @@ fn emit_app_output(rollup: &mut R, output: &AppOutput) -> trolley::Ro } } -/// Coerce a host `U256` block number to the scheduler's `u64` domain. Solidity -/// exposes block numbers as `uint256`; a value that does not fit `u64` is a -/// malformed host input for this prototype. Host-side, so it stays in the -/// harness rather than the pure scheduler library. -fn block_to_u64(block: U256) -> u64 { - u64::try_from(block).expect("block number does not fit u64") -} - -/// Coerce a host `U256` chain id to `u64` (same rationale as [`block_to_u64`]). -fn chain_id_to_u64(chain_id: U256) -> u64 { - u64::try_from(chain_id).expect("chain id does not fit u64") -} - #[cfg(test)] mod tests { use std::collections::VecDeque; + use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; - use alloy_primitives::{Address, U256}; - use app_core::application::{SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; - use trolley::{InputMetadata, RollupError}; + use alloy_primitives::{Address, address}; + use app_core::application::{ + DEVNET_SEQUENCER_ADDRESS, DepositNotice, Method, TransferNotice, WALLET_SCHEDULER_LIMITS, + WalletApp, WalletConfig, Withdrawal, + }; + use k256::ecdsa::SigningKey; + use rollups_types::alloy_sol_types::SolCall; + use rollups_types::{Erc20Deposit, Erc20Transfer}; + use sequencer_core::application::{Admission, AppError, AppOutputs, SchedulerLimits}; + use sequencer_core::batch::{Batch, Frame, WireUserOp}; + use sequencer_core::fee::fee_to_linear; + use sequencer_core::history::L2TxCount; + use sequencer_core::l2_tx::{Direct, UserOpTx}; + use sequencer_core::scheduler::SchedulerExport; + use sequencer_core::user_op::UserOp; + use trolley::RollupError; use super::*; + const CHAIN_ID: u64 = 31337; + const APP_CONTRACT: Address = address!("0xa5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5"); + const BOB: Address = address!("0x8888888888888888888888888888888888888888"); + /// The panic the loop ends with once the mock runs out of requests. + const END_OF_INPUTS: &str = "scheduler failed while reading next input"; + + #[derive(Debug, Clone, PartialEq, Eq)] + enum Emitted { + Notice(Vec), + Voucher { + destination: Address, + value: U256, + payload: Vec, + }, + Report(Vec), + } + + /// Serves scripted requests, then fails, which the loop treats as fatal. struct MockRollup { - inputs: VecDeque>, - reports: Arc>>>, + requests: VecDeque, + emitted: Arc>>, } impl MockRollup { - fn with_inputs( - inputs: Vec>, - ) -> (Self, Arc>>>) { - let reports = Arc::new(Mutex::new(Vec::new())); - ( - Self { - inputs: VecDeque::from(inputs), - reports: Arc::clone(&reports), - }, - reports, - ) + fn push(&self, emitted: Emitted) -> trolley::RollupResult<()> { + self.emitted + .lock() + .expect("poisoned emitted mutex") + .push(emitted); + Ok(()) } } impl Rollup for MockRollup { fn next_input(&mut self) -> trolley::RollupResult { - self.inputs - .pop_front() - .unwrap_or(Err(RollupError::CmtCallFailed { - operation: "next_input", - code: -1, - })) + self.requests.pop_front().ok_or(RollupError::CmtCallFailed { + operation: "next_input", + code: -22, + }) } fn revert(&mut self) -> ! { - panic!("mock rollup revert is not used in this test"); + panic!("the canonical guest never reverts"); } fn gio(&mut self, _domain: u16, _id: &[u8]) -> trolley::RollupResult<(u16, Vec)> { - unimplemented!("mock rollup gio is not used in this test"); + panic!("the canonical guest never issues a GIO request"); } - fn emit_voucher(&mut self, _voucher: &rollups_types::Voucher) -> trolley::RollupResult<()> { - unimplemented!("mock rollup emit_voucher is not used in this test"); + fn emit_voucher(&mut self, voucher: &Voucher) -> trolley::RollupResult<()> { + self.push(Emitted::Voucher { + destination: voucher.destination, + value: voucher.value, + payload: voucher.payload.to_vec(), + }) } - fn emit_notice(&mut self, _notice: &rollups_types::Notice) -> trolley::RollupResult<()> { - unimplemented!("mock rollup emit_notice is not used in this test"); + fn emit_notice(&mut self, notice: &Notice) -> trolley::RollupResult<()> { + self.push(Emitted::Notice(notice.payload.to_vec())) } fn emit_report(&mut self, report: &[u8]) -> trolley::RollupResult<()> { - self.reports - .lock() - .expect("poisoned reports mutex") - .push(report.to_vec()); - Ok(()) + self.push(Emitted::Report(report.to_vec())) + } + } + + /// Runs the loop over `requests` and returns what it emitted and the + /// message it halted with. + fn run( + app: A, + config: SchedulerConfig, + requests: Vec, + ) -> (Vec, String) { + let emitted = Arc::new(Mutex::new(Vec::new())); + let rollup = MockRollup { + requests: VecDeque::from(requests), + emitted: Arc::clone(&emitted), + }; + let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_scheduler_forever(rollup, app, config) + })) + .expect_err("the loop only ever ends by halting"); + let message = payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default(); + let emitted = emitted.lock().expect("poisoned emitted mutex").clone(); + (emitted, message) + } + + fn wallet_config() -> SchedulerConfig { + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS) + } + + /// Runs the devnet wallet through every request; it must not halt early. + fn run_wallet(script: Script) -> Vec { + let (emitted, halt) = run( + WalletApp::new(WalletConfig::devnet()), + wallet_config(), + script.requests, + ); + assert!(halt.starts_with(END_OF_INPUTS), "halted early: {halt}"); + emitted + } + + /// Requests in InputBox order: each advance gets the next index. + #[derive(Default)] + struct Script { + requests: Vec, + next_index: u64, + } + + impl Script { + fn advance(self, sender: Address, block: u64, payload: Vec) -> Self { + self.advance_at(sender, block, 0, payload) + } + + fn advance_at( + mut self, + sender: Address, + block: u64, + timestamp: u64, + payload: Vec, + ) -> Self { + let metadata = metadata(self.next_index, sender, block, timestamp); + self.next_index += 1; + self.requests + .push(RollupRequest::Advance { metadata, payload }); + self + } + + fn deposit(self, block: u64, owner: Address, amount: u64) -> Self { + self.deposit_with(block, owner, amount, Vec::new()) + } + + fn deposit_with( + self, + block: u64, + owner: Address, + amount: u64, + exec_layer_data: Vec, + ) -> Self { + let wallet = WalletConfig::devnet(); + let payload = Erc20Deposit { + token: wallet.supported_erc20_token, + sender: owner, + value: U256::from(amount), + exec_layer_data, + } + .encode(); + self.advance(wallet.erc20_portal_address, block, payload) + } + + fn batch(self, block: u64, batch: Batch) -> Self { + self.advance( + DEVNET_SEQUENCER_ADDRESS, + block, + ssz::Encode::as_ssz_bytes(&batch), + ) + } + + fn inspect(mut self, query: &[u8]) -> Self { + self.requests.push(RollupRequest::Inspect { + payload: query.to_vec(), + }); + self } } - fn metadata(sender: Address, block: u64) -> InputMetadata { + fn metadata(index: u64, sender: Address, block: u64, timestamp: u64) -> InputMetadata { InputMetadata { - chain_id: U256::from(1_u64), - app_contract: Address::ZERO, + chain_id: U256::from(CHAIN_ID), + app_contract: APP_CONTRACT, msg_sender: sender, block_number: U256::from(block), - block_timestamp: U256::ZERO, + block_timestamp: U256::from(timestamp), prev_randao: U256::ZERO, - index: U256::ZERO, + index: U256::from(index), } } - #[test] - fn run_scheduler_emits_exported_state_for_state_inspect() { - let inspect = RollupRequest::Inspect { - payload: STATE_INSPECT_QUERY.to_vec(), + fn batch(nonce: u64, frames: Vec) -> Batch { + Batch { nonce, frames } + } + + fn frame(safe_block: u64, fee_price: u16, user_ops: Vec) -> Frame { + Frame { + user_ops, + safe_block, + fee_price, + } + } + + fn key(byte: u8) -> SigningKey { + SigningKey::from_slice(&[byte; 32]).expect("signing key") + } + + fn address_of(key: &SigningKey) -> Address { + let point = key.verifying_key().to_encoded_point(false); + Address::from_raw_public_key(&point.as_bytes()[1..]) + } + + fn signed(key: &SigningKey, nonce: u32, max_fee: u16, method: &Method) -> WireUserOp { + let data = ssz::Encode::as_ssz_bytes(method); + let user_op = UserOp { + nonce, + max_fee, + data: data.clone().into(), }; - let terminal_err = Err(RollupError::CmtCallFailed { - operation: "next_input", - code: -22, - }); - let (rollup, reports) = MockRollup::with_inputs(vec![Ok(inspect), terminal_err]); - let expected = app_core::wallet_snapshot::encode(&WalletApp::new(WalletConfig::default())); - - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_scheduler_forever( - rollup, - WalletApp::new(WalletConfig::default()), - SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), - ) - })); + let domain = sequencer_core::build_input_domain(CHAIN_ID, APP_CONTRACT); + let signature = sequencer_core::test_support::sign_user_op(key, &domain, &user_op); + WireUserOp { + nonce, + max_fee, + data, + signature: signature.as_bytes().to_vec(), + } + } + + fn transfer(amount: u64) -> Method { + Method::Transfer(app_core::application::Transfer { + amount: U256::from(amount), + to: BOB, + }) + } + + fn deposit_notice(owner: Address, amount: u64) -> Emitted { + Emitted::Notice( + DepositNotice { + token: WalletConfig::devnet().supported_erc20_token, + sender: owner, + amount: U256::from(amount), + } + .abi_encode(), + ) + } + + fn transfer_notice(sender: Address, amount: u64) -> Emitted { + Emitted::Notice( + TransferNotice { + sender, + recipient: BOB, + amount: U256::from(amount), + } + .abi_encode(), + ) + } - assert!( - result.is_err(), - "scheduler loop should panic on rollup error" + fn invalid_batch() -> Emitted { + Emitted::Report(INVALID_BATCH_REPORT.to_vec()) + } + + fn scheduler_report(export: [u64; 6]) -> Emitted { + let [ + l1_input_count, + l2_tx_count, + next_batch_nonce, + clock, + first_queued_input, + fridge_len, + ] = export; + Emitted::Report( + SchedulerExport { + l1_input_count, + l2_tx_count, + next_batch_nonce, + clock, + first_queued_input, + fridge_len, + } + .encode() + .to_vec(), + ) + } + + /// The wallet the guest's `state` report holds, decoded. + fn reported_wallet(report: &Emitted) -> WalletApp { + let Emitted::Report(bytes) = report else { + panic!("expected a state report, got {report:?}"); + }; + app_core::wallet_snapshot::decode(bytes).expect("decode the state report") + } + + #[test] + fn the_state_query_reports_the_application_snapshot() { + let emitted = run_wallet(Script::default().inspect(STATE_INSPECT_QUERY).inspect(b"")); + let genesis = app_core::wallet_snapshot::encode(&WalletApp::new(WalletConfig::devnet())); + assert_eq!( + emitted, + [Emitted::Report(genesis.clone()), Emitted::Report(genesis)] ); - let reports = reports.lock().expect("poisoned reports mutex"); - assert!( - reports - .iter() - .any(|report| report.as_slice() == expected.as_slice()), - "missing state inspect report, got: {reports:?}" + } + + #[test] + fn the_scheduler_query_reports_zeros_at_genesis() { + let emitted = run_wallet(Script::default().inspect(SCHEDULER_INSPECT_QUERY)); + assert_eq!( + emitted, + [Emitted::Report(vec![0; SchedulerExport::ENCODED_LEN])] ); } + /// An unknown query gets a report and the guest keeps serving. #[test] - fn run_scheduler_reports_unsupported_inspect_query_without_panicking() { - // A non-"state" inspect payload must produce a graceful report, not a - // guest panic. The loop should survive the inspect and only panic when - // it later hits the terminal rollup error. - let inspect = RollupRequest::Inspect { - payload: b"balances".to_vec(), + fn an_unsupported_query_is_reported_without_halting() { + let emitted = run_wallet( + Script::default() + .inspect(b"balances") + .inspect(b"scheduler ") + .inspect(SCHEDULER_INSPECT_QUERY), + ); + let unsupported = Emitted::Report(UNSUPPORTED_QUERY_REPORT.to_vec()); + assert_eq!( + emitted, + [unsupported.clone(), unsupported, scheduler_report([0; 6])] + ); + } + + #[test] + fn a_failing_state_encode_is_reported_without_halting() { + let probe = ProbeApp { + snapshot_fails: true, + ..ProbeApp::default() }; - let terminal_err = Err(RollupError::CmtCallFailed { - operation: "next_input", - code: -22, - }); - let (rollup, reports) = MockRollup::with_inputs(vec![Ok(inspect), terminal_err]); + let script = Script::default() + .inspect(STATE_INSPECT_QUERY) + .inspect(SCHEDULER_INSPECT_QUERY); + let (emitted, halt) = run(probe, probe_config(), script.requests); + assert!(halt.starts_with(END_OF_INPUTS), "halted early: {halt}"); + let reason = AppError::Internal { + reason: "snapshot refused".to_string(), + }; + assert_eq!( + emitted, + [ + Emitted::Report(format!("inspect failed: {reason}").into_bytes()), + scheduler_report([0; 6]), + ] + ); + } - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_scheduler_forever( - rollup, - WalletApp::new(WalletConfig::default()), - SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), - ) - })); + #[test] + fn a_queued_direct_shows_in_the_scheduler_report() { + let alice = address_of(&key(1)); + let emitted = run_wallet( + Script::default() + .deposit(10, alice, 250) + .inspect(SCHEDULER_INSPECT_QUERY), + ); + // Queued, not executed: no notice, and the L2 tx count stays at zero. + assert_eq!(emitted, [scheduler_report([1, 0, 0, 0, 0, 1])]); + } - assert!( - result.is_err(), - "scheduler loop should panic only on the terminal rollup error" + /// A direct one byte over the cap never joins the fridge, so a drain + /// passes it by; one exactly at the cap executes. + #[test] + fn a_direct_over_the_payload_cap_never_executes() { + let alice = address_of(&key(1)); + let cap = usize::try_from(WALLET_SCHEDULER_LIMITS.max_direct_payload_bytes).unwrap(); + let at_cap = vec![0; cap - rollups_types::ERC20_DEPOSIT_PREFIX_BYTES]; + let over_cap = vec![0; at_cap.len() + 1]; + let emitted = run_wallet( + Script::default() + .deposit_with(10, alice, 7, over_cap) + .inspect(SCHEDULER_INSPECT_QUERY) + .deposit_with(10, alice, 5, at_cap) + .batch(11, batch(0, vec![frame(10, 0, vec![])])) + .inspect(SCHEDULER_INSPECT_QUERY) + .inspect(STATE_INSPECT_QUERY), ); - let reports = reports.lock().expect("poisoned reports mutex"); - assert!( - reports - .iter() - .any(|report| report.as_slice() == b"unsupported inspect query"), - "expected a graceful unsupported-query report, got: {reports:?}" + assert_eq!(emitted.len(), 4, "{emitted:?}"); + assert_eq!(emitted[0], scheduler_report([1, 0, 0, 0, 1, 0])); + assert_eq!(emitted[1], deposit_notice(alice, 5)); + assert_eq!(emitted[2], scheduler_report([3, 1, 1, 10, 3, 0])); + assert_eq!( + reported_wallet(&emitted[3]).current_user_balance(alice), + U256::from(5) ); } + /// A frame drains the deposits of its own safe block, strictly below the + /// batch's inclusion block, then runs its ops: outputs in that order. #[test] - fn run_scheduler_emits_report_for_invalid_batch_before_rollup_error() { - let sequencer = SEPOLIA_SEQUENCER_ADDRESS; - let invalid_batch_input = RollupRequest::Advance { - metadata: metadata(sequencer, 10), - payload: vec![0xFF, 0xEE, 0xDD], - }; - let terminal_err = Err(RollupError::CmtCallFailed { - operation: "next_input", - code: -22, + fn a_strict_drain_batch_executes_the_deposit_then_the_ops() { + let alice_key = key(2); + let alice = address_of(&alice_key); + let withdrawal = Method::Withdrawal(Withdrawal { + amount: U256::from(100), }); - let (rollup, reports) = - MockRollup::with_inputs(vec![Ok(invalid_batch_input), terminal_err]); - - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_scheduler_forever( - rollup, - WalletApp::new(WalletConfig::default()), - SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), + let emitted = run_wallet( + Script::default() + .deposit(10, alice, 250) + .batch( + 11, + batch( + 0, + vec![frame(10, 0, vec![signed(&alice_key, 0, 0, &withdrawal)])], + ), + ) + .inspect(SCHEDULER_INSPECT_QUERY), + ); + let voucher = Emitted::Voucher { + destination: WalletConfig::devnet().supported_erc20_token, + value: U256::ZERO, + payload: Erc20Transfer { + recipient: alice, + amount: U256::from(100), + } + .abi_encode(), + }; + assert_eq!( + emitted, + [ + deposit_notice(alice, 250), + voucher, + scheduler_report([2, 2, 1, 10, 2, 0]), + ] + ); + } + + #[test] + fn an_undecodable_batch_is_reported() { + let emitted = run_wallet( + Script::default() + .advance(DEVNET_SEQUENCER_ADDRESS, 10, vec![0xFF, 0xEE, 0xDD]) + .inspect(SCHEDULER_INSPECT_QUERY), + ); + assert_eq!( + emitted, + [invalid_batch(), scheduler_report([1, 0, 0, 0, 1, 0])] + ); + } + + /// The backstop drains an overdue deposit before the batch's gates run, + /// so its notice leaves before the reject's report; the reject consumes + /// no batch nonce. + #[test] + fn a_rejected_batch_reports_after_its_backstop_outputs() { + let alice = address_of(&key(3)); + let overdue = 1 + WALLET_SCHEDULER_LIMITS.max_wait_blocks; + let emitted = run_wallet( + Script::default() + .deposit(1, alice, 250) + .batch(overdue, batch(0, vec![frame(overdue, 0, vec![])])) + .batch(overdue, batch(0, vec![frame(overdue - 1, 0, vec![])])) + .inspect(SCHEDULER_INSPECT_QUERY), + ); + assert_eq!( + emitted, + [ + deposit_notice(alice, 250), + invalid_batch(), + scheduler_report([3, 1, 1, overdue - 1, 3, 0]), + ] + ); + } + + /// Gate f: once a frame raised the clock, a batch whose first frame lies + /// below it is rejected, and a retry at the clock is accepted. + #[test] + fn a_first_frame_below_the_clock_is_rejected() { + let alice = address_of(&key(4)); + let emitted = run_wallet( + Script::default() + .deposit(10, alice, 100) + .batch(12, batch(0, vec![frame(11, 0, vec![])])) + .deposit(12, alice, 50) + .batch(13, batch(1, vec![frame(10, 0, vec![])])) + .batch(13, batch(1, vec![frame(12, 0, vec![])])), + ); + assert_eq!( + emitted, + [ + deposit_notice(alice, 100), + invalid_batch(), + deposit_notice(alice, 50), + ] + ); + } + + /// An op its sender cannot pay in full still executes: it pays what the + /// sender has, consumes the nonce, and its method runs on what is left. + /// A later op at the same nonce is skipped. + #[test] + fn an_underfunded_op_executes_and_consumes_its_nonce() { + let alice_key = key(5); + let alice = address_of(&alice_key); + let fee_price = 200; + let gas = u64::try_from(fee_to_linear(fee_price)).unwrap(); + let base_gas = u64::try_from(fee_to_linear(0)).unwrap(); + assert!(gas > 1, "the deposit must be nonzero yet below the fee"); + let emitted = run_wallet( + Script::default() + .deposit(10, alice, gas - 1) + .batch( + 11, + batch( + 0, + vec![frame( + 10, + fee_price, + vec![signed(&alice_key, 0, fee_price, &transfer(1))], + )], + ), + ) + .deposit(12, alice, 50) + .batch( + 13, + batch( + 1, + vec![frame( + 12, + 0, + vec![ + signed(&alice_key, 0, 0, &transfer(1)), + signed(&alice_key, 1, 0, &transfer(1)), + ], + )], + ), + ) + .inspect(SCHEDULER_INSPECT_QUERY) + .inspect(STATE_INSPECT_QUERY), + ); + assert_eq!(emitted.len(), 5, "{emitted:?}"); + assert_eq!( + emitted[..3], + [ + deposit_notice(alice, gas - 1), + deposit_notice(alice, 50), + transfer_notice(alice, 1), + ] + ); + // Four executed: two deposits, the underfunded op and the nonce-1 op. + assert_eq!(emitted[3], scheduler_report([4, 4, 2, 12, 4, 0])); + let wallet = reported_wallet(&emitted[4]); + assert_eq!(wallet.current_user_nonce(alice), 2); + assert_eq!( + wallet.current_user_balance(alice), + U256::from(50 - base_gas - 1) + ); + assert_eq!(wallet.current_user_balance(BOB), U256::from(1)); + assert_eq!( + wallet.current_user_balance(DEVNET_SEQUENCER_ADDRESS), + U256::from(gas - 1 + base_gas) + ); + } + + /// Each direct reaches the application with its own InputBox index and + /// block timestamp, not the draining batch's. + #[test] + fn index_and_timestamp_reach_the_application() { + let script = Script::default() + .advance_at(BOB, 4, 1_000, b"a".to_vec()) + .inspect(SCHEDULER_INSPECT_QUERY) + .advance_at( + DEVNET_SEQUENCER_ADDRESS, + 5, + 1_500, + ssz::Encode::as_ssz_bytes(&batch(0, vec![])), ) - })); + .advance_at(BOB, 5, 2_000, b"b".to_vec()) + .advance_at( + DEVNET_SEQUENCER_ADDRESS, + 7, + 3_000, + ssz::Encode::as_ssz_bytes(&batch(1, vec![frame(5, 0, vec![])])), + ) + .inspect(STATE_INSPECT_QUERY); + let (emitted, halt) = run(ProbeApp::default(), probe_config(), script.requests); + assert!(halt.starts_with(END_OF_INPUTS), "halted early: {halt}"); + assert_eq!( + emitted, + [ + scheduler_report([1, 0, 0, 0, 0, 1]), + Emitted::Notice(b"a".to_vec()), + Emitted::Notice(b"b".to_vec()), + Emitted::Report(b"0@1000;2@2000;".to_vec()), + ] + ); + } - assert!( - result.is_err(), - "scheduler loop should panic on rollup error" + /// An application error halts the guest before any output of that input + /// leaves, including those of the transactions that ran before it. + #[test] + fn an_application_error_halts_before_the_input_emits() { + let script = Script::default() + .advance(BOB, 1, b"ok".to_vec()) + .advance(BOB, 1, b"boom".to_vec()) + .advance( + DEVNET_SEQUENCER_ADDRESS, + 2, + ssz::Encode::as_ssz_bytes(&batch(0, vec![frame(1, 0, vec![])])), + ); + let (emitted, halt) = run(ProbeApp::default(), probe_config(), script.requests); + assert_eq!( + halt, + "canonical application execution failed: internal: refusing boom" ); - let reports = reports.lock().expect("poisoned reports mutex"); - assert!( - reports + assert!(emitted.is_empty(), "{emitted:?}"); + } + + /// No canonical successor exists for metadata beyond `u64`: the guest + /// halts before the scheduler sees the input. + #[test] + fn metadata_beyond_u64_halts() { + let too_big = U256::from(u64::MAX) + U256::from(1); + /// A field's name, and how to set it. + type Case = (&'static str, fn(&mut InputMetadata, U256)); + let cases: [Case; 4] = [ + ("input index", |m, v| m.index = v), + ("block number", |m, v| m.block_number = v), + ("block timestamp", |m, v| m.block_timestamp = v), + ("chain id", |m, v| m.chain_id = v), + ]; + for (field, set) in cases { + let mut metadata = metadata(0, BOB, 1, 0); + set(&mut metadata, too_big); + let requests = vec![RollupRequest::Advance { + metadata, + payload: b"x".to_vec(), + }]; + let (emitted, halt) = run(ProbeApp::default(), probe_config(), requests); + assert_eq!(halt, format!("{field} does not fit u64")); + assert!(emitted.is_empty(), "{field}: {emitted:?}"); + } + } + + fn probe_config() -> SchedulerConfig { + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS) + } + + /// Records each direct's index and timestamp, which the wallet ignores, + /// and fails on the payload `boom`. + #[derive(Default)] + struct ProbeApp { + seen: Vec<(u64, u64)>, + count: L2TxCount, + snapshot_fails: bool, + } + + impl Application for ProbeApp { + fn user_nonce(&self, _sender: Address) -> Result { + Ok(0) + } + + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { + unreachable!("the probe's scripts carry no user ops") + } + + fn apply_direct(&mut self, direct: &Direct) -> Result { + if direct.payload == b"boom" { + return Err(AppError::Internal { + reason: "refusing boom".to_string(), + }); + } + self.seen.push((direct.input_index, direct.timestamp)); + self.count.advance(); + Ok(vec![AppOutput::Notice(direct.payload.clone())]) + } + + fn l2_tx_count(&self) -> L2TxCount { + self.count + } + + fn max_method_payload_bytes() -> usize { + 0 + } + + fn scheduler_limits() -> SchedulerLimits { + WALLET_SCHEDULER_LIMITS + } + + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + Ok(Admission::Admit) + } + + fn from_dump(_prefix: &Path) -> Result { + unreachable!("the guest never loads a dump") + } + + fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { + unreachable!("the guest never writes a dump") + } + + fn state_file_in_dump(prefix: &Path) -> PathBuf { + prefix.to_path_buf() + } + + fn from_canonical_state( + _state_file: &Path, + _l2_tx_count: L2TxCount, + ) -> Result { + unreachable!("the guest starts from genesis") + } + } + + impl CanonicalState for ProbeApp { + fn canonical_snapshot_bytes(&self) -> Result, AppError> { + if self.snapshot_fails { + return Err(AppError::Internal { + reason: "snapshot refused".to_string(), + }); + } + Ok(self + .seen .iter() - .any(|report| report.as_slice() == b"scheduler dropped invalid batch"), - "missing invalid batch report, got: {reports:?}" - ); + .map(|(index, timestamp)| format!("{index}@{timestamp};")) + .collect::() + .into_bytes()) + } } } diff --git a/examples/canonical-test/Cargo.toml b/examples/canonical-test/Cargo.toml index 452900b0..d16296b0 100644 --- a/examples/canonical-test/Cargo.toml +++ b/examples/canonical-test/Cargo.toml @@ -16,6 +16,8 @@ path = "src/main.rs" [dependencies] app-core = { path = "../app-core" } canonical-app = { path = "../canonical-app" } +# Setup's base loader, run against real machines. +sequencer = { path = "../../sequencer" } sequencer-core = { path = "../../sequencer-core", features = ["test-support"] } ssz = { workspace = true } testsi = { workspace = true } @@ -23,3 +25,4 @@ rollups-types = { workspace = true, features = ["test-support"] } k256 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tempfile = { workspace = true } diff --git a/examples/canonical-test/src/export.rs b/examples/canonical-test/src/export.rs new file mode 100644 index 00000000..c99322dd --- /dev/null +++ b/examples/canonical-test/src/export.rs @@ -0,0 +1,278 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Setup's base loader against real machines: `read_machine_export` must read +//! the template image as genesis, and a machine advanced and stored as exactly +//! what a native `CanonicalRunner` over the same inputs exports, without +//! changing the stored directory. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use app_core::application::{ + DEVNET_SEQUENCER_ADDRESS, Method, Transfer, WALLET_SCHEDULER_LIMITS, WalletApp, WalletConfig, + Withdrawal, +}; +use canonical_app::{STATE_INSPECT_QUERY, SchedulerConfig}; +use rollups_types::Erc20Deposit; +use rollups_types::alloy_primitives::{Address, U256, address}; +use sequencer::machine::read_machine_export; +use sequencer_core::application::{Application, CanonicalState}; +use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::history::L2TxCount; +use sequencer_core::scheduler::{CanonicalRunner, L1Input, SchedulerExport, TxFate}; +use testsi::{InputBuilder, TestResult}; + +use crate::{ + TEST_CHAIN_ID, TEST_DAPP_ADDRESS, address_from_signing_key, devnet_machine, machine_image_path, + signed_user_op, signing_key, +}; + +const BOB: Address = address!("0x8888888888888888888888888888888888888888"); + +/// The template image is the genesis base: six zeros and the genesis wallet, +/// and reading it leaves every file of the image as it was. +#[testsi::test_dapp(kind("export"))] +pub fn export_of_the_template_image_is_genesis() -> TestResult { + let image = machine_image_path(); + let before = directory_listing(&image); + let scratch = tempfile::tempdir()?; + let state_file = scratch.path().join("state"); + + let export = read_machine_export(&image, &state_file)?; + + assert_eq!(export, SchedulerExport::decode(&[0; 48])?); + let genesis = WalletApp::new(WalletConfig::devnet()); + assert_eq!( + std::fs::read(&state_file)?, + genesis.canonical_snapshot_bytes()? + ); + let projected = WalletApp::from_canonical_state(&state_file, L2TxCount::ZERO)?; + assert_eq!( + projected.canonical_snapshot_bytes()?, + genesis.canonical_snapshot_bytes()? + ); + assert_eq!(directory_listing(&image), before, "the image was written"); + Ok(()) +} + +/// A machine advanced through a mixed run and stored exports exactly what the +/// native runner holds after the same inputs, and the wallet accepts the +/// state at the exported count, and no other. +#[testsi::test_dapp(kind("export"))] +pub fn export_of_a_stored_machine_matches_the_native_runner() -> TestResult { + let inputs = mixed_run(); + let mut runner = CanonicalRunner::genesis( + WalletApp::new(WalletConfig::devnet()), + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS), + ); + let mut machine = devnet_machine()?; + let mut skipped = 0; + for (index, input) in inputs.iter().enumerate() { + let native = runner + .process_input(input.l1_input(index)) + .expect("the native run never fails"); + skipped += native + .txs + .iter() + .filter(|tx| matches!(tx.fate, TxFate::Skipped(_))) + .count(); + let (outputs, _reports) = machine.advance_state(input.to_machine())?; + assert_eq!( + outputs.list().len(), + native.outputs.len(), + "input {index}: the machine and the native runner disagree" + ); + } + let expected = runner.export(); + // The run must cover what a base can hold: directs still queued, a drained + // one, executed ops and a skipped one. + assert_eq!(skipped, 1, "the run skips one op"); + assert_eq!( + expected, + SchedulerExport { + l1_input_count: 6, + l2_tx_count: 3, + next_batch_nonce: 1, + clock: 11, + first_queued_input: 3, + fridge_len: 2, + } + ); + + let scratch = tempfile::tempdir()?; + let stored = scratch.path().join("machine"); + machine.store(&stored)?; + drop(machine); + let before = directory_listing(&stored); + let state_file = scratch.path().join("state"); + + let export = read_machine_export(&stored, &state_file)?; + + assert_eq!(export, expected); + let native_state = runner + .inspect(STATE_INSPECT_QUERY) + .expect("the wallet encodes its state"); + assert_eq!(std::fs::read(&state_file)?, native_state); + let count = L2TxCount::new(export.l2_tx_count); + let projected = WalletApp::from_canonical_state(&state_file, count)?; + assert_eq!(projected.canonical_snapshot_bytes()?, native_state); + assert_eq!(projected.l2_tx_count(), count); + let off_by_one = L2TxCount::new(export.l2_tx_count + 1); + assert!(WalletApp::from_canonical_state(&state_file, off_by_one).is_err()); + + // A second read finds the same machine: the first one changed nothing. + let again = scratch.path().join("state-again"); + assert_eq!(read_machine_export(&stored, &again)?, expected); + assert_eq!(std::fs::read(&again)?, native_state); + assert_eq!( + directory_listing(&stored), + before, + "the machine was written" + ); + Ok(()) +} + +/// A directory that holds no machine is a load error, and no state is written. +#[testsi::test_dapp(kind("export"))] +pub fn export_of_a_missing_machine_is_a_load_error() -> TestResult { + let scratch = tempfile::tempdir()?; + let state_file = scratch.path().join("state"); + let err = read_machine_export(scratch.path(), &state_file).expect_err("no machine there"); + assert!( + matches!(err, sequencer::machine::MachineExportError::Load { .. }), + "{err}" + ); + assert!(!state_file.exists()); + Ok(()) +} + +/// Six inputs. The first batch drains Alice's deposit by its frame, runs her +/// transfer, skips its replay and runs her withdrawal; the over-cap deposit +/// never joins; a rejected batch changes nothing; two deposits above the +/// clock stay queued. +fn mixed_run() -> Vec { + let token = WalletConfig::devnet().supported_erc20_token; + let alice_key = signing_key(41); + let alice = address_from_signing_key(&alice_key); + let carol = address!("0xc0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0"); + let cap = usize::try_from(WALLET_SCHEDULER_LIMITS.max_direct_payload_bytes) + .expect("the cap fits usize"); + let over_cap = vec![0_u8; cap + 1 - rollups_types::ERC20_DEPOSIT_PREFIX_BYTES]; + let method = |method: Method| ssz::Encode::as_ssz_bytes(&method); + let transfer = method(Method::Transfer(Transfer { + amount: U256::from(100_u64), + to: BOB, + })); + let ops: Vec = vec![ + signed_user_op(&alice_key, 0, transfer.clone()), + signed_user_op(&alice_key, 0, transfer), + signed_user_op( + &alice_key, + 1, + method(Method::Withdrawal(Withdrawal { + amount: U256::from(50_u64), + })), + ), + ]; + let batch = Batch { + nonce: 0, + frames: vec![Frame { + user_ops: ops, + safe_block: 11, + fee_price: 0, + }], + }; + vec![ + Step::deposit(10, 1_000, token, alice, 1_000, Vec::new()), + Step::deposit(10, 1_000, token, carol, 7, over_cap), + Step::submitter(12, 1_024, ssz::Encode::as_ssz_bytes(&batch)), + Step::deposit(12, 1_024, token, carol, 77, Vec::new()), + Step::submitter(13, 1_036, vec![0xff, 0xee, 0xdd]), + Step::deposit(13, 1_036, token, alice, 5, Vec::new()), + ] +} + +/// One input, as both the machine and the native runner receive it. +struct Step { + sender: Address, + block: u64, + timestamp: u64, + payload: Vec, +} + +impl Step { + fn deposit( + block: u64, + timestamp: u64, + token: Address, + owner: Address, + amount: u64, + exec_layer_data: Vec, + ) -> Self { + let payload = Erc20Deposit { + token, + sender: owner, + value: U256::from(amount), + exec_layer_data, + } + .encode(); + Self { + sender: WalletConfig::devnet().erc20_portal_address, + block, + timestamp, + payload, + } + } + + fn submitter(block: u64, timestamp: u64, payload: Vec) -> Self { + Self { + sender: DEVNET_SEQUENCER_ADDRESS, + block, + timestamp, + payload, + } + } + + fn l1_input(&self, index: usize) -> L1Input { + L1Input { + index: u64::try_from(index).expect("the index fits u64"), + sender: self.sender, + block: self.block, + timestamp: self.timestamp, + chain_id: TEST_CHAIN_ID, + app_contract: TEST_DAPP_ADDRESS, + payload: self.payload.clone(), + } + } + + fn to_machine(&self) -> InputBuilder { + let block = usize::try_from(self.block).expect("the block fits usize"); + let timestamp = usize::try_from(self.timestamp).expect("the timestamp fits usize"); + InputBuilder::from_address(self.sender) + .at_block(block) + .with_block_timestamp(timestamp) + .with_payload(&self.payload) + } +} + +/// Every file under `dir`, with its length and modification time: a store, +/// a shared load or any write moves one of them. +fn directory_listing(dir: &Path) -> BTreeMap { + let mut listing = BTreeMap::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(&next).expect("read the machine directory") { + let entry = entry.expect("read a directory entry"); + let metadata = entry.metadata().expect("read an entry's metadata"); + if metadata.is_dir() { + pending.push(entry.path()); + } else { + let modified = metadata.modified().expect("read a modification time"); + listing.insert(entry.path(), (metadata.len(), modified)); + } + } + } + listing +} diff --git a/examples/canonical-test/src/golden.rs b/examples/canonical-test/src/golden.rs index 204655fc..e8645970 100644 --- a/examples/canonical-test/src/golden.rs +++ b/examples/canonical-test/src/golden.rs @@ -1,27 +1,38 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Golden guest cases from `tests/fixtures/consensus_vectors_v1.json`: pinned -//! payloads go into the built image unchanged (nothing is signed or encoded here) -//! and its raw outputs must equal the pinned bytes; a mismatch is a finding. -//! Reports are diagnostics, not consensus, and are not compared. +//! Golden guest cases from `tests/fixtures/consensus_vectors_v2.json`: every +//! guest scenario replayed in the built image from genesis. Pinned payloads go +//! in unchanged (nothing is signed or encoded here), and the raw outputs, the +//! `scheduler` report after every input and the final `state` report must equal +//! the pinned bytes; a mismatch is a finding. A rejected batch's report is a +//! diagnostic, so only whether one appears is checked. use std::collections::BTreeMap; -use rollups_types::alloy_primitives::{Address, Bytes, U256}; +use app_core::application::{DEVNET_SEQUENCER_ADDRESS, WalletApp}; +use canonical_app::{INVALID_BATCH_REPORT, SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY}; +use rollups_types::alloy_primitives::{Address, B256, Bytes, U256, keccak256}; +use sequencer_core::scheduler::{SchedulerConfig, SchedulerExport}; use serde::Deserialize; use testsi::{InputBuilder, Output, TestResult}; -use crate::{TEST_CHAIN_ID, TEST_DAPP_ADDRESS, devnet_machine}; +use crate::{TEST_CHAIN_ID, TEST_DAPP_ADDRESS, devnet_machine, inspect_report}; const CONSENSUS_VECTORS: &str = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../../tests/fixtures/consensus_vectors_v1.json" + "/../../tests/fixtures/consensus_vectors_v2.json" )); /// The pinned output that marks the signed transfer as included. const SIGNED_TRANSFER_NOTICE: &str = "transfer_notice_alice_bob"; +/// The scenarios the earlier template held, restated under strict frames. +const RESTATED: [&str; 4] = ["deposit", "transfer", "withdrawal", "spill_batch"]; + +/// Scenarios named with this prefix carry one signature variant each. +const SIGNATURE_PREFIX: &str = "signature_"; + #[derive(Debug, Deserialize)] struct ConsensusVectors { signature: PinnedSignature, @@ -44,6 +55,7 @@ struct PinnedVariant { struct GuestVectors { chain_id: u64, app_contract: Address, + config: PinnedConfig, senders: BTreeMap, first_input_envelope: Bytes, payloads: BTreeMap, @@ -51,6 +63,14 @@ struct GuestVectors { scenarios: Vec, } +#[derive(Debug, Deserialize)] +struct PinnedConfig { + sequencer_address: Address, + max_wait_blocks: u64, + fridge_bound: u64, + max_direct_payload_bytes: u64, +} + #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(rename_all = "snake_case")] enum PinnedOutput { @@ -66,14 +86,57 @@ enum PinnedOutput { struct Scenario { name: String, steps: Vec, + state: Option, + state_keccak256: Option, } +/// What the machine can show of a step. The handed-out transactions and the +/// reject reason are not observable here; the native and C checks hold them. #[derive(Debug, Deserialize)] struct Step { from: String, - block: usize, + block: u64, + timestamp: u64, payload: String, + outcome: Outcome, outputs: Vec, + scheduler: PinnedExport, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Outcome { + DirectQueued, + DirectDropped, + BatchAccepted, + BatchStale, + BatchRejected, +} + +#[derive(Debug, Deserialize)] +struct PinnedExport { + l1_input_count: u64, + l2_tx_count: u64, + next_batch_nonce: u64, + clock: u64, + first_queued_input: u64, + fridge_len: u64, +} + +impl PinnedExport { + /// The 48-byte `scheduler` report these fields encode to. + fn report(&self) -> Vec { + SchedulerExport { + l1_input_count: self.l1_input_count, + l2_tx_count: self.l2_tx_count, + next_batch_nonce: self.next_batch_nonce, + clock: self.clock, + first_queued_input: self.first_queued_input, + fridge_len: self.fridge_len, + } + .encode() + .to_vec() + } } impl GuestVectors { @@ -86,20 +149,20 @@ impl GuestVectors { fn input(&self, step: &Step) -> InputBuilder { InputBuilder::from_address(self.senders[&step.from]) - .at_block(step.block) + .at_block(usize::try_from(step.block).expect("a pinned block fits usize")) + .with_block_timestamp( + usize::try_from(step.timestamp).expect("a pinned timestamp fits usize"), + ) .with_payload(&self.payloads[&step.payload]) } - /// Replays `name` on a fresh machine and compares every step's outputs. - fn run(&self, name: &str) -> TestResult { - assert_eq!( - (self.chain_id, self.app_contract), - (TEST_CHAIN_ID, TEST_DAPP_ADDRESS), - "the golden inputs were pinned against the devnet machine's chain and app" - ); + /// Replays `scenario` on a fresh machine and compares every step. + fn run(&self, scenario: &Scenario) -> TestResult { + let name = &scenario.name; let mut machine = devnet_machine()?; - for (index, step) in self.scenario(name).steps.iter().enumerate() { - let (outputs, _reports) = machine.advance_state(self.input(step))?; + for (index, step) in scenario.steps.iter().enumerate() { + // testsi turns a rejected input into an error: the guest accepts all. + let (outputs, reports) = machine.advance_state(self.input(step))?; let actual: Vec = outputs.list().iter().map(pinned_form).collect(); let expected: Vec = step .outputs @@ -107,6 +170,26 @@ impl GuestVectors { .map(|output| self.outputs[output].clone()) .collect(); assert_eq!(actual, expected, "{name}, step {index} ({})", step.payload); + + let expected_reports: &[&[u8]] = if step.outcome == Outcome::BatchRejected { + &[INVALID_BATCH_REPORT] + } else { + &[] + }; + assert_eq!(reports, expected_reports, "{name}, step {index} reports"); + + assert_eq!( + inspect_report(&mut machine, SCHEDULER_INSPECT_QUERY)?, + step.scheduler.report(), + "{name}, step {index}: scheduler report" + ); + } + + let state = inspect_report(&mut machine, STATE_INSPECT_QUERY)?; + match (&scenario.state, &scenario.state_keccak256) { + (Some(pinned), None) => assert_eq!(state, pinned.as_ref(), "{name}: state report"), + (None, Some(pinned)) => assert_eq!(keccak256(&state), *pinned, "{name}: state report"), + _ => panic!("{name} must pin its state one way"), } Ok(()) } @@ -124,56 +207,66 @@ fn pinned_form(output: &Output) -> PinnedOutput { } fn consensus_vectors() -> ConsensusVectors { - serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v1.json") + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json") } +/// The image runs the devnet template on the devnet chain and app, so the +/// pinned deployment must be that one. #[testsi::test_dapp(kind("golden"))] -pub fn golden_deposit_emits_the_pinned_notice() -> TestResult { +pub fn golden_vectors_pin_the_devnet_template() -> TestResult { let guest = consensus_vectors().guest; + assert_eq!( + (guest.chain_id, guest.app_contract), + (TEST_CHAIN_ID, TEST_DAPP_ADDRESS), + "the golden inputs were pinned against the devnet machine's chain and app" + ); + let pinned = &guest.config; + assert_eq!( + SchedulerConfig { + sequencer_address: pinned.sequencer_address, + max_wait_blocks: pinned.max_wait_blocks, + fridge_bound: pinned.fridge_bound, + max_direct_payload_bytes: pinned.max_direct_payload_bytes, + }, + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS) + ); + assert_eq!(guest.senders["sequencer"], DEVNET_SEQUENCER_ADDRESS); + // testsi wraps each payload in an EvmAdvance at test time; pinning the // first envelope tells a host encoder drift apart from a guest change. - let first = &guest.scenario("deposit").steps[0]; + let first = &guest.scenarios[0].steps[0]; assert_eq!( guest .input(first) .encode(TEST_CHAIN_ID as usize, U256::ZERO, TEST_DAPP_ADDRESS), guest.first_input_envelope.as_ref() ); - guest.run("deposit") -} - -#[testsi::test_dapp(kind("golden"))] -pub fn golden_signed_transfer_emits_the_pinned_notice() -> TestResult { - consensus_vectors().guest.run("transfer") -} - -#[testsi::test_dapp(kind("golden"))] -pub fn golden_withdrawal_emits_the_pinned_voucher() -> TestResult { - consensus_vectors().guest.run("withdrawal") + Ok(()) } #[testsi::test_dapp(kind("golden"))] -pub fn golden_spill_batch_emits_the_pinned_outputs() -> TestResult { - consensus_vectors().guest.run("spill_batch") +pub fn golden_restated_scenarios_emit_the_pinned_outputs() -> TestResult { + let guest = consensus_vectors().guest; + for name in RESTATED { + guest.run(guest.scenario(name))?; + } + Ok(()) } /// Each signature variant rides in its own batch after a deposit, followed by /// the signer's correctly signed op at the same nonce: a variant that runs as -/// the signer emits the transfer notice and leaves the follow-up with nothing; -/// any other emits nothing and the follow-up shows the signer's nonce was not -/// consumed. The image is thus checked for running each variant as the signer -/// or not; which other sender a variant recovers is pinned natively, in -/// sequencer-core. +/// the signer emits the transfer notice and leaves the follow-up skipped; any +/// other leaves the signer's nonce for the follow-up. Which sender a variant +/// recovers is pinned natively; here the image is held to the pinned outputs. #[testsi::test_dapp(kind("golden"))] pub fn golden_signature_variants_match_the_pinned_guest_outcome() -> TestResult { let vectors = consensus_vectors(); for variant in &vectors.signature.variants { - let name = format!("signature_{}", variant.name); - vectors.guest.run(&name)?; - + let name = format!("{SIGNATURE_PREFIX}{}", variant.name); + let scenario = vectors.guest.scenario(&name); // The pinned outputs agree with the pinned recovery on whether the op // runs as the signer. - let included = vectors.guest.scenario(&name).steps[1] + let included = scenario.steps[1] .outputs .iter() .any(|output| output == SIGNED_TRANSFER_NOTICE); @@ -183,5 +276,26 @@ pub fn golden_signature_variants_match_the_pinned_guest_outcome() -> TestResult "{name}" ); } + for scenario in &vectors.guest.scenarios { + if scenario.name.starts_with(SIGNATURE_PREFIX) { + vectors.guest.run(scenario)?; + } + } + Ok(()) +} + +/// Every other scenario: the gates, the clock, the backstop, the payload cap +/// and the skipped, executed and failed user op. The nonce-exhausted skip has +/// no scenario; the executor's unit tests hold it. +#[testsi::test_dapp(kind("golden"))] +pub fn golden_scheduler_rules_match_the_pinned_steps() -> TestResult { + let guest = consensus_vectors().guest; + for scenario in &guest.scenarios { + if !RESTATED.contains(&scenario.name.as_str()) + && !scenario.name.starts_with(SIGNATURE_PREFIX) + { + guest.run(scenario)?; + } + } Ok(()) } diff --git a/examples/canonical-test/src/main.rs b/examples/canonical-test/src/main.rs index aaae8f88..c01b9662 100644 --- a/examples/canonical-test/src/main.rs +++ b/examples/canonical-test/src/main.rs @@ -1,27 +1,32 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +mod export; mod golden; use std::path::PathBuf; use app_core::application::{ - DEVNET_SEQUENCER_ADDRESS, DepositNotice, Method, Transfer, TransferNotice, WalletConfig, - Withdrawal, + DEVNET_SEQUENCER_ADDRESS, DepositNotice, Method, Transfer, TransferNotice, + WALLET_SCHEDULER_LIMITS, WalletApp, WalletConfig, Withdrawal, +}; +use canonical_app::{ + INVALID_BATCH_REPORT, SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, SchedulerConfig, + UNSUPPORTED_QUERY_REPORT, }; -use canonical_app::SchedulerConfig; use k256::ecdsa::SigningKey; use rollups_types::alloy_primitives::{Address, U256, address}; use rollups_types::alloy_sol_types::{Eip712Domain, SolCall}; use rollups_types::{Erc20Deposit, Erc20Transfer}; use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::scheduler::SchedulerExport; use sequencer_core::test_support::sign_user_op; use sequencer_core::user_op::UserOp; +use testsi::libtest_mimic::Failed; use testsi::{InputBuilder, Machine, MachineBuilder, OutputsForInput, TestResult}; testsi::testsi_main!(); -const INVALID_BATCH_REPORT: &[u8] = b"scheduler dropped invalid batch"; const TEST_CHAIN_ID: u64 = 31337; const TEST_DAPP_ADDRESS: Address = Address::ZERO; @@ -86,8 +91,9 @@ pub fn scheduler_rejected_batch_does_not_consume_nonce() -> TestResult { #[testsi::test_dapp(kind("scheduler"))] pub fn scheduler_stale_batch_is_skipped_without_consuming_nonce() -> TestResult { let mut machine = devnet_machine()?; - let stale_trigger_block = - SchedulerConfig::new(DEVNET_SEQUENCER_ADDRESS).max_wait_blocks as usize + 1; + let max_wait_blocks = + SchedulerConfig::for_application::(DEVNET_SEQUENCER_ADDRESS).max_wait_blocks; + let stale_trigger_block = usize::try_from(max_wait_blocks).expect("window fits usize") + 1; // Stale batch (nonce 0, safe_block 1, inclusion block > max_wait_blocks) → skipped silently. let (outputs, reports) = machine.advance_state(batch_input( @@ -140,6 +146,24 @@ pub fn scheduler_reports_safe_block_above_inclusion_block_from_guest() -> TestRe Ok(()) } +/// Strict frames: a frame's safe block must lie below its batch's inclusion +/// block, so a frame at the inclusion block itself is rejected too. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_reports_safe_block_at_inclusion_block_from_guest() -> TestResult { + let mut machine = devnet_machine()?; + + let (outputs, reports) = + machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + assert_invalid_batch_step(&outputs, &reports); + + // The reject consumed no batch nonce: the same batch one block later runs. + let (outputs, reports) = + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; + assert_no_outputs_or_reports(&outputs, &reports); + assert_scheduler_report(&mut machine, [2, 0, 1, 10, 2, 0])?; + Ok(()) +} + #[testsi::test_dapp(kind("scheduler"))] pub fn scheduler_emits_deposit_notice_from_guest() -> TestResult { let mut machine = devnet_machine()?; @@ -156,7 +180,7 @@ pub fn scheduler_emits_deposit_notice_from_guest() -> TestResult { assert_no_outputs_or_reports(&outputs, &reports); let (outputs, reports) = - machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; assert_deposit_notice( &outputs, &reports, @@ -186,7 +210,7 @@ pub fn scheduler_emits_transfer_notice_from_guest() -> TestResult { assert_no_outputs_or_reports(&outputs, &reports); let (outputs, reports) = - machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; assert_deposit_notice( &outputs, &reports, @@ -227,7 +251,7 @@ pub fn scheduler_emits_withdrawal_voucher_from_guest() -> TestResult { assert_no_outputs_or_reports(&outputs, &reports); let (outputs, reports) = - machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; assert_deposit_notice( &outputs, &reports, @@ -287,7 +311,7 @@ pub fn scheduler_fee_arithmetic_matches_host_from_guest() -> TestResult { assert_no_outputs_or_reports(&outputs, &reports); } let (outputs, reports) = - machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; assert!(reports.is_empty(), "drain reports: {reports:?}"); assert_eq!( outputs.list().len(), @@ -335,10 +359,274 @@ pub fn scheduler_fee_arithmetic_matches_host_from_guest() -> TestResult { Ok(()) } +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_reports_its_export_after_a_deposit_and_its_drain() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + let amount = U256::from(250_u64); + assert_scheduler_report(&mut machine, [0; 6])?; + + let (outputs, reports) = machine.advance_state(portal_input(10, token, depositor, amount))?; + assert_no_outputs_or_reports(&outputs, &reports); + // Queued, not executed: the L2 tx count stays at zero. + assert_scheduler_report(&mut machine, [1, 0, 0, 0, 0, 1])?; + + let (outputs, reports) = + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; + assert_deposit_notice(&outputs, &reports, token, depositor, amount); + assert_scheduler_report(&mut machine, [2, 1, 1, 10, 2, 0])?; + Ok(()) +} + +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_answers_the_state_query_as_before() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + let amount = U256::from(250_u64); + + let genesis = app_core::wallet_snapshot::encode(&WalletApp::new(WalletConfig::devnet())); + assert_eq!(inspect_report(&mut machine, STATE_INSPECT_QUERY)?, genesis); + assert_eq!(inspect_report(&mut machine, b"")?, genesis); + assert_eq!( + inspect_report(&mut machine, b"balances")?, + UNSUPPORTED_QUERY_REPORT + ); + + machine.advance_state(portal_input(10, token, depositor, amount))?; + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; + let wallet = reported_wallet(&mut machine)?; + assert_eq!(wallet.current_user_balance(depositor), amount); + Ok(()) +} + +/// A direct one byte over the cap never joins the fridge, so the drain passes +/// it by; one exactly at the cap executes. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_never_executes_a_direct_over_the_payload_cap() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + let cap = usize::try_from(WALLET_SCHEDULER_LIMITS.max_direct_payload_bytes) + .expect("the cap fits usize"); + let at_cap = vec![0_u8; cap - rollups_types::ERC20_DEPOSIT_PREFIX_BYTES]; + let over_cap = vec![0_u8; at_cap.len() + 1]; + + let (outputs, reports) = machine.advance_state(portal_input_with( + 10, + token, + depositor, + U256::from(7_u64), + over_cap, + ))?; + assert_no_outputs_or_reports(&outputs, &reports); + assert_scheduler_report(&mut machine, [1, 0, 0, 0, 1, 0])?; + + let (outputs, reports) = machine.advance_state(portal_input_with( + 10, + token, + depositor, + U256::from(5_u64), + at_cap, + ))?; + assert_no_outputs_or_reports(&outputs, &reports); + + let (outputs, reports) = + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; + assert_deposit_notice(&outputs, &reports, token, depositor, U256::from(5_u64)); + assert_scheduler_report(&mut machine, [3, 1, 1, 10, 3, 0])?; + let wallet = reported_wallet(&mut machine)?; + assert_eq!(wallet.current_user_balance(depositor), U256::from(5_u64)); + Ok(()) +} + +/// An op its sender cannot pay in full still executes: it pays what the sender +/// has, consumes the nonce, and its transfer fails on the empty balance. A +/// later op at the same nonce is skipped; the next nonce runs. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_executes_an_underfunded_op_and_consumes_its_nonce() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let bob = address!("0x8888888888888888888888888888888888888888"); + let alice_key = signing_key(21); + let alice = address_from_signing_key(&alice_key); + let fee_price: u16 = 200; + let gas = sequencer_core::fee::fee_to_linear(fee_price); + let base_gas = sequencer_core::fee::fee_to_linear(0); + let one = U256::from(1_u64); + assert!(gas > one, "the deposit must be nonzero yet below the fee"); + let transfer = || { + ssz::Encode::as_ssz_bytes(&Method::Transfer(Transfer { + amount: one, + to: bob, + })) + }; + + machine.advance_state(portal_input(10, token, alice, gas - one))?; + let underfunded = signed_user_op_with_fee(&alice_key, 0, fee_price, transfer()); + let (outputs, reports) = machine.advance_state(batch_input( + 11, + batch_with_frame_fee(0, 10, fee_price, vec![underfunded]), + ))?; + // Only the deposit's notice: the op ran, but its transfer failed. + assert_deposit_notice(&outputs, &reports, token, alice, gas - one); + + let top_up = U256::from(50_u64); + machine.advance_state(portal_input(12, token, alice, top_up))?; + let replayed = signed_user_op(&alice_key, 0, transfer()); + let next = signed_user_op(&alice_key, 1, transfer()); + let (outputs, reports) = machine.advance_state(batch_input( + 13, + batch_with_frame(1, 12, vec![replayed, next]), + ))?; + assert!(reports.is_empty(), "expected no reports, got {reports:?}"); + assert_eq!(outputs.list().len(), 2, "got {:?}", outputs.list()); + let deposit = DepositNotice::abi_decode(&outputs[0].expect_notice().payload) + .expect("decode deposit notice"); + assert_eq!((deposit.sender, deposit.amount), (alice, top_up)); + let transferred = TransferNotice::abi_decode(&outputs[1].expect_notice().payload) + .expect("decode transfer notice"); + assert_eq!( + ( + transferred.sender, + transferred.recipient, + transferred.amount + ), + (alice, bob, one) + ); + + // Four executed: two deposits, the underfunded op and the nonce-1 op. + assert_scheduler_report(&mut machine, [4, 4, 2, 12, 4, 0])?; + let wallet = reported_wallet(&mut machine)?; + assert_eq!(wallet.current_user_nonce(alice), 2); + assert_eq!(wallet.current_user_balance(alice), top_up - base_gas - one); + assert_eq!(wallet.current_user_balance(bob), one); + assert_eq!( + wallet.current_user_balance(DEVNET_SEQUENCER_ADDRESS), + gas - one + base_gas + ); + Ok(()) +} + +/// Gate f: once a frame raised the clock (to 11), a batch whose first frame lies +/// below it is rejected without consuming its nonce, and a retry above the +/// clock runs and drains the later deposit. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_reports_first_frame_below_the_clock_from_guest() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + + machine.advance_state(portal_input(10, token, depositor, U256::from(100_u64)))?; + let (outputs, reports) = + machine.advance_state(batch_input(12, batch_with_safe_blocks(0, &[11])))?; + assert_deposit_notice(&outputs, &reports, token, depositor, U256::from(100_u64)); + + let (outputs, reports) = + machine.advance_state(portal_input(12, token, depositor, U256::from(50_u64)))?; + assert_no_outputs_or_reports(&outputs, &reports); + let (outputs, reports) = + machine.advance_state(batch_input(13, batch_with_safe_blocks(1, &[10])))?; + assert_invalid_batch_step(&outputs, &reports); + + let (outputs, reports) = + machine.advance_state(batch_input(13, batch_with_safe_blocks(1, &[12])))?; + assert_deposit_notice(&outputs, &reports, token, depositor, U256::from(50_u64)); + assert_scheduler_report(&mut machine, [5, 2, 2, 12, 5, 0])?; + Ok(()) +} + +/// A frame drains the deposits of its own safe block, strictly below the +/// batch's inclusion block, then runs its ops: the deposit's notice leaves +/// before the op's voucher, in the one input. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_strict_drain_runs_the_deposit_then_the_op() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let alice_key = signing_key(31); + let alice = address_from_signing_key(&alice_key); + let deposit = U256::from(250_u64); + let amount = U256::from(100_u64); + + machine.advance_state(portal_input(10, token, alice, deposit))?; + let withdrawal = signed_user_op( + &alice_key, + 0, + ssz::Encode::as_ssz_bytes(&Method::Withdrawal(Withdrawal { amount })), + ); + let (outputs, reports) = + machine.advance_state(batch_input(11, batch_with_frame(0, 10, vec![withdrawal])))?; + assert!(reports.is_empty(), "expected no reports, got {reports:?}"); + assert_eq!(outputs.list().len(), 2, "got {:?}", outputs.list()); + let notice = DepositNotice::abi_decode(&outputs[0].expect_notice().payload) + .expect("decode deposit notice"); + assert_eq!((notice.sender, notice.amount), (alice, deposit)); + let voucher = outputs[1].expect_voucher(); + assert_eq!(voucher.destination, token); + let paid = Erc20Transfer::abi_decode(&voucher.payload).expect("decode withdrawal voucher"); + assert_eq!((paid.recipient, paid.amount), (alice, amount)); + assert_scheduler_report(&mut machine, [2, 2, 1, 10, 2, 0])?; + Ok(()) +} + +/// The backstop drains an overdue deposit before a batch's gates run, so the +/// notice leaves before the reject's report, in the one input; the reject +/// consumes no batch nonce. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_rejected_batch_reports_after_its_backstop_outputs() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + let amount = U256::from(250_u64); + let overdue = 1 + WALLET_SCHEDULER_LIMITS.max_wait_blocks; + let overdue_block = usize::try_from(overdue).expect("the block fits usize"); + + machine.advance_state(portal_input(1, token, depositor, amount))?; + let (outputs, reports) = machine.advance_state(batch_input(overdue_block, empty_batch(1)))?; + assert_deposit_notice_then_invalid_batch(&outputs, &reports, depositor, amount); + // The drain raised the clock to the deposit's block, and no further. + assert_scheduler_report(&mut machine, [2, 1, 0, 1, 2, 0])?; + + let (outputs, reports) = machine.advance_state(batch_input( + overdue_block, + batch_with_safe_blocks(0, &[overdue - 1]), + ))?; + assert_no_outputs_or_reports(&outputs, &reports); + assert_scheduler_report(&mut machine, [3, 1, 1, overdue - 1, 3, 0])?; + Ok(()) +} + +/// Only the exact query names are answered; the machine keeps serving after +/// any other, and an inspect changes no state. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_answers_an_unknown_query_with_a_report() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let depositor = address!("0x7777777777777777777777777777777777777777"); + + machine.advance_state(portal_input(10, token, depositor, U256::from(9_u64)))?; + for query in [&b"scheduler "[..], b"Scheduler", b"state\0", &[0xff; 64]] { + assert_eq!( + inspect_report(&mut machine, query)?, + UNSUPPORTED_QUERY_REPORT, + "{query:?}" + ); + } + assert_scheduler_report(&mut machine, [1, 0, 0, 0, 0, 1])?; + let (outputs, reports) = + machine.advance_state(batch_input(11, batch_with_safe_blocks(0, &[10])))?; + assert_deposit_notice(&outputs, &reports, token, depositor, U256::from(9_u64)); + Ok(()) +} + +/// The devnet template image, as `just canonical-build-machine-image` stores it. +fn machine_image_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../canonical-app/out/canonical-machine-image") +} + fn devnet_machine() -> Result> { - let machine_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../canonical-app/out/canonical-machine-image"); - let machine = MachineBuilder::load_from(machine_path) + let machine = MachineBuilder::load_from(machine_image_path()) .at_chain(TEST_CHAIN_ID as usize) .deployed_at(TEST_DAPP_ADDRESS) .no_console_putchar(false) @@ -393,6 +681,16 @@ fn sequencer_input(block: usize, payload: &[u8]) -> InputBuilder { } fn portal_input(block: usize, token: Address, sender: Address, value: U256) -> InputBuilder { + portal_input_with(block, token, sender, value, Vec::new()) +} + +fn portal_input_with( + block: usize, + token: Address, + sender: Address, + value: U256, + exec_layer_data: Vec, +) -> InputBuilder { let wallet_config = WalletConfig::devnet(); InputBuilder::from_address(wallet_config.erc20_portal_address) .at_block(block) @@ -401,7 +699,7 @@ fn portal_input(block: usize, token: Address, sender: Address, value: U256) -> I token, sender, value, - exec_layer_data: Vec::new(), + exec_layer_data, } .encode(), ) @@ -454,6 +752,40 @@ fn batch_with_frame_fee( } } +/// The single report an inspect query leaves. +fn inspect_report(machine: &mut Machine, query: &[u8]) -> Result, Failed> { + let mut reports = machine.inspect(query)?; + assert_eq!( + reports.len(), + 1, + "expected one report for {query:?}, got {reports:?}" + ); + Ok(reports.remove(0)) +} + +/// The `scheduler` report, in the export's field order: L1 input count, L2 tx +/// count, next batch nonce, clock, first queued input, fridge length. +fn assert_scheduler_report(machine: &mut Machine, expected: [u64; 6]) -> TestResult { + let report = inspect_report(machine, SCHEDULER_INSPECT_QUERY)?; + let export = SchedulerExport::decode(&report).expect("decode the scheduler report"); + let actual = [ + export.l1_input_count, + export.l2_tx_count, + export.next_batch_nonce, + export.clock, + export.first_queued_input, + export.fridge_len, + ]; + assert_eq!(actual, expected, "scheduler report"); + Ok(()) +} + +/// The wallet the guest's `state` report holds, decoded. +fn reported_wallet(machine: &mut Machine) -> Result { + let report = inspect_report(machine, STATE_INSPECT_QUERY)?; + Ok(app_core::wallet_snapshot::decode(&report).expect("decode the state report")) +} + fn assert_invalid_batch_step(outputs: &OutputsForInput, reports: &[Vec]) { assert!( outputs.list().is_empty(), @@ -468,6 +800,19 @@ fn assert_invalid_batch_step(outputs: &OutputsForInput, reports: &[Vec]) { ); } +fn assert_deposit_notice_then_invalid_batch( + outputs: &OutputsForInput, + reports: &[Vec], + sender: Address, + amount: U256, +) { + assert_eq!(reports, [INVALID_BATCH_REPORT], "reports"); + assert_eq!(outputs.list().len(), 1, "got {:?}", outputs.list()); + let notice = DepositNotice::abi_decode(&outputs[0].expect_notice().payload) + .expect("decode deposit notice"); + assert_eq!((notice.sender, notice.amount), (sender, amount)); +} + fn assert_no_outputs_or_reports(outputs: &OutputsForInput, reports: &[Vec]) { assert!( outputs.list().is_empty(), diff --git a/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs index 9921e177..9607d3ba 100644 --- a/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs +++ b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs @@ -18,7 +18,7 @@ async fn main() -> std::process::ExitCode { ) .init(); - // `setup` is the only subcommand that constructs a genesis app; the - // closure runs only on that path. + // Only a setup without `--base` builds the genesis application; a setup + // from a stored machine builds it from the machine's state. sequencer::run_main(|| Ok(WalletApp::new(WalletConfig::devnet()))).await } diff --git a/examples/wallet-sequencer/src/main.rs b/examples/wallet-sequencer/src/main.rs index 491fa25d..912b277d 100644 --- a/examples/wallet-sequencer/src/main.rs +++ b/examples/wallet-sequencer/src/main.rs @@ -18,7 +18,7 @@ async fn main() -> std::process::ExitCode { ) .init(); - // `setup` is the only subcommand that constructs a genesis app; the - // closure runs only on that path. + // Only a setup without `--base` builds the genesis application; a setup + // from a stored machine builds it from the machine's state. sequencer::run_main(|| Ok(WalletApp::new(WalletConfig::default()))).await } diff --git a/justfile b/justfile index 68283121..c0dbaee0 100644 --- a/justfile +++ b/justfile @@ -117,6 +117,11 @@ clippy: verify: fmt-check check test clippy +# Setup refuses a directory that holds anything, hence the rm. The rest of the +# configuration (RPC, chain id, addresses, submitter key) comes from the +# CARTESI_SEQUENCER_* variables. +# Set up a new data directory from genesis, then run. run addr="127.0.0.1:3000" data_dir="sequencer-data": rm -rf {{data_dir}} - CARTESI_SEQUENCER_HTTP_ADDR={{addr}} CARTESI_SEQUENCER_DATA_DIR={{data_dir}} cargo run -p wallet-sequencer --release + CARTESI_SEQUENCER_DATA_DIR={{data_dir}} cargo run -p wallet-sequencer --release -- setup + CARTESI_SEQUENCER_HTTP_ADDR={{addr}} CARTESI_SEQUENCER_DATA_DIR={{data_dir}} cargo run -p wallet-sequencer --release -- run diff --git a/openapi.yaml b/openapi.yaml index 526b4dc1..6e38fd83 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -17,8 +17,7 @@ info: Address casing differs by route. `sender` in `POST /tx` and `GET /nonce` responses and WebSocket messages, and `verifyingContract` in `GET /domain`, use EIP-55 checksum casing. Address fields in `/history` - and `sender` in `/historical-l1-inputs` use lowercase hex. Compare decoded - 20-byte addresses. + use lowercase hex. Compare decoded 20-byte addresses. All `u64` values are JSON numbers; clients must preserve integer precision. Integer formats are OpenAPI Format Registry names (`uint16`, @@ -43,7 +42,7 @@ tags: - name: ingress description: Public routes for wallets and submitters. - name: history - description: Internal. Application-history subscription, metadata, and historical L1 inputs. + description: Internal. Application-history subscription and metadata. - name: snapshots description: Internal. Operator state and snapshot downloads for the watchdog and indexers. - name: health @@ -107,10 +106,12 @@ paths: example: { ok: false, code: PAYLOAD_TOO_LARGE, message: request body too large } "422": description: | - The application rejected the op; no state changed and nothing was - persisted. Reasons: bad nonce, exhausted nonce, `max_fee` below - the open-frame fee, or insufficient balance for the fee. After a - bad-nonce rejection, query `GET /nonce` again. + The sequencer refused the op; no state changed and nothing was + persisted. Checked in order: `max_fee` below the open-frame fee, + bad nonce, exhausted nonce, then the application's admission + policy (insufficient balance for the fee, or another policy + reason, whose text is the message). After a bad-nonce rejection, + query `GET /nonce` again. content: application/json: schema: @@ -170,8 +171,10 @@ paths: description: | `next_nonce` is one past the sender's latest included op, or 0 if it has none. It counts soft-confirmed ops. The value is a hint, not a - reservation: it can go down after automatic recovery, and after an - operator rebuild a sender with no surviving op reads 0. + reservation; it does not go down within an era. It reads only + the era's live ops, so after a setup a sender whose last op lies in + the rebuilt history or below the base count reads 0; the `422` + bad-nonce message names the expected nonce. `4294967295` means the account is exhausted. Keep one op in flight per sender: sign `next_nonce`, submit, and @@ -262,20 +265,16 @@ paths: are not modelled as parameters. After the upgrade, the server sends JSON text frames shaped as - `BroadcastTxMessage`. The subscriber cap is 64. Shutdown or a feed - failure may disconnect without a Close frame; resume from the saved - claim. + `BroadcastTxMessage`. A direct input whose payload is over the + application's cap never executes and has no offset or message. The + subscriber cap is 64. Shutdown or a feed failure may disconnect + without a Close frame; resume from the saved claim. parameters: - name: era_id in: query required: true schema: $ref: "#/components/schemas/EraId" - - name: recovery_generation - in: query - required: true - schema: - $ref: "#/components/schemas/U64" - name: next_input in: query required: true @@ -289,8 +288,9 @@ paths: `BroadcastTxMessage` JSON documents. "400": description: | - Missing or malformed query fields, or a missing or invalid - upgrade header. The body is plain text, not the JSON error shape. + Missing, malformed or unknown query fields, or a missing or + invalid upgrade header. The body is plain text, not the JSON + error shape. content: text/plain: schema: @@ -313,10 +313,8 @@ paths: schema: $ref: "#/components/schemas/HistoryPolicyError" example: - code: STALE_GENERATION - current: - era_id: 22222222-2222-4222-8222-222222222222 - recovery_generation: 1 + code: ERA_CHANGED + era_id: 22222222-2222-4222-8222-222222222222 "429": description: Subscriber limit reached. content: @@ -337,12 +335,20 @@ paths: get: tags: [history] operationId: getHistory - summary: Deployment, history, baseline, and accepted-checkpoint metadata + summary: Deployment, history, era baseline, and accepted-checkpoint metadata description: | - One coherent view. `compatibility` is `null` unless `from_generation` - is supplied together with `era_id`. Unknown query fields are rejected. - An era mismatch (`409`) takes precedence over the `from_generation` - bound (`400`). + One coherent view. `history.available_from` is the era's base count; + the feed serves the rebuilt history from there up to + `baseline.rebuilt_history_end`, then the live rows. Inside an era + the history only appends, so a state saved at any count from + `available_from` to `head` resumes with its era id. An unknown query + parameter is `400`. + + A client holding state from an older era at count `X` adopts this + era when `available_from <= X <= baseline.rebuilt_history_end`: it + subscribes with `history.era_id` and `next_input = X`. Below the + range the era's base is newer than its state; above it, its state + holds rows that never landed by the stop block. parameters: - name: era_id in: query @@ -350,12 +356,6 @@ paths: description: Require this era; a mismatch returns `409 ERA_CHANGED`. schema: $ref: "#/components/schemas/EraId" - - name: from_generation - in: query - required: false - description: Requires `era_id`. A generation ahead of the current one is `400`. - schema: - $ref: "#/components/schemas/U64" responses: "200": description: History metadata. @@ -371,17 +371,18 @@ paths: app_deployment_block: 1 batch_submitter_address: "0x3333333333333333333333333333333333333333" history: - version: - era_id: 22222222-2222-4222-8222-222222222222 - recovery_generation: 0 + era_id: 22222222-2222-4222-8222-222222222222 available_from: 7 head: 7 baseline: l1_stop_block: 1240 l1_end_input_index: 8 next_batch_nonce: 2 - accepted_checkpoint: null - compatibility: null + rebuilt_history_end: 7 + accepted_checkpoint: + inclusion_block: 1240 + executed_input_count: 7 + next_batch_nonce: 2 "400": $ref: "#/components/responses/BadRequest" "409": @@ -394,94 +395,6 @@ paths: $ref: "#/components/schemas/ErrorResponse" example: { ok: false, code: UNAVAILABLE, message: history read unavailable } - /historical-l1-inputs: - servers: - - url: http://127.0.0.1:3001 - description: Default internal listener (`CARTESI_SEQUENCER_INTERNAL_HTTP_ADDR`). - get: - tags: [history] - operationId: getHistoricalL1Inputs - summary: Page through the era's immutable L1 input prefix - description: | - Serves InputBox inputs `[0, end_input_index)` through the era's - `l1_stop_block`. Provide exactly one of `next_input_index` or - `after_block`; continue with the returned `next_input_index`. Only - `next_input_index == end_input_index` means EOF; a short page does - not. A page may split a block, so continue with `next_input_index` - rather than `after_block`. Pages target 1 MiB of raw payloads; a - larger first input is returned alone. At most eight responses are in - flight. An interrupted body is a failed page. - - Malformed or unknown query fields and selector errors are `400` - first. An era mismatch (`409`) then precedes the limit-range and - position-bound checks (`400`). Recovery generation changes do not - invalidate pages; an era change does. - parameters: - - name: era_id - in: query - required: true - schema: - $ref: "#/components/schemas/EraId" - - name: next_input_index - in: query - required: false - description: | - Inclusive per-application InputBox index, starting at 0. At most - `end_input_index`: equal returns an empty completed page, larger - is `400`. - schema: - $ref: "#/components/schemas/U64" - - name: after_block - in: query - required: false - description: | - Seek to the first input strictly after this L1 block. At most - `l1_stop_block`: equal returns an empty completed page, larger is - `400`. - schema: - $ref: "#/components/schemas/U64" - - name: limit - in: query - required: false - description: Out-of-range values are `400`, not clamped. - schema: - type: integer - minimum: 1 - maximum: 256 - default: 256 - responses: - "200": - description: One page of historical inputs. - content: - application/json: - schema: - $ref: "#/components/schemas/HistoricalL1InputsPage" - example: - era_id: 22222222-2222-4222-8222-222222222222 - l1_stop_block: 1240 - end_input_index: 8 - next_input_index: 6 - items: - - input_index: 5 - sender: "0x3333333333333333333333333333333333333333" - payload: "0x00" - block_number: 1230 - block_timestamp: 1700014760 - transaction_hash: "0x4444444444444444444444444444444444444444444444444444444444444444" - "400": - $ref: "#/components/responses/BadRequest" - "409": - $ref: "#/components/responses/HistoryConflict" - "429": - description: Historical response limit reached. - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - example: { ok: false, code: OVERLOADED, message: historical response limit reached } - "503": - $ref: "#/components/responses/Unavailable" - /finalized_state/inclusion_block: servers: - url: http://127.0.0.1:3001 @@ -495,7 +408,8 @@ paths: "200": description: | The accepted checkpoint's L1 inclusion block and executed - application input count. Block `0` is the genesis checkpoint. + application input count. Until a batch of the era is accepted, + it is the era baseline at the stop block. content: application/json: schema: @@ -568,8 +482,6 @@ paths: $ref: "#/components/headers/XExecutedInputCount" X-History-Era: $ref: "#/components/headers/XHistoryEra" - X-Recovery-Generation: - $ref: "#/components/headers/XRecoveryGeneration" content: application/octet-stream: schema: @@ -593,7 +505,7 @@ paths: operationId: getLatestSnapshot summary: Download the newest restorable snapshot description: | - Streams a tar archive of the newest valid batch-close snapshot, or the + Streams a tar archive of the newest batch-close snapshot, or the era baseline: `info.toml` plus the application's opaque `state` file or directory. The snapshot may still be optimistic. Its headers supply the complete `/ws/subscribe` claim. The download holds a GC lease @@ -608,8 +520,6 @@ paths: $ref: "#/components/headers/XExecutedInputCount" X-History-Era: $ref: "#/components/headers/XHistoryEra" - X-Recovery-Generation: - $ref: "#/components/headers/XRecoveryGeneration" content: application/x-tar: schema: @@ -631,7 +541,7 @@ paths: description: | Streams the accepted snapshot as a tar archive, adding a `checkpoint.toml` receipt with its L1 inclusion block and next batch - nonce for trusted recovery. The download holds a GC lease until the + nonce: an operator backup, which no command consumes. The download holds a GC lease until the response ends or the client disconnects. The body has no `Content-Length`; a failure after the headers aborts the transfer, so treat an incomplete body as a failed download. @@ -645,8 +555,6 @@ paths: $ref: "#/components/headers/XExecutedInputCount" X-History-Era: $ref: "#/components/headers/XHistoryEra" - X-Recovery-Generation: - $ref: "#/components/headers/XRecoveryGeneration" content: application/x-tar: schema: @@ -694,8 +602,10 @@ paths: operationId: healthz summary: Health report description: | - Same status as `/readyz`. Does not certify L1 freshness, submitter - balance, or canonical agreement. + Same status as `/readyz`. Reports the landing lag and the safe + head's age for monitoring, with no threshold of its own; they do not + change the status. Does not certify submitter balance or canonical + agreement. responses: "200": description: Healthy. @@ -703,14 +613,24 @@ paths: application/json: schema: $ref: "#/components/schemas/HealthStatus" - example: { status: ok, inclusion_lane: ok } + example: + status: ok + inclusion_lane: ok + landing_lag_blocks: 40 + max_wait_blocks: 7200 + safe_head_age_seconds: 780 "503": description: Degraded. content: application/json: schema: $ref: "#/components/schemas/HealthStatus" - example: { status: degraded, inclusion_lane: stopped } + example: + status: degraded + inclusion_lane: stopped + landing_lag_blocks: 40 + max_wait_blocks: 7200 + safe_head_age_seconds: 780 components: headers: @@ -723,7 +643,8 @@ components: XInclusionBlock: description: | L1 block whose end the accepted checkpoint represents: where its batch - was included, or `0` for the genesis checkpoint. + was included, or the era's stop block before the era's first accepted + batch. required: true schema: type: string @@ -735,16 +656,10 @@ components: type: string pattern: "^[0-9]+$" XHistoryEra: - description: Era of the artifact's history version. + description: Era of the artifact's history; the `era_id` to subscribe with. required: true schema: $ref: "#/components/schemas/EraId" - XRecoveryGeneration: - description: Recovery generation of the artifact's history version. - required: true - schema: - type: string - pattern: "^[0-9]+$" responses: BadRequest: @@ -784,17 +699,14 @@ components: message: canonical divergence prevents accepted checkpoint selection HistoryConflict: description: | - The requested era is not the current era. These routes refuse only - on era; a generation change does not invalidate their results. + The requested era is not the current era. content: application/json: schema: $ref: "#/components/schemas/HistoryEraChanged" example: code: ERA_CHANGED - current: - era_id: 22222222-2222-4222-8222-222222222222 - recovery_generation: 0 + era_id: 22222222-2222-4222-8222-222222222222 schemas: U64: @@ -822,7 +734,7 @@ components: format: uuid pattern: "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" description: | - Hyphenated UUIDv4 identifying one setup/rebuild era. The server + Hyphenated UUIDv4 identifying the era one setup published. The server emits lowercase and accepts hex digits in either case. UserOp: @@ -955,26 +867,18 @@ components: message: type: string - HistoryVersion: - type: object - required: [era_id, recovery_generation] - description: Identity of locally available application history; claim both fields when resuming. - properties: - era_id: - $ref: "#/components/schemas/EraId" - recovery_generation: - $ref: "#/components/schemas/U64" - HistoryBounds: type: object - required: [version, available_from, head] + required: [era_id, available_from, head] properties: - version: - $ref: "#/components/schemas/HistoryVersion" + era_id: + allOf: + - $ref: "#/components/schemas/EraId" + description: The history's identity; claim it with the count when resuming. available_from: allOf: - $ref: "#/components/schemas/U64" - description: Baseline application count; entries `[available_from, head)` are available through WS. + description: The era's base count; entries `[available_from, head)` are available through WS. head: $ref: "#/components/schemas/U64" @@ -1000,20 +904,37 @@ components: HistoryBaseline: type: object - required: [l1_stop_block, l1_end_input_index, next_batch_nonce] + required: [l1_stop_block, l1_end_input_index, next_batch_nonce, rebuilt_history_end] + description: | + Where setup's rebuild of the era ended. Fixed for the era's lifetime. properties: l1_stop_block: - $ref: "#/components/schemas/U64" + allOf: + - $ref: "#/components/schemas/U64" + description: The safe block setup pinned after flushing the submitter's wallet; the rebuilt history ends with it. l1_end_input_index: allOf: - $ref: "#/components/schemas/U64" description: Exclusive end of the per-application InputBox prefix through the stop block. next_batch_nonce: - $ref: "#/components/schemas/U64" + allOf: + - $ref: "#/components/schemas/U64" + description: The scheduler's next batch nonce at the end of the stop block; the era's first batch carries it. + rebuilt_history_end: + allOf: + - $ref: "#/components/schemas/U64" + description: | + The application count at the end of the stop block: the rebuilt + history covers `[available_from, rebuilt_history_end)` and the + live rows follow. AcceptedCheckpoint: type: object required: [inclusion_block, executed_input_count, next_batch_nonce] + description: | + The latest accepted batch's checkpoint. Until a batch of the era is + accepted, it is the era baseline: the stop block, the rebuilt + history end, and the baseline's next batch nonce. properties: inclusion_block: $ref: "#/components/schemas/U64" @@ -1022,22 +943,9 @@ components: next_batch_nonce: $ref: "#/components/schemas/U64" - HistoryCompatibility: - type: object - required: [from_generation, preserved_input_count] - properties: - from_generation: - $ref: "#/components/schemas/U64" - preserved_input_count: - allOf: - - $ref: "#/components/schemas/U64" - description: | - Checkpoint counts up to and including this boundary preserve - their input prefix across every recovery since `from_generation`. - HistoryInfo: type: object - required: [deployment, history, baseline, accepted_checkpoint, compatibility] + required: [deployment, history, baseline, accepted_checkpoint] properties: deployment: $ref: "#/components/schemas/HistoryDeployment" @@ -1049,54 +957,37 @@ components: oneOf: - $ref: "#/components/schemas/AcceptedCheckpoint" - type: "null" - compatibility: - oneOf: - - $ref: "#/components/schemas/HistoryCompatibility" - - type: "null" HistoryPolicyError: description: Typed refusal of a history claim, discriminated by `code`. oneOf: - $ref: "#/components/schemas/HistoryEraChanged" - - $ref: "#/components/schemas/HistoryStaleGeneration" - $ref: "#/components/schemas/HistoryUnavailable" - $ref: "#/components/schemas/HistoryAheadOfHead" discriminator: propertyName: code mapping: ERA_CHANGED: "#/components/schemas/HistoryEraChanged" - STALE_GENERATION: "#/components/schemas/HistoryStaleGeneration" HISTORY_UNAVAILABLE: "#/components/schemas/HistoryUnavailable" AHEAD_OF_HEAD: "#/components/schemas/HistoryAheadOfHead" HistoryEraChanged: type: object - required: [code, current] + required: [code, era_id] description: | - The claimed era is not the current era. Old-era results cannot be - resumed or spliced into the new era: a replica rebootstraps from a - current snapshot, and a projection re-establishes correspondence with - the new baseline. + The claimed era is not the current era. A client holding state at + count `X` adopts the current era when + `available_from <= X <= baseline.rebuilt_history_end` (`GET /history`) + and subscribes with `era_id` at `X`. Otherwise it needs another + starting state: a replica rebootstraps from a current snapshot. properties: code: type: string const: ERA_CHANGED - current: - $ref: "#/components/schemas/HistoryVersion" - - HistoryStaleGeneration: - type: object - required: [code, current] - description: | - Same era, different recovery generation (normally an older one). - Select a compatible saved checkpoint through - `GET /history?era_id=&from_generation=`, or rebootstrap. - properties: - code: - type: string - const: STALE_GENERATION - current: - $ref: "#/components/schemas/HistoryVersion" + era_id: + allOf: + - $ref: "#/components/schemas/EraId" + description: The server's current era. HistoryUnavailable: type: object @@ -1120,55 +1011,6 @@ components: head: $ref: "#/components/schemas/U64" - HistoricalL1Input: - type: object - required: - - input_index - - sender - - payload - - block_number - - block_timestamp - - transaction_hash - properties: - input_index: - $ref: "#/components/schemas/U64" - sender: - allOf: - - $ref: "#/components/schemas/Address" - description: Authenticated InputBox sender, lowercase hex. - payload: - allOf: - - $ref: "#/components/schemas/HexBytes" - description: Original inner payload, including malformed or rejected batches. - block_number: - $ref: "#/components/schemas/U64" - block_timestamp: - allOf: - - $ref: "#/components/schemas/U64" - description: Unix seconds. - transaction_hash: - $ref: "#/components/schemas/Hash32" - - HistoricalL1InputsPage: - type: object - required: [era_id, l1_stop_block, end_input_index, next_input_index, items] - properties: - era_id: - $ref: "#/components/schemas/EraId" - l1_stop_block: - $ref: "#/components/schemas/U64" - end_input_index: - $ref: "#/components/schemas/U64" - next_input_index: - allOf: - - $ref: "#/components/schemas/U64" - description: Equality with `end_input_index` is the only EOF signal. - items: - type: array - maxItems: 256 - items: - $ref: "#/components/schemas/HistoricalL1Input" - InclusionBlockResponse: type: object required: [inclusion_block, executed_input_count] @@ -1193,11 +1035,31 @@ components: HealthStatus: type: object - required: [status, inclusion_lane] + required: + [status, inclusion_lane, landing_lag_blocks, max_wait_blocks, safe_head_age_seconds] properties: status: type: string enum: [ok, degraded] + landing_lag_blocks: + oneOf: + - $ref: "#/components/schemas/U64" + - type: "null" + description: | + `H + 1 - s` in L1 blocks: the age, at the block after the persisted + safe head, of the oldest batch L1 has not accepted, counted from + its first frame's safe block. The history is void when it reaches + `max_wait_blocks`. `null` until the first reading after start. + max_wait_blocks: + $ref: "#/components/schemas/U64" + safe_head_age_seconds: + oneOf: + - $ref: "#/components/schemas/U64" + - type: "null" + description: | + The wall clock less the safe block's L1 timestamp, at least 0. + It keeps growing while the provider is down, when the landing lag + freezes. `null` until the first reading after start. inclusion_lane: type: string enum: [ok, stopped] diff --git a/scripts/ci-c-application-smoke.sh b/scripts/ci-c-application-smoke.sh index d1a6b7f2..4af559a7 100644 --- a/scripts/ci-c-application-smoke.sh +++ b/scripts/ci-c-application-smoke.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash # Link an external engine into the generic host and an independent Cargo consumer. +# Both link the emulator through the LIBCARTESI_PATH / INCLUDECARTESI_PATH that the +# dev shell or CI's install-cartesi-machine action exports; the offline consumer +# build finds the emulator bindings' git checkout in the Cargo home the workspace +# build filled. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" diff --git a/scripts/generate-release-manifest.sh b/scripts/generate-release-manifest.sh index b6a62f12..425b0dd5 100755 --- a/scripts/generate-release-manifest.sh +++ b/scripts/generate-release-manifest.sh @@ -93,6 +93,7 @@ manifest = { "alignment": { "rule": "All artifacts sharing release_tag were built from git_commit with the same toolchain pins.", "watchdog_cm_bootstrap": f"Use canonical-machine-image--{tag}.tar.gz with CARTESI_WATCHDOG_CM_SNAPSHOT_DIR; cartesi-machine in the watchdog image matches CARTESI_MACHINE_VERSION.", + "sequencer_cm_link": f"wallet-sequencer links cartesi-machine {os.environ['CARTESI_MACHINE_VERSION']} statically; setup --base reads machines stored by that version.", }, } json.dump(manifest, sys.stdout, indent=2) diff --git a/sdk/guest/testsi/src/machine.rs b/sdk/guest/testsi/src/machine.rs index 0f86846f..f6f9ab90 100644 --- a/sdk/guest/testsi/src/machine.rs +++ b/sdk/guest/testsi/src/machine.rs @@ -9,7 +9,7 @@ use cartesi_machine::{ use rollups_types::alloy_primitives::{Address, U256}; use std::{ ops::ControlFlow, - path::PathBuf, + path::{Path, PathBuf}, time::{Duration, Instant}, }; use thiserror::Error; @@ -36,6 +36,8 @@ pub enum TestsiMachineError { InputRejected, #[error("machine is not awaiting an input (expected a manual rx_accepted yield): {state}")] NotAwaitingInput { state: String }, + #[error("an inspect query emitted {count} output(s); only reports are allowed")] + InspectEmittedOutputs { count: usize }, #[error("machine did not take the {len}-byte input (the cmio rx buffer holds 2 MiB)")] InputNotDelivered { len: usize }, #[error("machine execution timed out after {elapsed:?} (timeout {timeout:?})")] @@ -173,8 +175,44 @@ impl Machine { Ok((outputs, reports)) } - pub fn inspect(&self) -> Result<()> { - panic!("testsi::Machine does not support inspect() yet") + /// Sends an inspect query and returns its reports. Unlike a node's host, + /// which runs the query on a snapshot it discards, this runs it on the + /// live machine: fine for a guest whose inspect only reads, but the root + /// hash moves, so never use it on a machine whose hash must be kept. + pub fn inspect(&mut self, query: &[u8]) -> Result>> { + expect_awaiting_input(&mut self.cartesi_machine)?; + self.cartesi_machine + .send_cmio_response(cmio::CmioResponseReason::Inspect, query, None)?; + if self.cartesi_machine.iflags_y()? { + return Err(TestsiMachineError::InputNotDelivered { len: query.len() }); + } + + let mut outputs = OutputsForInput::default(); + let mut reports = Vec::new(); + let execution_started_at = Instant::now(); + while let ControlFlow::Continue(_) = run_machine_increment( + &mut self.cartesi_machine, + &mut outputs, + &mut reports, + execution_started_at, + self.builder.execution_timeout, + )? {} + + // An inspect has no outputs tree to write to. + if !outputs.list().is_empty() { + return Err(TestsiMachineError::InspectEmittedOutputs { + count: outputs.list().len(), + }); + } + Ok(reports) + } + + /// Stores the machine at `dir`, which must not exist, as a node stores + /// one between inputs. The store is not synced to disk. + pub fn store(&mut self, dir: &Path) -> Result<()> { + expect_awaiting_input(&mut self.cartesi_machine)?; + self.cartesi_machine.store(dir)?; + Ok(()) } } diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs index fc2dc5c9..d487f25e 100644 --- a/sdk/rust-client/src/errors.rs +++ b/sdk/rust-client/src/errors.rs @@ -109,3 +109,14 @@ pub enum HistoryReadError { #[error("invalid history response: {0}")] Decode(#[from] serde_json::Error), } + +/// Why a client could not adopt the sequencer's current era at its count. +#[derive(Debug, Error)] +pub enum AdoptEraError { + #[error(transparent)] + Read(#[from] HistoryReadError), + /// The count lies outside the era's adoptable range; the client needs + /// another starting state, not a retry. + #[error(transparent)] + Adopt(#[from] sequencer_core::history_api::AdoptError), +} diff --git a/sdk/rust-client/src/history.rs b/sdk/rust-client/src/history.rs index 35a5da2e..3678dda6 100644 --- a/sdk/rust-client/src/history.rs +++ b/sdk/rust-client/src/history.rs @@ -2,25 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) use crate::{ - EraId, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryInfo, HistoryPolicyError, - HistoryReadError, RecoveryGeneration, SequencerClient, + AdoptEraError, EraId, HistoryClaim, HistoryInfo, HistoryPolicyError, HistoryReadError, + L2TxCount, SequencerClient, }; impl SequencerClient { - /// Discover history, optionally requiring the era selected for bootstrap. - /// A compatibility query supplies both the checkpoint's era and its generation. + /// The claim that resumes the current era at a client's own count, after + /// a cockroach recovery replaced the client's era (its claim was refused + /// with `ERA_CHANGED`). Below the era's base count the base is newer than + /// the client's state; past its rebuilt history's end the state holds + /// rows that never landed. Both refusals need another starting state, and + /// neither check sees a state that is wrong below its count. + pub async fn adopt_current_era(&self, count: L2TxCount) -> Result { + Ok(self.history(None).await?.adopt_claim(count)?) + } + + /// Discover history, optionally requiring the era selected for bootstrap: + /// a different current era is the typed `ERA_CHANGED` refusal. pub async fn history( &self, expected_era: Option, - from_generation: Option, ) -> Result { - let mut query: Vec<_> = expected_era + let query: Vec<_> = expected_era .map(|era| ("era_id", era.to_string())) .into_iter() .collect(); - if let Some(generation) = from_generation { - query.push(("from_generation", generation.get().to_string())); - } let request = self .http_client .get(format!("{}/history", self.endpoint.trim_end_matches('/'))) @@ -29,37 +35,6 @@ impl SequencerClient { let body = read_history_response(request).await?; Ok(serde_json::from_str(&body)?) } - - /// Read one complete raw-input page under the configured request deadline. - /// Continue with its `next_input_index`; a short page is not necessarily EOF. - /// Use `with_request_timeout` when historical transfers need a longer deadline. - pub async fn historical_l1_inputs( - &self, - era: EraId, - start: HistoricalL1InputStart, - limit: Option, - ) -> Result { - let mut query = vec![("era_id", era.to_string())]; - query.push(match start { - HistoricalL1InputStart::NextInputIndex(index) => { - ("next_input_index", index.to_string()) - } - HistoricalL1InputStart::AfterBlock(block) => ("after_block", block.to_string()), - }); - if let Some(limit) = limit { - query.push(("limit", limit.to_string())); - } - let request = self - .http_client - .get(format!( - "{}/historical-l1-inputs", - self.endpoint.trim_end_matches('/') - )) - .query(&query) - .timeout(self.request_timeout); - let body = read_history_response(request).await?; - Ok(serde_json::from_str(&body)?) - } } async fn read_history_response( @@ -142,39 +117,21 @@ mod tests { "batch_submitter_address": "0x3333333333333333333333333333333333333333" }, "history": { - "version": { "era_id": ERA, "recovery_generation": 2 }, + "era_id": ERA, "available_from": 7, "head": 12 }, "baseline": { "l1_stop_block": 1240, "l1_end_input_index": 8, - "next_batch_nonce": 2 + "next_batch_nonce": 2, + "rebuilt_history_end": 9 }, "accepted_checkpoint": { "inclusion_block": 1250, "executed_input_count": 10, "next_batch_nonce": 3 - }, - "compatibility": null - }) - .to_string() - } - - fn page_body() -> String { - serde_json::json!({ - "era_id": ERA, - "l1_stop_block": 1240, - "end_input_index": 8, - "next_input_index": 6, - "items": [{ - "input_index": 5, - "sender": "0x3333333333333333333333333333333333333333", - "payload": "0x00ff80", - "block_number": 1230, - "block_timestamp": 1700014760_u64, - "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" - }] + } }) .to_string() } @@ -183,12 +140,12 @@ mod tests { async fn discovery_decodes_history_and_encodes_optional_era() { for expected_era in [None, Some(ERA.parse().unwrap())] { let (client, server) = serve_once(200, history_body()).await; - let info = client.history(expected_era, None).await.unwrap(); + let info = client.history(expected_era).await.unwrap(); assert_eq!(info.history.available_from.get(), 7); assert_eq!(info.history.head.get(), 12); - assert_eq!(info.history.version.era_id.to_string(), ERA); + assert_eq!(info.history.era_id.to_string(), ERA); assert_eq!(info.baseline.l1_stop_block, 1240); - assert!(info.compatibility.is_none()); + assert_eq!(info.baseline.rebuilt_history_end.get(), 9); let url = request_url(&server.await.unwrap()); assert_eq!(url.path(), "/history"); let query: Vec<_> = url.query_pairs().collect(); @@ -201,98 +158,62 @@ mod tests { } #[tokio::test] - async fn compatibility_encodes_checkpoint_generation_and_decodes_preserved_boundary() { - let mut body: serde_json::Value = serde_json::from_str(&history_body()).unwrap(); - body["compatibility"] = serde_json::json!({ - "from_generation": 1, - "preserved_input_count": 9 - }); - let (client, server) = serve_once(200, body.to_string()).await; - let info = client - .history(Some(ERA.parse().unwrap()), Some(RecoveryGeneration::new(1))) - .await - .unwrap(); - let compatibility = info.compatibility.unwrap(); - assert_eq!(compatibility.from_generation.get(), 1); - assert_eq!(compatibility.preserved_input_count.get(), 9); - assert_eq!(info.history.version.recovery_generation.get(), 2); - let url = request_url(&server.await.unwrap()); - assert_eq!(url.path(), "/history"); - assert_eq!( - url.query_pairs().collect::>(), - [ - ("era_id".into(), ERA.into()), - ("from_generation".into(), "1".into()) - ] - ); + async fn adoption_claims_the_current_era_only_from_the_base_through_the_rebuilt_end() { + let adopt = |count| async move { + let (client, server) = serve_once(200, history_body()).await; + let result = client.adopt_current_era(L2TxCount::new(count)).await; + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/history"); + assert!(url.query().is_none(), "adoption reads the current era"); + result + }; + for count in [7, 8, 9] { + let claim = adopt(count).await.unwrap(); + assert_eq!(claim.era_id.to_string(), ERA); + assert_eq!(claim.next_input.get(), count); + } + for count in [0, 6] { + assert!(matches!( + adopt(count).await, + Err(AdoptEraError::Adopt(crate::AdoptError::BelowBase { available_from })) + if available_from.get() == 7 + )); + } + // Rows the old instance served past the rebuilt end never landed, + // even though the new era's head is already past them. + for count in [10, 12] { + assert!(matches!( + adopt(count).await, + Err(AdoptEraError::Adopt(crate::AdoptError::PastRebuiltEnd { rebuilt_history_end })) + if rebuilt_history_end.get() == 9 + )); + } } #[tokio::test] - async fn pages_encode_one_selector_and_decode_original_binary_fields() { - for (start, limit, selector, value) in [ - ( - HistoricalL1InputStart::NextInputIndex(5), - Some(1), - "next_input_index", - "5", - ), - ( - HistoricalL1InputStart::AfterBlock(10), - None, - "after_block", - "10", - ), - ] { - let (client, server) = serve_once(200, page_body()).await; - let page = client - .historical_l1_inputs(ERA.parse().unwrap(), start, limit) - .await - .unwrap(); - assert_eq!(page.next_input_index, 6); - assert_eq!(page.items[0].input_index, 5); - assert_eq!(page.items[0].payload.as_ref(), &[0x00, 0xff, 0x80]); - assert_eq!(page.items[0].sender.as_slice(), &[0x33; 20]); - assert_eq!(page.items[0].transaction_hash.as_slice(), &[0x44; 32]); - let url = request_url(&server.await.unwrap()); - assert_eq!(url.path(), "/historical-l1-inputs"); - let query: std::collections::BTreeMap<_, _> = url.query_pairs().collect(); - assert_eq!(query.get("era_id").unwrap(), ERA); - assert_eq!(query.get(selector).unwrap(), value); - assert_eq!(query.len(), if limit.is_some() { 3 } else { 2 }); - if let Some(limit) = limit { - assert_eq!(query.get("limit").unwrap(), &limit.to_string()); - } - } + async fn adoption_reports_a_failed_history_read() { + let (client, server) = serve_once(503, "unavailable".into()).await; + assert!(matches!( + client.adopt_current_era(L2TxCount::new(7)).await, + Err(AdoptEraError::Read(HistoryReadError::Http { + status: 503, + .. + })) + )); + server.await.unwrap(); } #[tokio::test] - async fn era_change_is_typed_for_both_reads() { + async fn era_change_is_a_typed_refusal() { + let current = "11111111-1111-4111-8111-111111111111"; let policy = HistoryPolicyError::EraChanged { - current: crate::HistoryVersion { - era_id: "11111111-1111-4111-8111-111111111111".parse().unwrap(), - recovery_generation: sequencer_core::history::RecoveryGeneration::new(0), - }, + era_id: current.parse().unwrap(), }; - for page_request in [false, true] { - let (client, server) = serve_once(409, serde_json::to_string(&policy).unwrap()).await; - let result = if page_request { - client - .historical_l1_inputs( - ERA.parse().unwrap(), - HistoricalL1InputStart::NextInputIndex(0), - None, - ) - .await - .map(|_| ()) - } else { - client - .history(Some(ERA.parse().unwrap()), None) - .await - .map(|_| ()) - }; - assert!(matches!(result, Err(HistoryReadError::History(actual)) if actual == policy)); - server.await.unwrap(); - } + let body = serde_json::json!({ "code": "ERA_CHANGED", "era_id": current }).to_string(); + let (client, server) = serve_once(409, body).await; + let result = client.history(Some(ERA.parse().unwrap())).await; + assert!(matches!(result, Err(HistoryReadError::History(actual)) if actual == policy)); + server.await.unwrap(); } #[tokio::test] @@ -301,7 +222,7 @@ mod tests { let body = format!("refusal {status}"); let (client, server) = serve_once(status, body.clone()).await; assert!(matches!( - client.history(None, None).await, + client.history(None).await, Err(HistoryReadError::Http { status: actual_status, body: actual_body }) if actual_status == status && actual_body == body )); @@ -313,7 +234,7 @@ mod tests { async fn malformed_success_is_a_decode_error() { let (client, server) = serve_once(200, "{}".into()).await; assert!(matches!( - client.history(None, None).await, + client.history(None).await, Err(HistoryReadError::Decode(_)) )); server.await.unwrap(); @@ -321,39 +242,24 @@ mod tests { #[tokio::test] async fn history_deadline_covers_the_response_body() { - for page_request in [false, true] { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.unwrap(); - request_headers(&mut stream).await; - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{") - .await - .unwrap(); - std::future::pending::<()>().await; - }); - let client = SequencerClient::new(endpoint) - .unwrap() - .with_request_timeout(Duration::from_millis(100)); - let result = tokio::time::timeout(Duration::from_secs(5), async { - if page_request { - client - .historical_l1_inputs( - ERA.parse().unwrap(), - HistoricalL1InputStart::NextInputIndex(0), - None, - ) - .await - .map(|_| ()) - } else { - client.history(None, None).await.map(|_| ()) - } - }) + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + request_headers(&mut stream).await; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{") + .await + .unwrap(); + std::future::pending::<()>().await; + }); + let client = SequencerClient::new(endpoint) + .unwrap() + .with_request_timeout(Duration::from_millis(100)); + let result = tokio::time::timeout(Duration::from_secs(5), client.history(None)) .await .expect("the configured deadline must include body transfer"); - server.abort(); - assert!(matches!(result, Err(HistoryReadError::Request(error)) if error.is_timeout())); - } + server.abort(); + assert!(matches!(result, Err(HistoryReadError::Request(error)) if error.is_timeout())); } } diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs index 1f965fc0..51152fd6 100644 --- a/sdk/rust-client/src/lib.rs +++ b/sdk/rust-client/src/lib.rs @@ -5,17 +5,15 @@ mod errors; mod history; pub use errors::{ - ClientBuildError, HistoryReadError, QueryError, SnapshotError, SubmitRejected, SubmitTxError, - SubscribeError, + AdoptEraError, ClientBuildError, HistoryReadError, QueryError, SnapshotError, SubmitRejected, + SubmitTxError, SubscribeError, }; pub use sequencer_core::history::{ - EraId, ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, HistoryVersion, - RecoveryGeneration, + EraId, HistoryBounds, HistoryClaim, HistoryPolicyError, L2TxCount, }; pub use sequencer_core::history_api::{ - AcceptedCheckpoint, HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, - HistoryBaseline, HistoryCompatibility, HistoryDeployment, HistoryInfo, + AcceptedCheckpoint, AdoptError, HistoryBaseline, HistoryDeployment, HistoryInfo, }; use alloy_primitives::Address; @@ -130,8 +128,10 @@ impl SequencerClient { self.get_json("/fee", "").await } - /// The nonce `sender` signs next. A hint for one op in flight: re-query - /// after a `422` bad-nonce rejection, since recovery can lower it. + /// The nonce `sender` signs next. A hint for one op in flight: recovery + /// can lower it, and after a cockroach recovery it is stale for a sender + /// whose last op predates the new era's live rows. A `422` bad-nonce + /// rejection names the `expected` nonce, which corrects it. pub async fn get_nonce(&self, sender: Address) -> Result { self.get_json("/nonce", &format!("?sender={sender:#x}")) .await @@ -201,21 +201,13 @@ impl SequencerClient { SnapshotError::Metadata(error.to_string()) }, )?; - let generation: u64 = header("X-Recovery-Generation")? - .parse() - .map_err(|error: std::num::ParseIntError| SnapshotError::Metadata(error.to_string()))?; let count: u64 = header("X-Executed-Input-Count")? .parse() .map_err(|error: std::num::ParseIntError| SnapshotError::Metadata(error.to_string()))?; Ok(SnapshotResponse { claim: HistoryClaim { - version: HistoryVersion { - era_id, - recovery_generation: sequencer_core::history::RecoveryGeneration::new( - generation, - ), - }, - next_input: ExecutedInputCount::new(count), + era_id, + next_input: L2TxCount::new(count), }, response, }) @@ -223,7 +215,8 @@ impl SequencerClient { /// Bounds the handshake by the request timeout; callers own the stream. /// The feed is plain `ws://`: a client built from an `https://` endpoint - /// cannot subscribe. + /// cannot subscribe. A claim refused with `ERA_CHANGED` can resume + /// through [`Self::adopt_current_era`]. pub async fn subscribe(&self, claim: HistoryClaim) -> Result { let url = self.ws_subscribe_url(claim); let (stream, _response) = @@ -287,9 +280,8 @@ fn with_history_claim(ws_subscribe_url: &str, claim: HistoryClaim) -> String { '?' }; format!( - "{ws_subscribe_url}{separator}era_id={}&recovery_generation={}&next_input={}", - claim.version.era_id, - claim.version.recovery_generation.get(), + "{ws_subscribe_url}{separator}era_id={}&next_input={}", + claim.era_id, claim.next_input.get() ) } @@ -312,7 +304,7 @@ fn map_subscribe_error(error: tokio_tungstenite::tungstenite::Error) -> Subscrib #[cfg(test)] mod tests { use super::*; - use sequencer_core::history::{EraId, RecoveryGeneration}; + use sequencer_core::history::EraId; #[tokio::test] async fn fee_request_keeps_its_request_timeout() { @@ -381,7 +373,7 @@ mod tests { assert_ne!(count, 0, "complete request headers"); received += count; } - stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nX-History-Era: 00112233-4455-4677-8899-aabbccddeeff\r\nX-Recovery-Generation: 0\r\nX-Executed-Input-Count: 7\r\n\r\n").await.unwrap(); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nX-History-Era: 00112233-4455-4677-8899-aabbccddeeff\r\nX-Executed-Input-Count: 7\r\n\r\n").await.unwrap(); tokio::time::sleep(Duration::from_millis(300)).await; stream.write_all(b"dump").await.unwrap(); }); @@ -391,7 +383,13 @@ mod tests { ) .unwrap(); let snapshot = client.latest_snapshot().await.unwrap(); - assert_eq!(snapshot.claim.next_input.get(), 7); + assert_eq!( + snapshot.claim, + HistoryClaim { + era_id: "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(), + next_input: L2TxCount::new(7), + } + ); assert_eq!(snapshot.response.bytes().await.unwrap().as_ref(), b"dump"); server.await.unwrap(); } @@ -404,13 +402,10 @@ mod tests { .unwrap() .with_request_timeout(Duration::from_millis(100)); let claim = HistoryClaim { - version: HistoryVersion { - era_id: "00112233-4455-4677-8899-aabbccddeeff" - .parse::() - .unwrap(), - recovery_generation: RecoveryGeneration::new(0), - }, - next_input: ExecutedInputCount::new(0), + era_id: "00112233-4455-4677-8899-aabbccddeeff" + .parse::() + .unwrap(), + next_input: L2TxCount::new(0), }; let stalled_server = async { let (_stream, _) = listener.accept().await.unwrap(); @@ -487,25 +482,22 @@ mod tests { #[test] fn subscription_url_carries_the_exact_resume_claim() { let claim = HistoryClaim { - version: HistoryVersion { - era_id: "00112233-4455-4677-8899-aabbccddeeff" - .parse::() - .unwrap(), - recovery_generation: RecoveryGeneration::new(9), - }, - next_input: ExecutedInputCount::new(50_001), + era_id: "00112233-4455-4677-8899-aabbccddeeff" + .parse::() + .unwrap(), + next_input: L2TxCount::new(50_001), }; let client = SequencerClient::new("http://localhost:8080").unwrap(); assert_eq!( client.ws_subscribe_url(claim), - "ws://localhost:8080/ws/subscribe?era_id=00112233-4455-4677-8899-aabbccddeeff&recovery_generation=9&next_input=50001" + "ws://localhost:8080/ws/subscribe?era_id=00112233-4455-4677-8899-aabbccddeeff&next_input=50001" ); } #[test] fn typed_refusal_survives_an_http_body_in_a_later_packet() { let policy = HistoryPolicyError::AheadOfHead { - head: ExecutedInputCount::new(12), + head: L2TxCount::new(12), }; let response = tokio_tungstenite::tungstenite::http::Response::builder() .status(409) @@ -540,16 +532,13 @@ mod tests { let client = SequencerClient::new(format!("http://{address}")).unwrap(); let result = client .subscribe(HistoryClaim { - version: HistoryVersion { - era_id: "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(), - recovery_generation: RecoveryGeneration::new(0), - }, - next_input: ExecutedInputCount::new(40), + era_id: "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(), + next_input: L2TxCount::new(40), }) .await; assert!(matches!(result, Err(SubscribeError::History(HistoryPolicyError::HistoryUnavailable { available_from })) - if available_from == ExecutedInputCount::new(41))); + if available_from == L2TxCount::new(41))); server.await.unwrap(); } } diff --git a/sequencer-core/src/application/mod.rs b/sequencer-core/src/application/mod.rs index d4710716..fcbd7075 100644 --- a/sequencer-core/src/application/mod.rs +++ b/sequencer-core/src/application/mod.rs @@ -1,9 +1,8 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use crate::history::ExecutedInputCount; -use crate::l2_tx::DirectInput; -use crate::l2_tx::ValidUserOp; +use crate::history::L2TxCount; +use crate::l2_tx::{Direct, L2Tx, UserOpTx}; use crate::user_op::UserOp; use alloy_primitives::{Address, U256}; use std::fmt; @@ -19,21 +18,6 @@ pub enum AppError { Io(#[from] std::io::Error), } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ExecutionOutcome { - /// A canonical application input executed successfully. The receipt owns - /// its pre-execution history offset and any application outputs. - Included(ExecutedInput), - - Invalid(InvalidReason), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ValidationOutcome { - Accept, - Reject(InvalidReason), -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppOutput { Notice(Vec), @@ -46,171 +30,154 @@ pub enum AppOutput { pub type AppOutputs = Vec; -/// Canonical progress owned and persisted by the application alongside its state. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct ApplicationProgress { - executed_input_count: ExecutedInputCount, - last_executed_safe_block: u64, +/// The scheduler constants a template compiles in for its application. They +/// belong to the application: the window is how long it lets a sequencer hold +/// a direct back, and the other two bound what its directs may cost. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SchedulerLimits { + /// The staleness window, at least 1: a direct queued this many L1 blocks + /// is executed, and a batch whose first frame is this old is skipped. + pub max_wait_blocks: u64, + /// How many admitted directs the fridge may hold. + pub fridge_bound: u64, + /// A direct with a longer payload never joins the fridge. + pub max_direct_payload_bytes: u64, } -impl ApplicationProgress { - /// Construct a coherent application-history boundary: `None` when the - /// pair is incoherent (zero executed inputs with a nonzero clock), since - /// a nonzero clock proves that at least one input executed. Used when - /// restoring a dump or importing progress from a native application; - /// a genesis instance starts from `Default`. - pub const fn try_new( - executed_input_count: ExecutedInputCount, - last_executed_safe_block: u64, - ) -> Option { - if executed_input_count.get() == 0 && last_executed_safe_block != 0 { - return None; - } - Some(Self { - executed_input_count, - last_executed_safe_block, - }) - } - - pub const fn executed_input_count(self) -> ExecutedInputCount { - self.executed_input_count - } - - pub const fn last_executed_safe_block(self) -> u64 { - self.last_executed_safe_block - } - - /// Advance after one successful input, including an application-level no-op. - /// Panics if the history count has no representable successor. - pub fn advance(&mut self, safe_block: u64) { - *self = self - .checked_after_input(safe_block) - .expect("executed input count overflow: no canonical successor"); - } - - fn checked_after_input(self, safe_block: u64) -> Option { - Some(Self { - executed_input_count: self.executed_input_count.checked_next()?, - last_executed_safe_block: if self.last_executed_safe_block > safe_block { - self.last_executed_safe_block - } else { - safe_block - }, - }) - } +/// The lane's admission verdict. Policy, never consensus: the canonical +/// machine never asks, and the answer never changes what an included op does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Admission { + Admit, + Refuse(PolicyRefusal), } -/// One successfully executed canonical application input. +/// Why the operator does not want an op. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ExecutedInput { - /// The boundary before this input executed; this is its history offset. - pub offset: ExecutedInputCount, - pub outputs: AppOutputs, +pub enum PolicyRefusal { + /// The sender cannot pay the frame fee. "Fee" (not "gas"): the fee + /// tracks DA usage; compute metering, if it ever exists, will be a + /// separate concept. + Underfunded { required: U256, available: U256 }, + /// Any other operator policy; its text reaches the client verbatim. + Other(String), } +/// Why the executor skipped a user op. A skipped op changes nothing. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InvalidReason { - InvalidNonce { - expected: u32, - got: u32, - }, +pub enum SkipReason { + /// The op's nonce is not its sender's next. + WrongNonce { expected: u32, got: u32 }, + /// The sender's next nonce is `u32::MAX`, which has no successor. + NonceExhausted, +} + +/// Why the lane refuses a live user op. Its text is the `POST /tx` 422 body, +/// which clients parse, so every variant prints exactly today's wording. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Refusal { /// Both values are log-space exponents (base 129/128). - InvalidMaxFee { + FeeCapBelowPrice { max_fee: u16, - base_fee: u16, - }, - /// The sender's expected nonce is `u32::MAX`, which has no successor. - NonceExhausted, - /// Sender cannot pay the frame fee. "Fee" (not "gas"): the current fee - /// tracks DA usage; compute metering, if it ever exists, will be a - /// separate concept. - InsufficientFeeBalance { - required: U256, - available: U256, + price: u16, }, + Skip(SkipReason), + Policy(PolicyRefusal), } -impl fmt::Display for InvalidReason { +impl fmt::Display for PolicyRefusal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::InvalidNonce { expected, got } => { + Self::Underfunded { + required, + available, + } => write!( + f, + "insufficient balance for fee: required {required}, available {available}" + ), + Self::Other(reason) => f.write_str(reason), + } + } +} + +impl fmt::Display for SkipReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongNonce { expected, got } => { write!(f, "bad nonce: expected {expected}, got {got}") } - Self::InvalidMaxFee { max_fee, base_fee } => { - write!(f, "max fee {max_fee} below base fee {base_fee}") - } Self::NonceExhausted => write!(f, "nonce {} has no successor", u32::MAX), - Self::InsufficientFeeBalance { - required, - available, - } => { - write!( - f, - "insufficient balance for fee: required {required}, available {available}" - ) + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::FeeCapBelowPrice { max_fee, price } => { + write!(f, "max fee {max_fee} below base fee {price}") } + Self::Skip(reason) => reason.fmt(f), + Self::Policy(reason) => reason.fmt(f), } } } +/// What became of one L2 transaction handed to [`execute`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Executed { + /// It executed; `index` is the L2 tx count before it, its index in the history. + Done { + index: L2TxCount, + outputs: AppOutputs, + }, + /// A user op whose nonce is not its sender's next. Nothing changed. + Skipped(SkipReason), +} + /// Deterministic application state with exclusive ownership and thread transfer. +/// +/// The application executes every transaction it is handed and stores each +/// sender's next nonce; shared code ([`execute`]) decides which user ops run, +/// so an application bug can change what an op does but not whether it runs. pub trait Application: Send + Sized { - /// Maximum user-op method payload size, stable for this implementation. - /// Zero permits only empty method payloads. + // Execution: the canonical machine, the sequencer and every replica. + + /// The sender's next nonce; zero for an address that has sent nothing. + /// Must not change state. [`execute`] compares it with the op's nonce. + fn user_nonce(&self, sender: Address) -> Result; + + /// Execute a user op [`execute`] found at its sender's next nonce. It never + /// refuses: whatever the op does or pays, even a fee the sender cannot + /// cover (charging what it has, a saturating subtraction, is recommended), + /// the sender's nonce and the count each advance by exactly one. + /// `AppError` is fatal: callers discard the instance. + fn apply_user_op(&mut self, op: &UserOpTx) -> Result; + + /// Execute a direct. Ignored or malformed payloads still count: the count + /// advances by exactly one and no nonce moves. `AppError` is fatal. + fn apply_direct(&mut self, direct: &Direct) -> Result; + + /// The L2 tx count: starts at zero and advances by exactly one per + /// executed transaction, so an application at `X` executes history entry + /// `X` next. It must survive dump round-trips. + fn l2_tx_count(&self) -> L2TxCount; + + // Admission: the sequencer's lane only. Policy, never consensus. + + /// Maximum user-op method payload size, stable for this implementation, + /// which only the lane enforces. Zero permits only empty method payloads. fn max_method_payload_bytes() -> usize; - /// Pure validation predicate over current app state: the op's nonce - /// equals the sender's expected nonce (user replay protection), and the - /// sender covers the fee. Must not mutate state. - /// [`validate_and_execute_user_op`] enforces the protocol - /// `max_fee >= current_fee` guard before calling here, and rejects an - /// accepted op at nonce `u32::MAX` after. Rejection leaves the app - /// unchanged; `AppError` is fatal and defines no successor. - fn validate_user_op( - &self, - sender: Address, - user_op: &UserOp, - current_fee: u16, - ) -> Result; - - /// Apply a validated user op and advance progress exactly once on success, - /// using `safe_block` for the clock. Also advance the sender's nonce by - /// exactly one; the sequencer derives `GET /nonce` from this rule (see the - /// application contract). Included business failures and no-ops also - /// advance both. `AppError` is fatal: callers discard the instance. - /// Execution callers use [`execute_valid_user_op`] to check the transition. - fn apply_valid_user_op( - &mut self, - user_op: &ValidUserOp, - safe_block: u64, - ) -> Result; - - /// Apply a direct input and advance progress exactly once on success, - /// using its L1 block number for the clock. Ignored or malformed inputs - /// still count. Never changes a user nonce. Execution callers use - /// [`execute_direct_input`] to check the transition; `AppError` requires - /// discarding the instance. - fn apply_direct_input(&mut self, input: &DirectInput) -> Result; - - /// Return the progress embedded in the application's logical state. - fn progress(&self) -> ApplicationProgress; - - /// The app's safe-block clock: the maximum block carried by any input - /// this instance has executed (frame safe blocks for user ops, L1 - /// inclusion blocks for direct inputs), or 0 if nothing executed. - /// Recovery reads this as `A`, the safe block a checkpoint state reflects; - /// it must survive dump round-trips. - fn last_executed_safe_block(&self) -> u64 { - self.progress().last_executed_safe_block() - } + /// The template's scheduler constants for this application. + fn scheduler_limits() -> SchedulerLimits; - /// Canonical application-history boundary. Starts at zero and advances by - /// exactly one after each successful user-op or direct-input execution; an - /// application at `X` is ready to consume history input `X`. It must - /// survive dump round-trips. - fn executed_input_count(&self) -> ExecutedInputCount { - self.progress().executed_input_count() - } + /// Does the operator want this op in a frame at this fee? Must not change + /// state and must not check the nonce, which [`check_live_user_op`] does + /// first. The canonical machine never asks, so a guest need not link it. + fn admit_user_op(&self, sender: Address, op: &UserOp, fee: u16) -> Result; + + // Durable state: the sequencer and replicas. // Genesis construction stays on the concrete type: its inputs depend on // the application. The host manages the resulting instance through dumps. @@ -224,7 +191,7 @@ pub trait Application: Send + Sized { /// Write a complete recovery dump at the absent path `prefix`, which may /// be a file or directory. A subsequent [`Application::from_dump`] must - /// rehydrate equivalent logical state, including progress. Creating the + /// rehydrate equivalent logical state, including the count. Creating the /// dump must preserve the live instance's logical state; later execution /// of that instance must not change the dump. All checkpoint-owned artifacts /// must reside at or beneath `prefix`; discarding them uses ordinary @@ -255,186 +222,212 @@ pub trait Application: Send + Sized { /// is a pure function of `prefix`: callers may invoke it without /// loading the dump or instantiating the Application. fn state_file_in_dump(prefix: &Path) -> PathBuf; + + /// Build an instance from a canonical machine's comparison bytes, held in + /// `state_file`, at the machine's count, whether or not those bytes hold + /// it. A dump of the result holds the same comparison bytes. + fn from_canonical_state(state_file: &Path, l2_tx_count: L2TxCount) -> Result; } /// Canonical inspection for applications hosted by the shared Rust scheduler. /// Applications using another canonical runtime need not implement this trait. pub trait CanonicalState { - /// Deterministic bytes matching the state file in a dump of this state. + /// Deterministic bytes matching the state file in a dump of this state; + /// [`Application::from_canonical_state`] on them yields an equal instance. fn canonical_snapshot_bytes(&self) -> Result, AppError>; } -/// Validate and execute a live user op: protocol guards, app validation, execution. -/// -/// Live inclusion and the canonical scheduler use this boundary. Trusted -/// replay uses [`execute_valid_user_op`] with the persisted validation result. -pub fn validate_and_execute_user_op( - app: &mut A, - sender: Address, - user_op: &UserOp, - current_fee: u16, - safe_block: u64, -) -> Result { - // Protocol invariant: max_fee must cover the current frame fee. - if user_op.max_fee < current_fee { - return Ok(ExecutionOutcome::Invalid(InvalidReason::InvalidMaxFee { - max_fee: user_op.max_fee, - base_fee: current_fee, - })); +/// The nonce rule, shared by [`execute`] and [`check_live_user_op`] so the +/// lane refuses exactly what the canonical machine would skip. The mismatch +/// comes first, so a stale op carrying `u32::MAX` reads as a wrong nonce. +fn nonce_skip(expected: u32, got: u32) -> Option { + if got != expected { + Some(SkipReason::WrongNonce { expected, got }) + } else if expected == u32::MAX { + Some(SkipReason::NonceExhausted) + } else { + None } +} - if let ValidationOutcome::Reject(reason) = app.validate_user_op(sender, user_op, current_fee)? { - return Ok(ExecutionOutcome::Invalid(reason)); - } - // Protocol invariant: the nonce rule gives `u32::MAX` no successor. Checked - // after validation so any other op carrying it gets the app's bad-nonce - // diagnostics; only a sender actually at `u32::MAX` is exhausted. - if user_op.nonce == u32::MAX { - return Ok(ExecutionOutcome::Invalid(InvalidReason::NonceExhausted)); +/// The one way a transaction reaches the application, in the canonical +/// machine, a rebuild, the lane and its replay alike. A direct always +/// executes; a user op executes only at its sender's next nonce. Wrong +/// application bookkeeping after a successful hook fails loud. +pub fn execute(app: &mut A, tx: &L2Tx) -> Result { + match tx { + L2Tx::Direct(direct) => { + let index = preflight_count(app); + let outputs = app.apply_direct(direct)?; + assert_count_advanced(app, index); + // A direct's no-nonce rule is not checked: the executor cannot + // enumerate the senders whose nonces it might have moved. + Ok(Executed::Done { index, outputs }) + } + L2Tx::UserOp(op) => { + let expected = app.user_nonce(op.sender)?; + if let Some(skip) = nonce_skip(expected, op.nonce) { + return Ok(Executed::Skipped(skip)); + } + let nonce_after = expected + .checked_add(1) + .expect("nonce_skip lets no op through at u32::MAX"); + let index = preflight_count(app); + let outputs = app.apply_user_op(op)?; + assert_count_advanced(app, index); + assert_eq!( + app.user_nonce(op.sender)?, + nonce_after, + "an executed user op must advance its sender's nonce exactly once" + ); + Ok(Executed::Done { index, outputs }) + } } - - let valid = ValidUserOp { - sender, - fee: current_fee, - data: user_op.data.to_vec(), - }; - execute_valid_user_op(app, &valid, safe_block).map(ExecutionOutcome::Included) } -/// Execute an already-validated user op and verify the application-owned progress. -/// The caller is responsible for supplying a valid op, including its fee. -pub fn execute_valid_user_op( - app: &mut A, - user_op: &ValidUserOp, - safe_block: u64, -) -> Result { - execute_and_verify_progress(app, safe_block, |app| { - app.apply_valid_user_op(user_op, safe_block) - }) +/// The lane's pre-check of a live op, in the order whose texts clients see: +/// the fee cap, then the executor's own nonce rule, then admission. `None` +/// means the op is wanted and [`execute`] will run it, not skip it. +pub fn check_live_user_op( + app: &A, + sender: Address, + op: &UserOp, + frame_fee: u16, +) -> Result, AppError> { + if op.max_fee < frame_fee { + return Ok(Some(Refusal::FeeCapBelowPrice { + max_fee: op.max_fee, + price: frame_fee, + })); + } + if let Some(skip) = nonce_skip(app.user_nonce(sender)?, op.nonce) { + return Ok(Some(Refusal::Skip(skip))); + } + match app.admit_user_op(sender, op, frame_fee)? { + Admission::Admit => Ok(None), + Admission::Refuse(policy) => Ok(Some(Refusal::Policy(policy))), + } } -/// Execute one direct input and verify the application-owned progress. -pub fn execute_direct_input( - app: &mut A, - input: &DirectInput, -) -> Result { - execute_and_verify_progress(app, input.block_number, |app| app.apply_direct_input(input)) +/// The count before execution. Panics before the hook runs when the count +/// has no successor, so an exhausted history never mutates the application. +fn preflight_count(app: &A) -> L2TxCount { + let count = app.l2_tx_count(); + count + .checked_next() + .expect("L2 tx count overflow: no canonical successor"); + count } -fn execute_and_verify_progress( - app: &mut A, - safe_block: u64, - apply: F, -) -> Result -where - A: Application, - F: FnOnce(&mut A) -> Result, -{ - let progress_before = app.progress(); - let progress_after = progress_before - .checked_after_input(safe_block) - .expect("executed input count overflow: no canonical successor"); - - let outputs = apply(app)?; +fn assert_count_advanced(app: &A, before: L2TxCount) { assert_eq!( - app.progress(), - progress_after, - "successful application execution must advance progress exactly once" + Some(app.l2_tx_count()), + before.checked_next(), + "successful application execution must advance the count exactly once" ); - - Ok(ExecutedInput { - offset: progress_before.executed_input_count(), - outputs, - }) } #[cfg(test)] mod tests { use super::*; + use std::cell::Cell; + /// One sender's worth of state with knobs for every way an application can + /// misbehave. The nonce is shared by all senders, which suffices here. struct ProgressApp { - progress: ApplicationProgress, + count: L2TxCount, + nonce: u32, applied: u64, - reject: bool, - fail_validation: bool, + admit_calls: Cell, + nonce_steps: u32, + count_steps: usize, + fail_nonce: bool, fail_apply: bool, - advance_count: usize, - clock_override: Option, + admission: Admission, } impl ProgressApp { fn new(count: u64) -> Self { Self { - progress: ApplicationProgress::try_new(ExecutedInputCount::new(count), 0) - .expect("coherent progress"), + count: L2TxCount::new(count), + nonce: 0, applied: 0, - reject: false, - fail_validation: false, + admit_calls: Cell::new(0), + nonce_steps: 1, + count_steps: 1, + fail_nonce: false, fail_apply: false, - advance_count: 1, - clock_override: None, + admission: Admission::Admit, } } - fn apply(&mut self, safe_block: u64) -> Result { + fn assert_usable(&self) { + assert!( + !(self.fail_apply && self.applied > 0), + "failed instance is unusable" + ); + } + + fn apply(&mut self) -> Result { self.applied += 1; if self.fail_apply { return Err(AppError::Internal { reason: "execution failed".into(), }); } - for _ in 0..self.advance_count { - self.progress - .advance(self.clock_override.unwrap_or(safe_block)); + for _ in 0..self.count_steps { + self.count.advance(); } Ok(Vec::new()) } } impl Application for ProgressApp { - fn max_method_payload_bytes() -> usize { - 0 - } - - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result { - if self.fail_validation { + fn user_nonce(&self, _sender: Address) -> Result { + self.assert_usable(); + if self.fail_nonce { return Err(AppError::Internal { - reason: "validation failed".into(), + reason: "nonce read failed".into(), }); } - if self.reject { - Ok(ValidationOutcome::Reject(InvalidReason::InvalidNonce { - expected: 1, - got: 0, - })) - } else { - Ok(ValidationOutcome::Accept) - } + Ok(self.nonce) } - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - self.apply(safe_block) + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { + let outputs = self.apply()?; + self.nonce = self.nonce.wrapping_add(self.nonce_steps); + Ok(outputs) } - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { - self.apply(input.block_number) + fn apply_direct(&mut self, _direct: &Direct) -> Result { + self.apply() } - fn progress(&self) -> ApplicationProgress { - assert!( - !(self.fail_apply && self.applied > 0), - "failed instance is unusable" - ); - self.progress + fn l2_tx_count(&self) -> L2TxCount { + self.assert_usable(); + self.count + } + + fn max_method_payload_bytes() -> usize { + 0 + } + + fn scheduler_limits() -> SchedulerLimits { + SchedulerLimits { + max_wait_blocks: 1, + fridge_bound: 0, + max_direct_payload_bytes: 0, + } + } + + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + self.admit_calls.set(self.admit_calls.get() + 1); + Ok(self.admission.clone()) } fn from_dump(_prefix: &Path) -> Result { @@ -446,148 +439,336 @@ mod tests { fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } + fn from_canonical_state( + _state_file: &Path, + _l2_tx_count: L2TxCount, + ) -> Result { + unreachable!("not used") + } } - fn user_op() -> UserOp { + fn user_op(nonce: u32, max_fee: u16) -> UserOp { UserOp { - nonce: 0, - max_fee: 0, + nonce, + max_fee, data: Vec::new().into(), } } - fn direct(block_number: u64) -> DirectInput { - DirectInput { + fn user_tx(nonce: u32) -> L2Tx { + L2Tx::UserOp(UserOpTx::new(Address::ZERO, &user_op(nonce, 0), 0, 9)) + } + + fn direct(block: u64) -> L2Tx { + L2Tx::Direct(Direct { + input_index: 0, sender: Address::ZERO, - block_number, + block, + timestamp: 0, payload: Vec::new(), + }) + } + + fn underfunded() -> PolicyRefusal { + PolicyRefusal::Underfunded { + required: U256::from(10), + available: U256::from(3), } } + fn panic_message(run: impl FnOnce()) -> String { + let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(run)) + .expect_err("the run must panic"); + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default() + } + #[test] - fn native_progress_defines_receipt_offsets_and_monotonic_clock() { + fn refusal_texts_are_byte_identical_to_the_served_422_bodies() { + let cases = [ + ( + Refusal::Skip(SkipReason::WrongNonce { + expected: 3, + got: 7, + }), + "bad nonce: expected 3, got 7", + ), + ( + Refusal::FeeCapBelowPrice { + max_fee: 5, + price: 9, + }, + "max fee 5 below base fee 9", + ), + ( + Refusal::Skip(SkipReason::NonceExhausted), + "nonce 4294967295 has no successor", + ), + ( + Refusal::Policy(underfunded()), + "insufficient balance for fee: required 10, available 3", + ), + ( + Refusal::Policy(PolicyRefusal::Other("sponsor quota spent".into())), + "sponsor quota spent", + ), + ]; + for (refusal, text) in cases { + assert_eq!(refusal.to_string(), text); + } + assert_eq!( + SkipReason::NonceExhausted.to_string(), + "nonce 4294967295 has no successor" + ); + assert_eq!( + underfunded().to_string(), + "insufficient balance for fee: required 10, available 3" + ); + } + + #[test] + fn the_count_defines_indexes_and_only_user_ops_move_the_nonce() { let mut app = ProgressApp::new(0); - let ExecutionOutcome::Included(user) = - validate_and_execute_user_op(&mut app, Address::ZERO, &user_op(), 0, 9).unwrap() - else { - panic!("user op should be included") + let Executed::Done { index, .. } = execute(&mut app, &user_tx(0)).unwrap() else { + panic!("a user op at its sender's nonce executes") }; - assert_eq!(user.offset, ExecutedInputCount::ZERO); - assert_eq!(app.executed_input_count(), ExecutedInputCount::new(1)); - assert_eq!(app.last_executed_safe_block(), 9); - - for (offset, block, clock) in [(1, 12, 12), (2, 10, 12)] { - let receipt = execute_direct_input(&mut app, &direct(block)).unwrap(); - assert_eq!(receipt.offset, ExecutedInputCount::new(offset)); + assert_eq!(index, L2TxCount::ZERO); + assert_eq!(app.l2_tx_count(), L2TxCount::new(1)); + assert_eq!(app.nonce, 1); + + for expected in [1, 2] { + let Executed::Done { index, outputs } = execute(&mut app, &direct(12)).unwrap() else { + panic!("a direct always executes") + }; + assert_eq!(index, L2TxCount::new(expected)); assert!( - receipt.outputs.is_empty(), - "included no-ops still advance progress" + outputs.is_empty(), + "executed no-ops still advance the count" ); + assert_eq!(app.l2_tx_count(), L2TxCount::new(expected + 1)); + } + assert_eq!(app.nonce, 1, "a direct never moves a nonce"); + } + + #[test] + fn a_skipped_op_changes_nothing_and_never_applies() { + for (app_nonce, op_nonce, reason) in [ + ( + 1, + 0, + SkipReason::WrongNonce { + expected: 1, + got: 0, + }, + ), + (u32::MAX, u32::MAX, SkipReason::NonceExhausted), + ] { + let mut app = ProgressApp::new(7); + app.nonce = app_nonce; assert_eq!( - app.executed_input_count(), - ExecutedInputCount::new(offset + 1) + execute(&mut app, &user_tx(op_nonce)).unwrap(), + Executed::Skipped(reason) ); - assert_eq!(app.last_executed_safe_block(), clock); + assert_eq!(app.applied, 0); + assert_eq!(app.l2_tx_count(), L2TxCount::new(7)); + assert_eq!(app.nonce, app_nonce); } } #[test] - fn rejection_and_fatal_validation_do_not_execute() { + fn a_skip_needs_no_count_successor() { + let mut app = ProgressApp::new(u64::MAX); + app.nonce = 1; + assert!(matches!( + execute(&mut app, &user_tx(0)).unwrap(), + Executed::Skipped(SkipReason::WrongNonce { .. }) + )); + } + + #[test] + fn a_nonce_read_error_propagates_without_executing() { let mut app = ProgressApp::new(7); - app.reject = true; + app.fail_nonce = true; assert!(matches!( - validate_and_execute_user_op(&mut app, Address::ZERO, &user_op(), 0, 9).unwrap(), - ExecutionOutcome::Invalid(InvalidReason::InvalidNonce { .. }) + execute(&mut app, &user_tx(0)), + Err(AppError::Internal { reason }) if reason == "nonce read failed" )); - app.fail_validation = true; assert!(matches!( - validate_and_execute_user_op(&mut app, Address::ZERO, &user_op(), 0, 9), - Err(AppError::Internal { reason }) if reason == "validation failed" + check_live_user_op(&app, Address::ZERO, &user_op(0, 0), 0), + Err(AppError::Internal { reason }) if reason == "nonce read failed" )); assert_eq!(app.applied, 0); - assert_eq!(app.executed_input_count(), ExecutedInputCount::new(7)); + assert_eq!(app.admit_calls.get(), 0); + assert_eq!(app.l2_tx_count(), L2TxCount::new(7)); } #[test] - fn protocol_max_fee_guard_precedes_application_validation() { + fn mismatch_is_reported_before_exhaustion() { let mut app = ProgressApp::new(0); - app.fail_validation = true; - assert!(matches!( - validate_and_execute_user_op(&mut app, Address::ZERO, &user_op(), 1, 9).unwrap(), - ExecutionOutcome::Invalid(InvalidReason::InvalidMaxFee { - max_fee: 0, - base_fee: 1 - }) - )); + let stale = SkipReason::WrongNonce { + expected: 0, + got: u32::MAX, + }; + assert_eq!( + execute(&mut app, &user_tx(u32::MAX)).unwrap(), + Executed::Skipped(stale), + "an op carrying u32::MAX for a fresh sender is a wrong nonce" + ); + assert_eq!( + check_live_user_op(&app, Address::ZERO, &user_op(u32::MAX, 0), 0).unwrap(), + Some(Refusal::Skip(stale)) + ); + + app.nonce = u32::MAX; + assert_eq!( + execute(&mut app, &user_tx(3)).unwrap(), + Executed::Skipped(SkipReason::WrongNonce { + expected: u32::MAX, + got: 3 + }), + "an exhausted sender's stale op still reads as a wrong nonce" + ); + assert_eq!( + execute(&mut app, &user_tx(u32::MAX)).unwrap(), + Executed::Skipped(SkipReason::NonceExhausted) + ); + // The lane refuses what the executor would skip, before admission. + assert_eq!( + check_live_user_op(&app, Address::ZERO, &user_op(u32::MAX, 0), 0).unwrap(), + Some(Refusal::Skip(SkipReason::NonceExhausted)) + ); + assert_eq!(app.admit_calls.get(), 0); assert_eq!(app.applied, 0); } #[test] - fn protocol_exhausted_nonce_guard_follows_application_validation() { - let mut exhausted = user_op(); - exhausted.nonce = u32::MAX; + fn the_fee_cap_is_checked_before_the_nonce_and_admission() { + let app = ProgressApp { + fail_nonce: true, + admission: Admission::Refuse(underfunded()), + ..ProgressApp::new(0) + }; + assert_eq!( + check_live_user_op(&app, Address::ZERO, &user_op(5, 0), 1).unwrap(), + Some(Refusal::FeeCapBelowPrice { + max_fee: 0, + price: 1 + }) + ); + assert_eq!(app.admit_calls.get(), 0); + } - let mut app = ProgressApp::new(0); - app.reject = true; - assert!( - matches!( - validate_and_execute_user_op(&mut app, Address::ZERO, &exhausted, 0, 9).unwrap(), - ExecutionOutcome::Invalid(InvalidReason::InvalidNonce { .. }) - ), - "an app rejection keeps its bad-nonce diagnostics" + #[test] + fn the_nonce_is_checked_before_admission() { + let app = ProgressApp { + nonce: 1, + admission: Admission::Refuse(underfunded()), + ..ProgressApp::new(0) + }; + assert_eq!( + check_live_user_op(&app, Address::ZERO, &user_op(0, 1), 1).unwrap(), + Some(Refusal::Skip(SkipReason::WrongNonce { + expected: 1, + got: 0 + })) ); + assert_eq!(app.admit_calls.get(), 0); + } - let mut app = ProgressApp::new(0); + #[test] + fn an_admission_refusal_surfaces_without_executing() { + let mut app = ProgressApp { + admission: Admission::Refuse(underfunded()), + ..ProgressApp::new(4) + }; assert_eq!( - validate_and_execute_user_op(&mut app, Address::ZERO, &exhausted, 0, 9).unwrap(), - ExecutionOutcome::Invalid(InvalidReason::NonceExhausted), - "an op the app accepts at u32::MAX is still never included" + check_live_user_op(&app, Address::ZERO, &user_op(0, 1), 1).unwrap(), + Some(Refusal::Policy(underfunded())) ); + assert_eq!(app.admit_calls.get(), 1); assert_eq!(app.applied, 0); + assert_eq!(app.l2_tx_count(), L2TxCount::new(4)); + assert_eq!(app.nonce, 0); + + app.admission = Admission::Admit; + assert_eq!( + check_live_user_op(&app, Address::ZERO, &user_op(0, 1), 1).unwrap(), + None + ); } #[test] - fn execution_error_is_propagated_without_reading_the_failed_instance() { - let mut app = ProgressApp::new(7); - app.fail_apply = true; + fn execute_never_consults_admission() { + let mut app = ProgressApp { + admission: Admission::Refuse(underfunded()), + ..ProgressApp::new(0) + }; assert!(matches!( - execute_direct_input(&mut app, &direct(12)), - Err(AppError::Internal { reason }) if reason == "execution failed" + execute(&mut app, &user_tx(0)).unwrap(), + Executed::Done { .. } )); - assert_eq!(app.applied, 1); + assert_eq!(app.admit_calls.get(), 0); + assert_eq!(app.nonce, 1); + } + + #[test] + fn execution_error_is_propagated_without_reading_the_failed_instance() { + for tx in [direct(12), user_tx(0)] { + let mut app = ProgressApp::new(7); + app.fail_apply = true; + assert!(matches!( + execute(&mut app, &tx), + Err(AppError::Internal { reason }) if reason == "execution failed" + )); + assert_eq!(app.applied, 1); + } } #[test] fn count_exhaustion_fails_before_application_mutation() { - let mut app = ProgressApp::new(u64::MAX); - let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = execute_direct_input(&mut app, &direct(1)); - })); - assert!(panic.is_err()); - assert_eq!(app.applied, 0, "overflow must preflight the app hook"); - assert_eq!( - app.executed_input_count(), - ExecutedInputCount::new(u64::MAX) - ); + for tx in [direct(1), user_tx(0)] { + let mut app = ProgressApp::new(u64::MAX); + let message = panic_message(|| { + let _ = execute(&mut app, &tx); + }); + assert_eq!(message, "L2 tx count overflow: no canonical successor"); + assert_eq!(app.applied, 0, "overflow must preflight the app hook"); + assert_eq!(app.l2_tx_count(), L2TxCount::new(u64::MAX)); + } } #[test] - fn successful_hooks_must_report_the_exact_successor() { - for (advances, clock_override) in [(0, None), (2, None), (1, Some(99))] { - let mut app = ProgressApp::new(1); - app.advance_count = advances; - app.clock_override = clock_override; - let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = execute_direct_input(&mut app, &direct(12)); - })); - assert!(panic.is_err(), "wrong application progress must fail loud"); + fn successful_hooks_must_advance_the_count_exactly_once() { + for steps in [0, 2] { + for tx in [direct(12), user_tx(0)] { + let mut app = ProgressApp::new(1); + app.count_steps = steps; + let message = panic_message(|| { + let _ = execute(&mut app, &tx); + }); + assert!( + message.contains("must advance the count exactly once"), + "{message}" + ); + } } } #[test] - fn zero_count_rejects_nonzero_safe_block_clock() { - assert!(ApplicationProgress::try_new(ExecutedInputCount::ZERO, 1).is_none()); - assert!(ApplicationProgress::try_new(ExecutedInputCount::ZERO, 0).is_some()); - assert!(ApplicationProgress::try_new(ExecutedInputCount::new(1), 7).is_some()); + fn successful_user_ops_must_advance_the_nonce_exactly_once() { + for steps in [0, 2] { + let mut app = ProgressApp::new(1); + app.nonce_steps = steps; + let message = panic_message(|| { + let _ = execute(&mut app, &user_tx(0)); + }); + assert!( + message.contains("must advance its sender's nonce exactly once"), + "{message}" + ); + } } } diff --git a/sequencer-core/src/broadcast.rs b/sequencer-core/src/broadcast.rs index 54ffa20d..0c9d6953 100644 --- a/sequencer-core/src/broadcast.rs +++ b/sequencer-core/src/broadcast.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use crate::l2_tx::{DirectInput, ValidUserOp}; +use crate::l2_tx::{Direct, UserOpTx}; use serde::{Deserialize, Serialize}; /// One transaction from the sequencer's replay-then-live feed. @@ -53,40 +53,36 @@ impl BroadcastTxMessage { } } - pub fn from_user_op( - offset: u64, - user_op: ValidUserOp, - nonce: u32, - safe_block: u64, - batch_nonce: u64, - ) -> Self { + /// The feed keeps its wire names: the record's `block` is the frame's + /// `safe_block` here. + pub fn from_user_op(offset: u64, op: &UserOpTx, batch_nonce: u64) -> Self { Self::UserOp { offset, - sender: user_op.sender.to_string(), - nonce, - fee: user_op.fee, - data: alloy_primitives::hex::encode_prefixed(user_op.data.as_slice()), - safe_block, + sender: op.sender.to_string(), + nonce: op.nonce, + fee: op.fee, + data: alloy_primitives::hex::encode_prefixed(op.data.as_slice()), + safe_block: op.block, batch_nonce, } } - pub fn from_direct_input( + /// The feed keeps its wire names: the record's `block` and `timestamp` + /// are `block_number` and `block_timestamp` here. + pub fn from_direct( offset: u64, - direct: DirectInput, - input_index: u64, + direct: &Direct, batch_nonce: u64, - block_timestamp: u64, transaction_hash: alloy_primitives::B256, ) -> Self { Self::DirectInput { offset, sender: direct.sender.to_string(), - block_number: direct.block_number, + block_number: direct.block, payload: alloy_primitives::hex::encode_prefixed(direct.payload.as_slice()), - input_index, + input_index: direct.input_index, batch_nonce, - block_timestamp, + block_timestamp: direct.timestamp, transaction_hash: alloy_primitives::hex::encode_prefixed(transaction_hash), } } diff --git a/sequencer-core/src/consensus_vectors.rs b/sequencer-core/src/consensus_vectors.rs index cc75a98c..840c3b1a 100644 --- a/sequencer-core/src/consensus_vectors.rs +++ b/sequencer-core/src/consensus_vectors.rs @@ -1,10 +1,11 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Byte-level consensus vectors from `tests/fixtures/consensus_vectors_v1.json`. +//! Byte-level consensus vectors from `tests/fixtures/consensus_vectors_v2.json`. //! They were pinned once, before any toolchain or dependency move, and are never //! regenerated: a mismatch means consensus bytes changed and is a finding. -//! Signature, SSZ and fee cases are asserted next to the code they pin. +//! Signature, SSZ and fee cases are asserted next to the code they pin; the +//! guest scenarios are checked where the wallet and the machine are. use alloy_primitives::{Address, B256, Bytes, U64, U256, keccak256}; use alloy_sol_types::{Eip712Domain, SolStruct}; @@ -16,11 +17,17 @@ use crate::user_op::UserOp; const CONSENSUS_VECTORS: &str = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../tests/fixtures/consensus_vectors_v1.json" + "/../tests/fixtures/consensus_vectors_v2.json" )); pub(crate) fn vectors() -> ConsensusVectors { - serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v1.json") + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json") +} + +/// The file untyped, for the scheduler-only scenarios, which the scheduler's +/// tests replay field by field. +pub(crate) fn raw_vectors() -> serde_json::Value { + serde_json::from_str(CONSENSUS_VECTORS).expect("parse consensus_vectors_v2.json") } #[derive(Debug, Deserialize)] @@ -170,7 +177,7 @@ pub(crate) struct FeeCase { pub linear: U256, } -/// The guest scenarios' input payloads; `canonical-test` replays them. +/// The guest scenarios' input payloads, of which the SSZ case is one. #[derive(Debug, Deserialize)] pub(crate) struct GuestPayloads { pub payloads: BTreeMap, diff --git a/sequencer-core/src/history.rs b/sequencer-core/src/history.rs index abdf938e..f7b439e7 100644 --- a/sequencer-core/src/history.rs +++ b/sequencer-core/src/history.rs @@ -1,25 +1,25 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! External history identity and version coordinates. +//! External history identity and coordinates: the era and the L2 tx count. use serde::{Deserialize, Serialize}; use std::{fmt, str::FromStr}; use thiserror::Error; -/// Boundary before the next canonical application input executes. +/// The L2 tx count: how many L2 transactions have executed. /// -/// If the application is at `X`, history entry `X` is the next input and a -/// successful execution moves the boundary to `X + 1`. Arithmetic is kept +/// If the application is at `X`, history entry `X` executes next and a +/// successful execution moves the count to `X + 1`. Arithmetic is kept /// behind checked methods so this coordinate cannot be confused with a /// physical SQLite cursor or advanced with wrapping/saturating math. #[derive( Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, )] #[serde(transparent)] -pub struct ExecutedInputCount(u64); +pub struct L2TxCount(u64); -impl ExecutedInputCount { +impl L2TxCount { pub const ZERO: Self = Self(0); pub const fn new(value: u64) -> Self { @@ -40,9 +40,18 @@ impl ExecutedInputCount { None => None, } } + + /// Advance by one executed transaction, as an application does on every + /// success. A count with no successor has no canonical next state, so + /// this panics rather than wrap. + pub fn advance(&mut self) { + *self = self + .checked_next() + .expect("L2 tx count overflow: no canonical successor"); + } } -/// One durable setup/rebuild era. +/// One durable era, published by setup. /// /// The bytes must carry the RFC 4122 UUIDv4 version and variant bits. Text and /// JSON use the canonical lowercase hyphenated representation. @@ -138,63 +147,40 @@ pub enum EraIdParseError { InvalidVariant, } -/// Monotonic soft-history revision within one [`EraId`]. -#[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, -)] -#[serde(transparent)] -pub struct RecoveryGeneration(u64); - -impl RecoveryGeneration { - pub const fn new(value: u64) -> Self { - Self(value) - } - - pub const fn get(self) -> u64 { - self.0 - } -} - -/// Equality/discontinuity token for locally available application history. -/// Consumers must claim both fields when resuming application history. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct HistoryVersion { - pub era_id: EraId, - pub recovery_generation: RecoveryGeneration, -} - /// The history a consumer holds and the next application input it can execute. +/// +/// An era is the history's whole identity: inside one era the history only +/// appends, so a claim needs no revision besides its era. A client of an older +/// era adopts a new one only through +/// [`crate::history_api::HistoryInfo::adopt_claim`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryClaim { - pub version: HistoryVersion, - pub next_input: ExecutedInputCount, + pub era_id: EraId, + pub next_input: L2TxCount, } /// One coherent view of the locally available canonical history. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryBounds { - pub version: HistoryVersion, - pub available_from: ExecutedInputCount, - pub head: ExecutedInputCount, + pub era_id: EraId, + pub available_from: L2TxCount, + pub head: L2TxCount, } impl HistoryBounds { /// Validate identity before position: equal counts cannot resume a different - /// history. Every count in the inclusive range is admissible; `head` waits - /// for the next input. + /// history by themselves. A client of another era first checks its count + /// against the new era's rebuilt range and claims the current era + /// ([`crate::history_api::HistoryInfo::adopt_claim`]). Every count in the + /// inclusive range is admissible; `head` waits for the next input. pub fn validate(&self, claim: HistoryClaim) -> Result<(), HistoryPolicyError> { assert!( self.available_from <= self.head, "available history base exceeds its head" ); - if claim.version.era_id != self.version.era_id { + if claim.era_id != self.era_id { return Err(HistoryPolicyError::EraChanged { - current: self.version, - }); - } - if claim.version.recovery_generation != self.version.recovery_generation { - return Err(HistoryPolicyError::StaleGeneration { - current: self.version, + era_id: self.era_id, }); } if claim.next_input < self.available_from { @@ -212,14 +198,13 @@ impl HistoryBounds { #[derive(Debug, Clone, Copy, PartialEq, Eq, Error, Serialize, Deserialize)] #[serde(tag = "code", rename_all = "SCREAMING_SNAKE_CASE")] pub enum HistoryPolicyError { + /// The claim names another era; `era_id` is the server's current one. #[error("history era changed")] - EraChanged { current: HistoryVersion }, - #[error("history generation changed")] - StaleGeneration { current: HistoryVersion }, + EraChanged { era_id: EraId }, #[error("requested input precedes locally available history")] - HistoryUnavailable { available_from: ExecutedInputCount }, + HistoryUnavailable { available_from: L2TxCount }, #[error("requested input is ahead of the history head")] - AheadOfHead { head: ExecutedInputCount }, + AheadOfHead { head: L2TxCount }, } #[cfg(test)] @@ -240,12 +225,16 @@ mod tests { "\"00112233-4455-4677-8899-aabbccddeeff\"" ); let claim = HistoryClaim { - version: HistoryVersion { - era_id: era, - recovery_generation: RecoveryGeneration::new(7), - }, - next_input: ExecutedInputCount::new(u64::MAX), + era_id: era, + next_input: L2TxCount::new(u64::MAX), }; + assert_eq!( + serde_json::to_value(claim).unwrap(), + serde_json::json!({ + "era_id": "00112233-4455-4677-8899-aabbccddeeff", + "next_input": u64::MAX + }) + ); assert_eq!( serde_json::from_str::(&serde_json::to_string(&claim).unwrap()).unwrap(), claim @@ -260,34 +249,21 @@ mod tests { #[test] fn history_policy_errors_preserve_literal_wire_codes_and_fields() { - let current = HistoryVersion { - era_id: CANONICAL.parse().unwrap(), - recovery_generation: RecoveryGeneration::new(7), - }; + let era_id: EraId = CANONICAL.parse().unwrap(); for (error, json) in [ ( - HistoryPolicyError::EraChanged { current }, - serde_json::json!({ - "code": "ERA_CHANGED", - "current": { "era_id": CANONICAL, "recovery_generation": 7 } - }), - ), - ( - HistoryPolicyError::StaleGeneration { current }, - serde_json::json!({ - "code": "STALE_GENERATION", - "current": { "era_id": CANONICAL, "recovery_generation": 7 } - }), + HistoryPolicyError::EraChanged { era_id }, + serde_json::json!({ "code": "ERA_CHANGED", "era_id": CANONICAL }), ), ( HistoryPolicyError::HistoryUnavailable { - available_from: ExecutedInputCount::new(41), + available_from: L2TxCount::new(41), }, serde_json::json!({ "code": "HISTORY_UNAVAILABLE", "available_from": 41 }), ), ( HistoryPolicyError::AheadOfHead { - head: ExecutedInputCount::new(50), + head: L2TxCount::new(50), }, serde_json::json!({ "code": "AHEAD_OF_HEAD", "head": 50 }), ), @@ -324,72 +300,57 @@ mod tests { } #[test] - fn executed_input_count_advances_checked() { - assert_eq!( - ExecutedInputCount::ZERO.checked_next(), - Some(ExecutedInputCount::new(1)) - ); - assert_eq!(ExecutedInputCount::new(u64::MAX).checked_next(), None); - assert_eq!( - ExecutedInputCount::new(7).checked_add(5), - Some(ExecutedInputCount::new(12)) - ); + fn l2_tx_count_advances_checked() { + assert_eq!(L2TxCount::ZERO.checked_next(), Some(L2TxCount::new(1))); + assert_eq!(L2TxCount::new(u64::MAX).checked_next(), None); + assert_eq!(L2TxCount::new(7).checked_add(5), Some(L2TxCount::new(12))); + + let mut count = L2TxCount::new(u64::MAX - 1); + count.advance(); + assert_eq!(count, L2TxCount::new(u64::MAX)); + } + + #[test] + #[should_panic(expected = "L2 tx count overflow: no canonical successor")] + fn l2_tx_count_advance_fails_loud_without_a_successor() { + L2TxCount::new(u64::MAX).advance(); } fn history_bounds(base: u64, head: u64) -> HistoryBounds { HistoryBounds { - version: HistoryVersion { - era_id: EraId::from_bytes(CANONICAL_BYTES).unwrap(), - recovery_generation: RecoveryGeneration::new(4), - }, - available_from: ExecutedInputCount::new(base), - head: ExecutedInputCount::new(head), + era_id: EraId::from_bytes(CANONICAL_BYTES).unwrap(), + available_from: L2TxCount::new(base), + head: L2TxCount::new(head), } } #[test] - fn history_claim_checks_era_before_generation_and_position() { + fn history_bounds_json_pins_the_era_and_both_counts() { + assert_eq!( + serde_json::to_value(history_bounds(7, 12)).unwrap(), + serde_json::json!({ "era_id": CANONICAL, "available_from": 7, "head": 12 }) + ); + } + + #[test] + fn history_claim_checks_era_before_position() { let bounds = history_bounds(41, 45); let mut other_era = CANONICAL_BYTES; other_era[0] ^= 1; let other_era = EraId::from_bytes(other_era).unwrap(); - for (generation, next_input) in [(4, 43), (3, 0), (u64::MAX, u64::MAX)] { + for next_input in [43, 0, u64::MAX] { assert_eq!( bounds.validate(HistoryClaim { - version: HistoryVersion { - era_id: other_era, - recovery_generation: RecoveryGeneration::new(generation), - }, - next_input: ExecutedInputCount::new(next_input), + era_id: other_era, + next_input: L2TxCount::new(next_input), }), Err(HistoryPolicyError::EraChanged { - current: bounds.version, + era_id: bounds.era_id, }) ); } } - #[test] - fn history_claim_requires_equal_generation_before_checking_position() { - let bounds = history_bounds(41, 45); - for generation in [0, 3, 5, u64::MAX] { - for next_input in [0, 43, u64::MAX] { - assert_eq!( - bounds.validate(HistoryClaim { - version: HistoryVersion { - recovery_generation: RecoveryGeneration::new(generation), - ..bounds.version - }, - next_input: ExecutedInputCount::new(next_input), - }), - Err(HistoryPolicyError::StaleGeneration { - current: bounds.version, - }) - ); - } - } - } - #[test] fn history_claim_rejects_unavailable_and_future_counts() { let bounds = history_bounds(41, 45); @@ -403,8 +364,8 @@ mod tests { }; assert_eq!( bounds.validate(HistoryClaim { - version: bounds.version, - next_input: ExecutedInputCount::new(next_input), + era_id: bounds.era_id, + next_input: L2TxCount::new(next_input), }), Err(expected) ); @@ -425,8 +386,8 @@ mod tests { let bounds = history_bounds(base, head); assert_eq!( bounds.validate(HistoryClaim { - version: bounds.version, - next_input: ExecutedInputCount::new(next_input), + era_id: bounds.era_id, + next_input: L2TxCount::new(next_input), }), Ok(()) ); @@ -438,8 +399,8 @@ mod tests { fn incoherent_history_bounds_fail_loud() { let bounds = history_bounds(42, 41); let _ = bounds.validate(HistoryClaim { - version: bounds.version, - next_input: ExecutedInputCount::new(41), + era_id: bounds.era_id, + next_input: L2TxCount::new(41), }); } } diff --git a/sequencer-core/src/history_api.rs b/sequencer-core/src/history_api.rs index d8312fb4..70a76aa5 100644 --- a/sequencer-core/src/history_api.rs +++ b/sequencer-core/src/history_api.rs @@ -1,16 +1,13 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Historical scheduler inputs and their handoff to the application feed. +//! `/history`'s metadata, and how a client of an older era adopts the current one. -use alloy_primitives::{Address, B256, Bytes}; +use alloy_primitives::Address; use serde::{Deserialize, Serialize}; -use crate::history::{EraId, ExecutedInputCount, HistoryBounds, RecoveryGeneration}; - -pub const HISTORICAL_INPUT_MAX_ITEMS: usize = 256; -/// A larger first input is served alone, preserving progress through any history. -pub const HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES: usize = 1024 * 1024; +use crate::history::{HistoryBounds, HistoryClaim, L2TxCount}; +use thiserror::Error; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryDeployment { @@ -21,18 +18,24 @@ pub struct HistoryDeployment { pub batch_submitter_address: Address, } +/// Where the era's rebuild ended. Its history starts at `available_from`, the +/// base's count, and setup rebuilt it from L1 through the stop block. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryBaseline { pub l1_stop_block: u64, /// Exclusive end of the per-application InputBox prefix through the stop block. pub l1_end_input_index: u64, + /// The scheduler's next batch nonce at the stop block. pub next_batch_nonce: u64, + /// The L2 tx count at the end of the stop block: the rebuilt history ends + /// here and the live rows start. + pub rebuilt_history_end: L2TxCount, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct AcceptedCheckpoint { pub inclusion_block: u64, - pub executed_input_count: ExecutedInputCount, + pub executed_input_count: L2TxCount, pub next_batch_nonce: u64, } @@ -41,41 +44,125 @@ pub struct HistoryInfo { pub deployment: HistoryDeployment, pub history: HistoryBounds, pub baseline: HistoryBaseline, - /// The accepted boundary shares `history.version`; it does not certify client state. + /// The accepted boundary belongs to `history.era_id`; it does not certify client state. pub accepted_checkpoint: Option, - pub compatibility: Option, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct HistoryCompatibility { - pub from_generation: RecoveryGeneration, - /// Checkpoint counts up to and including this boundary preserve their input prefix. - pub preserved_input_count: ExecutedInputCount, +impl HistoryInfo { + /// The claim a client holding state at `count` from another era resumes + /// this one with. Every count from the base through the rebuilt history's + /// end is one this era's history reaches through rows that landed on L1; + /// past that end the client holds rows that never landed. + pub fn adopt_claim(&self, count: L2TxCount) -> Result { + if count < self.history.available_from { + return Err(AdoptError::BelowBase { + available_from: self.history.available_from, + }); + } + if count > self.baseline.rebuilt_history_end { + return Err(AdoptError::PastRebuiltEnd { + rebuilt_history_end: self.baseline.rebuilt_history_end, + }); + } + Ok(HistoryClaim { + era_id: self.history.era_id, + next_input: count, + }) + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HistoricalL1InputStart { - NextInputIndex(u64), - AfterBlock(u64), +/// Why a client's count cannot adopt the current era. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum AdoptError { + /// The era starts after the client's state: recover from an older base. + #[error("count is below the era's base count {available_from}", available_from = available_from.get())] + BelowBase { available_from: L2TxCount }, + /// The client's state holds rows that never landed by the stop block: + /// restore an older state. + #[error("count is past the era's rebuilt history end {rebuilt_history_end}", rebuilt_history_end = rebuilt_history_end.get())] + PastRebuiltEnd { rebuilt_history_end: L2TxCount }, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HistoricalL1Input { - pub input_index: u64, - pub sender: Address, - /// Original inner application/batch payload, including malformed batches. - pub payload: Bytes, - pub block_number: u64, - pub block_timestamp: u64, - pub transaction_hash: B256, -} +#[cfg(test)] +mod tests { + use super::*; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HistoricalL1InputsPage { - pub era_id: EraId, - pub l1_stop_block: u64, - pub end_input_index: u64, - /// Only equality with `end_input_index` establishes completion, not page length. - pub next_input_index: u64, - pub items: Vec, + fn info(base: u64, rebuilt_end: u64, head: u64) -> HistoryInfo { + HistoryInfo { + deployment: HistoryDeployment { + chain_id: 1, + app_address: Address::repeat_byte(1), + input_box_address: Address::repeat_byte(2), + app_deployment_block: 0, + batch_submitter_address: Address::repeat_byte(3), + }, + history: HistoryBounds { + era_id: "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(), + available_from: L2TxCount::new(base), + head: L2TxCount::new(head), + }, + baseline: HistoryBaseline { + l1_stop_block: 50, + l1_end_input_index: 9, + next_batch_nonce: 4, + rebuilt_history_end: L2TxCount::new(rebuilt_end), + }, + accepted_checkpoint: None, + } + } + + #[test] + fn a_count_from_the_base_through_the_rebuilt_end_adopts_the_current_era() { + let info = info(41, 45, 60); + for count in [41, 43, 45] { + let claim = info.adopt_claim(L2TxCount::new(count)).unwrap(); + assert_eq!(claim.era_id, info.history.era_id); + assert_eq!(claim.next_input, L2TxCount::new(count)); + assert_eq!(info.history.validate(claim), Ok(())); + } + let empty_rebuild = self::info(41, 41, 41); + assert!(empty_rebuild.adopt_claim(L2TxCount::new(41)).is_ok()); + } + + #[test] + fn a_count_outside_the_rebuilt_range_cannot_adopt() { + let info = info(41, 45, 60); + for count in [0, 40] { + assert_eq!( + info.adopt_claim(L2TxCount::new(count)), + Err(AdoptError::BelowBase { + available_from: L2TxCount::new(41) + }) + ); + } + // Live rows past the rebuilt end were served by this era, not landed + // before its stop block, so an older era's state there is not canonical. + for count in [46, 60, u64::MAX] { + assert_eq!( + info.adopt_claim(L2TxCount::new(count)), + Err(AdoptError::PastRebuiltEnd { + rebuilt_history_end: L2TxCount::new(45) + }) + ); + } + } + + #[test] + fn the_baseline_carries_the_rebuilt_end_on_the_wire() { + let baseline = info(41, 45, 60).baseline; + let json = serde_json::to_value(baseline).unwrap(); + assert_eq!( + json, + serde_json::json!({ + "l1_stop_block": 50, + "l1_end_input_index": 9, + "next_batch_nonce": 4, + "rebuilt_history_end": 45 + }) + ); + assert_eq!( + serde_json::from_value::(json).unwrap(), + baseline + ); + } } diff --git a/sequencer-core/src/l2_tx.rs b/sequencer-core/src/l2_tx.rs index 77df26b0..6d64d172 100644 --- a/sequencer-core/src/l2_tx.rs +++ b/sequencer-core/src/l2_tx.rs @@ -1,26 +1,58 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +use crate::user_op::UserOp; use alloy_primitives::Address; +/// What the scheduler hands out and the application executes. The same record +/// serves the canonical machine, a rebuild, the lane and its replay, so every +/// caller runs the same executor checks on it. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct DirectInput { +pub enum L2Tx { + Direct(Direct), + UserOp(UserOpTx), +} + +/// A direct input the scheduler drained from its fridge. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Direct { + /// Per-application InputBox index of the input that carried it. + pub input_index: u64, pub sender: Address, - pub block_number: u64, + /// Its L1 inclusion block. + pub block: u64, + /// Unix timestamp, in seconds, of its L1 inclusion block. + pub timestamp: u64, pub payload: Vec, } -#[derive(Debug, Clone)] -pub struct ValidUserOp { +/// A user op whose signature the scheduler recovered and whose fee cap covers +/// the frame's price. It keeps its signed nonce so the executor, not the +/// application, decides whether it runs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserOpTx { + /// The signer the scheduler recovered. pub sender: Address, - /// Log-space fee exponent (base 129/128) committed by the sequencer for this frame. - /// Use [`crate::fee::fee_to_linear`] to convert to a linear amount for charging. + /// The nonce the sender signed. + pub nonce: u32, + /// The frame's log-space fee price (base 129/128), which the op's cap + /// covers. Use [`crate::fee::fee_to_linear`] for a linear amount. pub fee: u16, + /// The frame's safe block. + pub block: u64, pub data: Vec, } -#[derive(Debug, Clone)] -pub enum SequencedL2Tx { - UserOp(ValidUserOp), - Direct(DirectInput), +impl UserOpTx { + /// Build the record from a signed op and its frame, so the lane, the + /// scheduler and replays all build it the same way. + pub fn new(sender: Address, op: &UserOp, fee: u16, block: u64) -> Self { + Self { + sender, + nonce: op.nonce, + fee, + block, + data: op.data.to_vec(), + } + } } diff --git a/sequencer-core/src/lib.rs b/sequencer-core/src/lib.rs index b9a1ea7f..ac80ea7e 100644 --- a/sequencer-core/src/lib.rs +++ b/sequencer-core/src/lib.rs @@ -20,10 +20,6 @@ pub mod scheduler; pub mod test_support; pub mod user_op; -/// Maximum number of L1 blocks a batch can wait before the scheduler considers it stale. -/// Shared between the scheduler (canonical-app) and the sequencer (batch submitter, startup detection). -pub const MAX_WAIT_BLOCKS: u64 = 1200; - /// EIP-712 domain name shared between sequencer and scheduler. pub const DOMAIN_NAME: &str = "CartesiAppSequencer"; diff --git a/sequencer-core/src/protocol.rs b/sequencer-core/src/protocol.rs index 5f672b70..1bd68b8c 100644 --- a/sequencer-core/src/protocol.rs +++ b/sequencer-core/src/protocol.rs @@ -1,359 +1,87 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Protocol timing: the scheduler's staleness rule and its off-chain acceptance -//! predicate, which must agree with the canonical fold (I1), plus sequencer-only -//! knobs for when the sequencer stops issuing soft confirmations against stale -//! or unknowable L1 state. +//! The scheduler's staleness rule and its off-chain acceptance predicate, +//! which must agree with the canonical scheduler (I1), plus the frame clock. use crate::batch::Batch; use alloy_primitives::Address; -use thiserror::Error; -/// Error surfaced by [`ProtocolTiming::try_new`] when timing fields would -/// produce an unusable danger threshold. +/// Advance logical frame time after this many newly-safe L1 blocks. /// -/// Returning a typed error lets the runtime surface the misconfiguration at -/// config-parse time through the structured `RunError` taxonomy. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum ProtocolTimingError { - /// `preemptive_margin_blocks == 0` — zero-width preemptive zone defeats - /// the entire margin design (recovery only ever fires at the absolute max, - /// with no operator runway). Always an operator misconfig. - #[error("preemptive_margin_blocks must be greater than zero")] - MarginZero, - /// `preemptive_margin_blocks >= max_wait_blocks` — the danger threshold - /// would be 0, making preemptive recovery indistinguishable from hard - /// staleness. The margin is supposed to be operator runway *before* - /// hitting `MAX_WAIT_BLOCKS`, so this is always an operator misconfig. - #[error( - "preemptive_margin_blocks ({margin}) must be strictly less than \ - max_wait_blocks ({max_wait})" - )] - MarginNotLessThanMaxWait { margin: u64, max_wait: u64 }, - /// `l1_read_stale_after_blocks == 0` would make the L1 view unusable - /// immediately, so the sequencer could never start. - #[error("l1_read_stale_after_blocks must be greater than zero")] - ReadStaleAfterZero, - /// The read-staleness threshold must fire strictly before the write danger - /// threshold. Equality would have both refusal arms (L1ViewStale and - /// EstimatedBatchInDanger) trip at the same point, defeating the - /// "stale fires first" design property. - #[error( - "l1_read_stale_after_blocks ({read_stale_after}) must be strictly \ - less than danger_threshold ({danger_threshold})" - )] - ReadStaleAfterPastDanger { - read_stale_after: u64, - danger_threshold: u64, - }, - /// A zero block-time estimate makes the wall-clock fallback undefined. - #[error("seconds_per_block must be greater than zero")] - SecondsPerBlockZero, - /// The configured read-staleness horizon must fit in the `u64` Unix-time - /// arithmetic used by the detector. - #[error( - "l1_read_stale_after_blocks ({read_stale_after}) * seconds_per_block \ - ({seconds_per_block}) exceeds u64::MAX seconds" - )] - ReadStaleWindowOverflow { - read_stale_after: u64, - seconds_per_block: u64, - }, -} - -/// The local wall clock predates the persisted safe-head progress baseline. +/// Protocol-visible, deliberately not configurable: user ops validate at +/// their frame's safe block, so this value is the application-clock +/// granularity (about a minute on mainnet), a product semantics decision +/// — not a lane implementation detail (prose owner: +/// `docs/protocol/scheduler-semantics.md`, frame-clock policy). The +/// observed tip is used directly, so delayed or epoch-sized observations +/// create one frame and never synthesize missed intermediate ticks. +pub const FRAME_CLOCK_INTERVAL_SAFE_BLOCKS: u64 = 5; + +/// Off-chain simulation of the scheduler's batch-acceptance predicate, under +/// the application's staleness window `max_wait_blocks`. /// -/// This is an unusable timing environment, not a storage invariant violation: -/// callers must stop issuing soft confirmations until the clock catches up or -/// a new safe-head advance establishes a usable baseline. -#[derive(Debug, Error, PartialEq, Eq)] -#[error( - "current wall clock ({now_ms} ms) predates last safe-head progress \ - ({last_safe_progress_ms} ms)" -)] -pub struct WallClockRegression { - pub now_ms: u64, - pub last_safe_progress_ms: u64, -} - -/// Time-based protocol parameters: scheduler-mirroring `max_wait_blocks` -/// plus sequencer-side preemptive-recovery tuning. No identity (the -/// batch-submitter address is passed separately to predicates that need it). +/// Returns `Some(AcceptedBatch)` iff the scheduler would accept the input +/// at the given `expected_nonce`. The caller threads `expected_nonce` +/// across a stream of inputs, advancing by one on each `Some`. /// -/// Construct via [`ProtocolTiming::try_new`]: it checks the margin/stale -/// relationships once, and the private fields keep every value valid -/// afterwards. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ProtocolTiming { - /// `MAX_WAIT_BLOCKS` — after this many L1 blocks, the scheduler skips a - /// submitted batch as stale. +/// Rejection paths (wrong sender, SSZ decode failure, stale by inclusion, +/// nonce mismatch) return `None` without advancing — matching what the +/// scheduler does on-chain. +/// +/// **Deliberately omitted rejects: a self-trust.** The canonical scheduler +/// also rejects a batch with a frame whose `safe_block` is not strictly +/// below the inclusion block (`SafeBlockNotBelowInclusionBlock`), with +/// safe blocks that decrease across frames (`NonMonotonicSafeBlocks`), or +/// with a fee above `MAX_EXPONENT` (`FeePriceOutOfRange`): gate d. Gate f +/// rejects one whose first frame lies below the scheduler clock +/// (`FirstFrameBelowClock`). This predicate re-checks none of them. A +/// correct submitter never trips them: it opens frames in order, only at +/// safe heads it has already seen, so its batch lands in a later block. +/// Gate f belongs to the same omission for a further reason: the clock +/// rises with every executed direct, backstop and overflow drains +/// included, so the predicate could know it only by replaying the +/// directs. An overflow drain can still make gate f reject a sealed batch; +/// the watchdog's divergence latch reports that, and a manual recovery +/// answers it. The I1 duality test pins every omission as a *documented* +/// asymmetry; mirror them here if the submitter stops being trusted. +pub fn scheduler_accepts( + batch_submitter: Address, max_wait_blocks: u64, - /// How many blocks before `max_wait_blocks` the sequencer triggers - /// preemptive recovery. Sequencer-local; must be strictly less than - /// `max_wait_blocks` (enforced by [`ProtocolTiming::try_new`]). - preemptive_margin_blocks: u64, - /// How long the L1 safe block itself may be stale, measured in assumed L1 - /// blocks. Sequencer-local; startup refuses once the safe block timestamp - /// is older than this threshold. - l1_read_stale_after_blocks: u64, - /// Wall-clock estimate of L1 block time, used as a fallback when the L1 - /// safe head appears frozen. Sequencer-local. - seconds_per_block: u64, -} - -impl ProtocolTiming { - /// Advance logical frame time after this many newly-safe L1 blocks. - /// - /// Protocol-visible, deliberately not configurable: user ops validate at - /// their frame's safe block, so this value is the application-clock - /// granularity (about a minute on mainnet), a product semantics decision - /// — not a lane implementation detail (prose owner: - /// `docs/protocol/scheduler-semantics.md`, frame-clock policy). The - /// observed tip is used directly, so delayed or epoch-sized observations - /// create one frame and never synthesize missed intermediate ticks. - pub const FRAME_CLOCK_INTERVAL_SAFE_BLOCKS: u64 = 5; - - /// Validated constructor. Rejects timing configurations that would - /// produce an unusable danger threshold or a degenerate margin. - pub fn try_new( - max_wait_blocks: u64, - preemptive_margin_blocks: u64, - l1_read_stale_after_blocks: u64, - seconds_per_block: u64, - ) -> Result { - if preemptive_margin_blocks == 0 { - return Err(ProtocolTimingError::MarginZero); - } - if preemptive_margin_blocks >= max_wait_blocks { - return Err(ProtocolTimingError::MarginNotLessThanMaxWait { - margin: preemptive_margin_blocks, - max_wait: max_wait_blocks, - }); - } - if l1_read_stale_after_blocks == 0 { - return Err(ProtocolTimingError::ReadStaleAfterZero); - } - if seconds_per_block == 0 { - return Err(ProtocolTimingError::SecondsPerBlockZero); - } - let danger_threshold = max_wait_blocks - preemptive_margin_blocks; - if l1_read_stale_after_blocks >= danger_threshold { - return Err(ProtocolTimingError::ReadStaleAfterPastDanger { - read_stale_after: l1_read_stale_after_blocks, - danger_threshold, - }); - } - l1_read_stale_after_blocks - .checked_mul(seconds_per_block) - .ok_or(ProtocolTimingError::ReadStaleWindowOverflow { - read_stale_after: l1_read_stale_after_blocks, - seconds_per_block, - })?; - Ok(Self { - max_wait_blocks, - preemptive_margin_blocks, - l1_read_stale_after_blocks, - seconds_per_block, - }) - } - - /// The scheduler staleness window ([`crate::MAX_WAIT_BLOCKS`] in - /// production). - pub const fn max_wait_blocks(&self) -> u64 { - self.max_wait_blocks - } - - /// Blocks before [`Self::max_wait_blocks`] at which preemptive recovery - /// triggers. - pub const fn preemptive_margin_blocks(&self) -> u64 { - self.preemptive_margin_blocks - } - - /// How many assumed L1 blocks the safe block itself may be stale. - pub const fn l1_read_stale_after_blocks(&self) -> u64 { - self.l1_read_stale_after_blocks - } - - /// Wall-clock estimate of L1 block time, in seconds. - pub const fn seconds_per_block(&self) -> u64 { - self.seconds_per_block - } - - /// The block-age threshold at which preemptive recovery triggers. - pub fn danger_threshold(&self) -> u64 { - self.max_wait_blocks - self.preemptive_margin_blocks - } - - /// Wall-clock age, in seconds, after which the L1 safe block is too old - /// for the sequencer to trust its L1 view. - pub fn l1_read_stale_after_secs(&self) -> u64 { - self.l1_read_stale_after_blocks - .checked_mul(self.seconds_per_block) - .expect("validated L1 read-staleness window must fit in u64 seconds") - } - - /// Whether the safe block timestamp is too old to support recovery or - /// continued soft confirmations. `None` means the view is unknown and is - /// treated as unusable. - pub fn l1_view_is_stale(&self, safe_block_timestamp_secs: Option, now_ms: u64) -> bool { - let Some(timestamp_secs) = safe_block_timestamp_secs else { - return true; - }; - let now_secs = now_ms / 1000; - let Some(age_secs) = now_secs.checked_sub(timestamp_secs) else { - // Ahead-of-clock is not staleness — the observation is the - // freshest possible. Whether the local *clock* is usable against - // it is a separate fault ([`Self::clock_cannot_age_l1_view`]), - // deliberately checked after the observed danger arms. - return false; - }; - age_secs >= self.l1_read_stale_after_secs() - } - - /// Whether the local clock is a full block-time or more behind the - /// persisted L1 safe-block timestamp — i.e. it cannot age the view at - /// all. Sub-block ahead-ness is ordinary NTP-scale skew against a - /// block-granular timestamp (usable, age 0) and is tolerated. - /// - /// This is a clock fault, not a view fault: `check_danger` evaluates it - /// only after the observed danger arms, which are pure block arithmetic - /// and must not be suppressed by a broken local clock. - pub fn clock_cannot_age_l1_view( - &self, - safe_block_timestamp_secs: Option, - now_ms: u64, - ) -> bool { - let Some(timestamp_secs) = safe_block_timestamp_secs else { - return false; - }; - // Compare at millisecond precision: flooring `now_ms` to seconds - // would make the full-block refusal fire up to 999 ms early, - // contradicting the sub-block tolerance. Saturating multiplies are - // safe: a saturated ahead-side only widens toward refusal for - // timestamps that are absurd anyway. - let timestamp_ms = timestamp_secs.saturating_mul(1000); - match timestamp_ms.checked_sub(now_ms) { - Some(ahead_ms) => ahead_ms >= self.seconds_per_block.saturating_mul(1000), - None => false, - } - } - - /// Wall-clock-adjusted danger threshold, used when the L1 safe head may be - /// stale. - /// - /// Translates "wall-clock time elapsed since the last L1 safe-head - /// observation" into "blocks the safe head is presumed to be behind", and - /// shaves that off the strict [`Self::danger_threshold`]. Returns `None` - /// when the wall-clock arm should be skipped: - /// - /// - `last_safe_progress_ms` is `None` — no baseline to extrapolate from. - /// - Less than one block-time has elapsed — adjustment would be 0, so the - /// strict check covers this case directly. - /// - /// Returns `Ok(Some(adjusted))` where - /// `adjusted = danger_threshold − (elapsed_secs / seconds_per_block)`, - /// saturating at 0. A `now_ms` less than one block-time behind the - /// baseline is treated as elapsed 0 — the estimate quantizes elapsed time - /// into whole blocks, so a sub-block regression cannot change it and is - /// ordinary clock-step noise. Returns [`WallClockRegression`] only - /// for larger regressions, which genuinely invalidate the extrapolation; - /// callers must treat that environment as unusable rather than silently - /// reporting no elapsed time. - pub fn wall_clock_adjusted_danger_threshold( - &self, - last_safe_progress_ms: Option, - now_ms: u64, - ) -> Result, WallClockRegression> { - let Some(last) = last_safe_progress_ms else { - return Ok(None); - }; - let elapsed_ms = match now_ms.checked_sub(last) { - Some(elapsed) => elapsed, - None => { - let regression_secs = (last - now_ms) / 1000; - if regression_secs < self.seconds_per_block { - return Ok(None); - } - return Err(WallClockRegression { - now_ms, - last_safe_progress_ms: last, - }); - } - }; - let elapsed_secs = elapsed_ms / 1000; - let missed = elapsed_secs / self.seconds_per_block; - if missed == 0 { - return Ok(None); - } - Ok(Some(self.danger_threshold().saturating_sub(missed))) - } - - /// Scheduler's staleness predicate: a batch is stale when - /// `inclusion_block - first_frame_safe_block >= max_wait_blocks`. - /// Off-chain mirror of the scheduler's gate e (both call [`age_exceeds`]); - /// used by [`Self::scheduler_accepts`]. - pub fn is_scheduler_stale(&self, inclusion_block: u64, first_frame_safe_block: u64) -> bool { - age_exceeds( - inclusion_block, + input: SafeInputView<'_>, + expected_nonce: u64, +) -> Option { + if input.sender != batch_submitter { + return None; + } + let batch = ::from_ssz_bytes(input.payload).ok()?; + let first_frame_safe_block = batch.frames.first().map(|f| f.safe_block).unwrap_or(0); + // Gate e: a batch that lands `max_wait_blocks` or more after its first + // frame's safe block is stale. + if !batch.frames.is_empty() + && age_exceeds( + input.inclusion_block, first_frame_safe_block, - self.max_wait_blocks, + max_wait_blocks, ) - } - - /// Off-chain simulation of the scheduler's batch-acceptance predicate. - /// - /// Returns `Some(AcceptedBatch)` iff the scheduler would accept the input - /// at the given `expected_nonce`. The caller threads `expected_nonce` - /// across a stream of inputs, advancing by one on each `Some`. - /// - /// Rejection paths (wrong sender, SSZ decode failure, stale by inclusion, - /// nonce mismatch) return `None` without advancing — matching what the - /// scheduler does on-chain. - /// - /// **Deliberately omitted structural rejects.** The canonical fold's - /// `batch_reject_reason_for_block` additionally rejects a batch whose frames - /// have a `safe_block` above the inclusion block - /// (`SafeBlockAboveInclusionBlock`) or that decrease across frames - /// (`NonMonotonicSafeBlocks`), or whose fee exceeds `MAX_EXPONENT` - /// (`FeePriceOutOfRange`). This predicate does not re-check those: a - /// well-formed batch submitter never builds such a batch, so they are - /// reachable only under a buggy or forked submitter — already outside the - /// duality boundary. The I1 duality test pins this as a *documented* - /// asymmetry; mirror those checks here if the submitter ever stops being - /// trusted to be well-formed. - pub fn scheduler_accepts( - &self, - batch_submitter: Address, - input: SafeInputView<'_>, - expected_nonce: u64, - ) -> Option { - if input.sender != batch_submitter { - return None; - } - let batch = ::from_ssz_bytes(input.payload).ok()?; - let first_frame_safe_block = batch.frames.first().map(|f| f.safe_block).unwrap_or(0); - if !batch.frames.is_empty() - && self.is_scheduler_stale(input.inclusion_block, first_frame_safe_block) - { - return None; - } - if batch.nonce != expected_nonce { - return None; - } - Some(AcceptedBatch { - safe_input_index: input.safe_input_index, - nonce: batch.nonce, - first_frame_safe_block, - inclusion_block: input.inclusion_block, - }) - } + { + return None; + } + if batch.nonce != expected_nonce { + return None; + } + Some(AcceptedBatch { + safe_input_index: input.safe_input_index, + nonce: batch.nonce, + first_frame_safe_block, + inclusion_block: input.inclusion_block, + }) } /// The one staleness formula, `now - start >= window`. Saturating keeps the /// consensus path total when `start` is ahead of `now`. Callers: the /// scheduler's gate e and overdue-direct backstop, the off-chain -/// [`ProtocolTiming::is_scheduler_stale`], and the danger check. +/// [`scheduler_accepts`], and the sequencer's void-history check. pub fn age_exceeds(now: u64, start: u64, window: u64) -> bool { now.saturating_sub(start) >= window } @@ -384,10 +112,10 @@ mod tests { const SUBMITTER: Address = Address::repeat_byte(0xAA); const OTHER: Address = Address::repeat_byte(0xBB); - const MAX_WAIT: u64 = crate::MAX_WAIT_BLOCKS; + const MAX_WAIT: u64 = 1200; - fn timing() -> ProtocolTiming { - ProtocolTiming::try_new(MAX_WAIT, 75, 900, 12).expect("valid test timing") + fn accepts(input: SafeInputView<'_>, expected_nonce: u64) -> Option { + scheduler_accepts(SUBMITTER, MAX_WAIT, input, expected_nonce) } fn encode(batch: &Batch) -> Vec { @@ -405,230 +133,6 @@ mod tests { } } - #[test] - fn danger_threshold_is_max_wait_minus_margin() { - assert_eq!(timing().danger_threshold(), MAX_WAIT - 75); - } - - #[test] - fn l1_read_stale_after_secs_uses_configured_block_time() { - assert_eq!(timing().l1_read_stale_after_secs(), 900 * 12); - } - - #[test] - fn l1_view_is_stale_without_timestamp() { - assert!(timing().l1_view_is_stale(None, 1_000_000)); - } - - #[test] - fn l1_view_is_stale_at_threshold() { - let cfg = timing(); - let timestamp_secs = 1_000; - let before = (timestamp_secs + cfg.l1_read_stale_after_secs() - 1) * 1000; - let at = (timestamp_secs + cfg.l1_read_stale_after_secs()) * 1000; - assert!(!cfg.l1_view_is_stale(Some(timestamp_secs), before)); - assert!(cfg.l1_view_is_stale(Some(timestamp_secs), at)); - } - - #[test] - fn l1_view_is_stale_never_fires_on_a_future_timestamp() { - // Ahead-of-clock is not staleness — the observation is the freshest - // possible. Clock usability is `clock_cannot_age_l1_view`'s job. - assert!(!timing().l1_view_is_stale(Some(1_001), 1_000_000)); - assert!(!timing().l1_view_is_stale(Some(1_012), 1_000_000)); - assert!( - !timing().l1_view_is_stale(Some(1_001), 1_001_000), - "equality is a usable zero-age observation" - ); - } - - #[test] - fn clock_cannot_age_l1_view_tolerates_sub_block_skew() { - let cfg = timing(); - // No view at all: nothing for the clock to age — not a clock fault. - assert!(!cfg.clock_cannot_age_l1_view(None, 1_000_000)); - // Behind or equal: ordinary aging, never a clock fault. - assert!(!cfg.clock_cannot_age_l1_view(Some(900), 1_000_000)); - assert!(!cfg.clock_cannot_age_l1_view(Some(1_000), 1_000_000)); - // Sub-block ahead-ness is NTP-scale skew against a block-granular - // timestamp: usable, age 0. - assert!(!cfg.clock_cannot_age_l1_view(Some(1_011), 1_000_000)); - // Millisecond precision at the boundary: 11.999 s and 11.001 s ahead - // are both sub-block; exactly 12.000 s refuses. - assert!(!cfg.clock_cannot_age_l1_view(Some(1_012), 1_000_001)); - assert!(!cfg.clock_cannot_age_l1_view(Some(1_012), 1_000_999)); - assert!(cfg.clock_cannot_age_l1_view(Some(1_012), 1_000_000)); - } - - #[test] - fn wall_clock_adjusted_threshold_returns_none_without_baseline() { - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(None, 1_000_000), - Ok(None), - ); - } - - #[test] - fn wall_clock_adjusted_threshold_returns_none_below_one_block_time() { - // 11 seconds elapsed, seconds_per_block = 12 → missed = 0. - let last = 1_000_000; - let now = last + 11_000; - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), now), - Ok(None), - ); - } - - #[test] - fn wall_clock_adjusted_threshold_rejects_regression() { - let last = 1_000_000; - // Sub-block regressions are quantization noise: identical outcome to - // elapsed = 0 for a block-granular estimate (clock steps are - // legitimate). - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), last - 1), - Ok(None), - "a sub-block clock step must not be a fault" - ); - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), last - 11_999), - Ok(None), - "just under one block-time of regression is still noise" - ); - // A regression of one block-time or more genuinely invalidates the - // extrapolation. - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), last - 12_000), - Err(WallClockRegression { - now_ms: last - 12_000, - last_safe_progress_ms: last, - }), - ); - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), last), - Ok(None), - "equality is a usable zero-elapsed baseline" - ); - } - - #[test] - fn wall_clock_adjusted_threshold_shaves_one_block_per_block_time() { - // 25 blocks of elapsed time (300s) → adjusted = danger_threshold - 25. - let last = 1_000_000; - let now = last + 300_000; - let cfg = timing(); - assert_eq!( - cfg.wall_clock_adjusted_danger_threshold(Some(last), now), - Ok(Some(cfg.danger_threshold() - 25)), - ); - } - - #[test] - fn wall_clock_adjusted_threshold_saturates_at_zero() { - // Wildly long outage — adjustment shaves more blocks than the threshold. - let last = 0; - let now = u64::MAX / 2; - assert_eq!( - timing().wall_clock_adjusted_danger_threshold(Some(last), now), - Ok(Some(0)), - ); - } - - #[test] - fn try_new_rejects_margin_equal_to_max_wait() { - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, MAX_WAIT, 1, 12), - Err(ProtocolTimingError::MarginNotLessThanMaxWait { - margin: MAX_WAIT, - max_wait: MAX_WAIT, - }), - ); - } - - #[test] - fn try_new_rejects_margin_greater_than_max_wait() { - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, MAX_WAIT + 1, 1, 12), - Err(ProtocolTimingError::MarginNotLessThanMaxWait { - margin: MAX_WAIT + 1, - max_wait: MAX_WAIT, - }), - ); - } - - #[test] - fn try_new_accepts_max_margin_admitting_valid_stale() { - // margin = MAX_WAIT - 2 is the largest margin that still admits a - // valid stale value (stale must satisfy 0 < stale < danger, and - // danger = MAX_WAIT - margin, so danger >= 2 is required for stale = 1 - // to be valid). - let cfg = ProtocolTiming::try_new(MAX_WAIT, MAX_WAIT - 2, 1, 12) - .expect("margin admitting stale = 1 must be accepted"); - assert_eq!(cfg.danger_threshold(), 2); - } - - #[test] - fn try_new_rejects_zero_margin() { - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, 0, 1, 12), - Err(ProtocolTimingError::MarginZero), - ); - } - - #[test] - fn try_new_rejects_zero_l1_read_stale_after() { - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, 75, 0, 12), - Err(ProtocolTimingError::ReadStaleAfterZero), - ); - } - - #[test] - fn try_new_rejects_zero_seconds_per_block() { - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, 75, 1, 0), - Err(ProtocolTimingError::SecondsPerBlockZero), - ); - } - - #[test] - fn try_new_rejects_overflowing_read_staleness_window() { - assert_eq!( - ProtocolTiming::try_new(u64::MAX, 1, u64::MAX - 2, 2), - Err(ProtocolTimingError::ReadStaleWindowOverflow { - read_stale_after: u64::MAX - 2, - seconds_per_block: 2, - }), - ); - } - - #[test] - fn try_new_rejects_l1_read_stale_after_past_danger() { - let danger_threshold = MAX_WAIT - 75; - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, 75, danger_threshold + 1, 12), - Err(ProtocolTimingError::ReadStaleAfterPastDanger { - read_stale_after: danger_threshold + 1, - danger_threshold, - }), - ); - } - - #[test] - fn try_new_rejects_l1_read_stale_after_equal_to_danger() { - // Strict less-than: equality means L1ViewStale and - // EstimatedBatchInDanger fire at the same point, defeating the - // "stale fires first" design property. - let danger_threshold = MAX_WAIT - 75; - assert_eq!( - ProtocolTiming::try_new(MAX_WAIT, 75, danger_threshold, 12), - Err(ProtocolTimingError::ReadStaleAfterPastDanger { - read_stale_after: danger_threshold, - danger_threshold, - }), - ); - } - #[test] fn age_exceeds_saturates_on_underflow() { assert!(!age_exceeds(5, 10, 1)); @@ -636,36 +140,6 @@ mod tests { assert!(!age_exceeds(1199, 0, 1200)); } - // ── ProtocolTiming::is_scheduler_stale direct boundary tests ────────── - // - // Indirectly covered by `scheduler_accepts_boundary_just_below_stale`, but - // the staleness predicate is load-bearing on its own (the scheduler skips - // submissions that trip it) and deserves direct tests that don't go through - // SSZ decoding. - - #[test] - fn is_scheduler_stale_reports_false_below_threshold() { - // age = inclusion - first = MAX_WAIT - 1, strictly below. - assert!(!timing().is_scheduler_stale(MAX_WAIT, 1)); - // age = 0 (safe head right at the first frame). - assert!(!timing().is_scheduler_stale(100, 100)); - } - - #[test] - fn is_scheduler_stale_reports_true_at_and_past_threshold() { - // age = MAX_WAIT exactly — `>=` comparison trips. - assert!(timing().is_scheduler_stale(MAX_WAIT, 0)); - // age = MAX_WAIT + 1, clearly past. - assert!(timing().is_scheduler_stale(MAX_WAIT + 1, 0)); - } - - #[test] - fn is_scheduler_stale_saturates_when_first_frame_is_ahead() { - // Degenerate input: safe head is behind the first frame's safe_block. - // Saturating subtraction yields 0, strictly below threshold — never stale. - assert!(!timing().is_scheduler_stale(50, 100)); - } - #[test] fn scheduler_accepts_fresh_batch_with_matching_nonce() { let payload = encode(&single_frame_batch(3, 100)); @@ -675,9 +149,8 @@ mod tests { payload: payload.as_slice(), inclusion_block: 500, }; - let accepted = timing() - .scheduler_accepts(SUBMITTER, input, 3) - .expect("matching nonce + fresh inclusion should be accepted"); + let accepted = + accepts(input, 3).expect("matching nonce + fresh inclusion should be accepted"); assert_eq!(accepted.safe_input_index, 7); assert_eq!(accepted.nonce, 3); assert_eq!(accepted.first_frame_safe_block, 100); @@ -693,7 +166,7 @@ mod tests { payload: payload.as_slice(), inclusion_block: 0, }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_none()); + assert!(accepts(input, 0).is_none()); } #[test] @@ -705,7 +178,7 @@ mod tests { payload: payload.as_slice(), inclusion_block: MAX_WAIT, }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_none()); + assert!(accepts(input, 0).is_none()); } #[test] @@ -717,7 +190,22 @@ mod tests { payload: payload.as_slice(), inclusion_block: MAX_WAIT, }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_some()); + assert!(accepts(input, 0).is_some()); + } + + #[test] + fn scheduler_accepts_a_first_frame_ahead_of_inclusion_as_fresh() { + // Degenerate input: the first frame is ahead of the inclusion block. + // Saturating subtraction yields age 0, never stale; gate d, which + // the predicate omits, is what rejects it on chain. + let payload = encode(&single_frame_batch(0, 100)); + let input = SafeInputView { + safe_input_index: 0, + sender: SUBMITTER, + payload: payload.as_slice(), + inclusion_block: 50, + }; + assert!(accepts(input, 0).is_some()); } #[test] @@ -729,8 +217,8 @@ mod tests { payload: payload.as_slice(), inclusion_block: 200, }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 3).is_none()); - assert!(timing().scheduler_accepts(SUBMITTER, input, 1).is_none()); + assert!(accepts(input, 3).is_none()); + assert!(accepts(input, 1).is_none()); } #[test] @@ -741,7 +229,7 @@ mod tests { payload: &[0xFF, 0xEE, 0xDD], inclusion_block: 0, }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_none()); + assert!(accepts(input, 0).is_none()); } #[test] @@ -756,6 +244,6 @@ mod tests { payload: payload.as_slice(), inclusion_block: MAX_WAIT.saturating_mul(10), }; - assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_some()); + assert!(accepts(input, 0).is_some()); } } diff --git a/sequencer-core/src/scheduler/driver.rs b/sequencer-core/src/scheduler/driver.rs new file mode 100644 index 00000000..0cf0e6d4 --- /dev/null +++ b/sequencer-core/src/scheduler/driver.rs @@ -0,0 +1,638 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The canonical loop: feed an input to the scheduler, then run each +//! transaction it hands out through the executor. The guest harness and the +//! tests drive an application through it, and it answers the machine's export. + +use super::{InputOutcome, L1Input, Scheduled, Scheduler, SchedulerConfig, SchedulerExport}; +use crate::application::{ + AppError, AppOutputs, Application, CanonicalState, Executed, SkipReason, execute, +}; +use crate::history::L2TxCount; + +/// The `state` inspect query: the application's comparison bytes, whole. An +/// empty query means the same. +pub const STATE_INSPECT_QUERY: &[u8] = b"state"; + +/// The `scheduler` inspect query: the [`SchedulerExport`] report, 48 bytes. +pub const SCHEDULER_INSPECT_QUERY: &[u8] = b"scheduler"; + +/// An inspect query the machine cannot answer. It gets a report, never a halt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InspectError { + UnsupportedQuery, + Application(String), +} + +/// What one input did, in execution order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Advanced { + pub outcome: InputOutcome, + /// Every executed transaction's outputs, concatenated. + pub outputs: AppOutputs, + /// Every handed-out transaction, and whether it executed. + pub txs: Vec, +} + +/// One handed-out transaction and its fate; its outputs are in +/// [`Advanced::outputs`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TxRecord { + pub scheduled: Scheduled, + pub fate: TxFate, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TxFate { + /// It executed; `index` is the L2 tx count before it, its index in the history. + Done { index: L2TxCount }, + /// The executor skipped a user op for its nonce. Nothing changed. + Skipped(SkipReason), +} + +/// A scheduler and the application it feeds, kept in step input by input. +pub struct CanonicalRunner { + scheduler: Scheduler, + app: A, + /// Set by an `AppError`. The scheduler alone cannot tell: the failing + /// transaction may have been its input's last. + failed: bool, +} + +impl CanonicalRunner { + /// The genesis machine: both sides at zero. + pub fn genesis(app: A, config: SchedulerConfig) -> Self { + assert_eq!( + app.l2_tx_count(), + L2TxCount::ZERO, + "a genesis application must start at l2_tx_count = 0" + ); + Self::from_parts(Scheduler::genesis(config), app) + } + + /// Pair a scheduler with an application at the same point, such as a + /// resumed scheduler and the application projected from the same machine. + pub fn from_parts(scheduler: Scheduler, app: A) -> Self { + Self { + scheduler, + app, + failed: false, + } + } + + pub fn into_parts(self) -> (Scheduler, A) { + (self.scheduler, self.app) + } + + pub fn scheduler(&self) -> &Scheduler { + &self.scheduler + } + + pub fn app(&self) -> &A { + &self.app + } + + pub fn app_mut(&mut self) -> &mut A { + &mut self.app + } + + /// The machine's `scheduler` report. + pub fn export(&self) -> SchedulerExport { + SchedulerExport::new(self.scheduler.state(), self.app.l2_tx_count()) + } + + /// Feed one input and execute every transaction it hands out. An + /// `AppError` is fatal: it leaves the input half done, so the runner must + /// be discarded, and a later `process_input` panics. + pub fn process_input(&mut self, input: L1Input) -> Result { + assert!( + !self.failed, + "an application error left this runner half done: discard it" + ); + let outcome = self.scheduler.feed(input); + let mut outputs = Vec::new(); + let mut txs = Vec::new(); + while let Some(scheduled) = self.scheduler.next_tx() { + let executed = + execute(&mut self.app, &scheduled.tx).inspect_err(|_| self.failed = true)?; + let fate = match executed { + Executed::Done { + index, + outputs: tx_outputs, + } => { + outputs.extend(tx_outputs); + TxFate::Done { index } + } + Executed::Skipped(reason) => TxFate::Skipped(reason), + }; + txs.push(TxRecord { scheduled, fate }); + } + Ok(Advanced { + outcome, + outputs, + txs, + }) + } +} + +impl CanonicalRunner { + /// Answer an inspect query: the application's comparison bytes, unchanged + /// from today, or the scheduler's report, which the harness owns for every + /// application. + pub fn inspect(&self, query: &[u8]) -> Result, InspectError> { + if query.is_empty() || query == STATE_INSPECT_QUERY { + self.app + .canonical_snapshot_bytes() + .map_err(|err| InspectError::Application(err.to_string())) + } else if query == SCHEDULER_INSPECT_QUERY { + Ok(self.export().encode().to_vec()) + } else { + Err(InspectError::UnsupportedQuery) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::application::{Admission, AppOutput, PolicyRefusal, SchedulerLimits}; + use crate::l2_tx::{Direct, L2Tx, UserOpTx}; + use crate::scheduler::fixtures::*; + use crate::scheduler::{BatchRejectReason, SchedulerState}; + use crate::user_op::UserOp; + use alloy_primitives::{Address, U256}; + use std::collections::BTreeMap; + use std::path::{Path, PathBuf}; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Event { + UserOp(u8), + Direct(u8), + } + + /// A small fee-charging application. It charges what the sender has, as + /// the trait recommends, so an underfunded op still executes. + #[derive(Default)] + struct RecordingApp { + events: Vec, + balances: BTreeMap, + nonces: BTreeMap, + count: L2TxCount, + fail_on: Option, + } + + impl RecordingApp { + fn balance_of(&self, sender: Address) -> U256 { + self.balances.get(&sender).copied().unwrap_or_default() + } + + fn credit(&mut self, sender: Address, amount: u64) { + let balance = self.balance_of(sender).saturating_add(U256::from(amount)); + self.balances.insert(sender, balance); + } + + fn record(&mut self, event: Event) -> Result { + if self.fail_on == Some(event) { + return Err(AppError::Internal { + reason: format!("refusing {event:?}"), + }); + } + self.events.push(event); + self.count.advance(); + let output = match event { + Event::UserOp(marker) => vec![b'u', marker], + Event::Direct(marker) => vec![b'd', marker], + }; + Ok(vec![AppOutput::Notice(output)]) + } + } + + impl Application for RecordingApp { + fn user_nonce(&self, sender: Address) -> Result { + Ok(self.nonces.get(&sender).copied().unwrap_or(0)) + } + + fn apply_user_op(&mut self, op: &UserOpTx) -> Result { + let outputs = self.record(Event::UserOp(op.data.first().copied().unwrap_or(0)))?; + let fee = crate::fee::fee_to_linear(op.fee); + let balance = self.balance_of(op.sender); + self.balances.insert(op.sender, balance.saturating_sub(fee)); + let nonce = self.user_nonce(op.sender)?; + self.nonces.insert(op.sender, nonce + 1); + Ok(outputs) + } + + fn apply_direct(&mut self, direct: &Direct) -> Result { + self.record(Event::Direct(direct.payload.first().copied().unwrap_or(0))) + } + + fn l2_tx_count(&self) -> L2TxCount { + self.count + } + + fn max_method_payload_bytes() -> usize { + 1 + 32 + 20 + } + + fn scheduler_limits() -> SchedulerLimits { + SchedulerLimits { + max_wait_blocks: 5, + fridge_bound: 7, + max_direct_payload_bytes: 9, + } + } + + fn admit_user_op( + &self, + sender: Address, + _op: &UserOp, + fee: u16, + ) -> Result { + let required = crate::fee::fee_to_linear(fee); + let available = self.balance_of(sender); + Ok(if available < required { + Admission::Refuse(PolicyRefusal::Underfunded { + required, + available, + }) + } else { + Admission::Admit + }) + } + + fn from_dump(_prefix: &Path) -> Result { + unimplemented!("not used") + } + + fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { + unimplemented!("not used") + } + + fn state_file_in_dump(prefix: &Path) -> PathBuf { + prefix.to_path_buf() + } + + fn from_canonical_state( + _state_file: &Path, + _l2_tx_count: L2TxCount, + ) -> Result { + unimplemented!("not used") + } + } + + impl CanonicalState for RecordingApp { + fn canonical_snapshot_bytes(&self) -> Result, AppError> { + Ok(format!("events:{}", self.events.len()).into_bytes()) + } + } + + fn runner(app: RecordingApp, max_wait_blocks: u64) -> CanonicalRunner { + CanonicalRunner::genesis(app, config(max_wait_blocks)) + } + + fn advance( + runner: &mut CanonicalRunner, + mut input: L1Input, + ) -> Result { + input.index = runner.scheduler().state().l1_input_count; + runner.process_input(input) + } + + fn fates(advanced: &Advanced) -> Vec { + advanced.txs.iter().map(|record| record.fate).collect() + } + + #[test] + fn for_application_takes_every_constant_from_the_application() { + assert_eq!( + SchedulerConfig::for_application::(SEQUENCER), + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks: 5, + fridge_bound: 7, + max_direct_payload_bytes: 9, + } + ); + } + + #[test] + #[should_panic(expected = "a genesis application must start at l2_tx_count = 0")] + fn genesis_rejects_an_application_past_zero() { + let app = RecordingApp { + count: L2TxCount::new(1), + ..RecordingApp::default() + }; + let _ = runner(app, 100); + } + + /// The scheduler withholds only the op whose cap is below the price; the + /// executor skips a wrong nonce; an underfunded op executes and counts, + /// which turns a later op at the same nonce into a wrong-nonce skip. + #[test] + fn the_executor_skips_wrong_nonces_and_an_underfunded_op_executes() { + let mut runner = runner(RecordingApp::default(), 100); + let signer = key(9); + let sender = address_of(&signer); + // fee_to_linear(0) = 1: enough for the cheapest fee only. + runner.app_mut().credit(sender, 1); + let frames = vec![ + frame(1, 1, vec![signed(&signer, 1, 10, vec![1])]), + frame(1, 5, vec![signed(&signer, 0, 0, vec![2])]), + frame(1, 1000, vec![signed(&signer, 0, 1000, vec![3])]), + frame(1, 0, vec![signed(&signer, 0, 10, vec![4])]), + ]; + let advanced = advance(&mut runner, batch(2, 0, frames)).expect("advance"); + assert_eq!(advanced.outcome, InputOutcome::BatchAccepted); + assert_eq!( + fates(&advanced), + [ + TxFate::Skipped(SkipReason::WrongNonce { + expected: 0, + got: 1 + }), + TxFate::Done { + index: L2TxCount::ZERO + }, + TxFate::Skipped(SkipReason::WrongNonce { + expected: 1, + got: 0 + }), + ] + ); + assert_eq!(runner.app().events, [Event::UserOp(3)]); + assert_eq!(runner.app().balance_of(sender), U256::ZERO); + assert_eq!(runner.app().user_nonce(sender).unwrap(), 1); + assert_eq!(advanced.outputs, [AppOutput::Notice(vec![b'u', 3])]); + } + + #[test] + fn outputs_follow_execution_order() { + let mut runner = runner(RecordingApp::default(), 5); + let signer = key(11); + for input in [direct(1, 1), direct(3, 2), direct(4, 3)] { + let advanced = advance(&mut runner, input).expect("queue"); + assert!(advanced.txs.is_empty()); + } + let frames = vec![ + frame(3, 0, vec![signed(&signer, 0, 0, vec![10])]), + frame(4, 0, vec![signed(&signer, 1, 0, vec![11])]), + ]; + let advanced = advance(&mut runner, batch(6, 0, frames)).expect("advance"); + let notices: Vec> = advanced + .outputs + .iter() + .map(|output| match output { + AppOutput::Notice(bytes) => bytes.clone(), + AppOutput::Voucher { .. } => panic!("no vouchers"), + }) + .collect(); + // The backstop's direct, then frame 1's drain and op, then frame 2's. + assert_eq!( + notices, + [ + vec![b'd', 1], + vec![b'd', 2], + vec![b'u', 10], + vec![b'd', 3], + vec![b'u', 11] + ] + ); + let executed: Vec = (0..5) + .map(|index| TxFate::Done { + index: L2TxCount::new(index), + }) + .collect(); + assert_eq!(fates(&advanced), executed); + assert!( + advanced + .txs + .iter() + .all(|record| record.scheduled.batch_nonce == 0) + ); + assert_eq!(runner.app().l2_tx_count(), L2TxCount::new(5)); + } + + #[test] + fn an_application_error_stops_the_input_before_later_transactions() { + let app = RecordingApp { + fail_on: Some(Event::UserOp(1)), + ..RecordingApp::default() + }; + let mut runner = runner(app, 100); + let signer = key(17); + let ops = vec![ + signed(&signer, 0, 1, vec![1]), + signed(&signer, 1, 1, vec![2]), + ]; + let error = advance(&mut runner, batch(2, 0, vec![frame(1, 0, ops)])) + .expect_err("an application error is fatal"); + let AppError::Internal { reason } = error else { + panic!("unexpected application error: {error}"); + }; + assert_eq!(reason, "refusing UserOp(1)"); + assert!(runner.app().events.is_empty()); + assert_eq!(runner.app().l2_tx_count(), L2TxCount::ZERO); + } + + #[test] + fn a_backstop_direct_error_is_fatal() { + let app = RecordingApp { + fail_on: Some(Event::Direct(1)), + ..RecordingApp::default() + }; + let mut runner = runner(app, 5); + advance(&mut runner, direct(1, 1)).expect("queue"); + let error = advance(&mut runner, direct(6, 2)).expect_err("the overdue direct fails"); + let AppError::Internal { reason } = error else { + panic!("unexpected application error: {error}"); + }; + assert_eq!(reason, "refusing Direct(1)"); + assert!(runner.app().events.is_empty()); + } + + fn panic_message(run: impl FnOnce()) -> String { + let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(run)) + .expect_err("the run must panic"); + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default() + } + + /// Whether the failing transaction was its input's last or not, the + /// runner refuses the next input rather than run a discarded instance. + #[test] + fn a_runner_cannot_continue_after_an_error() { + for (label, failing) in [ + ("a transaction left", 1), + ("the input's last transaction", 2), + ] { + let app = RecordingApp { + fail_on: Some(Event::Direct(failing)), + ..RecordingApp::default() + }; + let mut runner = runner(app, 100); + for marker in [1, 2] { + advance(&mut runner, direct(1, marker)).expect("queue"); + } + advance(&mut runner, batch(2, 0, vec![empty_frame(1)])).expect_err(label); + let events = runner.app().events.clone(); + let message = panic_message(|| { + let _ = advance(&mut runner, direct(3, 3)); + }); + assert_eq!( + message, "an application error left this runner half done: discard it", + "{label}" + ); + assert_eq!(runner.app().events, events, "{label}"); + } + } + + /// The backstop queues an overdue direct before gate b's preflight halts + /// on the exhausted batch nonce; the halt still comes before it executes. + #[test] + fn batch_nonce_overflow_halts_before_a_backstop_direct_executes() { + let mut queued = direct(1, 1); + queued.index = 0; + let state = SchedulerState { + l1_input_count: 1, + next_batch_nonce: u64::MAX, + clock: 0, + first_queued_input: 0, + fridge_len: 1, + }; + let scheduler = Scheduler::resume(config(5), state, [queued]).expect("resume"); + let mut runner = CanonicalRunner::from_parts(scheduler, RecordingApp::default()); + let op = signed(&key(19), 0, 0, vec![7]); + let message = panic_message(|| { + let _ = advance( + &mut runner, + batch(10, u64::MAX, vec![frame(9, 0, vec![op])]), + ); + }); + assert_eq!(message, "batch nonce overflow: no canonical successor"); + assert!(runner.app().events.is_empty()); + assert_eq!(runner.app().l2_tx_count(), L2TxCount::ZERO); + } + + #[test] + fn inspect_answers_the_state_and_scheduler_queries() { + let mut runner = runner(RecordingApp::default(), 100); + advance(&mut runner, direct(1, 7)).expect("queue"); + // The state reflects executed transactions, not the fridge. + assert_eq!( + runner.inspect(STATE_INSPECT_QUERY), + Ok(b"events:0".to_vec()) + ); + assert_eq!(runner.inspect(b""), Ok(b"events:0".to_vec())); + + advance(&mut runner, batch(2, 0, vec![empty_frame(1)])).expect("drain"); + advance(&mut runner, direct(3, 8)).expect("queue"); + assert_eq!( + runner.inspect(STATE_INSPECT_QUERY), + Ok(b"events:1".to_vec()) + ); + let report = runner.inspect(SCHEDULER_INSPECT_QUERY).expect("report"); + assert_eq!( + SchedulerExport::decode(&report), + Ok(SchedulerExport { + l1_input_count: 3, + l2_tx_count: 1, + next_batch_nonce: 1, + clock: 1, + first_queued_input: 2, + fridge_len: 1, + }) + ); + assert_eq!(runner.export().encode().to_vec(), report); + + for query in [&b"balances"[..], b"State", b"scheduler ", b"state\0"] { + assert_eq!(runner.inspect(query), Err(InspectError::UnsupportedQuery)); + } + } + + /// Two machines from the same export, one carried on and one rebuilt from + /// its parts, execute the rest identically. + #[test] + fn a_runner_resumed_from_its_parts_matches_the_original() { + let signer = key(12); + let mut original = runner(RecordingApp::default(), 100); + let mut inputs = [ + direct(1, 1), + batch( + 3, + 0, + vec![frame(2, 0, vec![signed(&signer, 0, 0, vec![5])])], + ), + direct(4, 2), + direct(4, 3), + ]; + for (index, input) in inputs.iter_mut().enumerate() { + input.index = index as u64; + original.process_input(input.clone()).expect("advance"); + } + let export = original.export(); + let state: SchedulerState = export.scheduler_state(); + let queued = inputs[state.first_queued_input as usize..].to_vec(); + let scheduler = + Scheduler::resume(*original.scheduler().config(), state, queued).expect("resume"); + let app = RecordingApp { + events: original.app().events.clone(), + nonces: original.app().nonces.clone(), + count: L2TxCount::new(export.l2_tx_count), + ..RecordingApp::default() + }; + let mut resumed = CanonicalRunner::from_parts(scheduler, app); + let rest = [ + batch( + 6, + 1, + vec![frame(4, 0, vec![signed(&signer, 1, 0, vec![6])])], + ), + direct(7, 4), + batch(9, 1, vec![empty_frame(8)]), + ]; + for input in rest { + assert_eq!( + advance(&mut resumed, input.clone()).expect("resumed"), + advance(&mut original, input).expect("original") + ); + } + assert_eq!(resumed.export(), original.export()); + let (_, app) = resumed.into_parts(); + assert_eq!( + app.events, + [ + Event::Direct(1), + Event::UserOp(5), + Event::Direct(2), + Event::Direct(3), + Event::UserOp(6), + ] + ); + } + + #[test] + fn a_rejected_batch_still_reports_its_backstop_outputs() { + let mut runner = runner(RecordingApp::default(), 5); + advance(&mut runner, direct(1, 1)).expect("queue"); + let advanced = advance(&mut runner, batch(8, 0, vec![empty_frame(8)])).expect("advance"); + assert_eq!( + advanced.outcome, + InputOutcome::BatchRejected(BatchRejectReason::SafeBlockNotBelowInclusionBlock) + ); + assert_eq!(advanced.outputs, [AppOutput::Notice(vec![b'd', 1])]); + assert!(matches!( + advanced.txs.as_slice(), + [TxRecord { + scheduled: Scheduled { + batch_nonce: 0, + tx: L2Tx::Direct(_) + }, + fate: TxFate::Done { .. } + }] + )); + } +} diff --git a/sequencer-core/src/scheduler/fold.rs b/sequencer-core/src/scheduler/fold.rs deleted file mode 100644 index 5ae7f820..00000000 --- a/sequencer-core/src/scheduler/fold.rs +++ /dev/null @@ -1,736 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! The recovery replay/fold engine: reconstruct logical app state and -//! the resume batch nonce from a trusted checkpoint plus an L1 input range. -//! -//! `(machine@checkpoint, L1 range) → (machine@S', resume nonce N)`. The engine -//! is a thin, deterministic driver over the [`Scheduler`] library: it seeds the -//! fridge from the `(A, B]` directs (step 3), replays the `(B, C]` stream -//! (step 4), drains the leftover fridge at `C` (step 5), and returns the -//! advanced [`Application`] state plus the scheduler's resume nonce. -//! -//! Read-only and pure: it never touches L1 or storage. The caller sources the -//! inputs (`setup --recovery` from the synced `safe_inputs`; a future standalone -//! tool from raw L1) and persists the result. Determinism is inherited from the -//! scheduler (FIFO fridge, fixed-width arithmetic, no iteration-order leakage); -//! the engine's only added obligation is that the caller feed inputs in -//! ascending L1 order within `(A, C]` (seeds `(A, B]` before replay `(B, C]`). -//! That contract is enforced fail-loud with always-on assertions — a violation -//! would silently drop a direct and yield a wrong recovered state. - -use alloy_primitives::Address; -use alloy_sol_types::Eip712Domain; - -use super::{Scheduler, SchedulerConfig, SchedulerInput}; -use crate::application::{AppError, Application}; - -/// One reconstructed L1 input for the fold, ordered ascending by inclusion -/// block (ties broken by safe-input index at the source). Mirrors -/// [`SchedulerInput`] minus the EIP-712 `domain` — that is one -/// deployment-wide constant the engine clones onto each input rather than -/// storing per row. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FoldInput { - pub sender: Address, - pub inclusion_block: u64, - pub payload: Vec, -} - -/// Run recovery steps 3–5 over a trusted checkpoint and an L1 range, -/// returning `(S', N')` = (advanced app state, resume batch nonce). -/// -/// - `app` / `checkpoint_nonce`: the trusted checkpoint machine `S` at block -/// `B` and its scheduler nonce `N` (metadata — the bare-metal app cannot -/// recompute it, so the engine is *told* it via `resume_at`). -/// - `seeds`: directs reconstructed from `(A, B]`, with sequencer-sourced -/// batches already dropped by the caller. The checkpoint must account for -/// every direct through its application clock `A`: the exporter verifies no -/// pending canonical direct has inclusion block `<= A`. `A < B` alone cannot -/// establish this after faulty sequencing. Seeds arrive in ascending L1 order. -/// - `replay`: the full `(B, C]` stream (batches + directs) in L1 order. The -/// scheduler classifies each input (batch iff `sender == sequencer_address`), -/// force-executes overdue directs on arrival, applies accepted batches, -/// drains covered fridge directs, and advances the nonce to `N'`. -/// - `stop_block`: `C`, the stable stopping block. The terminal drain runs here. -/// -/// `checkpoint_nonce` is trusted unchallenged (the operator's checkpoint is -/// trusted input; there is no cheap on-chain oracle to validate it against). -/// Notices/vouchers produced during the fold are dropped: -/// recovery reconstructs *logical* state (balances, nonces), not the L1-facing -/// output stream. -pub fn fold_replay( - app: A, - checkpoint_nonce: u64, - config: SchedulerConfig, - domain: Eip712Domain, - seeds: S, - replay: R, - stop_block: u64, -) -> Result<(A, u64), AppError> -where - A: Application, - S: IntoIterator, - R: IntoIterator, -{ - let sequencer_address = config.sequencer_address; - let mut scheduler = Scheduler::resume_at(app, config, checkpoint_nonce); - - // The engine's input contract (ascending L1 order, seeds in (A, B] before - // replay in (B, C], everything `<= C`) is enforced fail-loud with always-on - // `assert!` rather than `debug_assert!`: a violation means the caller sourced - // the wrong inputs, and the failure mode is silent state divergence (a - // dropped direct → a wrong recovered `S'` that still *looks* valid). For a - // recovery engine that is the worst outcome, and the checks are negligible - // against a once-per-recovery fold, so we pay for them in release too. - - // Step 3 — reconstruct the fridge as of B from the (A, B] directs. - let mut last_seed_block: Option = None; - for seed in seeds { - assert!( - last_seed_block.is_none_or(|prev| seed.inclusion_block >= prev), - "fold seeds must arrive in ascending inclusion-block order" - ); - assert!( - seed.inclusion_block <= stop_block, - "fold seed at block {} is above stop block {stop_block}", - seed.inclusion_block, - ); - // Seeds are reconstructed `(A, B]` *directs* — enqueue_direct skips - // sender classification, so a mis-sourced `sender == sequencer_address` - // row would be silently filed as a direct instead of a batch (a wrong - // S'). Fail loud, matching the ordering/range asserts above. - assert!( - seed.sender != sequencer_address, - "fold seed must be a direct input, not a sequencer batch (sender == sequencer_address)" - ); - last_seed_block = Some(seed.inclusion_block); - scheduler.enqueue_direct(seed.sender, seed.inclusion_block, seed.payload); - } - - // Step 4 — replay (B, C] through the scheduler. - let mut last_replay_block: Option = None; - for input in replay { - if last_replay_block.is_none() { - // The first replay input opens `(B, C]`, which is strictly disjoint - // from the seeds' `(A, B]`: a seed block is `≤ B`, a replay block is - // `> B`, so the first replay block must come *strictly after* the - // last seeded direct. (A shared boundary block would let the FIFO - // fridge interleave the two ranges out of L1 order, since all seeds - // are enqueued before any replay input regardless of within-block - // index — see the disjoint-range contract in the docstring.) - assert!( - last_seed_block.is_none_or(|seed| input.inclusion_block > seed), - "fold replay must start strictly after the last seeded direct (ranges are disjoint)" - ); - } - assert!( - last_replay_block.is_none_or(|prev| input.inclusion_block >= prev), - "fold replay inputs must arrive in ascending inclusion-block order" - ); - // A batch above C would execute without leaving anything in the fridge - // for the post-drain check below to catch. - assert!( - input.inclusion_block <= stop_block, - "fold replay input at block {} is above stop block {stop_block}", - input.inclusion_block, - ); - last_replay_block = Some(input.inclusion_block); - scheduler.process_input(SchedulerInput { - sender: input.sender, - inclusion_block: input.inclusion_block, - domain: domain.clone(), - payload: input.payload, - })?; - } - - // Step 5 — drain the leftover fridge at C. Every still-queued direct has - // `inclusion_block <= C`, so this executes them all; the booting run, which - // starts past C with no fridge, would otherwise never see them. - scheduler.drain_covered_at(stop_block)?; - - // Backstop for the range asserts above: after draining at C the fridge - // must be empty, or `finish` would discard a direct silently. - assert_eq!( - scheduler.queued_direct_len(), - 0, - "fold left {} direct(s) undrained at stop block {stop_block}: an input \ - had inclusion_block > C (caller sourced inputs outside (A, C])", - scheduler.queued_direct_len(), - ); - - Ok(scheduler.finish()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::application::{AppError, AppOutputs, ApplicationProgress, ValidationOutcome}; - use crate::batch::{Batch, Frame}; - use crate::l2_tx::{DirectInput, ValidUserOp}; - use crate::user_op::UserOp; - use alloy_primitives::address; - - const SEQUENCER: Address = address!("0x1111111111111111111111111111111111111111"); - const DIRECT_SENDER: Address = address!("0x2222222222222222222222222222222222222222"); - const MAX_WAIT: u64 = 100; - - fn config() -> SchedulerConfig { - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: MAX_WAIT, - } - } - - fn domain() -> Eip712Domain { - crate::build_input_domain(1, Address::ZERO) - } - - /// Minimal `Application` double for the engine tests: records the markers of - /// executed direct inputs (first payload byte) in order, and advances a - /// safe-block clock. No user ops are exercised here — the engine drives the - /// scheduler over directs and empty/cover-only batches, so the user-op path - /// is trivial. Snapshot lifecycle is unused by the fold. - #[derive(Default, Clone)] - struct FoldApp { - executed_directs: Vec, - progress: ApplicationProgress, - fail_direct: Option, - } - - impl Application for FoldApp { - fn max_method_payload_bytes() -> usize { - 1 + 32 + 20 - } - - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result { - Ok(ValidationOutcome::Accept) - } - - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - self.progress.advance(safe_block); - Ok(Vec::new()) - } - - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { - let marker = input.payload.first().copied().unwrap_or(0); - if self.fail_direct == Some(marker) { - return Err(AppError::Internal { - reason: format!("refusing direct {marker}"), - }); - } - self.executed_directs.push(marker); - self.progress.advance(input.block_number); - Ok(Vec::new()) - } - - fn progress(&self) -> ApplicationProgress { - self.progress - } - - fn from_dump(_prefix: &std::path::Path) -> Result { - unimplemented!("FoldApp does not participate in snapshot lifecycle") - } - fn create_dump(&mut self, _prefix: &std::path::Path) -> Result<(), AppError> { - unimplemented!("FoldApp does not participate in snapshot lifecycle") - } - fn state_file_in_dump(_prefix: &std::path::Path) -> std::path::PathBuf { - unimplemented!("FoldApp does not participate in snapshot lifecycle") - } - } - - fn direct(sender: Address, block: u64, marker: u8) -> FoldInput { - FoldInput { - sender, - inclusion_block: block, - payload: vec![marker], - } - } - - /// A batch input (from the sequencer) carrying `batch`, SSZ-encoded. - fn batch(block: u64, batch: Batch) -> FoldInput { - FoldInput { - sender: SEQUENCER, - inclusion_block: block, - payload: ssz::Encode::as_ssz_bytes(&batch), - } - } - - /// An empty batch (no frames) at `nonce` — a scheduler no-op that only - /// advances the expected nonce. - fn empty_batch(block: u64, nonce: u64) -> FoldInput { - batch( - block, - Batch { - nonce, - frames: vec![], - }, - ) - } - - /// A batch at `nonce` with one user-op-free frame at `safe_block` — covers - /// (drains) fridge directs with `inclusion_block <= safe_block` without - /// needing signed user ops. - fn cover_batch(block: u64, nonce: u64, safe_block: u64) -> FoldInput { - batch( - block, - Batch { - nonce, - frames: vec![Frame { - user_ops: vec![], - safe_block, - fee_price: 0, - }], - }, - ) - } - - #[test] - fn empty_fold_returns_checkpoint_state_and_nonce() { - let (app, n) = fold_replay( - FoldApp::default(), - 7, - config(), - domain(), - Vec::new(), - Vec::new(), - 1_000, - ) - .expect("empty fold"); - assert_eq!(n, 7, "no batches ⇒ nonce stays at the checkpoint"); - assert!(app.executed_directs.is_empty()); - } - - #[test] - fn fold_replay_propagates_application_error() { - let app = FoldApp { - fail_direct: Some(0xEE), - ..FoldApp::default() - }; - let error = match fold_replay( - app, - 0, - config(), - domain(), - vec![direct(DIRECT_SENDER, 10, 0xEE)], - Vec::new(), - 10, - ) { - Err(error) => error, - Ok(_) => panic!("application error must abort recovery fold"), - }; - - let AppError::Internal { reason } = error else { - panic!("unexpected application error: {error}"); - }; - assert_eq!(reason, "refusing direct 238"); - } - - #[test] - fn resume_nonce_advances_from_checkpoint_through_replayed_batches() { - // Start at N0 = 5 (proves resume_at is honored, not 0); replay three - // empty batches at the expected nonces ⇒ N' = 8. - let replay = vec![ - empty_batch(100, 5), - empty_batch(110, 6), - empty_batch(120, 7), - ]; - let (_app, n) = fold_replay( - FoldApp::default(), - 5, - config(), - domain(), - Vec::new(), - replay, - 1_000, - ) - .expect("replay empty batches"); - assert_eq!(n, 8, "three accepted batches advance N from 5 to 8"); - } - - #[test] - fn seeded_fridge_directs_drain_on_a_covering_batch_frame() { - // Seed two (A,B] directs; replay a batch whose frame safe_block covers - // both ⇒ they execute in FIFO order, before the nonce advances. - let seeds = vec![ - direct(DIRECT_SENDER, 10, 0xAA), - direct(DIRECT_SENDER, 20, 0xBB), - ]; - let replay = vec![cover_batch(30, 0, 20)]; - let (app, n) = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - replay, - 1_000, - ) - .expect("drain seeded directs"); - assert_eq!( - app.executed_directs, - vec![0xAA, 0xBB], - "directs drain in L1 order" - ); - assert_eq!(n, 1, "the covering batch advances the nonce"); - } - - #[test] - fn step5_drains_all_leftover_directs_even_when_not_overdue() { - // A direct seeded at block 25, no covering batch, C = 30, MAX_WAIT = 100. - // The direct is NOT overdue at C (25 + 100 > 30), but it IS covered by C, - // so step 5 must still execute it — proving drain-covered-at-C, not - // drain-overdue. (An overdue-only drain would lose it.) - let seeds = vec![direct(DIRECT_SENDER, 25, 0xCC)]; - let (app, _n) = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - Vec::new(), - 30, - ) - .expect("terminal drain"); - assert_eq!( - app.executed_directs, - vec![0xCC], - "a non-overdue leftover direct ≤ C must be drained at step 5" - ); - } - - #[test] - fn fold_is_deterministic_over_identical_inputs() { - let seeds = vec![ - direct(DIRECT_SENDER, 5, 0x01), - direct(DIRECT_SENDER, 15, 0x02), - ]; - let replay = vec![cover_batch(40, 0, 15), empty_batch(50, 1)]; - let run = || { - fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds.clone(), - replay.clone(), - 100, - ) - }; - let (app_a, n_a) = run().expect("first deterministic fold"); - let (app_b, n_b) = run().expect("second deterministic fold"); - assert_eq!(app_a.executed_directs, app_b.executed_directs); - assert_eq!( - app_a.last_executed_safe_block(), - app_b.last_executed_safe_block() - ); - assert_eq!(n_a, n_b); - } - - // --- fail-loud contract guards (always-on, not debug-only) --- - - #[test] - #[should_panic(expected = "is above stop block")] - fn replay_input_past_stop_block_panics_instead_of_silently_dropping() { - // A direct at block 50 with C = 30 would never be drained by step 5, and - // the booting run would never see it. The range assert rejects it before - // it reaches the fridge. - let replay = vec![direct(DIRECT_SENDER, 50, 0xDD)]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - Vec::new(), - replay, - 30, - ); - } - - #[test] - #[should_panic(expected = "fold replay input at block 50 is above stop block 30")] - fn replay_batch_past_stop_block_panics() { - // A batch leaves nothing in the fridge, so only the range assert sees it. - let replay = vec![empty_batch(50, 0)]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - Vec::new(), - replay, - 30, - ); - } - - #[test] - #[should_panic(expected = "fold seed at block 50 is above stop block 30")] - fn seed_past_stop_block_panics() { - let seeds = vec![direct(DIRECT_SENDER, 50, 0x01)]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - Vec::new(), - 30, - ); - } - - #[test] - #[should_panic(expected = "seeds must arrive in ascending")] - fn out_of_order_seeds_panic() { - let seeds = vec![ - direct(DIRECT_SENDER, 20, 0x01), - direct(DIRECT_SENDER, 10, 0x02), - ]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - Vec::new(), - 100, - ); - } - - #[test] - #[should_panic(expected = "replay inputs must arrive in ascending")] - fn out_of_order_replay_panics() { - let replay = vec![ - direct(DIRECT_SENDER, 100, 0x01), - direct(DIRECT_SENDER, 50, 0x02), - ]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - Vec::new(), - replay, - 200, - ); - } - - #[test] - #[should_panic(expected = "replay must start strictly after")] - fn replay_starting_before_last_seed_panics() { - // Seed a direct at B-ish (block 50); a replay input at block 10 precedes - // it, which would interleave the (A,B] and (B,C] ranges out of L1 order. - let seeds = vec![direct(DIRECT_SENDER, 50, 0x01)]; - let replay = vec![direct(DIRECT_SENDER, 10, 0x02)]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - replay, - 100, - ); - } - - #[test] - #[should_panic(expected = "replay must start strictly after")] - fn replay_sharing_the_last_seed_block_panics() { - // The (A,B] seeds and (B,C] replay are strictly disjoint: a replay input - // at the SAME block as the last seed violates the contract (and would - // risk FIFO interleaving), so the boundary assert is strict (`>`). - let seeds = vec![direct(DIRECT_SENDER, 30, 0x01)]; - let replay = vec![direct(DIRECT_SENDER, 30, 0x02)]; - let _ = fold_replay( - FoldApp::default(), - 0, - config(), - domain(), - seeds, - replay, - 100, - ); - } - - #[test] - fn earlier_checkpoint_recovers_a_direct_hidden_below_a_later_checkpoint_clock() { - let feed = |scheduler: &mut Scheduler, inputs: Vec| { - for input in inputs { - scheduler - .process_input(SchedulerInput { - sender: input.sender, - inclusion_block: input.inclusion_block, - domain: domain(), - payload: input.payload, - }) - .expect("canonical execution"); - } - }; - let mut canonical = Scheduler::new(FoldApp::default(), config()); - feed( - &mut canonical, - vec![ - direct(DIRECT_SENDER, 5, 1), - cover_batch(9, 0, 5), - direct(DIRECT_SENDER, 9, 2), - ], - ); - - let checkpoint = canonical.app.clone(); - let checkpoint_nonce = canonical.next_expected_batch_nonce(); - let a = checkpoint.last_executed_safe_block(); - assert_eq!(a, 5); - assert!( - canonical - .direct_q - .iter() - .all(|input| input.block_number > a), - "the earlier checkpoint is eligible, with a nonempty seed queue" - ); - assert_eq!(canonical.queued_direct_len(), 1); - - // Equality is canonically valid even though honest live sequencing - // cannot submit a batch into its own already-safe block. - let replay = vec![ - direct(DIRECT_SENDER, 10, 3), - cover_batch(10, 1, 10), - direct(DIRECT_SENDER, 10, 4), - empty_batch(11, 2), - ]; - feed(&mut canonical, replay.clone()); - let later_a = canonical.app.last_executed_safe_block(); - assert_eq!(later_a, 10); - assert!(later_a < 11, "the later checkpoint passes the scalar bound"); - assert_eq!(canonical.next_expected_batch_nonce(), 3); - assert_eq!(canonical.app.executed_directs, vec![1, 2, 3]); - assert_eq!(canonical.queued_direct_len(), 1); - assert_eq!(canonical.direct_q[0].payload, vec![4]); - assert_eq!(canonical.direct_q[0].block_number, later_a); - // Its pending direct is excluded by (A,B], so the exporter must refuse - // this later checkpoint. Recovery can use the eligible earlier one. - - canonical.drain_covered_at(11).expect("terminal drain"); - let (expected, expected_nonce) = canonical.finish(); - let (recovered, recovered_nonce) = fold_replay( - checkpoint, - checkpoint_nonce, - config(), - domain(), - vec![direct(DIRECT_SENDER, 9, 2)], - replay, - 11, - ) - .expect("recovery from eligible checkpoint"); - assert_eq!(recovered.executed_directs, vec![1, 2, 3, 4]); - assert_eq!(recovered.executed_directs, expected.executed_directs); - assert_eq!(recovered.progress(), expected.progress()); - assert_eq!(recovered_nonce, expected_nonce); - } - - #[test] - fn fold_replay_reconstructs_identical_state_to_a_live_run() { - // Differential: splitting the (genesis, C] stream at an intermediate B - // and running fold_replay over (checkpoint@B, (A,B] seeds, (B,C] replay) - // must reconstruct EXACTLY the (state, nonce) a single uninterrupted live - // run produces. This is the property the (C,H1] bug violated — and - // idempotence (the existing `fold_is_deterministic` test) cannot catch a - // systematic reconstruction error, because it only re-runs fold against - // itself. - - // Full input stream over (genesis, C=30], deliberately shaped so block B - // = 15 leaves a non-empty (A,B] fridge (the seed path): - // block 5 direct(1) — executed pre-A by the batch@10 - // block 10 batch nonce0 sb5 — drains direct(1); A := 5 - // block 12 direct(2) — UNDRAINED at B=15 ⇒ a seed in (A=5, B=15] - // block 20 batch nonce1 sb12 — drains the seed direct(2) - // block 25 direct(3) — drained only by the terminal drain at C - let covering_batch = |nonce: u64, safe_block: u64| { - batch( - if nonce == 0 { 10 } else { 20 }, - Batch { - nonce, - frames: vec![Frame { - user_ops: vec![], - safe_block, - fee_price: 0, - }], - }, - ) - }; - let pre_b = || { - vec![ - direct(DIRECT_SENDER, 5, 1), - covering_batch(0, 5), - direct(DIRECT_SENDER, 12, 2), - ] - }; - let post_b = || vec![covering_batch(1, 12), direct(DIRECT_SENDER, 25, 3)]; - let stop = 30; - - let feed = |scheduler: &mut Scheduler, inputs: Vec| { - for input in inputs { - scheduler.process_input(SchedulerInput { - sender: input.sender, - inclusion_block: input.inclusion_block, - domain: domain(), - payload: input.payload, - })?; - } - Ok::<(), AppError>(()) - }; - - // LIVE: one scheduler over the whole stream, terminal-drained at C. - let mut live = Scheduler::new(FoldApp::default(), config()); - feed(&mut live, pre_b()).expect("live pre-checkpoint feed"); - feed(&mut live, post_b()).expect("live post-checkpoint feed"); - live.drain_covered_at(stop).expect("live terminal drain"); - let (live_app, live_nonce) = live.finish(); - - // CHECKPOINT @ B: a scheduler over (genesis, B] then `finish` WITHOUT a - // drain — the undrained (A,B] directs are dropped from S and re-supplied - // as seeds. This is exactly the checkpoint contract: every batch ≤ B - // applied, no (A,B] direct executed yet. - let mut at_b = Scheduler::new(FoldApp::default(), config()); - feed(&mut at_b, pre_b()).expect("checkpoint feed"); - let (checkpoint_app, checkpoint_nonce) = at_b.finish(); - let a = checkpoint_app.last_executed_safe_block(); - assert!(a < 12 && 12 <= 15, "the seed direct(2) must sit in (A, B]"); - - // RECOVERY: fold the checkpoint over the (A,B] seeds + (B,C] replay to C. - let seeds = vec![direct(DIRECT_SENDER, 12, 2)]; - let (recovered_app, recovered_nonce) = fold_replay( - checkpoint_app, - checkpoint_nonce, - config(), - domain(), - seeds, - post_b(), - stop, - ) - .expect("recovery fold"); - - assert_eq!( - recovered_nonce, live_nonce, - "resume nonce N' must equal the live run's nonce" - ); - assert_eq!( - recovered_app.executed_directs, live_app.executed_directs, - "executed directs (content AND order) must equal the live run" - ); - assert_eq!( - recovered_app.last_executed_safe_block(), - live_app.last_executed_safe_block(), - "the safe-block clock must equal the live run" - ); - // Sanity: the stream actually executed all three directs (not a trivial pass). - assert_eq!(recovered_app.executed_directs, vec![1, 2, 3]); - } -} diff --git a/sequencer-core/src/scheduler/mod.rs b/sequencer-core/src/scheduler/mod.rs index dafca910..620002c8 100644 --- a/sequencer-core/src/scheduler/mod.rs +++ b/sequencer-core/src/scheduler/mod.rs @@ -1,1446 +1,2270 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -pub mod fold; - -pub use fold::{FoldInput, fold_replay}; - -use crate::application::{AppError, AppOutputs, Application, CanonicalState, ExecutionOutcome}; +//! The canonical scheduler: it turns an application's L1 inputs, in InputBox +//! order, into the one sequence of L2 transactions the application executes. +//! +//! It reads nothing from the application, neither its state nor its code: the +//! fridge admits by sender and payload length, and a user op is handed out +//! once its sender recovers and its cap covers the frame's price. That is what +//! lets a recovery schedule a whole range before any application runs. The +//! nonce is the executor's check ([`crate::application::execute`]); +//! [`driver::CanonicalRunner`] joins the two for the guest and the tests. +//! +//! The rules are those of the scheduler contract +//! (docs/protocol/scheduler-semantics.md), in its order. + +pub mod driver; + +pub use driver::{ + Advanced, CanonicalRunner, InspectError, SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, TxFate, + TxRecord, +}; + +use crate::application::Application; use crate::batch::{Batch, Frame, WireUserOp}; -use crate::history::ExecutedInputCount; -use crate::l2_tx::DirectInput; +use crate::history::L2TxCount; +use crate::l2_tx::{Direct, L2Tx, UserOpTx}; use crate::protocol::age_exceeds; use alloy_primitives::{Address, Signature}; -use alloy_sol_types::Eip712Domain; -use alloy_sol_types::SolStruct; +use alloy_sol_types::{Eip712Domain, SolStruct}; use std::collections::VecDeque; +use thiserror::Error; -#[derive(Debug, Clone, PartialEq, Eq)] +/// Deployment constants, compiled into the template. Every field is explicit: +/// a default would let a build silently run another deployment's rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SchedulerConfig { - /// L1 address whose inputs are trusted as sequencer batches; every other - /// sender is a direct input. This is deployment/app data (which key the - /// sequencer submits batches from), supplied by the app binary — the - /// protocol library never hardcodes a concrete sequencer address. + /// The batch submitter: its inputs are batches, every other sender's are + /// directs. Deployment data, never hardcoded in the protocol library. pub sequencer_address: Address, + /// At least 1, which genesis and resume assert: a zero window would let + /// the backstop drain the arriving block and queue a direct at the clock. pub max_wait_blocks: u64, + /// The fridge is full at this many queued directs. + pub fridge_bound: u64, + /// A direct with a longer payload never joins the fridge. + pub max_direct_payload_bytes: u64, } impl SchedulerConfig { - /// Production config for `sequencer_address`, pinning the staleness window - /// to the protocol's [`crate::MAX_WAIT_BLOCKS`]. Tests that need a custom - /// window construct the struct directly. - pub const fn new(sequencer_address: Address) -> Self { + /// The production configuration: the submitter, and the constants the + /// application's template compiles in. + pub fn for_application(sequencer_address: Address) -> Self { + let limits = A::scheduler_limits(); Self { sequencer_address, - max_wait_blocks: crate::MAX_WAIT_BLOCKS, + max_wait_blocks: limits.max_wait_blocks, + fridge_bound: limits.fridge_bound, + max_direct_payload_bytes: limits.max_direct_payload_bytes, } } } +/// One InputBox input, as `EvmAdvance` delivers it. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct SchedulerInput { +pub struct L1Input { + /// Per-application InputBox index. + pub index: u64, pub sender: Address, - pub inclusion_block: u64, - pub domain: Eip712Domain, + /// Its inclusion block. + pub block: u64, + pub timestamp: u64, + /// With `app_contract`, the EIP-712 domain a batch's ops are signed under. + pub chain_id: u64, + pub app_contract: Address, pub payload: Vec, } +/// All that carries over between inputs, except the queued directs' bytes, +/// which L1 holds. It is the scheduler's part of the machine's export. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProcessOutcome { - DirectEnqueued, - BatchExecuted, - BatchSkippedStale, - BatchRejected(BatchRejectReason), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProcessResult { - pub outcome: ProcessOutcome, - pub outputs: AppOutputs, +pub struct SchedulerState { + /// Inputs consumed: the scheduler's position in the InputBox. + pub l1_input_count: u64, + pub next_batch_nonce: u64, + /// The latest block the scheduler has executed at. + pub clock: u64, + /// Index of the oldest queued direct; `l1_input_count` when none is queued. + pub first_queued_input: u64, + /// How many directs are queued, to check a re-supplied range against. + pub fridge_len: u64, } -impl ProcessResult { - fn new(outcome: ProcessOutcome, outputs: AppOutputs) -> Self { - Self { outcome, outputs } - } - - fn without_outputs(outcome: ProcessOutcome) -> Self { - Self::new(outcome, Vec::new()) - } -} - -/// Inspect query accepted by the scheduler's state export endpoint. -pub const STATE_INSPECT_QUERY: &[u8] = b"state"; - -#[derive(Debug, PartialEq, Eq)] -pub enum InspectError { - UnsupportedQuery, - Application(String), +/// What became of one input. The reject reason is diagnostic: every rejected +/// or stale batch has the same effect, none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InputOutcome { + DirectQueued, + /// Its payload is over the cap: it never executes and never counts. + DirectDropped, + BatchAccepted, + BatchStale, + BatchRejected(BatchRejectReason), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BatchRejectReason { + /// Gate a. DecodeFailed, + /// Gate b. WrongNonce { expected: u64, got: u64 }, - SafeBlockAboveInclusionBlock, + /// Gate d: a frame's safe block must lie strictly below the inclusion block. + SafeBlockNotBelowInclusionBlock, + /// Gate d. NonMonotonicSafeBlocks, + /// Gate d. FeePriceOutOfRange, + /// Gate f: the first frame lies below the scheduler clock. + FirstFrameBelowClock { safe_block: u64, clock: u64 }, } -#[derive(Debug)] -pub struct Scheduler { - app: A, - config: SchedulerConfig, - direct_q: VecDeque, - next_expected_batch_nonce: u64, +/// One handed-out transaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Scheduled { + /// The scheduler's next batch nonce when it handed this out: the batch + /// whose frame holds or drained it, or the next batch to come. An accepted + /// batch's nonce advances only after its last transaction. + pub batch_nonce: u64, + pub tx: L2Tx, +} + +/// Why [`Scheduler::resume`] refused a state and its re-supplied inputs. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ResumeError { + #[error("first queued input {first_queued_input} is past the L1 input count {l1_input_count}")] + FirstQueuedAfterCount { + first_queued_input: u64, + l1_input_count: u64, + }, + #[error("re-supplied input out of sequence: expected index {expected}, got {got}")] + OutOfSequence { expected: u64, got: u64 }, + #[error("re-supplied input {index} is at or past the L1 input count {l1_input_count}")] + BeyondInputCount { index: u64, l1_input_count: u64 }, + #[error("re-supplied inputs stop at {next}, before the L1 input count {l1_input_count}")] + MissingInputs { next: u64, l1_input_count: u64 }, + #[error( + "re-supplied input {index} at block {block} is below the previous input's block {previous_block}" + )] + BlockDecreases { + index: u64, + block: u64, + previous_block: u64, + }, + #[error("first queued input {index} is not an admitted direct")] + FirstQueuedNotADirect { index: u64 }, + #[error("queued direct {index} at block {block} is not above the clock {clock}")] + DirectNotAboveClock { index: u64, block: u64, clock: u64 }, + #[error("the state queues {expected} directs, but the range admits {admitted}")] + FridgeLenMismatch { expected: u64, admitted: u64 }, +} + +/// The machine's `scheduler` inspect report: the scheduler's state and the +/// application's count, as six little-endian `u64` in field order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SchedulerExport { + /// InputBox inputs the machine has consumed: its position. + pub l1_input_count: u64, + /// L2 transactions it has executed: the base count of a recovery from it. + pub l2_tx_count: u64, + pub next_batch_nonce: u64, + /// The latest block the scheduler has executed at. + pub clock: u64, + /// Index of the oldest queued direct; the L1 input count when none is. + pub first_queued_input: u64, + /// How many directs are queued. + pub fridge_len: u64, } -impl Scheduler { - /// Construct a scheduler that begins expecting batch nonce - /// `next_expected_batch_nonce`. The genesis scheduler starts at 0 - /// ([`Scheduler::new`]); the recovery fold engine resumes at the - /// checkpoint's batch nonce, which the bare-metal app cannot recompute — it - /// rides as checkpoint metadata. - /// Sets the nonce field directly (not via the private advance). - pub fn resume_at(app: A, config: SchedulerConfig, next_expected_batch_nonce: u64) -> Self { +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[error("a scheduler export is {expected} bytes, got {got}", expected = SchedulerExport::ENCODED_LEN)] +pub struct SchedulerExportLengthError { + pub got: usize, +} + +impl SchedulerExport { + pub const ENCODED_LEN: usize = 48; + + pub fn new(state: SchedulerState, l2_tx_count: L2TxCount) -> Self { Self { - app, - config, - direct_q: VecDeque::new(), - next_expected_batch_nonce, + l1_input_count: state.l1_input_count, + l2_tx_count: l2_tx_count.get(), + next_batch_nonce: state.next_batch_nonce, + clock: state.clock, + first_queued_input: state.first_queued_input, + fridge_len: state.fridge_len, } } - pub fn new(app: A, config: SchedulerConfig) -> Self { - assert_eq!( - app.executed_input_count(), - ExecutedInputCount::ZERO, - "a genesis scheduler application must start at executed_input_count = 0" - ); - Self::resume_at(app, config, 0) + /// The scheduler's part, as [`Scheduler::resume`] takes it. + pub fn scheduler_state(&self) -> SchedulerState { + SchedulerState { + l1_input_count: self.l1_input_count, + next_batch_nonce: self.next_batch_nonce, + clock: self.clock, + first_queued_input: self.first_queued_input, + fridge_len: self.fridge_len, + } } - /// Number of directs still queued in the fridge. The recovery fold asserts - /// this is zero after draining at the stop block `C`: any leftover direct - /// means an input arrived with `inclusion_block > C` (a caller contract - /// violation), which would otherwise be silently dropped by [`finish`]. - pub fn queued_direct_len(&self) -> usize { - self.direct_q.len() + fn fields(&self) -> [u64; 6] { + [ + self.l1_input_count, + self.l2_tx_count, + self.next_batch_nonce, + self.clock, + self.first_queued_input, + self.fridge_len, + ] } - /// The batch nonce the scheduler next expects (and that a resumed sequencer - /// submits at). The recovery fold reads it via [`Scheduler::finish`] as the - /// resume nonce `N`. - pub fn next_expected_batch_nonce(&self) -> u64 { - self.next_expected_batch_nonce + pub fn encode(&self) -> [u8; Self::ENCODED_LEN] { + let mut bytes = [0_u8; Self::ENCODED_LEN]; + let (chunks, _) = bytes.as_chunks_mut::<8>(); + for (chunk, field) in chunks.iter_mut().zip(self.fields()) { + *chunk = field.to_le_bytes(); + } + bytes } - /// Seed the fridge (direct-input queue) with a reconstructed direct. The - /// recovery fold uses this to rebuild the scheduler's `(A, B]` fridge before - /// replaying `(B, C]`. Callers MUST enqueue in ascending - /// L1 order (`inclusion_block`) so the FIFO drain matches the on-chain order. - pub fn enqueue_direct(&mut self, sender: Address, inclusion_block: u64, payload: Vec) { - self.direct_q.push_back(DirectInput { - sender, - block_number: inclusion_block, - payload, - }); + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() != Self::ENCODED_LEN { + return Err(SchedulerExportLengthError { got: bytes.len() }); + } + let (chunks, _) = bytes.as_chunks::<8>(); + let mut fields = chunks.iter().map(|chunk| u64::from_le_bytes(*chunk)); + let mut next = || fields.next().expect("48 bytes hold six fields"); + Ok(Self { + l1_input_count: next(), + l2_tx_count: next(), + next_batch_nonce: next(), + clock: next(), + first_queued_input: next(), + fridge_len: next(), + }) } +} - /// Execute every fridge direct covered by `safe_block` (those with - /// `inclusion_block <= safe_block`), returning their outputs. The recovery - /// fold calls this once at the stopping block `C`: every - /// still-queued direct has `inclusion_block <= C`, so this drains them all — - /// exactly what the booting run's first frame at a safe block `>= C` would - /// do, except the booting run (bare-metal, no fridge) never sees them. - pub fn drain_covered_at(&mut self, safe_block: u64) -> Result { - let mut outputs = Vec::new(); - self.drain_directs_safe_at(safe_block, &mut outputs)?; - Ok(outputs) - } +/// The free-standing canonical scheduler. Feed it one input, then pull that +/// input's transactions with [`Scheduler::next_tx`] until it returns `None`. +/// +/// `feed` decides everything eagerly (gates, drains, the clock, the nonce), +/// so [`Scheduler::state`] already describes the scheduler after the input; +/// only signature recovery waits for `next_tx`, one op at a time. +#[derive(Debug, Clone)] +pub struct Scheduler { + config: SchedulerConfig, + /// Admitted directs not yet handed out, oldest first. + fridge: VecDeque, + l1_input_count: u64, + next_batch_nonce: u64, + clock: u64, + cursor: Cursor, +} - /// Consume the scheduler, returning the advanced application state `S'` and - /// the resume batch nonce `N`: `S'` is the folded app state, `N` - /// the nonce the new sequencer resumes submitting at. - pub fn finish(self) -> (A, u64) { - (self.app, self.next_expected_batch_nonce) - } +/// The current input's transactions not yet handed out, in execution order. +#[derive(Debug, Clone, Default)] +struct Cursor { + batch_nonce: u64, + /// The accepted batch's signature domain; `None` for any other input. + domain: Option, + steps: VecDeque, +} - /// Watchdog / CM `inspect_state` hook: the app's canonical snapshot bytes - /// for the `/finalized_state` byte-compare. `pub` because the canonical-app - /// harness is a separate crate. - pub fn inspect_state(&self, query: &[u8]) -> Result, InspectError> - where - A: CanonicalState, - { - if !query.is_empty() && query != STATE_INSPECT_QUERY { - return Err(InspectError::UnsupportedQuery); - } +#[derive(Debug, Clone)] +enum Step { + Direct(Direct), + /// An op of an accepted frame, still to pass the cap and signature checks. + UserOp { + op: WireUserOp, + fee_price: u16, + safe_block: u64, + }, +} - self.app - .canonical_snapshot_bytes() - .map_err(|err| InspectError::Application(err.to_string())) +impl Scheduler { + pub fn genesis(config: SchedulerConfig) -> Self { + assert_window(&config); + Self { + config, + fridge: VecDeque::new(), + l1_input_count: 0, + next_batch_nonce: 0, + clock: 0, + cursor: Cursor::default(), + } } - pub fn process_input(&mut self, input: SchedulerInput) -> Result { - // Execute overdue directs before any input to keep backstop semantics explicit. - let mut outputs = Vec::new(); - self.force_execute_overdue(input.inclusion_block, &mut outputs)?; - - if input.sender != self.config.sequencer_address { - self.direct_q.push_back(DirectInput { - sender: input.sender, - block_number: input.inclusion_block, - payload: input.payload, + /// Resume from an exported state. `queued` re-supplies every input from + /// `first_queued_input` up to `l1_input_count`, in order. The fridge is a + /// queue, so re-admitting the directs among them by the same rule rebuilds + /// it; the checks refuse a range that does not match the state. + pub fn resume( + config: SchedulerConfig, + state: SchedulerState, + queued: impl IntoIterator, + ) -> Result { + assert_window(&config); + if state.first_queued_input > state.l1_input_count { + return Err(ResumeError::FirstQueuedAfterCount { + first_queued_input: state.first_queued_input, + l1_input_count: state.l1_input_count, }); - Ok(ProcessResult::new(ProcessOutcome::DirectEnqueued, outputs)) - } else { - let batch_result = - self.process_batch_payload(input.inclusion_block, &input.domain, &input.payload)?; - outputs.extend(batch_result.outputs); - Ok(ProcessResult::new(batch_result.outcome, outputs)) } + let mut scheduler = Self { + config, + fridge: VecDeque::new(), + l1_input_count: state.l1_input_count, + next_batch_nonce: state.next_batch_nonce, + clock: state.clock, + cursor: Cursor::default(), + }; + let mut next = state.first_queued_input; + // InputBox order never lowers the block. A range that does would put a + // newer direct before an older one, and drains stop at the front. + let mut previous_block = 0; + for input in queued { + if input.index != next { + return Err(ResumeError::OutOfSequence { + expected: next, + got: input.index, + }); + } + if input.index >= state.l1_input_count { + return Err(ResumeError::BeyondInputCount { + index: input.index, + l1_input_count: state.l1_input_count, + }); + } + if input.block < previous_block { + return Err(ResumeError::BlockDecreases { + index: input.index, + block: input.block, + previous_block, + }); + } + previous_block = input.block; + let admitted = scheduler.admits(&input); + if input.index == state.first_queued_input && !admitted { + return Err(ResumeError::FirstQueuedNotADirect { index: input.index }); + } + // Below the count, so the successor exists. + next = input.index + 1; + if admitted { + if input.block <= state.clock { + return Err(ResumeError::DirectNotAboveClock { + index: input.index, + block: input.block, + clock: state.clock, + }); + } + scheduler.fridge.push_back(into_direct(input)); + } + } + if next != state.l1_input_count { + return Err(ResumeError::MissingInputs { + next, + l1_input_count: state.l1_input_count, + }); + } + let admitted = scheduler.fridge_len(); + if admitted != state.fridge_len { + return Err(ResumeError::FridgeLenMismatch { + expected: state.fridge_len, + admitted, + }); + } + Ok(scheduler) } - fn process_batch_payload( - &mut self, - inclusion_block: u64, - domain: &Eip712Domain, - payload: &[u8], - ) -> Result { - let Ok(batch): Result = ssz::Decode::from_ssz_bytes(payload) else { - return Ok(ProcessResult::without_outputs( - ProcessOutcome::BatchRejected(BatchRejectReason::DecodeFailed), - )); - }; + pub fn config(&self) -> &SchedulerConfig { + &self.config + } - if batch.nonce != self.next_expected_batch_nonce { - return Ok(ProcessResult::without_outputs( - ProcessOutcome::BatchRejected(BatchRejectReason::WrongNonce { - expected: self.next_expected_batch_nonce, - got: batch.nonce, - }), - )); + /// The scheduler once the current input's transactions are all handed out. + pub fn state(&self) -> SchedulerState { + SchedulerState { + l1_input_count: self.l1_input_count, + next_batch_nonce: self.next_batch_nonce, + clock: self.clock, + first_queued_input: self + .fridge + .front() + .map_or(self.l1_input_count, |direct| direct.input_index), + fridge_len: self.fridge_len(), } + } - let Some((frame_head, frame_tail)) = batch.frames.split_first() else { - let next_nonce = self.checked_next_batch_nonce(); - self.next_expected_batch_nonce = next_nonce; - return Ok(ProcessResult::without_outputs( - ProcessOutcome::BatchExecuted, - )); + /// Take the next input and say what became of it. Its transactions follow + /// from [`Scheduler::next_tx`]. Panics on misuse: an index other than + /// `l1_input_count`, or a previous input with transactions left. + pub fn feed(&mut self, input: L1Input) -> InputOutcome { + assert_eq!( + input.index, self.l1_input_count, + "inputs must arrive in InputBox order" + ); + assert!( + self.cursor.steps.is_empty(), + "the previous input still has transactions to hand out" + ); + self.l1_input_count = self + .l1_input_count + .checked_add(1) + .expect("L1 input count overflow: no canonical successor"); + self.cursor = Cursor { + batch_nonce: self.next_batch_nonce, + domain: None, + steps: VecDeque::new(), }; - if let Some(reason) = - self.batch_reject_reason_for_block(inclusion_block, frame_head, frame_tail) - { - return Ok(ProcessResult::without_outputs( - ProcessOutcome::BatchRejected(reason), - )); - } + let max_wait_blocks = self.config.max_wait_blocks; + self.drain_front_while(|direct| age_exceeds(input.block, direct.block, max_wait_blocks)); - if age_exceeds( - inclusion_block, - frame_head.safe_block, - self.config.max_wait_blocks, - ) { - return Ok(ProcessResult::without_outputs( - ProcessOutcome::BatchSkippedStale, - )); + if input.sender == self.config.sequencer_address { + self.feed_batch(&input) + } else { + self.feed_direct(input) } + } - // Preflight nonce exhaustion before any frame can mutate application - // state. A batch at `u64::MAX` has no canonical successor. - let next_nonce = self.checked_next_batch_nonce(); - let mut outputs = Vec::new(); - for frame in &batch.frames { - self.drain_directs_safe_at(frame.safe_block, &mut outputs)?; - self.execute_frame_user_ops(domain, frame, &mut outputs)?; + /// The current input's next transaction, in execution order. + pub fn next_tx(&mut self) -> Option { + while let Some(step) = self.cursor.steps.pop_front() { + let tx = match step { + Step::Direct(direct) => L2Tx::Direct(direct), + Step::UserOp { + op, + fee_price, + safe_block, + } => { + // Both checks drop the op without a trace, so their order + // is unobservable; the cheap one goes first. + if op.max_fee < fee_price { + continue; + } + let domain = self + .cursor + .domain + .as_ref() + .expect("a user op is queued only with its batch's domain"); + let Some(sender) = recover_sender(domain, &op) else { + continue; + }; + L2Tx::UserOp(UserOpTx { + sender, + nonce: op.nonce, + fee: fee_price, + block: safe_block, + data: op.data, + }) + } + }; + return Some(Scheduled { + batch_nonce: self.cursor.batch_nonce, + tx, + }); } - - self.next_expected_batch_nonce = next_nonce; - Ok(ProcessResult::new(ProcessOutcome::BatchExecuted, outputs)) + None } - fn checked_next_batch_nonce(&self) -> u64 { - self.next_expected_batch_nonce - .checked_add(1) - .expect("batch nonce overflow: no canonical successor") + fn feed_direct(&mut self, input: L1Input) -> InputOutcome { + // The cap comes first: a direct that never joins drains nothing. + if !self.admits(&input) { + return InputOutcome::DirectDropped; + } + // Whole blocks, oldest first, so no direct at or below the clock stays + // queued; never the arriving block, so the bound is soft by one block. + while self.fridge_len() >= self.config.fridge_bound { + let Some(front_block) = self.fridge.front().map(|direct| direct.block) else { + break; + }; + if front_block >= input.block { + break; + } + self.drain_front_while(|direct| direct.block == front_block); + } + self.fridge.push_back(into_direct(input)); + InputOutcome::DirectQueued } - fn batch_reject_reason_for_block( - &self, - inclusion_block: u64, - head: &Frame, - tail: &[Frame], - ) -> Option { - if head.safe_block > inclusion_block { - return Some(BatchRejectReason::SafeBlockAboveInclusionBlock); + fn feed_batch(&mut self, input: &L1Input) -> InputOutcome { + let Ok(batch) = ::from_ssz_bytes(&input.payload) else { + return InputOutcome::BatchRejected(BatchRejectReason::DecodeFailed); + }; + if batch.nonce != self.next_batch_nonce { + return InputOutcome::BatchRejected(BatchRejectReason::WrongNonce { + expected: self.next_batch_nonce, + got: batch.nonce, + }); } - - let mut previous_safe_block = head.safe_block; - for frame in tail { - if frame.safe_block > inclusion_block { - return Some(BatchRejectReason::SafeBlockAboveInclusionBlock); - } else if frame.safe_block < previous_safe_block { - return Some(BatchRejectReason::NonMonotonicSafeBlocks); - } else { - previous_safe_block = frame.safe_block; - } + let Some(first_safe_block) = batch.frames.first().map(|frame| frame.safe_block) else { + self.next_batch_nonce = self.successor_batch_nonce(); + return InputOutcome::BatchAccepted; + }; + if let Some(reason) = frame_structure_reject(input.block, &batch.frames) { + return InputOutcome::BatchRejected(reason); + } + if age_exceeds(input.block, first_safe_block, self.config.max_wait_blocks) { + return InputOutcome::BatchStale; + } + if first_safe_block < self.clock { + return InputOutcome::BatchRejected(BatchRejectReason::FirstFrameBelowClock { + safe_block: first_safe_block, + clock: self.clock, + }); } - if std::iter::once(head) - .chain(tail) - .any(|frame| frame.fee_price > crate::fee::MAX_EXPONENT) - { - return Some(BatchRejectReason::FeePriceOutOfRange); + // Preflight before any frame is handed out: a batch at u64::MAX has no + // canonical successor. + let next_nonce = self.successor_batch_nonce(); + self.cursor.domain = Some(crate::build_input_domain( + input.chain_id, + input.app_contract, + )); + for frame in batch.frames { + let safe_block = frame.safe_block; + self.drain_front_while(|direct| direct.block <= safe_block); + self.raise_clock(safe_block); + for op in frame.user_ops { + self.cursor.steps.push_back(Step::UserOp { + op, + fee_price: frame.fee_price, + safe_block, + }); + } } + self.next_batch_nonce = next_nonce; + InputOutcome::BatchAccepted + } - None + fn successor_batch_nonce(&self) -> u64 { + self.next_batch_nonce + .checked_add(1) + .expect("batch nonce overflow: no canonical successor") } - /// Execute user-ops in a frame, skipping any whose `max_fee` is below the frame's `fee_price`. - /// - /// Both `max_fee` and `fee_price` are log-space exponents (base 129/128). - /// See [`crate::fee`] for conversion to linear amounts. - fn execute_frame_user_ops( - &mut self, - domain: &Eip712Domain, - frame: &Frame, - outputs: &mut AppOutputs, - ) -> Result<(), AppError> { - for user_op in &frame.user_ops { - // An unrecoverable signature is dropped silently (the scheduler is a - // pure deterministic fold; diagnostics would be a nondeterministic - // side effect at the library seam). - if let Some(sender) = self.recover_sender(domain, user_op) { - let plain = user_op.to_user_op(); - match crate::application::validate_and_execute_user_op( - &mut self.app, - sender, - &plain, - frame.fee_price, - frame.safe_block, - )? { - ExecutionOutcome::Included(executed) => outputs.extend(executed.outputs), - ExecutionOutcome::Invalid(_) => {} - } - } - } - Ok(()) + /// Whether a direct joins the fridge: by sender and payload length only, + /// never by content, so no application code runs. + fn admits(&self, input: &L1Input) -> bool { + input.sender != self.config.sequencer_address + && u64::try_from(input.payload.len()) + .is_ok_and(|len| len <= self.config.max_direct_payload_bytes) } - fn recover_sender(&self, domain: &Eip712Domain, wire_user_op: &WireUserOp) -> Option
{ - if wire_user_op.signature.len() != WireUserOp::SIGNATURE_BYTES { - return None; + /// Hand out queued directs from the front while `due` holds, raising the + /// clock to each one's block, as every drain does. + fn drain_front_while(&mut self, due: impl Fn(&Direct) -> bool) { + while self.fridge.front().is_some_and(&due) { + let direct = self.fridge.pop_front().expect("the front was just seen"); + self.raise_clock(direct.block); + self.cursor.steps.push_back(Step::Direct(direct)); } - let signature = Signature::from_raw(wire_user_op.signature.as_slice()).ok()?; - let user_op = wire_user_op.to_user_op(); - let signing_hash = user_op.eip712_signing_hash(domain); - signature.recover_address_from_prehash(&signing_hash).ok() } - fn drain_directs_safe_at( - &mut self, - safe_block: u64, - outputs: &mut AppOutputs, - ) -> Result<(), AppError> { - self.drain_front_while(outputs, |direct| direct.block_number <= safe_block) + fn raise_clock(&mut self, block: u64) { + self.clock = self.clock.max(block); } - fn force_execute_overdue( - &mut self, - current_block: u64, - outputs: &mut AppOutputs, - ) -> Result<(), AppError> { - let max_wait_blocks = self.config.max_wait_blocks; - self.drain_front_while(outputs, |direct| { - age_exceeds(current_block, direct.block_number, max_wait_blocks) - }) + fn fridge_len(&self) -> u64 { + u64::try_from(self.fridge.len()).expect("a fridge length fits in u64") } +} - /// Execute fridge directs from the front, in FIFO order, while `due` - /// holds. Each direct leaves the queue only after it executed. - fn drain_front_while( - &mut self, - outputs: &mut AppOutputs, - due: impl Fn(&DirectInput) -> bool, - ) -> Result<(), AppError> { - while let Some(front) = self.direct_q.front() { - if !due(front) { - break; - } - let executed = crate::application::execute_direct_input(&mut self.app, front)?; - outputs.extend(executed.outputs); - self.direct_q.pop_front(); +/// With a window of at least one block the backstop only reaches blocks below +/// the arriving input's, so it never raises the clock to a block that can +/// still queue a direct. A zero window is a build error, so it fails loud. +fn assert_window(config: &SchedulerConfig) { + assert!( + config.max_wait_blocks >= 1, + "max_wait_blocks must be at least 1" + ); +} + +/// Gate d, over every frame before any executes. Strict frames let a frame +/// drain every direct of its own safe block: all of them landed before its +/// batch. +fn frame_structure_reject(inclusion_block: u64, frames: &[Frame]) -> Option { + let mut previous_safe_block = 0; + for frame in frames { + if frame.safe_block >= inclusion_block { + return Some(BatchRejectReason::SafeBlockNotBelowInclusionBlock); } - Ok(()) + if frame.safe_block < previous_safe_block { + return Some(BatchRejectReason::NonMonotonicSafeBlocks); + } + previous_safe_block = frame.safe_block; + } + if frames + .iter() + .any(|frame| frame.fee_price > crate::fee::MAX_EXPONENT) + { + return Some(BatchRejectReason::FeePriceOutOfRange); + } + None +} + +/// The op's signer, or `None` when the op must be dropped: only a 65-byte +/// signature recovers, with alloy's acceptance set (see the scheduler page). +pub(crate) fn recover_sender(domain: &Eip712Domain, wire_user_op: &WireUserOp) -> Option
{ + if wire_user_op.signature.len() != WireUserOp::SIGNATURE_BYTES { + return None; + } + let signature = Signature::from_raw(wire_user_op.signature.as_slice()).ok()?; + let signing_hash = wire_user_op.to_user_op().eip712_signing_hash(domain); + signature.recover_address_from_prehash(&signing_hash).ok() +} + +fn into_direct(input: L1Input) -> Direct { + Direct { + input_index: input.index, + sender: input.sender, + block: input.block, + timestamp: input.timestamp, + payload: input.payload, } } +/// Builders shared by the scheduler's and the driver's tests. #[cfg(test)] -mod tests { +pub(crate) mod fixtures { use super::*; - use crate::application::{ApplicationProgress, ValidationOutcome}; use crate::user_op::UserOp; - use alloy_primitives::{U256, address}; + use alloy_primitives::address; use k256::ecdsa::SigningKey; - #[cfg(test)] - #[derive(Default)] - struct RecordingApp { - executed: Vec, - balances: std::collections::HashMap, - nonces: std::collections::HashMap, - progress: ApplicationProgress, - fail_on: Option, + pub(crate) const SEQUENCER: Address = address!("0x1111111111111111111111111111111111111111"); + pub(crate) const DIRECT_SENDER: Address = + address!("0x2222222222222222222222222222222222222222"); + pub(crate) const CHAIN_ID: u64 = 1; + pub(crate) const APP_CONTRACT: Address = Address::ZERO; + + /// A configuration whose bound and cap stay out of the way. + pub(crate) fn config(max_wait_blocks: u64) -> SchedulerConfig { + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks, + fridge_bound: 1_000, + max_direct_payload_bytes: 1_000, + } + } + + pub(crate) fn domain() -> Eip712Domain { + crate::build_input_domain(CHAIN_ID, APP_CONTRACT) + } + + pub(crate) fn key(seed: u8) -> SigningKey { + SigningKey::from_bytes((&[seed; 32]).into()).expect("signing key") } - #[cfg(test)] - #[derive(Debug, Clone, PartialEq, Eq)] - enum RecordedTx { - UserOp(u8), - Direct(u8), + pub(crate) fn address_of(key: &SigningKey) -> Address { + let verifying = key.verifying_key().to_encoded_point(false); + Address::from_raw_public_key(&verifying.as_bytes()[1..]) } - #[cfg(test)] - impl RecordingApp { - fn events(&self) -> &[RecordedTx] { - self.executed.as_slice() + pub(crate) fn sign_in( + domain: &Eip712Domain, + key: &SigningKey, + nonce: u32, + max_fee: u16, + data: Vec, + ) -> WireUserOp { + let user_op = UserOp { + nonce, + max_fee, + data: data.clone().into(), + }; + let signature = crate::test_support::sign_user_op(key, domain, &user_op); + WireUserOp { + nonce, + max_fee, + data, + signature: signature.as_bytes().to_vec(), } + } - fn balance_of(&self, sender: Address) -> U256 { - *self.balances.get(&sender).unwrap_or(&U256::ZERO) + pub(crate) fn signed(key: &SigningKey, nonce: u32, max_fee: u16, data: Vec) -> WireUserOp { + sign_in(&domain(), key, nonce, max_fee, data) + } + + pub(crate) fn frame(safe_block: u64, fee_price: u16, user_ops: Vec) -> Frame { + Frame { + user_ops, + safe_block, + fee_price, } + } + + pub(crate) fn empty_frame(safe_block: u64) -> Frame { + frame(safe_block, 0, Vec::new()) + } - fn nonce_of(&self, sender: Address) -> u32 { - self.nonces.get(&sender).copied().unwrap_or(0) + /// Inputs carry index 0 until fed; whoever feeds them assigns the index. + pub(crate) fn direct_from(sender: Address, block: u64, payload: Vec) -> L1Input { + L1Input { + index: 0, + sender, + block, + timestamp: block.saturating_mul(12), + chain_id: CHAIN_ID, + app_contract: APP_CONTRACT, + payload, } + } + + pub(crate) fn direct(block: u64, marker: u8) -> L1Input { + direct_from(DIRECT_SENDER, block, vec![marker]) + } + + pub(crate) fn raw_batch(block: u64, payload: Vec) -> L1Input { + direct_from(SEQUENCER, block, payload) + } + + pub(crate) fn batch(block: u64, nonce: u64, frames: Vec) -> L1Input { + raw_batch(block, ssz::Encode::as_ssz_bytes(&Batch { nonce, frames })) + } - fn credit(&mut self, sender: Address, amount: u64) { - let current = self.balance_of(sender); - self.balances - .insert(sender, current.saturating_add(U256::from(amount))); + /// A handed-out transaction by its marker: a direct's first payload byte + /// or an op's first data byte. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Tx { + D(u8), + Op(u8), + } + + pub(crate) fn short(tx: &L2Tx) -> Tx { + match tx { + L2Tx::Direct(direct) => Tx::D(direct.payload.first().copied().unwrap_or(0)), + L2Tx::UserOp(op) => Tx::Op(op.data.first().copied().unwrap_or(0)), } + } - fn with_progress(executed_input_count: u64, last_executed_safe_block: u64) -> Self { - Self { - progress: ApplicationProgress::try_new( - ExecutedInputCount::new(executed_input_count), - last_executed_safe_block, - ) - .expect("coherent progress"), - ..Self::default() - } + pub(crate) fn tx_block(tx: &L2Tx) -> u64 { + match tx { + L2Tx::Direct(direct) => direct.block, + L2Tx::UserOp(op) => op.block, } } +} - #[cfg(test)] - impl Application for RecordingApp { - // Mirrors the wallet app's method-payload cap (selector + amount + - // address). A local literal keeps sequencer-core free of an app-core - // dependency (which would invert the crate graph). - fn max_method_payload_bytes() -> usize { - 1 + 32 + 20 - } - - fn validate_user_op( - &self, - sender: Address, - user_op: &crate::user_op::UserOp, - current_fee: u16, - ) -> Result { - let expected_nonce = self.nonce_of(sender); - if user_op.nonce != expected_nonce { - return Ok(ValidationOutcome::Reject( - crate::application::InvalidReason::InvalidNonce { - expected: expected_nonce, - got: user_op.nonce, - }, - )); - } - if user_op.max_fee < current_fee { - return Ok(ValidationOutcome::Reject( - crate::application::InvalidReason::InvalidMaxFee { - max_fee: user_op.max_fee, - base_fee: current_fee, - }, - )); - } - let required = crate::fee::fee_to_linear(current_fee); - let balance = self.balance_of(sender); - if balance < required { - return Ok(ValidationOutcome::Reject( - crate::application::InvalidReason::InsufficientFeeBalance { - required, - available: balance, - }, - )); - } - Ok(ValidationOutcome::Accept) - } - - fn apply_valid_user_op( - &mut self, - user_op: &crate::l2_tx::ValidUserOp, - safe_block: u64, - ) -> Result { - let marker = user_op.data.first().copied().unwrap_or_default(); - let event = RecordedTx::UserOp(marker); - if self.fail_on.as_ref() == Some(&event) { - return Err(AppError::Internal { - reason: format!("refusing {event:?}"), - }); - } +#[cfg(test)] +mod tests { + use super::fixtures::*; + use super::*; + use std::collections::{BTreeMap, BTreeSet}; + + /// Drives a scheduler and, after every input, asserts the clock's split: + /// an admitted direct has been handed out exactly when its block is at or + /// below the clock, and the fridge and `state()` agree. It also asserts + /// that blocks never go backwards along the hand-out order. + struct Harness { + scheduler: Scheduler, + /// Every input it knows, by index, to re-supply on a resume. + inputs: BTreeMap, + /// `(index, block)` of every direct admitted since the harness began. + admitted: Vec<(u64, u64)>, + handed_out: BTreeSet, + last_block: u64, + } - let sender = user_op.sender; - let fee = crate::fee::fee_to_linear(user_op.fee); - let balance = self.balance_of(sender); - if balance < fee { - return Err(crate::application::AppError::Internal { - reason: "validated user op cannot pay fee".to_string(), - }); - } - self.balances.insert(sender, balance - fee); - let next_nonce = self.nonce_of(sender).wrapping_add(1); - self.nonces.insert(sender, next_nonce); - - self.executed.push(event); - self.progress.advance(safe_block); - Ok(Vec::new()) - } - - fn apply_direct_input( - &mut self, - input: &DirectInput, - ) -> Result { - let marker = input.payload.first().copied().unwrap_or(0); - let event = RecordedTx::Direct(marker); - if self.fail_on.as_ref() == Some(&event) { - return Err(AppError::Internal { - reason: format!("refusing {event:?}"), - }); - } - self.executed.push(event); - self.progress.advance(input.block_number); - Ok(Vec::new()) + impl Harness { + fn new(config: SchedulerConfig) -> Self { + Self::wrap(Scheduler::genesis(config), BTreeMap::new()) } - fn progress(&self) -> ApplicationProgress { - self.progress + fn resumed( + config: SchedulerConfig, + state: SchedulerState, + queued: Vec, + ) -> Result { + let scheduler = Scheduler::resume(config, state, queued.clone())?; + let inputs = queued.into_iter().map(|input| (input.index, input)); + Ok(Self::wrap(scheduler, inputs.collect())) } - fn from_dump(_prefix: &std::path::Path) -> Result { - unimplemented!("RecordingApp does not participate in snapshot lifecycle") + fn wrap(scheduler: Scheduler, inputs: BTreeMap) -> Self { + let admitted = scheduler + .fridge + .iter() + .map(|direct| (direct.input_index, direct.block)) + .collect(); + let last_block = scheduler.clock; + let harness = Self { + scheduler, + inputs, + admitted, + handed_out: BTreeSet::new(), + last_block, + }; + harness.assert_split(); + harness } - fn create_dump( - &mut self, - _prefix: &std::path::Path, - ) -> Result<(), crate::application::AppError> { - unimplemented!("RecordingApp does not participate in snapshot lifecycle") + fn state(&self) -> SchedulerState { + self.scheduler.state() } - fn state_file_in_dump(_prefix: &std::path::Path) -> std::path::PathBuf { - unimplemented!("RecordingApp does not participate in snapshot lifecycle") + fn feed(&mut self, mut input: L1Input) -> (InputOutcome, Vec) { + input.index = self.state().l1_input_count; + let (index, block) = (input.index, input.block); + self.inputs.insert(index, input.clone()); + let outcome = self.scheduler.feed(input); + if outcome == InputOutcome::DirectQueued { + self.admitted.push((index, block)); + } + let txs: Vec = std::iter::from_fn(|| self.scheduler.next_tx()).collect(); + for scheduled in &txs { + let block = tx_block(&scheduled.tx); + assert!( + block >= self.last_block, + "blocks went backwards: {block} after {}", + self.last_block + ); + self.last_block = block; + if let L2Tx::Direct(direct) = &scheduled.tx { + assert!( + self.handed_out.insert(direct.input_index), + "direct {} handed out twice", + direct.input_index + ); + } + } + self.assert_split(); + (outcome, txs) } - } - impl CanonicalState for RecordingApp { - fn canonical_snapshot_bytes(&self) -> Result, crate::application::AppError> { - Ok(format!("events:{}", self.executed.len()).into_bytes()) + /// Feed and keep only the outcome and the markers. + fn run(&mut self, input: L1Input) -> (InputOutcome, Vec) { + let (outcome, txs) = self.feed(input); + (outcome, txs.iter().map(|s| short(&s.tx)).collect()) } - } - - const SEQUENCER: Address = address!("0x1111111111111111111111111111111111111111"); - const DIRECT_SENDER: Address = address!("0x2222222222222222222222222222222222222222"); - const TEST_CHAIN_ID: u64 = 1; - const TEST_VERIFYING_CONTRACT: Address = Address::ZERO; - fn test_domain() -> Eip712Domain { - crate::build_input_domain(TEST_CHAIN_ID, TEST_VERIFYING_CONTRACT) - } + fn assert_split(&self) { + let state = self.state(); + for &(index, block) in &self.admitted { + assert_eq!( + self.handed_out.contains(&index), + block <= state.clock, + "direct {index} at block {block} is on the wrong side of clock {}", + state.clock + ); + } + let queued: Vec = self + .admitted + .iter() + .map(|&(index, _)| index) + .filter(|index| !self.handed_out.contains(index)) + .collect(); + let fridge: Vec = self + .scheduler + .fridge + .iter() + .map(|d| d.input_index) + .collect(); + assert_eq!( + queued, fridge, + "the fridge holds the admitted, unexecuted directs" + ); + assert_eq!(state.fridge_len, fridge.len() as u64); + assert_eq!( + state.first_queued_input, + fridge.first().copied().unwrap_or(state.l1_input_count) + ); + } - fn direct_input(block: u64, marker: u8) -> SchedulerInput { - SchedulerInput { - sender: DIRECT_SENDER, - inclusion_block: block, - domain: test_domain(), - payload: vec![marker], + /// A scheduler resumed from this one's state and its re-supplied inputs. + fn resume_here(&self) -> Self { + let state = self.state(); + let queued = (state.first_queued_input..state.l1_input_count) + .map(|index| self.inputs[&index].clone()) + .collect(); + Self::resumed(*self.scheduler.config(), state, queued) + .expect("a scheduler's own state resumes") } } - fn batch_input(block: u64, batch: Batch) -> SchedulerInput { - SchedulerInput { - sender: SEQUENCER, - inclusion_block: block, - domain: test_domain(), - payload: ssz::Encode::as_ssz_bytes(&batch), + fn bounded( + max_wait_blocks: u64, + fridge_bound: u64, + max_direct_payload_bytes: u64, + ) -> SchedulerConfig { + SchedulerConfig { + fridge_bound, + max_direct_payload_bytes, + ..config(max_wait_blocks) } } - fn process_ok( - scheduler: &mut Scheduler, - input: SchedulerInput, - ) -> ProcessResult { - scheduler - .process_input(input) - .expect("test application execution must succeed") + fn at_state(max_wait_blocks: u64, next_batch_nonce: u64, clock: u64) -> Harness { + let state = SchedulerState { + l1_input_count: 0, + next_batch_nonce, + clock, + first_queued_input: 0, + fridge_len: 0, + }; + Harness::resumed(config(max_wait_blocks), state, Vec::new()).expect("an empty resume") } - fn address_from_signing_key(signing_key: &SigningKey) -> Address { - let verifying = signing_key.verifying_key().to_encoded_point(false); - Address::from_raw_public_key(&verifying.as_bytes()[1..]) + fn rejected(reason: BatchRejectReason) -> InputOutcome { + InputOutcome::BatchRejected(reason) } - fn sign_wire_user_op( - domain: &Eip712Domain, - signing_key: &SigningKey, - nonce: u32, - max_fee: u16, - data: Vec, - ) -> WireUserOp { - let user_op = UserOp { - nonce, - max_fee, - data: data.clone().into(), - }; - let signature = crate::test_support::sign_user_op(signing_key, domain, &user_op); - - WireUserOp { - nonce, - max_fee, - data, - signature: signature.as_bytes().to_vec(), - } - } + // ── Batches and frames ──────────────────────────────────────────────── #[test] - fn batch_drains_safe_inputs_before_executing_user_ops() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn batch_drains_safe_directs_before_handing_out_user_ops() { + let mut h = Harness::new(config(100)); + assert_eq!(h.run(direct(10, 1)), (InputOutcome::DirectQueued, vec![])); + let op = signed(&key(1), 0, 1, vec![2]); + assert_eq!( + h.run(batch(20, 0, vec![frame(10, 0, vec![op])])), + (InputOutcome::BatchAccepted, vec![Tx::D(1), Tx::Op(2)]) ); + assert_eq!(h.state().fridge_len, 0); + assert_eq!(h.state().next_batch_nonce, 1); + } + #[test] + fn handed_out_ops_carry_the_frame_and_the_recovered_sender() { + let mut h = Harness::new(config(100)); + let signer = key(7); + let op = signed(&signer, 4, 9, vec![0xAB, 0xCD]); + let (_, txs) = h.feed(batch(20, 0, vec![frame(15, 6, vec![op])])); assert_eq!( - process_ok(&mut scheduler, direct_input(10, 1)).outcome, - ProcessOutcome::DirectEnqueued + txs, + [Scheduled { + batch_nonce: 0, + tx: L2Tx::UserOp(UserOpTx { + sender: address_of(&signer), + nonce: 4, + fee: 6, + block: 15, + data: vec![0xAB, 0xCD], + }), + }] ); + } - let signing_key = SigningKey::from_bytes((&[1_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - scheduler.app.credit(sender, 1); - - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key, - 0, - 1, - vec![2], - )], - safe_block: 10, - fee_price: 0, - }], - }; - + #[test] + fn handed_out_directs_carry_their_input_context() { + let mut h = Harness::new(config(100)); + h.feed(batch(5, 0, vec![])); + h.feed(direct_from(DIRECT_SENDER, 10, vec![3, 4])); + let (_, txs) = h.feed(batch(11, 1, vec![empty_frame(10)])); assert_eq!( - process_ok(&mut scheduler, batch_input(20, batch)).outcome, - ProcessOutcome::BatchExecuted + txs, + [Scheduled { + batch_nonce: 1, + tx: L2Tx::Direct(Direct { + input_index: 1, + sender: DIRECT_SENDER, + block: 10, + timestamp: 120, + payload: vec![3, 4], + }), + }] ); + } + + #[test] + fn pre_batch_backstop_hands_out_overdue_directs_before_user_ops() { + let mut h = Harness::new(config(5)); + h.feed(direct(1, 1)); + let op = signed(&key(2), 0, 1, vec![2]); assert_eq!( - scheduler.app.events(), - [RecordedTx::Direct(1), RecordedTx::UserOp(2)] + h.run(batch(6, 0, vec![frame(2, 0, vec![op])])), + (InputOutcome::BatchAccepted, vec![Tx::D(1), Tx::Op(2)]) ); - assert_eq!(scheduler.queued_direct_len(), 0); } #[test] - fn pre_batch_backstop_executes_overdue_directs_before_user_ops() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 5, - }, + fn stale_batch_is_skipped_without_consuming_the_nonce() { + let mut h = Harness::new(config(5)); + h.feed(direct(1, 9)); + let stale = batch( + 10, + 0, + vec![frame(4, 0, vec![signed(&key(3), 0, 1, vec![7])])], ); + // The backstop still runs: 10 - 1 >= 5. + assert_eq!(h.run(stale), (InputOutcome::BatchStale, vec![Tx::D(9)])); + assert_eq!(h.state().next_batch_nonce, 0); - process_ok(&mut scheduler, direct_input(1, 1)); - let signing_key = SigningKey::from_bytes((&[2_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - scheduler.app.credit(sender, 1); - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key, - 0, - 1, - vec![2], - )], - safe_block: 2, - fee_price: 0, - }], - }; - - process_ok(&mut scheduler, batch_input(6, batch)); - assert_eq!( - scheduler.app.events(), - [RecordedTx::Direct(1), RecordedTx::UserOp(2)] - ); - } - - #[test] - fn stale_batch_is_skipped_without_consuming_nonce() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 5, - }, - ); - - process_ok(&mut scheduler, direct_input(1, 9)); - let signing_key = SigningKey::from_bytes((&[3_u8; 32]).into()).expect("signing key"); - let stale_batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key, - 0, - 1, - vec![7], - )], - safe_block: 4, - fee_price: 0, - }], - }; - - let outcome = process_ok(&mut scheduler, batch_input(10, stale_batch)).outcome; - assert_eq!(outcome, ProcessOutcome::BatchSkippedStale); - assert_eq!(scheduler.app.events(), [RecordedTx::Direct(9)]); - // Stale batches do NOT consume the nonce — they are true no-ops in nonce space. - assert_eq!(scheduler.next_expected_batch_nonce(), 0); - - // The next valid batch reuses nonce 0. - let fresh_signing_key = - SigningKey::from_bytes((&[13_u8; 32]).into()).expect("fresh signing key"); - let fresh_sender = address_from_signing_key(&fresh_signing_key); - scheduler.app.credit(fresh_sender, 1); - let fresh_batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &fresh_signing_key, - 0, - 1, - vec![8], - )], - safe_block: 10, - fee_price: 0, - }], - }; - - assert_eq!( - process_ok(&mut scheduler, batch_input(10, fresh_batch)).outcome, - ProcessOutcome::BatchExecuted + let fresh = batch( + 10, + 0, + vec![frame(9, 0, vec![signed(&key(13), 0, 1, vec![8])])], ); + assert_eq!(h.run(fresh), (InputOutcome::BatchAccepted, vec![Tx::Op(8)])); + assert_eq!(h.state().next_batch_nonce, 1); } #[test] fn fee_bounds_reject_the_whole_batch_and_preserve_the_backstop() { use crate::fee::MAX_EXPONENT; - for fee in [MAX_EXPONENT + 1, u16::MAX] { for invalid_position in [0, 1] { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 10, - }, - ); - process_ok(&mut scheduler, direct_input(1, 1)); - process_ok(&mut scheduler, direct_input(8, 2)); - let key = SigningKey::from_bytes((&[1_u8; 32]).into()).unwrap(); - scheduler.app.credit(address_from_signing_key(&key), 1); - let frame = Frame { - safe_block: 8, - fee_price: 0, - user_ops: vec![sign_wire_user_op( - &test_domain(), - &key, - 0, - u16::MAX, - vec![3], - )], - }; - let mut frames = vec![frame.clone(), frame]; + let mut h = Harness::new(config(10)); + h.feed(direct(1, 1)); + h.feed(direct(8, 2)); + let op = signed(&key(1), 0, u16::MAX, vec![3]); + let mut frames = vec![frame(8, 0, vec![op.clone()]), frame(8, 0, vec![op])]; frames[invalid_position].fee_price = fee; + // Only the overdue direct: no frame drain, no op. assert_eq!( - process_ok(&mut scheduler, batch_input(11, Batch { nonce: 0, frames })).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::FeePriceOutOfRange), - ); - // Only the overdue direct executes: no frame drain or user op. - assert_eq!(scheduler.app.events(), [RecordedTx::Direct(1)]); - assert_eq!(scheduler.queued_direct_len(), 1); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); - assert_eq!( - process_ok( - &mut scheduler, - batch_input( - 11, - Batch { - nonce: 0, - frames: vec![Frame { - safe_block: 8, - fee_price: MAX_EXPONENT, - user_ops: vec![] - }], - } - ) + h.run(batch(11, 0, frames)), + ( + rejected(BatchRejectReason::FeePriceOutOfRange), + vec![Tx::D(1)] ) - .outcome, - ProcessOutcome::BatchExecuted, ); + assert_eq!(h.state().fridge_len, 1); + assert_eq!(h.state().next_batch_nonce, 0); + assert_eq!( - scheduler.app.events(), - [RecordedTx::Direct(1), RecordedTx::Direct(2)] + h.run(batch(11, 0, vec![frame(8, MAX_EXPONENT, vec![])])), + (InputOutcome::BatchAccepted, vec![Tx::D(2)]) ); - assert_eq!(scheduler.next_expected_batch_nonce(), 1); + assert_eq!(h.state().next_batch_nonce, 1); } } } #[test] - fn non_monotonic_safe_blocks_invalidate_batch() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - ); - - let signing_key_a = SigningKey::from_bytes((&[4_u8; 32]).into()).expect("signing key a"); - let signing_key_b = SigningKey::from_bytes((&[5_u8; 32]).into()).expect("signing key b"); - let invalid = Batch { - nonce: 0, - frames: vec![ - Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key_a, - 0, - 1, - vec![1], - )], - safe_block: 8, - fee_price: 0, - }, - Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key_b, - 0, - 1, - vec![2], - )], - safe_block: 7, - fee_price: 0, - }, - ], - }; - + fn non_monotonic_safe_blocks_invalidate_the_batch() { + let mut h = Harness::new(config(100)); + let frames = vec![ + frame(8, 0, vec![signed(&key(4), 0, 1, vec![1])]), + frame(7, 0, vec![signed(&key(5), 0, 1, vec![2])]), + ]; assert_eq!( - process_ok(&mut scheduler, batch_input(10, invalid)).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::NonMonotonicSafeBlocks) + h.run(batch(10, 0, frames)), + (rejected(BatchRejectReason::NonMonotonicSafeBlocks), vec![]) ); - assert!(scheduler.app.events().is_empty()); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); + assert_eq!(h.state().next_batch_nonce, 0); } #[test] - fn frame_safe_block_above_inclusion_block_invalidates_batch() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - ); + fn a_first_frame_not_below_the_inclusion_block_invalidates_the_batch() { + for safe_block in [11, 10] { + let mut h = Harness::new(config(100)); + let op = signed(&key(6), 0, 1, vec![9]); + assert_eq!( + h.run(batch(10, 0, vec![frame(safe_block, 0, vec![op])])), + ( + rejected(BatchRejectReason::SafeBlockNotBelowInclusionBlock), + vec![] + ), + "safe block {safe_block} at inclusion 10" + ); + assert_eq!(h.state().next_batch_nonce, 0); + } + } - let signing_key = SigningKey::from_bytes((&[6_u8; 32]).into()).expect("signing key"); - let invalid = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key, - 0, - 1, - vec![9], - )], - safe_block: 11, - fee_price: 0, - }], - }; + #[test] + fn a_tail_frame_not_below_the_inclusion_block_invalidates_the_batch() { + for safe_block in [11, 10] { + let mut h = Harness::new(config(100)); + h.feed(direct(5, 1)); + assert_eq!( + h.run(batch(10, 0, vec![empty_frame(5), empty_frame(safe_block)])), + ( + rejected(BatchRejectReason::SafeBlockNotBelowInclusionBlock), + vec![] + ), + "tail safe block {safe_block} at inclusion 10: nothing drains" + ); + assert_eq!(h.state().next_batch_nonce, 0); + assert_eq!(h.state().fridge_len, 1); + } + } + #[test] + fn frame_drain_includes_its_safe_block_and_nothing_after() { + let mut h = Harness::new(config(100)); + h.feed(direct(10, 1)); + h.feed(direct(11, 2)); assert_eq!( - process_ok(&mut scheduler, batch_input(10, invalid)).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::SafeBlockAboveInclusionBlock) + h.run(batch(12, 0, vec![empty_frame(10)])), + (InputOutcome::BatchAccepted, vec![Tx::D(1)]) ); - assert!(scheduler.app.events().is_empty()); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); + assert_eq!(h.state().fridge_len, 1); } #[test] - fn frame_drain_uses_consistent_inclusive_safe_block_rule() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn decode_failure_invalidates_the_batch_and_keeps_running() { + let mut h = Harness::new(config(100)); + assert_eq!( + h.run(raw_batch(10, vec![0xFF, 0xEE, 0xDD])), + (rejected(BatchRejectReason::DecodeFailed), vec![]) ); - - process_ok(&mut scheduler, direct_input(10, 1)); - process_ok(&mut scheduler, direct_input(11, 2)); - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![], - safe_block: 10, - fee_price: 0, - }], - }; - - process_ok(&mut scheduler, batch_input(12, batch)); - assert_eq!(scheduler.app.events(), [RecordedTx::Direct(1)]); - assert_eq!(scheduler.queued_direct_len(), 1); + assert_eq!(h.state().next_batch_nonce, 0); + assert_eq!(h.run(direct(11, 3)), (InputOutcome::DirectQueued, vec![])); + assert_eq!(h.state().fridge_len, 1); } #[test] - fn decode_failure_invalidates_batch_and_keeps_running() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn wrong_batch_nonce_is_rejected_without_consuming_the_nonce() { + let mut h = Harness::new(config(100)); + assert_eq!( + h.run(batch(2, 1, vec![empty_frame(1)])), + ( + rejected(BatchRejectReason::WrongNonce { + expected: 0, + got: 1 + }), + vec![] + ) ); + assert_eq!(h.state().next_batch_nonce, 0); + } - let bad_batch = SchedulerInput { - sender: SEQUENCER, - inclusion_block: 10, - domain: test_domain(), - payload: vec![0xFF, 0xEE, 0xDD], - }; + #[test] + fn empty_batches_are_accepted_at_any_age_and_hand_out_nothing() { + let mut h = Harness::new(config(100)); + h.feed(direct(10, 1)); assert_eq!( - process_ok(&mut scheduler, bad_batch).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::DecodeFailed) + h.run(batch(10, 0, vec![])), + (InputOutcome::BatchAccepted, vec![]) ); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); - + assert_eq!(h.state().next_batch_nonce, 1); + assert_eq!(h.state().clock, 0, "an empty batch raises nothing"); + // Far past the window: the backstop drains, and the batch is still accepted. assert_eq!( - process_ok(&mut scheduler, direct_input(11, 3)).outcome, - ProcessOutcome::DirectEnqueued + h.run(batch(10_000, 1, vec![])), + (InputOutcome::BatchAccepted, vec![Tx::D(1)]) ); - assert_eq!(scheduler.queued_direct_len(), 1); + assert_eq!(h.state().next_batch_nonce, 2); } #[test] fn backstop_drains_all_overdue_directs() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 5, - }, - ); - - process_ok(&mut scheduler, direct_input(1, 1)); - process_ok(&mut scheduler, direct_input(2, 2)); - process_ok(&mut scheduler, direct_input(8, 3)); - + let mut h = Harness::new(config(5)); + h.feed(direct(1, 1)); + h.feed(direct(2, 2)); assert_eq!( - scheduler.app.events(), - [RecordedTx::Direct(1), RecordedTx::Direct(2)] + h.run(direct(8, 3)), + (InputOutcome::DirectQueued, vec![Tx::D(1), Tx::D(2)]) ); - assert_eq!(scheduler.queued_direct_len(), 1); + assert_eq!(h.state().fridge_len, 1); + assert_eq!(h.state().clock, 2); } #[test] - fn invalid_signature_is_skipped() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - ); + fn backstop_runs_before_a_rejected_or_stale_batch() { + let wrong_nonce = batch(8, 9, vec![empty_frame(7)]); + let stale = batch(8, 0, vec![empty_frame(1)]); + for (label, input) in [("wrong nonce", wrong_nonce), ("stale", stale)] { + let mut h = Harness::new(config(5)); + h.feed(direct(1, 1)); + let (outcome, txs) = h.run(input); + assert!( + matches!( + outcome, + InputOutcome::BatchRejected(_) | InputOutcome::BatchStale + ), + "{label}: {outcome:?}" + ); + assert_eq!( + txs, + [Tx::D(1)], + "{label}: the overdue direct still executes" + ); + assert_eq!(h.state().next_batch_nonce, 0, "{label}"); + assert_eq!(h.state().clock, 1, "{label}: the backstop raises the clock"); + } + } - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![WireUserOp { - nonce: 0, - max_fee: 0, - data: vec![7], - signature: vec![0_u8; WireUserOp::SIGNATURE_BYTES], - }], - safe_block: 1, - fee_price: 0, - }], + #[test] + fn invalid_signatures_are_not_handed_out() { + let mut h = Harness::new(config(100)); + let good = signed(&key(8), 0, 0, vec![3]); + let zero = WireUserOp { + signature: vec![0; WireUserOp::SIGNATURE_BYTES], + data: vec![1], + ..good.clone() + }; + let short_signature = WireUserOp { + signature: good.signature[..64].to_vec(), + data: vec![2], + ..good.clone() }; - assert_eq!( - process_ok(&mut scheduler, batch_input(1, batch)).outcome, - ProcessOutcome::BatchExecuted + h.run(batch( + 2, + 0, + vec![frame(1, 0, vec![zero, short_signature, good])] + )), + (InputOutcome::BatchAccepted, vec![Tx::Op(3)]) ); - assert!(scheduler.app.events().is_empty()); - assert_eq!(scheduler.next_expected_batch_nonce(), 1); + assert_eq!(h.state().next_batch_nonce, 1); } #[test] - fn invalid_nonce_max_fee_or_balance_is_skipped() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - ); - let signing_key = SigningKey::from_bytes((&[9_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - // fee_to_linear(0) = 1, so credit 1 unit — just enough for the cheapest fee. - scheduler.app.credit(sender, 1); - - let bad_nonce = sign_wire_user_op(&test_domain(), &signing_key, 1, 10, vec![1]); - let bad_max_fee = sign_wire_user_op(&test_domain(), &signing_key, 0, 0, vec![2]); - // max_fee=1000 is high enough, but fee_to_linear(1000) ≈ 2397 > balance of 1. - let insufficient = sign_wire_user_op(&test_domain(), &signing_key, 0, 1000, vec![3]); - let valid = sign_wire_user_op(&test_domain(), &signing_key, 0, 10, vec![4]); - - let batch = Batch { - nonce: 0, - frames: vec![ - Frame { - user_ops: vec![bad_nonce], - safe_block: 1, - fee_price: 1, - }, - Frame { - user_ops: vec![bad_max_fee], - safe_block: 1, - fee_price: 5, - }, - Frame { - user_ops: vec![insufficient], - safe_block: 1, - fee_price: 1000, - }, - Frame { - user_ops: vec![valid], - safe_block: 1, - fee_price: 0, - }, - ], - }; - + fn a_fee_cap_below_the_price_is_not_handed_out() { + let mut h = Harness::new(config(100)); + let signer = key(9); + let frames = vec![ + frame( + 1, + 5, + vec![ + signed(&signer, 0, 4, vec![1]), + signed(&signer, 0, 5, vec![2]), + ], + ), + frame(1, 0, vec![signed(&signer, 0, 0, vec![3])]), + ]; + // The nonce is not the scheduler's check: all three carry nonce 0. assert_eq!( - process_ok(&mut scheduler, batch_input(1, batch)).outcome, - ProcessOutcome::BatchExecuted + h.run(batch(2, 0, frames)), + (InputOutcome::BatchAccepted, vec![Tx::Op(2), Tx::Op(3)]) ); - assert_eq!(scheduler.app.events(), [RecordedTx::UserOp(4)]); } #[test] - fn empty_batches_are_valid_noops() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, + fn batch_uses_the_input_domain_for_signature_recovery() { + let mut h = Harness::new(config(100)); + let signer = key(10); + let app_contract = alloy_primitives::address!("0x3333333333333333333333333333333333333333"); + let batch_domain = crate::build_input_domain(CHAIN_ID + 7, app_contract); + let op = sign_in(&batch_domain, &signer, 0, 1, vec![9]); + let mut input = batch(2, 0, vec![frame(1, 0, vec![op.clone()])]); + input.chain_id = CHAIN_ID + 7; + input.app_contract = app_contract; + let (outcome, txs) = h.feed(input); + assert_eq!(outcome, InputOutcome::BatchAccepted); + let [ + Scheduled { + tx: L2Tx::UserOp(tx), + .. }, - ); - - let batch = Batch { - nonce: 0, - frames: vec![], + ] = txs.as_slice() + else { + panic!("one op: {txs:?}") + }; + assert_eq!(tx.sender, address_of(&signer)); + + // The default domain recovers some other address from the same op. + let (_, txs) = h.feed(batch(3, 1, vec![frame(2, 0, vec![op])])); + let [ + Scheduled { + tx: L2Tx::UserOp(tx), + .. + }, + ] = txs.as_slice() + else { + panic!("one op: {txs:?}") }; + assert_ne!(tx.sender, address_of(&signer)); + } - assert_eq!( - process_ok(&mut scheduler, batch_input(10, batch)).outcome, - ProcessOutcome::BatchExecuted + #[test] + fn batch_nonce_overflow_panics_before_any_frame_is_handed_out() { + let full = batch( + 2, + u64::MAX, + vec![frame(1, 0, vec![signed(&key(18), 0, 1, vec![7])])], ); - assert!(scheduler.app.events().is_empty()); - assert_eq!(scheduler.next_expected_batch_nonce(), 1); + let empty = batch(2, u64::MAX, vec![]); + for input in [full, empty] { + let mut h = at_state(100, u64::MAX, 0); + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + h.scheduler.feed(input); + })) + .expect_err("nonce exhaustion must fail loud"); + let message = panic + .downcast_ref::() + .map(String::as_str) + .or_else(|| panic.downcast_ref::<&str>().copied()); + assert_eq!( + message, + Some("batch nonce overflow: no canonical successor") + ); + assert_eq!( + h.scheduler.next_tx(), + None, + "the preflight runs before the frame's op is queued" + ); + assert_eq!(h.state().next_batch_nonce, u64::MAX); + } } + // ── The clock and gate f ────────────────────────────────────────────── + #[test] - fn batch_uses_input_domain_for_signature_recovery() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn gate_f_rejects_a_first_frame_below_the_clock_and_accepts_at_it() { + let mut h = Harness::new(config(100)); + h.feed(batch(12, 0, vec![empty_frame(10)])); + assert_eq!(h.state().clock, 10); + h.feed(direct(13, 1)); + assert_eq!( + h.run(batch( + 14, + 1, + vec![frame(9, 0, vec![signed(&key(1), 0, 0, vec![2])])] + )), + ( + rejected(BatchRejectReason::FirstFrameBelowClock { + safe_block: 9, + clock: 10 + }), + vec![] + ) ); - let signing_key = SigningKey::from_bytes((&[10_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - scheduler.app.credit(sender, 1); - let batch_domain = crate::build_input_domain( - TEST_CHAIN_ID + 7, - address!("0x3333333333333333333333333333333333333333"), + assert_eq!(h.state().next_batch_nonce, 1); + assert_eq!(h.state().fridge_len, 1); + + assert_eq!( + h.run(batch( + 14, + 1, + vec![ + frame(10, 0, vec![signed(&key(1), 0, 0, vec![2])]), + empty_frame(13) + ] + )), + (InputOutcome::BatchAccepted, vec![Tx::Op(2), Tx::D(1)]) ); - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &batch_domain, - &signing_key, - 0, - 1, - vec![9], - )], - safe_block: 1, - fee_price: 0, - }], - }; + assert_eq!(h.state().clock, 13); + } - let input = SchedulerInput { - sender: SEQUENCER, - inclusion_block: 1, - domain: batch_domain, - payload: ssz::Encode::as_ssz_bytes(&batch), + /// A batch that fails several gates gets the first one's outcome. Stale + /// versus rejected is what the reference reports, so the order is pinned. + #[test] + fn the_first_failing_gate_decides_the_outcome() { + let wrong_nonce = rejected(BatchRejectReason::WrongNonce { + expected: 0, + got: 1, + }); + let out_of_range = Frame { + fee_price: crate::fee::MAX_EXPONENT + 1, + ..empty_frame(4) }; + // (label, clock, input, outcome), at max_wait 5. + let cases = [ + ("b before c", 0, batch(20, 1, vec![]), wrong_nonce), + ( + "b before d", + 0, + batch(12, 1, vec![empty_frame(12)]), + wrong_nonce, + ), + ( + "b before e", + 0, + batch(20, 1, vec![empty_frame(4)]), + wrong_nonce, + ), + ( + "b before f", + 10, + batch(12, 1, vec![empty_frame(9)]), + wrong_nonce, + ), + ( + "d before e: a tail frame at inclusion", + 0, + batch(20, 0, vec![empty_frame(4), empty_frame(20)]), + rejected(BatchRejectReason::SafeBlockNotBelowInclusionBlock), + ), + ( + "d before e: decreasing frames", + 0, + batch(20, 0, vec![empty_frame(4), empty_frame(3)]), + rejected(BatchRejectReason::NonMonotonicSafeBlocks), + ), + ( + "d before e: fee out of range", + 0, + batch(20, 0, vec![out_of_range]), + rejected(BatchRejectReason::FeePriceOutOfRange), + ), + ( + "d before f", + 10, + batch(12, 0, vec![empty_frame(9), empty_frame(8)]), + rejected(BatchRejectReason::NonMonotonicSafeBlocks), + ), + ( + "e before f", + 10, + batch(20, 0, vec![empty_frame(4)]), + InputOutcome::BatchStale, + ), + ]; + for (label, clock, input, outcome) in cases { + let mut h = at_state(5, 0, clock); + assert_eq!(h.run(input), (outcome, vec![]), "{label}"); + assert_eq!(h.state().next_batch_nonce, 0, "{label}"); + } + } + #[test] + fn the_clock_rises_with_frames_and_every_drain() { + let mut h = Harness::new(bounded(5, 100, 100)); + // A frame raises it, with or without directs. + h.feed(batch(12, 0, vec![empty_frame(10)])); + assert_eq!(h.state().clock, 10); + // A backstop drain raises it to the drained direct's block; the bound + // is out of reach, so no overflow drain could have done it. + h.feed(direct(20, 1)); assert_eq!( - process_ok(&mut scheduler, input).outcome, - ProcessOutcome::BatchExecuted + h.run(direct(25, 2)), + (InputOutcome::DirectQueued, vec![Tx::D(1)]) ); - assert_eq!(scheduler.app.events(), [RecordedTx::UserOp(9)]); + assert_eq!(h.state().clock, 20); + // An overflow drain raises it to the drained block: bound 1, and block + // 25 is one block old, far inside the window. + h.scheduler.config.fridge_bound = 1; + assert_eq!( + h.run(direct(26, 3)), + (InputOutcome::DirectQueued, vec![Tx::D(2)]) + ); + assert_eq!(h.state().clock, 25); + // A later frame raises it past the drained block. + h.feed(batch(28, 1, vec![empty_frame(27)])); + assert_eq!(h.state().clock, 27); } + // ── The fridge ──────────────────────────────────────────────────────── + #[test] - fn inspect_exports_application_state_for_state_query() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, + fn the_payload_cap_drops_a_direct_that_then_drains_nothing() { + let mut h = Harness::new(bounded(100, 2, 2)); + assert_eq!( + h.run(direct_from(DIRECT_SENDER, 1, vec![1, 1])), + (InputOutcome::DirectQueued, vec![]) + ); + assert_eq!(h.run(direct(2, 2)), (InputOutcome::DirectQueued, vec![])); + let full = h.state(); + assert_eq!(full.fridge_len, 2, "the fridge is full"); + + assert_eq!( + h.run(direct_from(DIRECT_SENDER, 3, vec![3, 3, 3])), + (InputOutcome::DirectDropped, vec![]) + ); + assert_eq!( + h.state(), + SchedulerState { + l1_input_count: full.l1_input_count + 1, + ..full }, + "a dropped direct counts as an input and changes nothing else" ); + // An empty payload is within any cap. assert_eq!( - process_ok(&mut scheduler, direct_input(1, 7)).outcome, - ProcessOutcome::DirectEnqueued + h.run(direct_from(DIRECT_SENDER, 4, vec![])), + (InputOutcome::DirectQueued, vec![Tx::D(1)]) ); - // Inspect reflects executed app state, not the direct-input queue. - let state = scheduler - .inspect_state(STATE_INSPECT_QUERY) - .expect("inspect state"); - assert_eq!(state, b"events:0"); + } - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![], - safe_block: 1, - fee_price: 0, - }], - }; + #[test] + fn overflow_drains_whole_blocks_oldest_first() { + let mut h = Harness::new(bounded(100, 3, 100)); + h.feed(direct(1, 1)); + h.feed(direct(1, 2)); + h.feed(direct(2, 3)); + // One entry would make room, but block 1 leaves whole. assert_eq!( - process_ok(&mut scheduler, batch_input(2, batch)).outcome, - ProcessOutcome::BatchExecuted + h.run(direct(3, 4)), + (InputOutcome::DirectQueued, vec![Tx::D(1), Tx::D(2)]) ); - - let state = scheduler - .inspect_state(STATE_INSPECT_QUERY) - .expect("inspect state after drain"); - assert_eq!(state, b"events:1"); + assert_eq!(h.state().clock, 1); + assert_eq!(h.state().fridge_len, 2); + assert_eq!(h.state().first_queued_input, 2); } #[test] - fn inspect_rejects_unsupported_query() { - let scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn overflow_never_drains_the_arriving_block() { + let mut h = Harness::new(bounded(100, 2, 100)); + h.feed(direct(5, 1)); + h.feed(direct(5, 2)); + // Only the arriving block remains: the direct joins over the bound. + assert_eq!(h.run(direct(5, 3)), (InputOutcome::DirectQueued, vec![])); + assert_eq!(h.state().fridge_len, 3); + assert_eq!(h.state().clock, 0); + // The next block drains all of block 5 at once. + assert_eq!( + h.run(direct(6, 4)), + ( + InputOutcome::DirectQueued, + vec![Tx::D(1), Tx::D(2), Tx::D(3)] + ) ); + assert_eq!(h.state().fridge_len, 1); + assert_eq!(h.state().clock, 5); + } + #[test] + fn a_zero_bound_drains_every_older_block_on_arrival() { + let mut h = Harness::new(bounded(100, 0, 100)); + assert_eq!(h.run(direct(1, 1)), (InputOutcome::DirectQueued, vec![])); + assert_eq!(h.run(direct(1, 2)), (InputOutcome::DirectQueued, vec![])); assert_eq!( - scheduler.inspect_state(b"balances"), - Err(InspectError::UnsupportedQuery) + h.run(direct(2, 3)), + (InputOutcome::DirectQueued, vec![Tx::D(1), Tx::D(2)]) ); } + /// A batch sealed before an overflow it could not predict: accepted with + /// the hand-out order the sequencer predicted when its first frame is at or + /// above the drained block, rejected by gate f when below. #[test] - fn wrong_batch_nonce_is_rejected_without_consuming_nonce() { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + fn an_overflow_keeps_the_order_of_a_later_batch_or_gets_it_rejected() { + let directs = [direct(1, 1), direct(2, 2), direct(3, 3), direct(4, 4)]; + let run_all = |fridge_bound: u64, safe_block: u64| { + let mut h = Harness::new(bounded(100, fridge_bound, 100)); + let mut stream = Vec::new(); + for input in directs.iter().cloned() { + stream.extend(h.feed(input).1); + } + let (outcome, txs) = h.feed(batch( + 6, + 0, + vec![frame(safe_block, 0, vec![signed(&key(3), 0, 0, vec![9])])], + )); + stream.extend(txs); + (outcome, stream) + }; + + // Bound 3: d@4 drains block 1 alone, clock 1. + let (outcome, overflowed) = run_all(3, 2); + assert_eq!(outcome, InputOutcome::BatchAccepted); + let (_, unbounded) = run_all(100, 2); + assert_eq!( + overflowed, unbounded, + "the same transactions, the same labels" + ); + assert_eq!( + overflowed.iter().map(|s| short(&s.tx)).collect::>(), + [Tx::D(1), Tx::D(2), Tx::Op(9)] ); - let batch = Batch { - nonce: 1, - frames: vec![Frame { - user_ops: vec![], + // Bound 2: d@3 drains block 1 and d@4 drains block 2, clock 2. + let (outcome, _) = run_all(2, 1); + assert_eq!( + outcome, + rejected(BatchRejectReason::FirstFrameBelowClock { safe_block: 1, - fee_price: 0, - }], - }; + clock: 2 + }) + ); + assert_eq!(run_all(100, 1).0, InputOutcome::BatchAccepted); + } + #[test] + fn batch_nonce_labels_name_the_nonce_at_hand_out() { + let mut h = at_state(5, 3, 0); + h.scheduler.config.fridge_bound = 2; + let labels = |txs: Vec| -> Vec<(Tx, u64)> { + txs.iter().map(|s| (short(&s.tx), s.batch_nonce)).collect() + }; + h.feed(direct(10, 1)); + h.feed(direct(11, 2)); + // Overflow drain on a direct: the next batch's nonce. + assert_eq!(labels(h.feed(direct(12, 3)).1), [(Tx::D(1), 3)]); + // Frame drain and op: the accepted batch's own nonce. + let op = signed(&key(4), 0, 0, vec![7]); assert_eq!( - process_ok(&mut scheduler, batch_input(1, batch)).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::WrongNonce { - expected: 0, - got: 1, - }) + labels(h.feed(batch(13, 3, vec![frame(11, 0, vec![op])])).1), + [(Tx::D(2), 3), (Tx::Op(7), 3)] + ); + assert_eq!(h.state().next_batch_nonce, 4); + // Backstop drain on a direct: the next batch's nonce. + assert_eq!(labels(h.feed(direct(20, 4)).1), [(Tx::D(3), 4)]); + // Backstop drain on an accepted batch: that batch's nonce. + assert_eq!( + labels(h.feed(batch(26, 4, vec![empty_frame(22)])).1), + [(Tx::D(4), 4)] ); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); - assert!(scheduler.app.events().is_empty()); + // Backstop drain on a rejected batch: the nonce the next batch needs. + h.feed(direct(30, 5)); + assert_eq!(labels(h.feed(batch(36, 9, vec![])).1), [(Tx::D(5), 5)]); + assert_eq!(h.state().next_batch_nonce, 5); } - // ── I1 duality: off-chain predicate vs canonical fold ───────────────── - // - // `ProtocolTiming::scheduler_accepts` (the off-chain gold-frontier predicate) - // and `Scheduler::process_input` (the canonical fold) are hand-maintained in - // separate files and MUST agree on accept-vs-reject for every input — the - // system's most load-bearing invariant (I1). Nothing exercised both until - // now. The one *documented* divergence is the predicate's structural-reject - // omission (it trusts the sequencer to emit well-formed batches), pinned - // explicitly at the end so any *other* drift fails this test. + // ── State and resume ────────────────────────────────────────────────── #[test] - #[should_panic( - expected = "a genesis scheduler application must start at executed_input_count = 0" - )] - fn genesis_scheduler_rejects_nonzero_application_progress() { - let _ = Scheduler::new( - RecordingApp::with_progress(1, 0), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, + fn state_names_the_first_queued_input() { + let mut h = Harness::new(config(100)); + let genesis = SchedulerState { + l1_input_count: 0, + next_batch_nonce: 0, + clock: 0, + first_queued_input: 0, + fridge_len: 0, + }; + assert_eq!(h.state(), genesis); + h.feed(batch(3, 0, vec![empty_frame(2)])); + h.feed(direct(4, 1)); + h.feed(direct(5, 2)); + assert_eq!( + h.state(), + SchedulerState { + l1_input_count: 3, + next_batch_nonce: 1, + clock: 2, + first_queued_input: 1, + fridge_len: 2, + } + ); + h.feed(batch(6, 1, vec![empty_frame(5)])); + assert_eq!( + h.state(), + SchedulerState { + l1_input_count: 4, + next_batch_nonce: 2, + clock: 5, + first_queued_input: 4, + fridge_len: 0, }, + "with nothing queued, the first queued input is the input count" ); } #[test] - fn application_error_is_fatal_before_later_user_ops_and_nonce_commit() { - let app = RecordingApp { - fail_on: Some(RecordedTx::UserOp(1)), - ..RecordingApp::default() - }; - let mut scheduler = Scheduler::new( - app, - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, + #[should_panic(expected = "inputs must arrive in InputBox order")] + fn feed_panics_on_an_index_gap() { + let mut scheduler = Scheduler::genesis(config(100)); + let mut input = direct(1, 1); + input.index = 1; + scheduler.feed(input); + } + + #[test] + #[should_panic(expected = "the previous input still has transactions to hand out")] + fn feed_panics_while_transactions_are_left() { + let mut scheduler = Scheduler::genesis(config(100)); + scheduler.feed(direct(1, 1)); + let mut input = batch(3, 0, vec![empty_frame(2)]); + input.index = 1; + scheduler.feed(input); + let mut next = direct(4, 2); + next.index = 2; + scheduler.feed(next); + } + + /// With a zero window the backstop would drain the arriving block, raise + /// the clock to it, and then queue a same-block direct at the clock. + #[test] + fn a_zero_window_is_refused_at_genesis_and_resume() { + let genesis = std::panic::catch_unwind(|| Scheduler::genesis(config(0))); + let resume = std::panic::catch_unwind(|| { + let state = Scheduler::genesis(config(1)).state(); + Scheduler::resume(config(0), state, Vec::new()) + }); + for (label, result) in [ + ("genesis", genesis.map(|_| ())), + ("resume", resume.map(|_| ())), + ] { + let panic = result.expect_err(label); + assert_eq!( + panic.downcast_ref::<&str>().copied(), + Some("max_wait_blocks must be at least 1"), + "{label}" + ); + } + // One block is enough: the arriving block never drains. + let mut h = Harness::new(config(1)); + h.feed(direct(10, 1)); + assert_eq!(h.run(direct(10, 2)), (InputOutcome::DirectQueued, vec![])); + assert_eq!( + h.run(direct(11, 3)), + (InputOutcome::DirectQueued, vec![Tx::D(1), Tx::D(2)]) ); + h.resume_here(); + } - let signing_key = SigningKey::from_bytes((&[17_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - scheduler.app.credit(sender, 2); - let batch = Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![ - sign_wire_user_op(&test_domain(), &signing_key, 0, 1, vec![1]), - sign_wire_user_op(&test_domain(), &signing_key, 1, 1, vec![2]), - ], - safe_block: 1, - fee_price: 0, - }], - }; + #[test] + fn resume_continues_the_batch_nonce() { + let mut h = at_state(100, 5, 0); + for (block, nonce) in [(100, 5), (110, 6), (120, 7)] { + assert_eq!( + h.run(batch(block, nonce, vec![])).0, + InputOutcome::BatchAccepted + ); + } + assert_eq!(h.state().next_batch_nonce, 8); + } - let error = scheduler - .process_input(batch_input(1, batch)) - .expect_err("application error must abort scheduler processing"); - let AppError::Internal { reason } = error else { - panic!("unexpected application error: {error}"); + #[test] + fn resumed_queued_directs_drain_on_the_first_covering_frame() { + let mut first = direct(10, 0xAA); + first.index = 3; + let mut gap = batch(15, 0, vec![]); + gap.index = 4; + let mut second = direct(20, 0xBB); + second.index = 5; + let state = SchedulerState { + l1_input_count: 6, + next_batch_nonce: 2, + clock: 9, + first_queued_input: 3, + fridge_len: 2, }; - assert_eq!(reason, "refusing UserOp(1)"); - assert!(scheduler.app.events().is_empty()); + let mut h = Harness::resumed(config(100), state, vec![first, gap, second]).expect("resume"); + assert_eq!(h.state(), state); assert_eq!( - scheduler.app.executed_input_count(), - ExecutedInputCount::ZERO + h.run(batch(30, 2, vec![empty_frame(20)])), + (InputOutcome::BatchAccepted, vec![Tx::D(0xAA), Tx::D(0xBB)]) ); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); + assert_eq!(h.state().next_batch_nonce, 3); } #[test] - fn overdue_direct_error_is_fatal_before_current_input_is_classified() { - let app = RecordingApp { - fail_on: Some(RecordedTx::Direct(1)), - ..RecordingApp::default() + fn resume_refuses_a_range_that_does_not_match_the_state() { + let indexed = |mut input: L1Input, index: u64| { + input.index = index; + input }; - let mut scheduler = Scheduler::new( - app, - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 5, - }, - ); + // Inputs 2..5: a direct, a batch, a dropped direct (cap 2), a direct. + let range = || { + vec![ + indexed(direct(10, 1), 2), + indexed(batch(11, 0, vec![]), 3), + indexed(direct_from(DIRECT_SENDER, 12, vec![0; 3]), 4), + indexed(direct(13, 2), 5), + ] + }; + let state = SchedulerState { + l1_input_count: 6, + next_batch_nonce: 0, + clock: 9, + first_queued_input: 2, + fridge_len: 2, + }; + let cfg = bounded(100, 10, 2); + let resume = |state: SchedulerState, queued: Vec| { + Scheduler::resume(cfg, state, queued).map(|s| s.state()) + }; + assert_eq!(resume(state, range()), Ok(state)); assert_eq!( - process_ok(&mut scheduler, direct_input(1, 1)).outcome, - ProcessOutcome::DirectEnqueued + resume( + SchedulerState { + first_queued_input: 7, + ..state + }, + Vec::new() + ), + Err(ResumeError::FirstQueuedAfterCount { + first_queued_input: 7, + l1_input_count: 6 + }) + ); + let mut gapped = range(); + gapped.remove(1); + assert_eq!( + resume(state, gapped), + Err(ResumeError::OutOfSequence { + expected: 3, + got: 4 + }) ); - let error = scheduler - .process_input(direct_input(6, 2)) - .expect_err("overdue direct failure must abort scheduler processing"); - let AppError::Internal { reason } = error else { - panic!("unexpected application error: {error}"); - }; - assert_eq!(reason, "refusing Direct(1)"); - assert!(scheduler.app.events().is_empty()); + let mut long = range(); + long.push(indexed(direct(14, 3), 6)); assert_eq!( - scheduler.app.executed_input_count(), - ExecutedInputCount::ZERO + resume(state, long), + Err(ResumeError::BeyondInputCount { + index: 6, + l1_input_count: 6 + }) + ); + let mut short_range = range(); + short_range.pop(); + assert_eq!( + resume(state, short_range), + Err(ResumeError::MissingInputs { + next: 5, + l1_input_count: 6 + }) + ); + assert_eq!( + resume( + SchedulerState { + first_queued_input: 3, + fridge_len: 1, + ..state + }, + range().split_off(1) + ), + Err(ResumeError::FirstQueuedNotADirect { index: 3 }) + ); + assert_eq!( + resume( + SchedulerState { + first_queued_input: 4, + fridge_len: 1, + ..state + }, + range().split_off(2) + ), + Err(ResumeError::FirstQueuedNotADirect { index: 4 }), + "a direct over the cap is not admitted" + ); + // Nothing queued, yet the state claims input 5: only an admitted + // direct may be the first queued input, so `state()` round-trips. + assert_eq!( + resume( + SchedulerState { + first_queued_input: 5, + fridge_len: 0, + ..state + }, + vec![indexed(batch(13, 0, vec![]), 5)] + ), + Err(ResumeError::FirstQueuedNotADirect { index: 5 }) + ); + let mut lowered = range(); + lowered[1].block = 9; + assert_eq!( + resume(state, lowered), + Err(ResumeError::BlockDecreases { + index: 3, + block: 9, + previous_block: 10 + }), + "any input, admitted or not, keeps the InputBox's block order" + ); + // Admitted directs out of block order: drains stop at the newer front. + assert_eq!( + resume( + SchedulerState { + l1_input_count: 2, + first_queued_input: 0, + ..state + }, + vec![indexed(direct(20, 1), 0), indexed(direct(15, 2), 1)] + ), + Err(ResumeError::BlockDecreases { + index: 1, + block: 15, + previous_block: 20 + }) + ); + assert_eq!( + resume(SchedulerState { clock: 10, ..state }, range()), + Err(ResumeError::DirectNotAboveClock { + index: 2, + block: 10, + clock: 10 + }) + ); + assert_eq!( + resume( + SchedulerState { + fridge_len: 3, + ..state + }, + range() + ), + Err(ResumeError::FridgeLenMismatch { + expected: 3, + admitted: 2 + }) + ); + assert_eq!( + resume( + SchedulerState { + first_queued_input: 6, + ..state + }, + Vec::new() + ), + Err(ResumeError::FridgeLenMismatch { + expected: 2, + admitted: 0 + }) ); - assert_eq!(scheduler.queued_direct_len(), 1); } - #[test] - fn batch_nonce_overflow_panics_before_application_mutation() { - let mut scheduler = Scheduler::resume_at( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - u64::MAX, - ); - let signing_key = SigningKey::from_bytes((&[18_u8; 32]).into()).expect("signing key"); - let sender = address_from_signing_key(&signing_key); - scheduler.app.credit(sender, 1); - let batch = Batch { - nonce: u64::MAX, - frames: vec![Frame { - user_ops: vec![sign_wire_user_op( - &test_domain(), - &signing_key, - 0, - 1, - vec![7], - )], - safe_block: 1, - fee_price: 0, - }], + /// A long scenario touching every rule: overflow, the cap, gate f, the + /// backstop, staleness, a soft bound, a decode failure, a wrong nonce. + fn mixed_scenario() -> (SchedulerConfig, Vec) { + let (k1, k2) = (key(21), key(22)); + let bad = WireUserOp { + signature: vec![0; WireUserOp::SIGNATURE_BYTES], + ..signed(&k2, 5, 0, vec![99]) }; + let inputs = vec![ + direct(10, 1), + direct(10, 2), + direct_from(DIRECT_SENDER, 11, vec![3, 0, 0]), + batch( + 12, + 0, + vec![ + frame(10, 1, vec![signed(&k1, 0, 2, vec![10])]), + frame(11, 0, vec![signed(&k2, 0, 0, vec![11]), bad]), + ], + ), + direct(12, 4), + direct(13, 5), + direct(13, 6), + direct(14, 7), + batch(15, 1, vec![empty_frame(11)]), + batch(15, 1, vec![frame(13, 0, vec![signed(&k1, 1, 0, vec![12])])]), + raw_batch(15, vec![0xFF]), + direct(16, 8), + batch(16, 2, vec![]), + direct(20, 9), + direct(23, 10), + batch(30, 3, vec![empty_frame(21)]), + batch(31, 3, vec![frame(26, 0, vec![signed(&k2, 1, 0, vec![13])])]), + direct(32, 11), + batch(33, 9, vec![empty_frame(32)]), + direct(33, 12), + batch( + 34, + 4, + vec![ + frame(32, 0, vec![signed(&k1, 2, 0, vec![14])]), + empty_frame(33), + ], + ), + direct(35, 13), + direct(35, 14), + direct(35, 15), + direct(35, 16), + direct(36, 17), + batch(37, 5, vec![empty_frame(36)]), + ]; + (bounded(6, 3, 2), inputs) + } - let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = scheduler.process_input(batch_input(1, batch)); - })); - assert!(panic.is_err(), "nonce exhaustion must fail loud"); - assert!(scheduler.app.events().is_empty()); + #[test] + fn the_mixed_scenario_hands_out_what_the_rules_say() { + let (cfg, inputs) = mixed_scenario(); + let mut h = Harness::new(cfg); + let runs: Vec<(InputOutcome, Vec)> = inputs.into_iter().map(|i| h.run(i)).collect(); + use InputOutcome::*; + use Tx::*; + let expected: Vec<(InputOutcome, Vec)> = vec![ + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectDropped, vec![]), + (BatchAccepted, vec![D(1), D(2), Op(10), Op(11)]), + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectQueued, vec![D(4)]), + ( + rejected(BatchRejectReason::FirstFrameBelowClock { + safe_block: 11, + clock: 12, + }), + vec![], + ), + (BatchAccepted, vec![D(5), D(6), Op(12)]), + (rejected(BatchRejectReason::DecodeFailed), vec![]), + (DirectQueued, vec![]), + (BatchAccepted, vec![]), + (DirectQueued, vec![D(7)]), + (DirectQueued, vec![D(8)]), + (BatchStale, vec![D(9), D(10)]), + (BatchAccepted, vec![Op(13)]), + (DirectQueued, vec![]), + ( + rejected(BatchRejectReason::WrongNonce { + expected: 4, + got: 9, + }), + vec![], + ), + (DirectQueued, vec![]), + (BatchAccepted, vec![D(11), Op(14), D(12)]), + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectQueued, vec![]), + (DirectQueued, vec![D(13), D(14), D(15), D(16)]), + (BatchAccepted, vec![D(17)]), + ]; + assert_eq!(runs, expected); assert_eq!( - scheduler.app.executed_input_count(), - ExecutedInputCount::ZERO + h.state(), + SchedulerState { + l1_input_count: 27, + next_batch_nonce: 6, + clock: 36, + first_queued_input: 27, + fridge_len: 0, + } ); - assert_eq!(scheduler.next_expected_batch_nonce(), u64::MAX); } #[test] - fn executed_input_count_overflow_panics_before_application_mutation() { - let mut scheduler = Scheduler::resume_at( - RecordingApp::with_progress(u64::MAX, 0), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - 0, - ); - assert_eq!( - process_ok(&mut scheduler, direct_input(1, 9)).outcome, - ProcessOutcome::DirectEnqueued - ); + fn a_resumed_scheduler_hands_out_what_the_original_does_at_every_point() { + let (cfg, inputs) = mixed_scenario(); + let mut original = Harness::new(cfg); + for split in 0..=inputs.len() { + let mut resumed = original.resume_here(); + assert_eq!(resumed.state(), original.state(), "split {split}"); + let mut reference = Harness { + scheduler: original.scheduler.clone(), + inputs: original.inputs.clone(), + admitted: original.admitted.clone(), + handed_out: original.handed_out.clone(), + last_block: original.last_block, + }; + for input in &inputs[split..] { + assert_eq!( + resumed.feed(input.clone()), + reference.feed(input.clone()), + "split {split}" + ); + } + assert_eq!(resumed.state(), reference.state(), "split {split}"); + if let Some(input) = inputs.get(split) { + original.feed(input.clone()); + } + } + } - let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = scheduler.drain_covered_at(1); - })); - assert!(panic.is_err(), "input-count exhaustion must fail loud"); - assert!(scheduler.app.events().is_empty()); - assert_eq!( - scheduler.app.executed_input_count(), - ExecutedInputCount::new(u64::MAX) - ); - assert_eq!(scheduler.queued_direct_len(), 1); + /// A fixed-seed linear congruential generator: reproducible, no dependency. + struct Lcg(u64); + + impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } } - const DUALITY_MAX_WAIT: u64 = 5; + #[test] + fn a_pseudo_random_run_keeps_the_split_and_resumes_equivalently() { + const INPUTS: usize = 3_000; + const RESUME_EVERY: usize = 97; + let max_wait = 8; + let cfg = bounded(max_wait, 4, 2); + let keys = [key(31), key(32)]; + let mut pool = Vec::new(); + for (k, signer) in keys.iter().enumerate() { + for nonce in 0..3 { + for max_fee in [0, 3] { + let marker = u8::try_from(pool.len()).unwrap() + 100 * u8::try_from(k).unwrap(); + pool.push(signed(signer, nonce, max_fee, vec![marker])); + } + } + } + pool.push(WireUserOp { + signature: vec![0; WireUserOp::SIGNATURE_BYTES], + ..pool[0].clone() + }); + + let mut rng = Lcg(0x5EED_CAFE); + let mut main = Harness::new(cfg); + let mut shadow = main.resume_here(); + let mut block = 10_u64; + let mut outcomes: BTreeMap = BTreeMap::new(); + let mut ops_handed_out = 0; + let mut over_bound = false; + let (mut backstop_drains, mut overflow_drains) = (0_u64, 0_u64); + for step in 0..INPUTS { + block += rng.below(3); + let input = if rng.chance(55) { + let sender = if rng.chance(85) { + DIRECT_SENDER + } else { + Address::repeat_byte(0x44) + }; + let len = rng.below(4) as usize; + let marker = u8::try_from(step % 256).unwrap(); + direct_from(sender, block, vec![marker; len]) + } else if rng.chance(4) { + raw_batch(block, vec![0xFF; rng.below(5) as usize]) + } else { + let expected = main.state().next_batch_nonce; + let nonce = if rng.chance(90) { + expected + } else { + expected + 1 + rng.below(2) + }; + let mut frames = Vec::new(); + let mut safe_block = if rng.chance(70) { + block.saturating_sub(1 + rng.below(3)) + } else { + block.saturating_sub(1 + rng.below(max_wait + 3)) + }; + for _ in 0..rng.below(4) { + if rng.chance(3) { + safe_block = block; + } else if rng.chance(3) { + safe_block = safe_block.saturating_sub(1); + } + let fee_price = if rng.chance(2) { + crate::fee::MAX_EXPONENT + 1 + } else { + u16::try_from(rng.below(4)).unwrap() + }; + let ops = (0..rng.below(3)) + .map(|_| pool[rng.below(pool.len() as u64) as usize].clone()) + .collect(); + frames.push(frame(safe_block, fee_price, ops)); + safe_block = (safe_block + rng.below(3)).min(block.saturating_sub(1)); + } + batch(block, nonce, frames) + }; + + let (input_block, is_direct) = (input.block, input.sender != SEQUENCER); + let (outcome, txs) = main.feed(input.clone()); + assert_eq!(shadow.feed(input), (outcome, txs.clone()), "step {step}"); + // Overdue directs sit at the front and the backstop runs first, so + // an overdue drain is the backstop's; on a direct, any other drain + // is the overflow's. + for scheduled in &txs { + if let L2Tx::Direct(drained) = &scheduled.tx { + if age_exceeds(input_block, drained.block, max_wait) { + backstop_drains += 1; + } else if is_direct { + overflow_drains += 1; + } + } + } + assert_eq!(shadow.state(), main.state(), "step {step}"); + let kind = format!("{outcome:?}"); + let kind = kind + .split([' ', '(', ')', '{']) + .filter(|word| !word.is_empty()) + .take(2) + .collect::>() + .join(" "); + *outcomes.entry(kind).or_default() += 1; + ops_handed_out += txs + .iter() + .filter(|s| matches!(s.tx, L2Tx::UserOp(_))) + .count(); + over_bound |= main.state().fridge_len > cfg.fridge_bound; + if step % RESUME_EVERY == 0 { + shadow = main.resume_here(); + } + } - fn duality_timing() -> crate::protocol::ProtocolTiming { - crate::protocol::ProtocolTiming::try_new(DUALITY_MAX_WAIT, 1, 1, 12) - .expect("valid duality timing") + // The run must have reached every rule it exists to exercise. + for kind in [ + "DirectQueued", + "DirectDropped", + "BatchAccepted", + "BatchStale", + "BatchRejected DecodeFailed", + "BatchRejected WrongNonce", + "BatchRejected SafeBlockNotBelowInclusionBlock", + "BatchRejected NonMonotonicSafeBlocks", + "BatchRejected FeePriceOutOfRange", + "BatchRejected FirstFrameBelowClock", + ] { + assert!( + outcomes.get(kind).is_some_and(|n| *n > 0), + "never saw {kind}: {outcomes:?}" + ); + } + assert!(ops_handed_out > 100, "{ops_handed_out} ops handed out"); + assert!(over_bound, "the soft bound never showed"); + assert!(backstop_drains > 0, "the backstop never drained"); + assert!(overflow_drains > 0, "the overflow never drained"); } - // A frame with no user ops — accept/reject is then purely about - // sender/nonce/structure/staleness, no app credit needed. - fn empty_frame(safe_block: u64) -> Frame { - Frame { - user_ops: vec![], - safe_block, - fee_price: 0, + // ── Export ──────────────────────────────────────────────────────────── + + #[test] + fn the_export_is_six_little_endian_fields_in_order() { + let export = SchedulerExport { + l1_input_count: 1, + l2_tx_count: 2, + next_batch_nonce: 3, + clock: 0x0102_0304_0506_0708, + first_queued_input: 5, + fridge_len: u64::MAX, + }; + let bytes = export.encode(); + let mut expected = Vec::new(); + for field in [1_u64, 2, 3, 0x0102_0304_0506_0708, 5, u64::MAX] { + expected.extend_from_slice(&field.to_le_bytes()); + } + assert_eq!(bytes.as_slice(), expected.as_slice()); + assert_eq!(&bytes[24..32], &[8, 7, 6, 5, 4, 3, 2, 1]); + assert_eq!(SchedulerExport::decode(&bytes), Ok(export)); + for len in [0, 47, 49] { + assert_eq!( + SchedulerExport::decode(&vec![0; len]), + Err(SchedulerExportLengthError { got: len }) + ); + } + let state = export.scheduler_state(); + assert_eq!(SchedulerExport::new(state, L2TxCount::new(2)), export); + } + + // ── Signatures and the off-chain predicate ──────────────────────────── + + /// What the guest's recovery makes of each pinned signature variant: the + /// sender it hands the op out as, or `None` when the op is dropped. The + /// ingress side of the same table is pinned in `api.rs`. + #[test] + fn signature_variants_recover_the_pinned_guest_sender() { + let vectors = crate::consensus_vectors::vectors(); + let pinned = &vectors.signature; + let domain = vectors.domain(&pinned.domain); + for case in &pinned.variants { + let wire_user_op = WireUserOp { + nonce: pinned.user_op.nonce, + max_fee: pinned.user_op.max_fee, + data: pinned.user_op.data.to_vec(), + signature: case.signature.to_vec(), + }; + assert_eq!( + recover_sender(&domain, &wire_user_op), + case.guest, + "{}", + case.name + ); } } - /// Run one input through BOTH sides at `expected_nonce`; return whether each - /// *accepted* it (canonical `BatchExecuted` / predicate `Some`). + // `protocol::scheduler_accepts` (the off-chain predicate) and the + // scheduler are maintained apart and must agree on accept-vs-reject (I1). + // The documented self-trust omissions are pinned at the end, so any other + // drift fails this test. + + const DUALITY_MAX_WAIT: u64 = 5; + + /// Whether each side accepts one input at `expected_nonce` and `clock`. fn duality_run( sender: Address, inclusion: u64, expected_nonce: u64, + clock: u64, payload: &[u8], ) -> (bool, bool) { - use crate::protocol::SafeInputView; - let mut scheduler = Scheduler::resume_at( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: DUALITY_MAX_WAIT, - }, - expected_nonce, - ); - let canonical_executed = process_ok( - &mut scheduler, - SchedulerInput { + let mut h = at_state(DUALITY_MAX_WAIT, expected_nonce, clock); + let mut input = raw_batch(inclusion, payload.to_vec()); + input.sender = sender; + let canonical = h.feed(input).0 == InputOutcome::BatchAccepted; + let offchain = crate::protocol::scheduler_accepts( + SEQUENCER, + DUALITY_MAX_WAIT, + crate::protocol::SafeInputView { + safe_input_index: 0, sender, + payload, inclusion_block: inclusion, - domain: test_domain(), - payload: payload.to_vec(), }, + expected_nonce, ) - .outcome - == ProcessOutcome::BatchExecuted; - let offchain_accepted = duality_timing() - .scheduler_accepts( - SEQUENCER, - SafeInputView { - safe_input_index: 0, - sender, - payload, - inclusion_block: inclusion, - }, - expected_nonce, - ) - .is_some(); - (canonical_executed, offchain_accepted) + .is_some(); + (canonical, offchain) } fn ssz(batch: &Batch) -> Vec { @@ -1449,235 +2273,293 @@ mod tests { #[test] fn scheduler_accepts_agrees_with_canonical_on_accept_reject() { - // (label, sender, inclusion, expected_nonce, payload) — structurally - // valid inputs where the two sides MUST agree. - let valid_fresh = ssz(&Batch { - nonce: 0, - frames: vec![empty_frame(10)], - }); - let empty_frames = ssz(&Batch { + let one_frame = |nonce: u64, safe_block: u64| { + ssz(&Batch { + nonce, + frames: vec![empty_frame(safe_block)], + }) + }; + let empty = ssz(&Batch { nonce: 0, frames: vec![], }); - let wrong_nonce = ssz(&Batch { - nonce: 1, - frames: vec![empty_frame(10)], - }); - let stale = ssz(&Batch { - nonce: 0, - frames: vec![empty_frame(1)], - }); - let fresh_at_boundary = ssz(&Batch { - nonce: 0, - frames: vec![empty_frame(10)], - }); - - let agree_cases: &[(&str, Address, u64, u64, &[u8])] = &[ - ("valid fresh, right nonce", SEQUENCER, 12, 0, &valid_fresh), + let agree: &[(&str, Address, u64, u64, Vec)] = &[ ( - "empty frames (no-op batch)", + "valid fresh, right nonce", SEQUENCER, - 100, + 12, 0, - &empty_frames, + one_frame(0, 10), ), - ("wrong nonce", SEQUENCER, 12, 0, &wrong_nonce), - // age = 10 - 1 = 9 >= MAX_WAIT(5): stale on both sides. - ("stale by first frame", SEQUENCER, 10, 0, &stale), - // wrong sender: canonical enqueues a direct, predicate rejects sender. - ("wrong sender", DIRECT_SENDER, 12, 0, &valid_fresh), - // garbage payload: DecodeFailed / decode error. - ("garbage payload", SEQUENCER, 1, 0, &[0xFF, 0xEE, 0xDD]), - // staleness boundary: age = 14 - 10 = 4 < 5 accepted. + ("empty frames", SEQUENCER, 100, 0, empty.clone()), + ("wrong nonce", SEQUENCER, 12, 0, one_frame(1, 10)), + ("stale by first frame", SEQUENCER, 10, 0, one_frame(0, 1)), + ("wrong sender", DIRECT_SENDER, 12, 0, one_frame(0, 10)), + ("garbage payload", SEQUENCER, 1, 0, vec![0xFF, 0xEE, 0xDD]), + ("fresh just under stale", SEQUENCER, 14, 0, one_frame(0, 10)), + ("stale at the boundary", SEQUENCER, 15, 0, one_frame(0, 10)), ( - "fresh just under stale", + "first frame at the clock", SEQUENCER, - 14, + 12, + 10, + one_frame(0, 10), + ), + ]; + for (label, sender, inclusion, clock, payload) in agree { + let (canonical, offchain) = duality_run(*sender, *inclusion, 0, *clock, payload); + assert_eq!( + canonical, offchain, + "I1 disagreement on `{label}`: canonical={canonical}, offchain={offchain}" + ); + } + + // The documented self-trust omissions: the scheduler rejects, the + // predicate accepts. If either side changes, these flip and force a + // deliberate update. + let omitted: &[(&str, u64, Vec)] = &[ + ( + "non-monotonic safe blocks", 0, - &fresh_at_boundary, + ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(8), empty_frame(7)], + }), ), - // age = 15 - 10 = 5 >= 5: stale. ( - "stale just at boundary", - SEQUENCER, - 15, + "fee price out of range", 0, - &fresh_at_boundary, + ssz(&Batch { + nonce: 0, + frames: vec![Frame { + fee_price: crate::fee::MAX_EXPONENT + 1, + ..empty_frame(10) + }], + }), ), + ("first frame above inclusion", 0, one_frame(0, 20)), + ("first frame at inclusion", 0, one_frame(0, 12)), + ( + "tail frame at inclusion", + 0, + ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(10), empty_frame(12)], + }), + ), + ("first frame below the clock", 11, one_frame(0, 10)), ]; - - for (label, sender, inclusion, expected, payload) in agree_cases { - let (canonical, offchain) = duality_run(*sender, *inclusion, *expected, payload); - assert_eq!( - canonical, offchain, - "I1 disagreement on `{label}`: canonical_executed={canonical}, \ - offchain_accepted={offchain} — the predicate and the canonical \ - fold must agree on accept/reject" + for (label, clock, payload) in omitted { + let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, *clock, payload); + assert!( + !canonical && offchain, + "`{label}`: expected canonical reject and predicate accept \ + (documented omission), got canonical={canonical} offchain={offchain}" ); } + } - // Documented exception — the predicate's structural-reject omission. The - // canonical fold rejects these (it checks frame structure); the predicate - // accepts them (self-trust: the sequencer never emits such batches). If - // either side changes, these asserts flip and force a deliberate update. - let non_monotonic = ssz(&Batch { - nonce: 0, - frames: vec![empty_frame(8), empty_frame(7)], - }); - let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, &non_monotonic); - assert!( - !canonical && offchain, - "non-monotonic safe_blocks: expected canonical reject + predicate \ - accept (documented structural omission), got canonical={canonical} \ - offchain={offchain}" - ); + // ── The scheduler-only conformance vectors ──────────────────────────── - let out_of_range_fee = ssz(&Batch { - nonce: 0, - frames: vec![Frame { - fee_price: crate::fee::MAX_EXPONENT + 1, - ..empty_frame(10) - }], - }); - let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, &out_of_range_fee); - assert!( - !canonical && offchain, - "fee range is a documented structural omission" - ); + /// Replays the vector file's scheduler-only section: the overflow drain and + /// resume, at a small fridge bound the wallet's template never reaches. + /// c-scheduler's conformance test holds scheduler.h to the same section. + mod scheduler_vectors { + use super::*; + use serde_json::{Value, json}; - let frame_above_inclusion = ssz(&Batch { - nonce: 0, - frames: vec![empty_frame(20)], - }); - let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, &frame_above_inclusion); - assert!( - !canonical && offchain, - "frame safe_block > inclusion: expected canonical reject + predicate \ - accept (documented structural omission), got canonical={canonical} \ - offchain={offchain}" - ); - } + fn text(value: &Value) -> &str { + value.as_str().expect("pinned string") + } - #[test] - fn force_execute_overdue_runs_before_a_rejected_or_stale_batch() { - // The backstop force-executes overdue fridge directs at the START of - // process_input — before the batch is even classified. So an overdue - // direct is drained even when the same tick's batch is rejected or - // skipped (a danger-zone shape). - for label in ["wrong-nonce", "stale"] { - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 5, - }, - ); - // A direct at block 1; by inclusion block 8 it is overdue (age 7 >= 5). - process_ok(&mut scheduler, direct_input(1, 1)); + fn number(value: &Value) -> u64 { + value.as_u64().expect("pinned number") + } - let batch = if label == "wrong-nonce" { - // expected 0, got 9 → BatchRejected(WrongNonce). - Batch { - nonce: 9, - frames: vec![Frame { - user_ops: vec![], - safe_block: 8, - fee_price: 0, - }], - } - } else { - // right nonce but a stale first frame (age 8 - 1 = 7 >= 5). - Batch { - nonce: 0, - frames: vec![Frame { - user_ops: vec![], - safe_block: 1, - fee_price: 0, - }], + fn address(value: &Value) -> Address { + text(value).parse().expect("pinned address") + } + + /// The file's inputs: senders and payloads by name from the guest section. + struct Inputs<'a> { + guest: &'a Value, + chain_id: u64, + app_contract: Address, + } + + impl Inputs<'_> { + fn input(&self, index: u64, record: &Value) -> L1Input { + L1Input { + index, + sender: address(&self.guest["senders"][text(&record["from"])]), + block: number(&record["block"]), + timestamp: number(&record["timestamp"]), + chain_id: self.chain_id, + app_contract: self.app_contract, + payload: alloy_primitives::hex::decode(text( + &self.guest["payloads"][text(&record["payload"])], + )) + .expect("pinned hex bytes"), } - }; + } + } - let outcome = process_ok(&mut scheduler, batch_input(8, batch)).outcome; - assert!( - matches!( - outcome, - ProcessOutcome::BatchRejected(_) | ProcessOutcome::BatchSkippedStale + fn rendered_state(state: SchedulerState) -> Value { + json!({ + "l1_input_count": state.l1_input_count, + "next_batch_nonce": state.next_batch_nonce, + "clock": state.clock, + "first_queued_input": state.first_queued_input, + "fridge_len": state.fridge_len, + }) + } + + fn pinned_state(pinned: &Value) -> SchedulerState { + SchedulerState { + l1_input_count: number(&pinned["l1_input_count"]), + next_batch_nonce: number(&pinned["next_batch_nonce"]), + clock: number(&pinned["clock"]), + first_queued_input: number(&pinned["first_queued_input"]), + fridge_len: number(&pinned["fridge_len"]), + } + } + + /// One fed input in the file's form: outcome, reject, txs and state. + fn rendered_step(h: &mut Harness, input: L1Input) -> Value { + let (outcome, txs) = h.feed(input); + let (name, reject) = match outcome { + InputOutcome::DirectQueued => ("direct_queued", None), + InputOutcome::DirectDropped => ("direct_dropped", None), + InputOutcome::BatchAccepted => ("batch_accepted", None), + InputOutcome::BatchStale => ("batch_stale", None), + InputOutcome::BatchRejected(reason) => ( + "batch_rejected", + Some(match reason { + BatchRejectReason::DecodeFailed => json!({"reason": "decode_failed"}), + BatchRejectReason::WrongNonce { expected, got } => { + json!({"reason": "wrong_nonce", "expected": expected, "got": got}) + } + BatchRejectReason::SafeBlockNotBelowInclusionBlock => { + json!({"reason": "safe_block_not_below_inclusion_block"}) + } + BatchRejectReason::NonMonotonicSafeBlocks => { + json!({"reason": "non_monotonic_safe_blocks"}) + } + BatchRejectReason::FeePriceOutOfRange => { + json!({"reason": "fee_price_out_of_range"}) + } + BatchRejectReason::FirstFrameBelowClock { safe_block, clock } => json!({ + "reason": "first_frame_below_clock", + "safe_block": safe_block, + "clock": clock, + }), + }), ), - "{label}: batch should be rejected/skipped, got {outcome:?}" - ); - assert_eq!( - scheduler.app.events(), - [RecordedTx::Direct(1)], - "{label}: the overdue direct must be force-executed before the rejected batch" - ); - assert_eq!( - scheduler.next_expected_batch_nonce(), - 0, - "{label}: a rejected/stale batch consumes no nonce" - ); + }; + let txs: Vec = txs + .iter() + .map(|scheduled| match &scheduled.tx { + L2Tx::Direct(direct) => { + json!({"direct": direct.input_index, "batch_nonce": scheduled.batch_nonce}) + } + L2Tx::UserOp(_) => panic!("the scheduler-only scenarios carry no user ops"), + }) + .collect(); + let mut step = json!({ + "outcome": name, + "txs": txs, + "scheduler": rendered_state(h.state()), + }); + if let Some(reject) = reject { + step["reject"] = reject; + } + step } - } - #[test] - fn multiframe_overheight_in_tail_frame_rejected_by_scheduler() { - // `batch_reject_reason_for_block` checks `safe_block <= inclusion` in - // BOTH the head and the tail frames; a tail frame above the inclusion - // block must reject the whole batch. - let mut scheduler = Scheduler::new( - RecordingApp::default(), - SchedulerConfig { - sequencer_address: SEQUENCER, - max_wait_blocks: 100, - }, - ); - let batch = Batch { - nonce: 0, - frames: vec![ - // head: 5 <= 10, valid. - Frame { - user_ops: vec![], - safe_block: 5, - fee_price: 0, - }, - // tail: 11 > 10 (the inclusion block) → reject. - Frame { - user_ops: vec![], - safe_block: 11, - fee_price: 0, - }, - ], - }; - assert_eq!( - process_ok(&mut scheduler, batch_input(10, batch)).outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::SafeBlockAboveInclusionBlock) - ); - assert_eq!(scheduler.next_expected_batch_nonce(), 0); - assert!(scheduler.app.events().is_empty()); - } + fn expected_step(step: &Value) -> Value { + let mut expected = json!({ + "outcome": step["outcome"].clone(), + "txs": step["txs"].clone(), + "scheduler": step["scheduler"].clone(), + }); + if let Some(reject) = step.get("reject") { + expected["reject"] = reject.clone(); + } + expected + } - /// What the guest's recovery makes of each pinned signature variant today: - /// the sender it executes the op as, or `None` when the op is dropped. The - /// ingress side of the same table is pinned in `api.rs`; `canonical-test` - /// replays every variant through the built image. - #[test] - fn signature_variants_recover_the_pinned_guest_sender() { - let vectors = crate::consensus_vectors::vectors(); - let pinned = &vectors.signature; - let domain = vectors.domain(&pinned.domain); - let scheduler = Scheduler::new(RecordingApp::default(), SchedulerConfig::new(SEQUENCER)); - for case in &pinned.variants { - let wire_user_op = WireUserOp { - nonce: pinned.user_op.nonce, - max_fee: pinned.user_op.max_fee, - data: pinned.user_op.data.to_vec(), - signature: case.signature.to_vec(), + fn refusal(error: &ResumeError) -> &'static str { + match error { + ResumeError::FirstQueuedAfterCount { .. } => "first_queued_after_count", + ResumeError::OutOfSequence { .. } => "out_of_sequence", + ResumeError::BeyondInputCount { .. } => "beyond_input_count", + ResumeError::MissingInputs { .. } => "missing_inputs", + ResumeError::BlockDecreases { .. } => "block_decreases", + ResumeError::FirstQueuedNotADirect { .. } => "first_queued_not_a_direct", + ResumeError::DirectNotAboveClock { .. } => "direct_not_above_clock", + ResumeError::FridgeLenMismatch { .. } => "fridge_len_mismatch", + } + } + + #[test] + fn the_reference_reproduces_every_scheduler_only_scenario() { + let vectors = crate::consensus_vectors::raw_vectors(); + let guest = &vectors["guest"]; + let inputs = Inputs { + guest, + chain_id: number(&guest["chain_id"]), + app_contract: address(&guest["app_contract"]), }; - assert_eq!( - scheduler.recover_sender(&domain, &wire_user_op), - case.guest, - "{}", - case.name - ); + let section = &vectors["scheduler_only"]; + let pinned_config = §ion["config"]; + let config = SchedulerConfig { + sequencer_address: address(&pinned_config["sequencer_address"]), + max_wait_blocks: number(&pinned_config["max_wait_blocks"]), + fridge_bound: number(&pinned_config["fridge_bound"]), + max_direct_payload_bytes: number(&pinned_config["max_direct_payload_bytes"]), + }; + + let scenarios = section["scenarios"].as_array().expect("pinned scenarios"); + assert!(!scenarios.is_empty()); + for scenario in scenarios { + let name = text(&scenario["name"]); + let mut h = Harness::new(config); + for (index, step) in (0_u64..).zip(scenario["steps"].as_array().expect("steps")) { + let rendered = rendered_step(&mut h, inputs.input(index, step)); + assert_eq!(rendered, expected_step(step), "{name}, step {index}"); + } + } + + let cases = section["resume"].as_array().expect("pinned resume cases"); + assert!(!cases.is_empty()); + for case in cases { + let name = text(&case["name"]); + let state = pinned_state(&case["state"]); + let queued = case["queued"] + .as_array() + .expect("pinned range") + .iter() + .map(|record| inputs.input(number(&record["index"]), record)) + .collect(); + let resumed = Harness::resumed(config, state, queued); + let mut h = match (resumed, text(&case["result"])) { + (Ok(h), "resumed") => h, + (Err(error), "refused") => { + assert_eq!(refusal(&error), text(&case["refusal"]), "{name}"); + continue; + } + (Ok(_), pinned) => panic!("{name}: pinned {pinned}, but it resumed"), + (Err(error), pinned) => panic!("{name}: pinned {pinned}, but {error}"), + }; + assert_eq!(h.state(), state, "{name}"); + let then = case["then"] + .as_array() + .expect("pinned steps after a resume"); + for (index, step) in (state.l1_input_count..).zip(then) { + let rendered = rendered_step(&mut h, inputs.input(index, step)); + assert_eq!(rendered, expected_step(step), "{name}, step {index}"); + } + } } } } diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 97215cbe..d821dcd9 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -37,6 +37,9 @@ clap = { workspace = true, features = ["env"] } cartesi-rollups-contracts = { workspace = true } async-trait = "0.1" sha2 = "0.10" +# Setup reads a base machine's export through the emulator. The same pin and +# feature as sdk/guest/testsi: one libcartesi, named by LIBCARTESI_PATH. +cartesi-machine = { git = "https://github.com/cartesi/dave", rev = "612c54cf22d73b364cd57a7058ba6aa449e5c77a", features = ["external_cartesi"] } [dev-dependencies] sequencer-core = { path = "../sequencer-core", features = ["test-support"] } diff --git a/sequencer/src/clock.rs b/sequencer/src/clock.rs index 41fa54db..08863251 100644 --- a/sequencer/src/clock.rs +++ b/sequencer/src/clock.rs @@ -6,8 +6,7 @@ use std::time::{Duration, SystemTime}; -/// Current wall-clock time as Unix-ms. Passed into -/// [`crate::storage::Storage::check_danger`] and friends. +/// Current wall-clock time as Unix-ms. pub fn unix_now_ms() -> u64 { let elapsed = SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/sequencer/src/commands/config.rs b/sequencer/src/commands/config.rs index 0cf26869..8239c382 100644 --- a/sequencer/src/commands/config.rs +++ b/sequencer/src/commands/config.rs @@ -11,7 +11,6 @@ use clap::{ArgGroup, Args}; use crate::commands::error::BootstrapError; use crate::l1::SubmitterKey; -use sequencer_core::protocol::{ProtocolTiming, ProtocolTimingError}; const DEFAULT_HTTP_ADDR: &str = "127.0.0.1:3000"; const DEFAULT_INTERNAL_HTTP_ADDR: &str = "127.0.0.1:3001"; @@ -19,7 +18,8 @@ const DEFAULT_DATA_DIR: &str = "sequencer-data"; const DB_FILENAME: &str = "sequencer.db"; /// Blocks a time-sealed batch needs after its open window to become -/// safe-accepted before danger: its first frame trails the safe head by under +/// safe-accepted inside the staleness window: its first frame trails the safe +/// head by under /// one frame-clock interval (5), Ethereum's safe head trails inclusion by up to /// two epochs (64), and the rest covers submitter pickup, inclusion and the dump. const BATCH_SUBMISSION_MARGIN_BLOCKS: u64 = 100; @@ -50,53 +50,16 @@ pub struct L1RpcArgs { pub allow_insecure_rpc: bool, } -/// Protocol-timing tuning knobs shared by `setup` and `run`. `setup` needs a -/// valid `ProtocolTiming` for the initial-sync writes (and validating it here -/// fails fast at setup, not just at run). +/// The L1 block-time estimate shared by `setup` and `run`. #[derive(Debug, Clone, Args)] pub struct TimingArgs { - /// Blocks before MAX_WAIT_BLOCKS to trigger preemptive recovery. - /// The danger threshold is MAX_WAIT_BLOCKS minus this margin. - /// Must be less than MAX_WAIT_BLOCKS (validated at startup). - /// - /// Default 300 gives ~1h of headroom before canonical expiry at 12s/block. - /// Detection starts recovery immediately; this is neither an operator - /// grace period nor a completion deadline. See `docs/recovery/README.md`. - #[arg( - long, - env = "CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS", - default_value = "300" - )] - pub preemptive_margin_blocks: u64, - - /// Blocks of safe-head age after which the L1 read view is considered too - /// stale to trust. Independent of the preemptive margin. Must be strictly - /// less than the danger threshold (validated at startup). - /// - /// Default 600 (~2h at 12s/block). - #[arg(long, env = "CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS", default_value = "600", value_parser = clap::value_parser!(u64).range(1..))] - pub l1_read_stale_after_blocks: u64, - - /// Assumed L1 block time in seconds. Used to estimate block progression - /// from wall-clock time when the L1 provider is unreachable. + /// Assumed L1 block time in seconds. Sets the flush's and the poster's + /// confirmation cadence, and converts `--max-batch-open-seconds` into + /// blocks for the batch-open check. #[arg(long, env = "CARTESI_SEQUENCER_SECONDS_PER_BLOCK", default_value = "12", value_parser = clap::value_parser!(u64).range(1..))] pub seconds_per_block: u64, } -impl TimingArgs { - /// Build a validated [`ProtocolTiming`]. Pure derivation — no I/O. - /// `max_wait_blocks` is the shared scheduler constant; the rest are the - /// operator-tunable CLI args. - pub fn protocol_timing(&self) -> Result { - ProtocolTiming::try_new( - sequencer_core::MAX_WAIT_BLOCKS, - self.preemptive_margin_blocks, - self.l1_read_stale_after_blocks, - self.seconds_per_block, - ) - } -} - /// L1 fee-oracle source selection. A fixed logarithmic price is intended for /// local development; public chains use a validated Uniswap V3 TWAP source. #[derive(Debug, Clone, Args)] @@ -218,8 +181,8 @@ impl FeeOracleArgs { } /// Batch-submitter signing-key source, shared by `run` and `flush-mempool` -/// (both sign L1 transactions). `setup` does NOT use this — it takes the -/// submitter address directly and never signs. +/// (both sign L1 transactions). `setup` takes the same two sources through +/// [`SetupKeyArgs`], where `--dry-run` may leave both out. #[derive(Debug, Clone, Args)] #[command(group( ArgGroup::new("batch_submitter_key_source") @@ -254,32 +217,51 @@ impl KeyArgs { &self.batch_submitter_private_key, &self.batch_submitter_private_key_file, ) - .map(|opt| opt.expect("batch submitter private key is required by CLI arg group")) + .map(|key| key.expect("batch submitter private key is required by CLI arg group")) } } -/// Resolve a batch-submitter key from an inline value or a key file (first line). -/// Returns `Ok(None)` when neither source is set — `KeyArgs` makes that -/// impossible via its required arg group, but `setup --recovery` validates the -/// "exactly one source" rule in code (the group can't be conditionally required -/// at the clap level), so it needs the `None` case. +/// `setup`'s signing-key source: the same arguments as [`KeyArgs`], required +/// unless `--dry-run`, which flushes nothing. +#[derive(Debug, Clone, Args)] +pub struct SetupKeyArgs { + /// Hex-encoded private key for the batch submitter. + #[arg( + long, + env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", + hide_env_values = true, + conflicts_with = "batch_submitter_private_key_file", + required_unless_present_any = ["batch_submitter_private_key_file", "dry_run"], + value_parser = parse_submitter_key + )] + batch_submitter_private_key: Option, + /// Path to a file whose first line contains the batch submitter private key. + #[arg( + long, + env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", + hide_env_values = true, + required_unless_present_any = ["batch_submitter_private_key", "dry_run"] + )] + batch_submitter_private_key_file: Option, +} + +/// The key from the file when one is named, otherwise the inline value. The +/// error carries the path, never the contents: a key file's bytes must not +/// reach a log line even when the read fails. fn resolve_key_source( inline: &Option, file: &Option, ) -> Result, BootstrapError> { - if let Some(file) = file { - // The error carries the path, never the contents: a key file's - // bytes must not reach a log line even when the read fails. - let contents = std::fs::read_to_string(file).map_err(|source| BootstrapError::KeyFile { - path: std::path::PathBuf::from(file), - source, - })?; - Ok(Some(SubmitterKey::new( - contents.lines().next().unwrap_or("").trim().to_string(), - ))) - } else { - Ok(inline.clone()) - } + let Some(file) = file else { + return Ok(inline.clone()); + }; + let contents = std::fs::read_to_string(file).map_err(|source| BootstrapError::KeyFile { + path: std::path::PathBuf::from(file), + source, + })?; + Ok(Some(SubmitterKey::new( + contents.lines().next().unwrap_or("").trim().to_string(), + ))) } /// Whether a key-file read failure is deterministic operator @@ -308,57 +290,11 @@ pub(crate) fn key_file_io_is_terminal(source: &std::io::Error) -> bool { ) } -/// Batch-submitter signing-key source for a context where the key is -/// *conditionally* required — `setup --recovery` needs it (to flush the wallet -/// nonce), plain `setup` must not carry it. clap can't express a -/// conditionally-required arg group, so unlike [`KeyArgs`] neither source is -/// required at the clap level; the owner validates presence + exclusivity via -/// [`is_present`](Self::is_present) / [`both_set`](Self::both_set). -#[derive(Debug, Clone, Args)] -pub struct OptionalKeyArgs { - /// Hex-encoded batch-submitter private key. - #[arg( - long, - env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", - hide_env_values = true, - value_parser = parse_submitter_key - )] - batch_submitter_private_key: Option, - /// Path to a file whose first line is the batch-submitter private key. - #[arg( - long, - env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", - hide_env_values = true - )] - batch_submitter_private_key_file: Option, -} - -impl OptionalKeyArgs { - /// At least one source is set. - fn is_present(&self) -> bool { - self.batch_submitter_private_key.is_some() - || self.batch_submitter_private_key_file.is_some() - } - - /// Both sources are set — the owner rejects this (at most one is allowed). - fn both_set(&self) -> bool { - self.batch_submitter_private_key.is_some() - && self.batch_submitter_private_key_file.is_some() - } - - /// Resolve the key if either source is set; `Ok(None)` when neither is. - fn resolve_if_present(&self) -> Result, BootstrapError> { - resolve_key_source( - &self.batch_submitter_private_key, - &self.batch_submitter_private_key_file, - ) - } -} - -/// `setup` — establish the deployment's timeless state: pin identity, do the -/// initial L1 sync, register the genesis finalized snapshot, and atomically -/// complete setup. L1-read-only: takes the batch-submitter address, not the -/// signing key. +/// `setup` — start a new era in an empty data directory: pin identity, flush +/// the submitter's wallet to pin the stop block, sync L1, rebuild the history +/// from the base (genesis, or a stored canonical machine) through the stop +/// block, and publish the era. `docs/recovery/cockroach.md` ("What setup does") +/// has the procedure. #[derive(Debug, Clone, Args)] pub struct SetupConfig { #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] @@ -371,38 +307,21 @@ pub struct SetupConfig { /// Application (EIP-712 verifying contract) address. #[arg(long, env = "CARTESI_SEQUENCER_APP_ADDRESS", value_parser = parse_address)] pub app_address: Address, - /// Address that submits batches. Pinned into the deployment identity. - /// `setup` never signs, so it takes the address (derivable from the key - /// without the secret) rather than the key itself. + /// Address that submits batches, pinned into the deployment identity. + /// The key must sign for it: setup's flush settles this account. #[arg(long, env = "CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS", value_parser = parse_address)] pub batch_submitter_address: Address, - /// L1 inclusion block `B` of the trusted checkpoint this setup boots from; - /// `0` is genesis. Plain `setup` uses `B` only as the lower bound of its - /// read-only detection scan: work a previous instance left past `B`, or an - /// unsettled wallet nonce, refuses with exit 40. `setup --recovery` loads the - /// checkpoint itself (`docs/recovery/cockroach.md`). - #[arg(long, env = "CARTESI_SEQUENCER_CHECKPOINT_BLOCK", default_value_t = 0)] - pub checkpoint_block: u64, - /// Recovery mode (cockroach recovery): rebuild this freshly-wiped DB from a - /// trusted checkpoint at `--checkpoint-block` instead of genesis bootstrap. - /// Requires `--checkpoint-dump-dir` and the - /// batch-submitter signing key (recovery flushes the wallet nonce, which - /// signs L1 no-ops). Plain `setup` stays L1-read-only and key-less. - #[arg(long, env = "CARTESI_SEQUENCER_RECOVERY", default_value_t = false)] - pub recovery: bool, - /// Directory of the trusted checkpoint dump (a finalized `dumps//` with - /// `state/` + `info.toml`) that `setup --recovery` boots the machine `S` - /// from. Required with `--recovery`; rejected without it. This is a - /// **sequencer** dump from `$CARTESI_SEQUENCER_DATA_DIR/dumps/`, not a - /// watchdog CM checkpoint under `.../checkpoints//`. - #[arg(long, env = "CARTESI_SEQUENCER_CHECKPOINT_DUMP_DIR", value_parser = parse_non_empty_string)] - pub checkpoint_dump_dir: Option, - /// Batch-submitter signing key, recovery-only (for the wallet-nonce flush). - /// Conditionally required: present iff `--recovery`. The two sources are - /// mutually exclusive and rejected on a non-recovery `setup` — see - /// [`SetupConfig::validate`]. + /// A stored canonical machine of the deployment's template, stopped + /// between inputs, to rebuild from. Without it the base is genesis. + #[arg(long, value_name = "MACHINE_DIR")] + pub base: Option, + /// Report the base's counts, the submitter's activity past it and the + /// rebuild's size, then exit. Needs no key and never touches the data + /// directory. + #[arg(long, default_value_t = false)] + pub dry_run: bool, #[command(flatten)] - key: OptionalKeyArgs, + key: SetupKeyArgs, #[command(flatten)] pub timing: TimingArgs, #[command(flatten)] @@ -414,50 +333,14 @@ impl SetupConfig { db_path_in(&self.data_dir) } - /// Validate the cross-field recovery constraints clap can't express (a - /// conditionally-required arg group). Returns the first violation as a - /// human-readable message; the caller maps it to a terminal bootstrap error. - /// - /// Recovery requires an exported checkpoint (a matching block + dump - /// dir) and the signing key; a plain `setup` must carry none of the - /// recovery-only inputs (key, dump dir). - pub fn validate(&self) -> Result<(), String> { - let key_present = self.key.is_present(); - if self.key.both_set() { - return Err( - "--batch-submitter-private-key and --batch-submitter-private-key-file \ - are mutually exclusive" - .to_string(), - ); - } - if self.recovery { - if self.checkpoint_dump_dir.is_none() { - return Err("--recovery requires --checkpoint-dump-dir".to_string()); - } - if !key_present { - return Err("--recovery requires the batch-submitter signing key \ - (--batch-submitter-private-key[-file]) to flush the wallet nonce" - .to_string()); - } - } else { - if self.checkpoint_dump_dir.is_some() { - return Err("--checkpoint-dump-dir is only valid with --recovery".to_string()); - } - if key_present { - return Err("plain `setup` is L1-read-only and takes no signing key; \ - pass the batch-submitter address, or use --recovery" - .to_string()); - } - } - Ok(()) - } - - /// Resolve the recovery signing key. Precondition: [`SetupConfig::validate`] - /// passed with `recovery == true` (so exactly one source is set). - pub fn resolve_recovery_key(&self) -> Result { - self.key - .resolve_if_present() - .map(|opt| opt.expect("recovery key presence is validated before resolve")) + /// The submitter key. Clap requires one unless `--dry-run`, which never + /// asks for it. + pub fn resolve_private_key(&self) -> Result { + resolve_key_source( + &self.key.batch_submitter_private_key, + &self.key.batch_submitter_private_key_file, + ) + .map(|key| key.expect("setup requires a batch submitter key unless --dry-run")) } } @@ -503,7 +386,8 @@ pub struct RunConfig { /// for a low-traffic deployment. Default 7200 (2h); operators trade latency /// against L1 cost, and tests set it small to seal batches promptly. /// Rounded up to blocks at `--seconds-per-block`, it must leave a - /// submission margin below the danger threshold (validated at startup). + /// submission margin below the application's staleness window + /// (validated at startup). #[arg( long, env = "CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS", @@ -530,26 +414,22 @@ impl RunConfig { std::time::Duration::from_secs(self.max_batch_open_seconds) } - /// Build a validated [`ProtocolTiming`] and refuse a batch-open window that - /// would carry a low-volume Tip into danger before it can be sealed, - /// submitted and safe-accepted. - pub fn protocol_timing(&self) -> Result { - let timing = self.timing.protocol_timing()?; + /// Refuse a batch-open window that would seal a low-volume batch too late + /// to land inside the application's staleness window `max_wait_blocks`. + pub fn check_batch_open_window(&self, max_wait_blocks: u64) -> Result<(), BootstrapError> { + let seconds_per_block = self.timing.seconds_per_block; // Rounding up counts a partial block against the operator. - let open_blocks = self - .max_batch_open_seconds - .div_ceil(timing.seconds_per_block()); - let danger_threshold = timing.danger_threshold(); - if open_blocks > danger_threshold.saturating_sub(BATCH_SUBMISSION_MARGIN_BLOCKS) { - return Err(BootstrapError::BatchOpenPastDanger { + let open_blocks = self.max_batch_open_seconds.div_ceil(seconds_per_block); + if open_blocks > max_wait_blocks.saturating_sub(BATCH_SUBMISSION_MARGIN_BLOCKS) { + return Err(BootstrapError::BatchOpenPastWindow { max_batch_open_seconds: self.max_batch_open_seconds, - seconds_per_block: timing.seconds_per_block(), + seconds_per_block, open_blocks, - danger_threshold, + max_wait_blocks, margin_blocks: BATCH_SUBMISSION_MARGIN_BLOCKS, }); } - Ok(timing) + Ok(()) } /// Resolve the batch submitter private key from either the inline value or a key file. @@ -620,7 +500,7 @@ mod tests { // Parse a full subcommand line through the top-level `Cli`, since the // per-subcommand configs derive `Args` (embeddable) not `Parser`. - const SETUP_ARGS: [&str; 10] = [ + const SETUP_ARGS_WITHOUT_KEY: [&str; 10] = [ "sequencer", "setup", "--eth-rpc-url", @@ -633,6 +513,12 @@ mod tests { "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", ]; + fn setup_args() -> Vec<&'static str> { + let mut args = SETUP_ARGS_WITHOUT_KEY.to_vec(); + args.extend(["--batch-submitter-private-key", TEST_KEY]); + args + } + const RUN_ARGS: [&str; 5] = [ "sequencer", "run", @@ -658,7 +544,7 @@ mod tests { } fn setup_config_from(extra: &[&str]) -> SetupConfig { - let mut args: Vec<&str> = SETUP_ARGS.to_vec(); + let mut args = setup_args(); args.extend_from_slice(extra); match Cli::try_parse_from(args).expect("parse setup").command { crate::harness::Command::Setup(c) => *c, @@ -667,7 +553,7 @@ mod tests { } #[test] - fn setup_requires_address_not_key() { + fn setup_takes_the_address_and_the_key() { let cfg = setup_config(); assert_eq!(cfg.chain_id, 31337); assert_eq!( @@ -676,124 +562,93 @@ mod tests { .parse::
() .unwrap() ); + assert_eq!(cfg.resolve_private_key().unwrap().expose_secret(), TEST_KEY); + assert_eq!(cfg.base, None); + assert!(!cfg.dry_run); // The default db path lives under the default data dir. assert!(cfg.db_path().ends_with("sequencer.db")); } + /// Setup flushes with the key, so it needs one; the dry run flushes + /// nothing and needs none. Both sources together are refused. #[test] - fn setup_checkpoint_block_defaults_to_genesis_and_parses() { - // Default = 0 (genesis bootstrap) when the flag is absent. - assert_eq!(setup_config().checkpoint_block, 0); - // Explicit value parses. - let mut args: Vec<&str> = SETUP_ARGS.to_vec(); - args.push("--checkpoint-block"); - args.push("4200"); - let cfg = match Cli::try_parse_from(args).expect("parse setup").command { - crate::harness::Command::Setup(c) => *c, - other => panic!("expected setup subcommand, got {other:?}"), + fn setup_requires_the_key_unless_dry_run() { + assert!( + Cli::try_parse_from(SETUP_ARGS_WITHOUT_KEY).is_err(), + "setup without a key must not parse" + ); + let mut dry_run = SETUP_ARGS_WITHOUT_KEY.to_vec(); + dry_run.push("--dry-run"); + let crate::harness::Command::Setup(cfg) = Cli::try_parse_from(dry_run) + .expect("a dry run needs no key") + .command + else { + panic!("expected setup subcommand"); }; - assert_eq!(cfg.checkpoint_block, 4200); - } + assert!(cfg.dry_run); - #[test] - fn plain_setup_passes_validation_and_is_not_recovery() { - let cfg = setup_config(); - assert!(!cfg.recovery); - assert!(cfg.checkpoint_dump_dir.is_none()); - cfg.validate().expect("plain setup is a valid config"); - } - - #[test] - fn setup_rejects_a_signing_key_without_recovery() { - // The key now parses (recovery needs it), but a non-recovery `setup` - // carrying one is rejected by cross-field validation, not at parse time. - let cfg = setup_config_from(&["--batch-submitter-private-key", TEST_KEY]); - let err = cfg.validate().expect_err("plain setup must reject a key"); + let mut both = setup_args(); + both.extend(["--batch-submitter-private-key-file", "/run/secrets/key"]); assert!( - err.contains("key-less") || err.contains("L1-read-only"), - "got: {err}" + Cli::try_parse_from(both).is_err(), + "two key sources must not parse" ); - } - #[test] - fn setup_rejects_dump_dir_without_recovery() { - let cfg = setup_config_from(&["--checkpoint-dump-dir", "/tmp/ckpt"]); - let err = cfg - .validate() - .expect_err("dump dir without --recovery must fail"); - assert!(err.contains("--checkpoint-dump-dir"), "got: {err}"); + let key_file = tempfile::NamedTempFile::new().expect("key file"); + std::fs::write(key_file.path(), format!("{TEST_KEY}\n")).expect("write key"); + let mut from_file = SETUP_ARGS_WITHOUT_KEY.to_vec(); + let path = key_file.path().to_str().expect("utf8 path"); + from_file.extend(["--batch-submitter-private-key-file", path]); + let crate::harness::Command::Setup(cfg) = Cli::try_parse_from(from_file) + .expect("a key file parses") + .command + else { + panic!("expected setup subcommand"); + }; + assert_eq!(cfg.resolve_private_key().unwrap().expose_secret(), TEST_KEY); } #[test] - fn recovery_requires_dump_dir_and_key() { - // --recovery alone (no dump dir / block / key) is invalid; each missing - // input is reported. - let bare = setup_config_from(&["--recovery"]); - assert!( - bare.validate() - .unwrap_err() - .contains("--checkpoint-dump-dir") + fn setup_parses_a_base_machine() { + let cfg = setup_config_from(&["--base", "/machines/epoch-41", "--dry-run"]); + assert_eq!( + cfg.base.as_deref(), + Some(std::path::Path::new("/machines/epoch-41")) ); - - let genesis = setup_config_from(&[ - "--recovery", - "--checkpoint-dump-dir", - "/tmp/ckpt", - "--batch-submitter-private-key", - TEST_KEY, - ]); - genesis - .validate() - .expect("genesis checkpoint is a valid recovery source"); - assert_eq!(genesis.checkpoint_block, 0); - - let no_key = setup_config_from(&[ - "--recovery", - "--checkpoint-dump-dir", - "/tmp/ckpt", - "--checkpoint-block", - "1200", - ]); - assert!(no_key.validate().unwrap_err().contains("signing key")); + assert!(cfg.dry_run); } + /// The checkpoint options of the removed recovery mode are gone: the base + /// machine names the position. #[test] - fn recovery_full_config_validates_and_resolves_key() { - let cfg = setup_config_from(&[ - "--recovery", - "--checkpoint-dump-dir", - "/tmp/ckpt", - "--checkpoint-block", - "1200", - "--batch-submitter-private-key", - TEST_KEY, - ]); - cfg.validate().expect("full recovery config is valid"); - assert!(cfg.recovery); - assert_eq!(cfg.checkpoint_block, 1200); - assert_eq!(cfg.checkpoint_dump_dir.as_deref(), Some("/tmp/ckpt")); - assert_eq!( - cfg.resolve_recovery_key() - .expect("resolve key") - .expose_secret(), - TEST_KEY - ); + fn setup_has_no_checkpoint_options() { + for extra in [ + &["--recovery"][..], + &["--checkpoint-dump-dir", "/tmp/dump"][..], + &["--checkpoint-block", "4200"][..], + ] { + let mut args = setup_args(); + args.extend_from_slice(extra); + Cli::try_parse_from(args).expect_err("a removed option must not parse"); + } } #[test] fn missing_key_file_is_terminal_and_names_the_path_not_the_contents() { - let cfg = setup_config_from(&[ - "--recovery", - "--checkpoint-dump-dir", - "/tmp/ckpt", - "--checkpoint-block", - "1200", + let args = [ + "sequencer", + "run", + "--eth-rpc-url", + "http://127.0.0.1:8545", "--batch-submitter-private-key-file", "/nonexistent/dir/submitter.key", - ]); - cfg.validate().expect("one key source is valid"); + ]; + let cfg = match Cli::try_parse_from(args).expect("parse run").command { + crate::harness::Command::Run(c) => *c, + other => panic!("expected run subcommand, got {other:?}"), + }; let err = cfg - .resolve_recovery_key() + .resolve_private_key() .expect_err("a missing key file must not resolve"); assert!( matches!(&err, BootstrapError::KeyFile { path, source } @@ -810,22 +665,6 @@ mod tests { ); } - #[test] - fn recovery_rejects_both_key_sources() { - let cfg = setup_config_from(&[ - "--recovery", - "--checkpoint-dump-dir", - "/tmp/ckpt", - "--checkpoint-block", - "1200", - "--batch-submitter-private-key", - TEST_KEY, - "--batch-submitter-private-key-file", - "/tmp/key", - ]); - assert!(cfg.validate().unwrap_err().contains("mutually exclusive")); - } - #[test] fn run_requires_a_key_and_drops_identity_args() { // Key required. @@ -986,77 +825,78 @@ mod tests { assert_eq!(run_config_from(&[]).timing.seconds_per_block, 12); } + /// The repair's tuning flags are gone with it. #[test] - fn run_default_l1_read_stale_after_blocks_is_600() { - assert_eq!(run_config_from(&[]).timing.l1_read_stale_after_blocks, 600); + fn run_and_setup_have_no_repair_flags() { + for flag in ["--preemptive-margin-blocks", "--l1-read-stale-after-blocks"] { + let mut run = RUN_ARGS.to_vec(); + run.extend([TEST_KEY, flag, "300"]); + Cli::try_parse_from(run).expect_err("a removed run flag must not parse"); + let mut setup = setup_args(); + setup.extend([flag, "300"]); + Cli::try_parse_from(setup).expect_err("a removed setup flag must not parse"); + } } - const _: () = assert!( - sequencer_core::MAX_WAIT_BLOCKS == 1200, - "batch-open window fixtures below assume the 1200-block window" - ); + /// The staleness window the fixtures below assume. + const WINDOW: u64 = 1200; #[test] - fn run_bounds_batch_open_window_by_danger_threshold_less_submission_margin() { - // Defaults: danger_threshold MAX_WAIT_BLOCKS (1200) - 300 = 900; budget 800 blocks = - // 9600 s at 12 s. + fn run_bounds_batch_open_window_by_the_window_less_submission_margin() { + // The window (1200) less the margin (100) leaves 1100 blocks = 13200 s + // at 12 s. run_config_from(&[]) - .protocol_timing() + .check_batch_open_window(WINDOW) .expect("default window fits"); - run_config_from(&["--max-batch-open-seconds", "9600"]) - .protocol_timing() + run_config_from(&["--max-batch-open-seconds", "13200"]) + .check_batch_open_window(WINDOW) .expect("boundary window fits"); - // 9601 s is 800.08 blocks; rounding up must refuse it. - let err = run_config_from(&["--max-batch-open-seconds", "9601"]) - .protocol_timing() + // 13201 s is 1100.08 blocks; rounding up must refuse it. + let err = run_config_from(&["--max-batch-open-seconds", "13201"]) + .check_batch_open_window(WINDOW) .expect_err("one second past the budget must be refused"); assert!( matches!( err, - BootstrapError::BatchOpenPastDanger { - max_batch_open_seconds: 9601, + BootstrapError::BatchOpenPastWindow { + max_batch_open_seconds: 13201, seconds_per_block: 12, - open_blocks: 801, - danger_threshold: 900, + open_blocks: 1101, + max_wait_blocks: WINDOW, margin_blocks: 100, } ), "{err:?}" ); + assert_eq!( + crate::commands::error::CommandError::from(err).exit_code(), + crate::commands::error::EXIT_TERMINAL + ); let err = run_config_from(&[ "--max-batch-open-seconds", "18446744073709551615", "--seconds-per-block", "1", ]) - .protocol_timing() + .check_batch_open_window(WINDOW) .expect_err("an unbounded window must be refused without overflow"); assert!( - matches!(err, BootstrapError::BatchOpenPastDanger { .. }), + matches!(err, BootstrapError::BatchOpenPastWindow { .. }), "{err:?}" ); } #[test] - fn run_refuses_every_batch_open_window_when_danger_threshold_is_below_the_margin() { - // danger_threshold MAX_WAIT_BLOCKS (1200) - 1150 = 50 < 100: no window can land before - // danger. - let err = run_config_from(&[ - "--preemptive-margin-blocks", - "1150", - "--l1-read-stale-after-blocks", - "10", - "--max-batch-open-seconds", - "1", - ]) - .protocol_timing() - .expect_err("no budget remains"); + fn run_refuses_every_batch_open_window_when_the_window_is_below_the_margin() { + let err = run_config_from(&["--max-batch-open-seconds", "1"]) + .check_batch_open_window(50) + .expect_err("no budget remains"); assert!( matches!( err, - BootstrapError::BatchOpenPastDanger { + BootstrapError::BatchOpenPastWindow { open_blocks: 1, - danger_threshold: 50, + max_wait_blocks: 50, .. } ), @@ -1064,19 +904,30 @@ mod tests { ); } + /// At the boundary window a low-volume batch, sealed by time, still has + /// the submission margin less the frame lag to land before its history + /// turns void. The database pins the wallet's window, so the boundary is + /// derived from it. #[test] - fn boundary_batch_open_window_seals_a_low_volume_tip_before_danger() { + fn boundary_batch_open_window_seals_a_low_volume_batch_inside_the_window() { use crate::storage::test_helpers::{SENDER_A, temp_db_with_default_deployment_identity}; - use crate::storage::{DangerStatus, SafeInputRange, Storage}; + use crate::storage::{SafeInputRange, Storage}; + use sequencer_core::protocol::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS; - let config = run_config_from(&["--max-batch-open-seconds", "9600"]); - let timing = config.protocol_timing().expect("boundary window fits"); + let window = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; + let boundary_seconds = ((window - BATCH_SUBMISSION_MARGIN_BLOCKS) + * run_config_from(&[]).timing.seconds_per_block) + .to_string(); + let config = run_config_from(&["--max-batch-open-seconds", &boundary_seconds]); + config + .check_batch_open_window(window) + .expect("boundary window fits"); let open_blocks = config .max_batch_open_seconds - .div_ceil(timing.seconds_per_block()); + .div_ceil(config.timing.seconds_per_block); // Worst case: the first frame trails the safe head by one block less // than the frame-clock interval. - let frame_lag = ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS - 1; + let frame_lag = FRAME_CLOCK_INTERVAL_SAFE_BLOCKS - 1; let first_frame = 10; let seal_safe_block = first_frame + frame_lag + open_blocks; @@ -1085,51 +936,29 @@ mod tests { let mut head = storage .initialize_open_state(first_frame, SafeInputRange::empty_at(0)) .expect("open Tip"); - let status_at = |storage: &mut Storage, safe_block: u64| { + let void_at = |storage: &mut Storage, safe_block: u64| { storage - .append_safe_inputs(safe_block, &[], SENDER_A, &timing) + .append_safe_inputs(safe_block, &[], SENDER_A) .expect("advance safe head"); - storage - .check_danger(&timing, crate::clock::unix_now_ms()) - .expect("check danger") + storage.terminal_fact().expect("terminal fact").is_some() }; - // Low volume: only the deadline seals the Tip, and it is not in danger yet. - assert_eq!(status_at(&mut storage, seal_safe_block), DangerStatus::Safe); - // Closure keeps the frame clock; the batch still ages from first_frame. + // Low volume: only the deadline seals the batch, and it can still land. + assert!(!void_at(&mut storage, seal_safe_block)); storage - .close_frame_and_batch(&mut head, first_frame) + .close_frame_and_batch(&mut head) .expect("seal by time"); - // The sealed batch keeps the margin, less the frame lag, to become accepted. - let last_safe = first_frame + timing.danger_threshold() - 1; - assert_eq!(last_safe - seal_safe_block, 100 - frame_lag - 1); - assert_eq!(status_at(&mut storage, last_safe), DangerStatus::Safe); + // The sealed batch keeps the margin, less the frame lag, to be accepted: + // the last safe head at which a later block can still accept it is + // `first_frame + window - 2`. + let last_live = first_frame + window - 2; assert_eq!( - status_at(&mut storage, last_safe + 1), - DangerStatus::ClosedBatchInDanger(0) - ); - } - - #[test] - fn timing_args_reject_zero_l1_read_stale_after_blocks() { - let err = Cli::try_parse_from([ - "sequencer", - "run", - "--eth-rpc-url", - "http://x", - "--batch-submitter-private-key", - TEST_KEY, - "--l1-read-stale-after-blocks", - "0", - ]) - .expect_err("l1_read_stale_after_blocks=0 must be rejected"); - let message = err.to_string(); - assert!( - message.contains("--l1-read-stale-after-blocks") - || message.contains("l1_read_stale_after_blocks"), - "error must name the offending field, got: {message}" + last_live - seal_safe_block, + BATCH_SUBMISSION_MARGIN_BLOCKS - frame_lag - 2 ); + assert!(!void_at(&mut storage, last_live)); + assert!(void_at(&mut storage, last_live + 1)); } /// Render `--help` for `subcmd` inside a child process whose env carries @@ -1210,9 +1039,10 @@ mod tests { !help.contains("SECRET_TOKEN_XYZ"), "{subcmd} --help must not print the RPC URL (may contain API tokens)" ); + // Every command signs, setup's flush included, so each names the key. assert!( help.contains("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"), - "{subcmd} --help should still mention the env var name" + "{subcmd} --help names the key env var" ); } } diff --git a/sequencer/src/commands/error.rs b/sequencer/src/commands/error.rs index e54d1f72..c789350d 100644 --- a/sequencer/src/commands/error.rs +++ b/sequencer/src/commands/error.rs @@ -13,13 +13,12 @@ use crate::ingress::inclusion_lane::{ use crate::l1::fee_oracle::worker::FeeOracleError; use crate::l1::reader::InputReaderError; use crate::l1::submitter::BatchSubmitterError; -use crate::recovery::{DangerDetectorError, RecoveryError}; +use crate::landing::LandingWatchError; use crate::storage::{ - DangerStatus, DeploymentIdentity, LifecycleError, StorageOpenError, + DeploymentIdentity, LifecycleError, OpenBatchError, StorageOpenError, TerminalFact, is_persistent_storage_error, is_persistent_storage_open_error, }; use sequencer_core::application::AppError; -use sequencer_core::protocol::ProtocolTimingError; // ── Top-level CommandError ──────────────────────────────────────────────── @@ -27,7 +26,7 @@ use sequencer_core::protocol::ProtocolTimingError; /// /// - `Bootstrap`: startup failures before runtime workers come up. /// - `Worker`: one of the runtime workers exited (server, inclusion lane, -/// input reader, batch submitter, danger detector, fee oracle). +/// input reader, batch submitter, landing watch, fee oracle). /// - `Io` / `Storage`: generic catch-alls used widely; not worth nesting. #[derive(Debug, Error)] pub enum CommandError { @@ -58,16 +57,15 @@ pub enum CommandError { // panic before the command harness). Diagnosed terminal runtime faults bypass // this projection with `abort()` (SIGABRT, 134). -/// Restart; a recovery boot follows and may take 15+ min (flush plus -/// safe-finality wait), so startup probes must allow for it. -pub const EXIT_RESTART_EXPECT_RECOVERY: u8 = 10; -/// Restart with backoff; the refusal clears when the L1 view freshens. +/// Restart with backoff; the refusal clears once the provider or the +/// contended resource recovers. pub const EXIT_RESTART_TRANSIENT: u8 = 20; /// Do not restart; page an operator. pub const EXIT_TERMINAL: u8 = 30; -/// Do not restart: a previous instance left work past the checkpoint, so wipe -/// the uncompleted data dir and run `setup --recovery`. -pub const EXIT_SETUP_NEEDS_RECOVERY: u8 = 40; +/// Do not restart: setup's escape check found the old instance's work past +/// the stop block. Stop the old instance, discard the data directory and run +/// setup again. +pub const EXIT_SETUP_ESCAPE: u8 = 40; /// Unclassified operational failure; restart with backoff. pub const EXIT_UNCLASSIFIED: u8 = 1; @@ -75,26 +73,27 @@ pub const EXIT_UNCLASSIFIED: u8 = 1; /// /// This is the single classification boundary shared by the terminal-fault /// black box and the process exit-code projection: only [`Self::Terminal`] -/// best-effort records a cause (and exits 30: do not restart, page). +/// exits 30 (do not restart, page) and best-effort records its cause, unless +/// the cause is a terminal fact ([`CommandError::is_terminal_fact`]). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum CommandFailureVerdict { - ExpectedRecovery, Retryable, Terminal, - SetupRecoveryRequired, + SetupEscape, Unclassified, } impl CommandFailureVerdict { - /// Whether the black box records a terminal cause (and the process - /// exits 30: do not restart, page an operator). + /// Whether the process exits 30 (do not restart, page an operator). The + /// black box records the cause of a terminal failure that is not a + /// terminal fact. pub(crate) const fn is_terminal(self) -> bool { matches!(self, Self::Terminal) } /// Exit-line severity: ERROR when the cause is a fault or unknown, WARN - /// when the verdict names a planned procedure (a recovery boot, a retry, - /// `setup --recovery`). Alert on the exit code, not the level. + /// when the verdict names a planned procedure (a retry, a setup + /// refusal). Alert on the exit code, not the level. pub(crate) const fn logs_at_error(self) -> bool { matches!(self, Self::Terminal | Self::Unclassified) } @@ -102,11 +101,10 @@ impl CommandFailureVerdict { /// The supervisor action this verdict asks for, named on the exit line. pub(crate) const fn supervisor_action(self) -> &'static str { match self { - Self::ExpectedRecovery => "restart; a recovery boot follows", Self::Retryable => "restart with backoff; alert if it persists", Self::Terminal => "do not restart; page an operator", - Self::SetupRecoveryRequired => { - "do not restart; wipe the data dir and run setup --recovery" + Self::SetupEscape => { + "do not restart; stop the old instance, discard this data directory, run setup again" } Self::Unclassified => "restart with backoff", } @@ -114,10 +112,9 @@ impl CommandFailureVerdict { const fn exit_code(self) -> u8 { match self { - Self::ExpectedRecovery => EXIT_RESTART_EXPECT_RECOVERY, Self::Retryable => EXIT_RESTART_TRANSIENT, Self::Terminal => EXIT_TERMINAL, - Self::SetupRecoveryRequired => EXIT_SETUP_NEEDS_RECOVERY, + Self::SetupEscape => EXIT_SETUP_ESCAPE, Self::Unclassified => EXIT_UNCLASSIFIED, } } @@ -128,9 +125,6 @@ impl CommandError { /// supervisor-facing projection. pub(crate) fn failure_verdict(&self) -> CommandFailureVerdict { match self { - CommandError::Worker(WorkerExit::DangerDetected { status }) => { - danger_failure_verdict(status) - } // Admission-fact refusals are terminal: the wrong command for // this database, the absorbing divergence, a malformed black // box. A plain storage failure underneath a lifecycle operation @@ -191,6 +185,24 @@ impl CommandError { } } + /// Whether this failure states a terminal fact: a recorded divergence or + /// a void history. The database holds a terminal fact and every `run` + /// states it, so the black box, which keeps causes that would otherwise + /// be lost, does not copy it. + pub(crate) fn is_terminal_fact(&self) -> bool { + matches!( + self, + CommandError::Bootstrap(BootstrapError::TerminalFact(_)) + | CommandError::Worker( + WorkerExit::TerminalFact(_) + | WorkerExit::Lane(WorkerStop::Source( + InclusionLaneError::CanonicalDivergence { .. } + )) + ) + | CommandError::Lifecycle(LifecycleError::CanonicalDivergence { .. }) + ) + } + /// Project this error onto the exit-code contract. Clean shutdown /// (exit 0) is handled by the caller — a `CommandError` is always a failure. pub fn exit_code(&self) -> u8 { @@ -200,33 +212,7 @@ impl CommandError { // Per-worker terminality lives as `is_terminal_invariant` on each worker's // own error type, beside its enum; `WorkerExit::is_terminal` below -// composes them, and takes danger exits from `danger_failure_verdict`. - -fn danger_failure_verdict(status: &DangerStatus) -> CommandFailureVerdict { - match status { - // A doomed closed batch / aging Tip: the next boot legitimately runs - // a slow recovery (flush + cascade). - DangerStatus::ClosedBatchInDanger(_) | DangerStatus::TipInDanger(_) => { - CommandFailureVerdict::ExpectedRecovery - } - // View-dependent refusals that self-heal once the provider recovers. - DangerStatus::L1ViewStale | DangerStatus::EstimatedBatchInDanger(_) => { - CommandFailureVerdict::Retryable - } - // The only genuinely terminal danger: canonical divergence. - DangerStatus::CanonicalDivergence(_) => CommandFailureVerdict::Terminal, - // `Safe` is never a detector exit. If it ever reaches here the danger - // classification is self-contradicting — page an operator (EXIT_TERMINAL) - // rather than silently restart-loop with backoff (EXIT_UNCLASSIFIED). - DangerStatus::Safe => { - debug_assert!( - false, - "danger_failure_verdict called with DangerStatus::Safe" - ); - CommandFailureVerdict::Terminal - } - } -} +// composes them. fn bootstrap_failure_verdict(err: &BootstrapError) -> CommandFailureVerdict { match err { @@ -239,17 +225,9 @@ fn bootstrap_failure_verdict(err: &BootstrapError) -> CommandFailureVerdict { BootstrapError::Flush(source) if source.is_terminal_invariant() => { CommandFailureVerdict::Terminal } - // The recovery controller owns classification. No raw provider, - // storage, flush, or phase error crosses this boundary. - BootstrapError::Recovery(source) => { - if source.is_retryable() { - CommandFailureVerdict::Retryable - } else { - CommandFailureVerdict::Terminal - } - } - // Setup's reader failures take the startup polarity. - BootstrapError::SetupInputReader(source) => { + // Setup's reader failures and run's startup sync take the startup + // polarity. + BootstrapError::SetupInputReader(source) | BootstrapError::StartupSync(source) => { if source.is_retryable_at_startup() { CommandFailureVerdict::Retryable } else { @@ -264,7 +242,7 @@ fn bootstrap_failure_verdict(err: &BootstrapError) -> CommandFailureVerdict { | BootstrapError::Identity( IdentityError::FirstBootRequiresL1 | IdentityError::DeploymentNotYetSafe { .. }, ) - | BootstrapError::DetectionWalletNonceRead { .. } + | BootstrapError::WalletNonceRead { .. } | BootstrapError::DataDirLocked { .. } | BootstrapError::FeeOracleTransient { .. } | BootstrapError::Flush(_) => CommandFailureVerdict::Retryable, @@ -273,19 +251,19 @@ fn bootstrap_failure_verdict(err: &BootstrapError) -> CommandFailureVerdict { // was never set up). BootstrapError::ChainIdMismatch { .. } | BootstrapError::RpcRedirect { .. } - | BootstrapError::InvalidProtocolTiming(_) - | BootstrapError::BatchOpenPastDanger { .. } + | BootstrapError::TerminalFact(_) + | BootstrapError::StartBatchNotOpened + | BootstrapError::BatchOpenPastWindow { .. } | BootstrapError::FeeOracleMisconfig { .. } | BootstrapError::FeeOracleFatal { .. } | BootstrapError::SignerMisconfig { .. } | BootstrapError::SetupNotComplete { .. } - | BootstrapError::CheckpointBeforeAppDeployment { .. } - | BootstrapError::SetupRecovery(_) - | BootstrapError::Identity(IdentityError::Mismatch { .. } | IdentityError::OrphanedState) => { + | BootstrapError::Setup(_) + | BootstrapError::Identity(IdentityError::Mismatch { .. }) => { CommandFailureVerdict::Terminal } - BootstrapError::SetupRefuse(_) => CommandFailureVerdict::SetupRecoveryRequired, + BootstrapError::SetupRefuse(_) => CommandFailureVerdict::SetupEscape, BootstrapError::OpenStorage(_) | BootstrapError::KeyFile { .. } => { CommandFailureVerdict::Unclassified } @@ -295,7 +273,8 @@ fn bootstrap_failure_verdict(err: &BootstrapError) -> CommandFailureVerdict { // ── Bootstrap-phase errors ───────────────────────────────────────────── /// Anything that can go wrong before runtime workers start: config validation, -/// deployment-identity guards, startup recovery, initial DB open. +/// deployment-identity guards, the startup terminal-fact check and sync, +/// initial DB open. #[derive(Debug, Error)] pub enum BootstrapError { #[error(transparent)] @@ -318,24 +297,19 @@ pub enum BootstrapError { followed, so configure the provider's final URL" )] RpcRedirect { status: u16 }, - /// Protocol-level config (`preemptive_margin_blocks` vs `max_wait_blocks`, - /// `l1_read_stale_after_blocks` vs `danger_threshold`) failed validation. - /// See [`ProtocolTimingError`]. - #[error(transparent)] - InvalidProtocolTiming(#[from] ProtocolTimingError), /// `max_batch_open_seconds`, in blocks, leaves less than the submission - /// margin below the danger threshold, so a low-volume Tip would reach - /// danger before it can be sealed, submitted and safe-accepted. + /// margin below the staleness window, so a low-volume batch could be + /// sealed too late to land: its history would be void. #[error( "max_batch_open_seconds ({max_batch_open_seconds}) spans {open_blocks} blocks at \ - seconds_per_block ({seconds_per_block}); it must be at most danger_threshold \ - ({danger_threshold}) minus the {margin_blocks}-block submission margin" + seconds_per_block ({seconds_per_block}); it must be at most max_wait_blocks \ + ({max_wait_blocks}) minus the {margin_blocks}-block submission margin" )] - BatchOpenPastDanger { + BatchOpenPastWindow { max_batch_open_seconds: u64, seconds_per_block: u64, open_blocks: u64, - danger_threshold: u64, + max_wait_blocks: u64, margin_blocks: u64, }, /// Setup-pinned fee oracle configuration or validation is invalid. @@ -365,9 +339,19 @@ pub enum BootstrapError { #[source] source: std::io::Error, }, - /// Startup recovery (or refusal) failed before runtime workers started. + /// A fact that ends the era holds at startup: a recorded divergence or a + /// void history. Every later `run` refuses on it. #[error(transparent)] - Recovery(#[from] RecoveryError), + TerminalFact(TerminalFact), + /// `run`'s startup sync failed with something other than an unreachable + /// provider (which falls back to the stored view), classified by + /// [`InputReaderError::is_retryable_at_startup`]. + #[error("startup L1 sync: {0}")] + StartupSync(InputReaderError), + /// Opening the start batch left no open batch: a broken postcondition, + /// rolled back. + #[error("opening the start batch left no open batch")] + StartBatchNotOpened, /// An input-reader failure in setup's L1 discovery (the chain-id check's /// client, the safe-head sample, the reader) or initial sync, classified by /// [`InputReaderError::is_retryable_at_startup`]. @@ -377,7 +361,7 @@ pub enum BootstrapError { #[error(transparent)] Identity(#[from] IdentityError), /// `run` (or `flush-mempool`) was invoked against a DB where `setup` - /// has not completed — its completion fact is absent (setup never ran, or + /// has not completed — its era is absent (setup never ran, or /// crashed midway), or its outputs are incomplete. The /// operator must run `setup` first; restarting `run` cannot self-heal. #[error("setup has not completed for data dir {data_dir} — run `setup` first")] @@ -394,120 +378,128 @@ pub enum BootstrapError { DataDirLocked { path: String }, /// The `flush-mempool` subcommand's flush failed (provider/transport). #[error("mempool flush failed: {0}")] - Flush(#[from] crate::recovery::FlushError), - /// `setup`'s detection gate could not read the batch-submitter wallet - /// nonce from L1 (the one live RPC the gate makes). Transient — the - /// operator retries once the provider recovers; the prior sync already - /// proved L1 reachable, so this is a hiccup, not a misconfig. - #[error("setup detection: could not read submitter wallet nonce from RPC: {message}")] - DetectionWalletNonceRead { message: String }, - /// `setup`'s read-only detection gate found a previous instance left work - /// this checkpoint cannot account for. Sticky: only wiping the uncompleted - /// data dir and running `setup --recovery` resolves it — a plain - /// `setup` restart re-detects and re-refuses. + Flush(#[from] crate::l1::flusher::FlushError), + /// `setup` could not read the batch-submitter wallet nonce for its escape + /// check. The sync before it proved L1 reachable, so this is a hiccup; but + /// the data directory already holds the sync, so the operator discards it + /// and runs setup again. + #[error("setup could not read the submitter wallet nonce from RPC: {message}")] + WalletNonceRead { message: String }, + /// `setup`'s escape check found the old instance's work past the stop + /// block. A restart cannot clear it. #[error(transparent)] SetupRefuse(#[from] SetupRefuse), - /// `setup --checkpoint-block` predates the application's deployment block: - /// a promotion cannot have landed before the application contract existed. - /// Operator misconfig; restarting cannot self-heal. + /// `setup` refused its directory, its base, or the history it rebuilt. + #[error(transparent)] + Setup(Box), +} + +/// `setup`'s escape check, after the flush and the sync through the stop +/// block: either finding means a transaction escaped the flush or the old +/// instance still runs, so the history setup would publish may be wrong. Exit +/// [`EXIT_SETUP_ESCAPE`]: stop the old instance and run setup again in an +/// empty directory. +#[derive(Debug, Error)] +pub enum SetupRefuse { + /// The submitter's wallet nonce is not settled at the synced safe block + /// (`pending > safe`) on the local provider. Local view only: a zombie + /// transaction dropped from this provider's pool evades it. #[error( - "checkpoint block {checkpoint_block} predates the application \ - deployment block {app_deployment_block}" + "the submitter's wallet is still unsettled after the flush (pending nonce {pending} > \ + safe nonce {safe}); stop the old instance, discard this data directory, run setup again" )] - CheckpointBeforeAppDeployment { - checkpoint_block: u64, - app_deployment_block: u64, + WalletUnsettledAfterFlush { pending: u64, safe: u64 }, + /// A submitter input landed past the stop block. + #[error( + "a submitter input (index {safe_input_index}, block {found_block}) landed past the stop \ + block {stop_block}; stop the old instance, discard this data directory, run setup again" + )] + SubmitterInputPastStop { + stop_block: u64, + found_block: u64, + safe_input_index: u64, }, - /// `setup --recovery` failed in a way only the operator can fix (bad config, - /// a checkpoint that can't be loaded or doesn't fit the chain, or a DB that - /// is already set up). Terminal — see [`SetupRecoveryError`]. - #[error(transparent)] - SetupRecovery(#[from] SetupRecoveryError), } -/// Terminal failures of the `setup --recovery` procedure — the ones an -/// operator must resolve. All map to [`EXIT_TERMINAL`]: a plain restart -/// re-runs the same bad inputs and re-fails identically. Other failures take -/// other paths: the flush as [`BootstrapError::Flush`], the post-flush -/// re-sync as a [`WorkerExit::InputReader`] exit, and a re-sync behind the -/// flush or a stopping block behind the checkpoint as a retryable -/// [`RecoveryError`]. +/// Why `setup` refuses, each terminal: the operator fixes the cause and runs +/// setup again in an empty directory. #[derive(Debug, Error)] -pub enum SetupRecoveryError { - /// Cross-field config validation failed (recovery missing its dump dir / - /// checkpoint block / key, or a plain `setup` carrying recovery-only args). - /// See [`crate::commands::config::SetupConfig::validate`]. - #[error("invalid recovery configuration: {message}")] - InvalidConfig { message: String }, - /// `setup --recovery` was invoked against a DB that is already set up. - /// Recovery is a strict one-shot on a freshly-wiped DB (wipe and re-run with - /// `--recovery`); re-pointing a live deployment at a different checkpoint - /// would strand its existing state. +pub enum SetupError { + /// Setup starts only in an empty directory: nothing already there can be + /// trusted to fit this run, a completed setup or a failed one. #[error( - "`setup --recovery` requires a freshly-wiped data dir, but this one is \ - already set up — wipe it and re-run" + "data directory {path} is not empty (it holds {entry}); setup runs only in an empty \ + directory" )] - AlreadySetUp, - /// The checkpoint dump could not be loaded (missing/corrupt `info.toml`, or - /// the app's `from_dump` failed). Operator must supply a valid **sequencer** - /// recovery export (`info.toml` + `checkpoint.toml` + `state/`), not a watchdog CM checkpoint. - #[error("failed to load checkpoint dump at {path}: {message}")] - CheckpointLoad { path: String, message: String }, - /// Outside empty genesis, A must precede B: equality can exclude pending - /// directs in B from the seed range. A < B alone does not certify queue - /// eligibility; the exporter must also verify no pending direct is at or - /// below A (docs/recovery/cockroach.md). + DataDirNotEmpty { path: String, entry: String }, + #[error("cannot read the base machine at {path}: {source}")] + BaseMachine { + path: std::path::PathBuf, + #[source] + source: crate::machine::MachineExportError, + }, + #[error("cannot build the application from the base machine's state at {path}: {source}")] + BaseApplication { + path: std::path::PathBuf, + #[source] + source: AppError, + }, + /// The base consumed inputs past the stop block, so its state is not at + /// or below the era's. #[error( - "checkpoint last-executed safe block {executed_safe_block} (A) must precede \ - checkpoint block {checkpoint_block} (B), except for empty genesis; \ - equality can omit pending same-block directs" + "the base consumed {l1_input_count} L1 inputs, but only {inputs_through_stop} lie at or \ + below the stop block {stop_block}" )] - CheckpointNotBeforeBlock { - executed_safe_block: u64, - checkpoint_block: u64, + BaseAheadOfStop { + l1_input_count: u64, + inputs_through_stop: u64, + stop_block: u64, }, - /// The input reader reported a successful post-flush sync without - /// persisting the safe-head observation that the recovery fold requires. + /// The export's queued directs are not the admitted directs above its + /// clock: the machine is not canonical, or its scheduler drained wrongly. #[error( - "post-flush L1 resync completed without a persisted safe-head observation: \ - internal storage invariant violation" + "the base's export disagrees with L1: among its {l1_input_count} inputs, {admitted} \ + admitted directs lie above its clock {clock}, the first at {first_admitted:?}, but it \ + queues {fridge_len} from input {first_queued_input}" )] - MissingResyncedSafeHead, -} - -/// `setup`'s read-only detection gate: the reasons a -/// fresh `setup` refuses because a *previous* instance left work past the -/// checkpoint. The gate runs before baseline publication and leaves setup -/// incomplete. Rebuild in a fresh data directory with `setup --recovery` to -/// flush/fold the outstanding batches; a plain `setup` restart re-detects and -/// re-refuses (hence [`EXIT_SETUP_NEEDS_RECOVERY`], not auto-recovery class 10). -/// -/// Both variants carry diagnostic fields for the refusal log line. -#[derive(Debug, Error)] -pub enum SetupRefuse { - /// Step 1: the batch-submitter wallet nonce is not settled - /// (`pending > safe`) on the local provider — a previous instance left - /// pending or mined-but-unsafe batch txs. Local-view only: a - /// zombie tx dropped from this provider's pool but alive elsewhere evades - /// this check; bounded at runtime by the content-identity check. + ExportDisagrees { + l1_input_count: u64, + clock: u64, + fridge_len: u64, + first_queued_input: u64, + admitted: u64, + first_admitted: Option, + }, + /// The scheduler refused the export with its queued range from L1. + #[error("the base's export disagrees with L1: {0}")] + ResumeRefused(#[from] sequencer_core::scheduler::ResumeError), + #[error("the rebuilt scheduler's clock {clock} is past the stop block {stop_block}")] + ClockPastStop { clock: u64, stop_block: u64 }, + /// The safe head the sync reached is below the stop block the flush saw: + /// a provider behind its own earlier answer. #[error( - "batch-submitter wallet nonce not settled (pending {pending} > safe \ - {safe}) — a previous instance left in-flight batch txs; wipe this \ - uncompleted data dir, then run `setup --recovery`" + "the sync reached safe block {synced_safe_block}, below the stop block {stop_block}; \ + discard this data directory and run setup again" )] - WalletNonceUnsettled { pending: u64, safe: u64 }, - /// Step 2: a batch-submitter tx exists in `(checkpoint_block, safe]` — a - /// previous instance already wrote batches past this checkpoint, so a - /// genesis-style bootstrap would silently diverge from canonical state. + SyncBehindStopBlock { + synced_safe_block: u64, + stop_block: u64, + }, + /// The application failed on a rebuilt transaction: the `position`-th + /// the scheduler handed out for input `l1_input_index`, skipped ones + /// included. Replaying a trusted machine to that input says whether the + /// application is dead on chain. #[error( - "batch-submitter input found at block {found_block} past checkpoint \ - block {checkpoint_block} (safe_input_index {safe_input_index}) — wipe \ - this uncompleted data dir, then run `setup --recovery`" + "the application failed on the rebuilt transaction at input {l1_input_index}, \ + position {position} ({transaction}): {source}" )] - BatchPastCheckpoint { - checkpoint_block: u64, - found_block: u64, - safe_input_index: u64, + HistoryExecution { + l1_input_index: u64, + position: u64, + /// A direct's input index, or a user op's sender and nonce. + transaction: String, + #[source] + source: AppError, }, } @@ -536,11 +528,6 @@ pub enum IdentityError { app_deployment_block: u64, safe_block: u64, }, - /// The DB has persisted state but no pinned identity. Binding the current - /// config now would silently inherit an unknown deployment's data. - /// Operator: confirm provenance or wipe the DB. - #[error("orphaned state: DB has persisted state but no deployment identity to claim it")] - OrphanedState, /// The pinned identity doesn't match the current config. /// /// `stored` and `expected` are boxed to keep `CommandError` under clippy's @@ -556,8 +543,8 @@ pub enum IdentityError { // ── Worker exits ─────────────────────────────────────────────────────── /// Which runtime worker exited, and why: one generic stop shape per worker, plus -/// the danger detector's deliberate `RecoveryRequired` trip, which is not an error -/// but exits the runtime so the orchestrator respawns into startup recovery. +/// the landing watch's terminal fact, which is not a worker error but ends the +/// era: the runtime stops terminally and every later `run` refuses. #[derive(Debug, Error)] pub enum WorkerExit { #[error("server: {0}")] @@ -568,12 +555,12 @@ pub enum WorkerExit { InputReader(WorkerStop), #[error("batch submitter: {0}")] BatchSubmitter(WorkerStop), - #[error("danger detector: {0}")] - DangerDetector(WorkerStop), + #[error("landing watch: {0}")] + LandingWatch(WorkerStop), #[error("fee oracle: {0}")] FeeOracle(WorkerStop), - #[error("danger detector: danger detected ({status:?}) — stopping for startup recovery")] - DangerDetected { status: DangerStatus }, + #[error("landing watch: {0}")] + TerminalFact(TerminalFact), } impl WorkerExit { @@ -597,14 +584,14 @@ impl WorkerExit { WorkerExit::BatchSubmitter(stop) => { stop.is_terminal_with(BatchSubmitterError::is_terminal_invariant) } - WorkerExit::DangerDetector(stop) => { - stop.is_terminal_with(DangerDetectorError::is_terminal_invariant) + WorkerExit::LandingWatch(stop) => { + stop.is_terminal_with(LandingWatchError::is_terminal_invariant) } WorkerExit::FeeOracle(stop) => { stop.is_terminal_with(FeeOracleError::is_terminal_invariant) } - // One home with the exit code, so the abort decision cannot drift. - WorkerExit::DangerDetected { status } => danger_failure_verdict(status).is_terminal(), + // A fact that ends the era: a divergence or a void history. + WorkerExit::TerminalFact(_) => true, } } } @@ -688,9 +675,7 @@ impl From for CommandError { } /// One shared classification for the keyed signer-provider constructor, so -/// `setup`, `flush-mempool` and any future keyed command agree. Run recovery -/// keeps its own Retry/Refuse polarity map, but its terminal/transient split must -/// agree with this one. +/// `setup`, `flush-mempool` and any future keyed command agree. impl From for BootstrapError { fn from(e: crate::l1::provider::VerifiedSignerProviderError) -> Self { use crate::l1::provider::VerifiedSignerProviderError as E; @@ -711,15 +696,23 @@ impl From for CommandError { } } -impl From for CommandError { - fn from(e: ProtocolTimingError) -> Self { - CommandError::Bootstrap(e.into()) +impl From for CommandError { + fn from(fact: TerminalFact) -> Self { + CommandError::Bootstrap(BootstrapError::TerminalFact(fact)) } } -impl From for CommandError { - fn from(e: RecoveryError) -> Self { - CommandError::Bootstrap(e.into()) +/// A storage failure keeps its persistence classification; the other two +/// arms are terminal refusals. +impl From for CommandError { + fn from(error: OpenBatchError) -> Self { + match error { + OpenBatchError::Storage(source) => CommandError::Storage(source), + OpenBatchError::Terminal(fact) => fact.into(), + OpenBatchError::OpenBatchMissing => { + CommandError::Bootstrap(BootstrapError::StartBatchNotOpened) + } + } } } @@ -729,8 +722,8 @@ impl From for CommandError { } } -impl From for CommandError { - fn from(e: crate::recovery::FlushError) -> Self { +impl From for CommandError { + fn from(e: crate::l1::flusher::FlushError) -> Self { CommandError::Bootstrap(BootstrapError::Flush(e)) } } @@ -741,9 +734,9 @@ impl From for CommandError { } } -impl From for CommandError { - fn from(e: SetupRecoveryError) -> Self { - CommandError::Bootstrap(BootstrapError::SetupRecovery(e)) +impl From for CommandError { + fn from(e: SetupError) -> Self { + CommandError::Bootstrap(BootstrapError::Setup(Box::new(e))) } } @@ -773,17 +766,28 @@ impl From for CommandError { mod tests { use super::*; use crate::l1::fee_oracle::worker::FeeOracleError; + use crate::l1::flusher::FlushError; use crate::l1::provider::VerifiedSignerProviderError; use crate::l1::submitter::BatchPosterError; use crate::l1::watermark::WalletNonceWatermarkError; - use crate::recovery::{ - FlushError, RecoveryError, RecoveryFailure, RecoveryRefusalReason, RecoveryRetryReason, - }; - use crate::storage::DeploymentIdentity; - use sequencer_core::protocol::ProtocolTimingError; - - fn danger(status: DangerStatus) -> CommandError { - CommandError::Worker(WorkerExit::DangerDetected { status }) + use crate::storage::{DeploymentIdentity, Divergence, LandingLag}; + + fn divergence() -> TerminalFact { + TerminalFact::Divergence(Divergence { + nonce: 0, + safe_input_index: 4, + }) + } + + fn void_history() -> TerminalFact { + TerminalFact::VoidHistory { + lag: LandingLag { + batch_nonce: 3, + first_frame_safe_block: 10, + safe_block: 1209, + }, + max_wait_blocks: 1200, + } } fn dummy_identity() -> DeploymentIdentity { @@ -794,6 +798,8 @@ mod tests { input_box_address: Address::repeat_byte(0x22), app_deployment_block: 0, batch_submitter_address: Address::repeat_byte(0x33), + max_direct_payload_bytes: 1024, + max_wait_blocks: 1200, fee_oracle: crate::storage::FeeOracleIdentity::Fixed { log_gas_price: 0 }, } } @@ -823,12 +829,8 @@ mod tests { }) } - fn recovery_retry(failure: impl Into) -> CommandError { - CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::retry(failure))) - } - - fn recovery_refuse(failure: impl Into) -> CommandError { - CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::refuse(failure))) + fn startup_sync(error: InputReaderError) -> CommandError { + CommandError::Bootstrap(BootstrapError::StartupSync(error)) } fn input_reader_worker(error: InputReaderError) -> CommandError { @@ -839,8 +841,8 @@ mod tests { CommandError::Worker(WorkerExit::BatchSubmitter(WorkerStop::Source(error))) } - fn detector_worker(error: DangerDetectorError) -> CommandError { - CommandError::Worker(WorkerExit::DangerDetector(WorkerStop::Source(error))) + fn landing_watch_worker(error: LandingWatchError) -> CommandError { + CommandError::Worker(WorkerExit::LandingWatch(WorkerStop::Source(error))) } fn fee_oracle_worker(error: FeeOracleError) -> CommandError { @@ -851,17 +853,6 @@ mod tests { /// assertion prints when its projection moves. type Row = (CommandError, &'static str); - /// Class 10: restart; the next boot legitimately runs a slow recovery. - fn expect_recovery() -> Vec { - vec![ - ( - danger(DangerStatus::ClosedBatchInDanger(0)), - "a doomed closed batch", - ), - (danger(DangerStatus::TipInDanger(3)), "an aging Tip"), - ] - } - fn setup_reader(error: InputReaderError) -> CommandError { CommandError::Bootstrap(BootstrapError::SetupInputReader(error)) } @@ -883,30 +874,18 @@ mod tests { "busy storage during setup's L1 sync", ), ( - danger(DangerStatus::L1ViewStale), - "a stale L1 view at runtime", - ), - ( - danger(DangerStatus::EstimatedBatchInDanger(2)), - "an estimated batch in danger", + startup_sync(InputReaderError::InconsistentL1Response("gap".into())), + "an inconsistent L1 response during run's startup sync", ), ( - recovery_retry(RecoveryRetryReason::L1ViewStale), - "a stale L1 view at startup recovery", + startup_sync(InputReaderError::Storage(busy())), + "busy storage during run's startup sync", ), ( - recovery_retry(RecoveryRetryReason::ResyncBehindFlushView { - resynced_safe_block: 1, - flush_observed_safe_block: 2, - }), - "a resync behind the flush's observed view", - ), - ( - recovery_retry(RecoveryRetryReason::CheckpointAheadOfStop { - checkpoint_block: 901, - stop_block: 899, - }), - "a valid manual recovery checkpoint ahead of the RPC stopping block", + startup_sync(InputReaderError::OpenStorage(StorageOpenError::Sqlite( + busy(), + ))), + "a busy open during run's startup sync remains restartable", ), ( CommandError::Bootstrap(BootstrapError::Identity( @@ -923,6 +902,12 @@ mod tests { )), "a setup before the deployment is safe", ), + ( + CommandError::Bootstrap(BootstrapError::WalletNonceRead { + message: "provider unavailable".into(), + }), + "the escape check's wallet nonce read failing", + ), ( CommandError::Bootstrap(BootstrapError::ChainIdRpc { message: "provider unavailable".into(), @@ -933,19 +918,6 @@ mod tests { CommandError::from(FlushError::Provider("x".into())), "the flush's provider", ), - // A flush failure surfaced via startup recovery lands in the same - // class as the flush-mempool subcommand's `FlushError`. - ( - recovery_retry(RecoveryFailure::Flush(FlushError::Provider("x".into()))), - "the same flush provider failure under startup recovery", - ), - // `bootstrap_failure_verdict` classifies `Recovery` on its - // retry/refuse wrapper alone; which wrapper a provider failure - // gets is pinned where it is built (`classify_signer_provider`). - ( - recovery_retry(RecoveryFailure::ProviderUnreachable("timeout".into())), - "the recovery flush's signer provider unreachable", - ), // Documented "may self-heal": the definition of the retryable class. ( CommandError::from(FeeOracleError::Transient("RPC unavailable".into())), @@ -957,17 +929,11 @@ mod tests { ))), "operational watermark contention remains retryable", ), - ( - recovery_retry(RecoveryFailure::OpenStorage(StorageOpenError::Sqlite( - busy(), - ))), - "a busy open during the startup sync remains restartable", - ), ] } - /// Class 30: do not restart; page. The state cannot self-heal, so the - /// black box records the cause (`is_terminal`) for exactly these rows. + /// Class 30: do not restart; page. The state cannot self-heal + /// (`is_terminal` for exactly these rows). fn terminal() -> Vec { let mut rows: Vec = vec![ ( @@ -1034,11 +1000,6 @@ mod tests { CommandError::from(VerifiedSignerProviderError::Create("bad key".into())), "a signer-construction misconfig", ), - // Retry/refuse is the wrapper's, pinned at `classify_signer_provider`. - ( - recovery_refuse(RecoveryFailure::SignerMisconfig("bad key".into())), - "the same signer misconfig under startup recovery", - ), ( CommandError::ReferencedSnapshotArtifact { path: "/durable/snapshot".into(), @@ -1051,7 +1012,9 @@ mod tests { "a mandatory durable row disappearing cannot self-heal on restart", ), ( - detector_worker(DangerDetectorError::Storage(rusqlite::Error::QueryReturnedNoRows)), + landing_watch_worker(LandingWatchError::Storage( + rusqlite::Error::QueryReturnedNoRows, + )), "typed persistent storage errors retain terminal classification through workers", ), ( @@ -1075,8 +1038,8 @@ mod tests { "a submitter storage-task panic", ), ( - detector_worker(DangerDetectorError::StorageTaskPanicked), - "a detector storage-task panic", + landing_watch_worker(LandingWatchError::StorageTaskPanicked), + "a landing-watch storage-task panic", ), ( submitter_worker(BatchSubmitterError::Poster(BatchPosterError::Watermark( @@ -1092,12 +1055,6 @@ mod tests { CommandError::from(FlushError::UnsignableTransaction("no signer".into())), "a flush no-op the submit provider cannot sign", ), - ( - recovery_refuse(RecoveryFailure::Flush(FlushError::UnsignableTransaction( - "no signer".into(), - ))), - "the same unsignable no-op under startup recovery", - ), ( submitter_worker(BatchSubmitterError::Poster( BatchPosterError::UnsignableTransaction("no signer".into()), @@ -1115,18 +1072,37 @@ mod tests { "an undecodable batch from our own submitter key", ), ( - recovery_refuse(RecoveryFailure::Flush(FlushError::Watermark( - persistent_watermark(), + CommandError::Worker(WorkerExit::TerminalFact(divergence())), + "canonical divergence at runtime", + ), + ( + CommandError::Worker(WorkerExit::TerminalFact(void_history())), + "a void history at runtime", + ), + ( + CommandError::Worker(WorkerExit::Lane(WorkerStop::Source( + InclusionLaneError::CanonicalDivergence { + nonce: 0, + safe_input_index: 4, + }, ))), - "the same watermark failure under startup recovery", + "canonical divergence at the lane's frontier read", ), ( - danger(DangerStatus::CanonicalDivergence(0)), - "canonical divergence at runtime", + CommandError::from(divergence()), + "canonical divergence at startup", + ), + ( + CommandError::from(void_history()), + "a void history at startup", + ), + ( + CommandError::from(OpenBatchError::Terminal(void_history())), + "a void history the start batch's transaction refuses", ), ( - recovery_refuse(RecoveryRefusalReason::CanonicalDivergence { nonce: 0 }), - "canonical divergence at startup recovery", + CommandError::from(OpenBatchError::OpenBatchMissing), + "a start batch whose transaction left no open batch", ), ( CommandError::Bootstrap(BootstrapError::SetupNotComplete { @@ -1143,27 +1119,14 @@ mod tests { "an RPC endpoint that redirects setup's chain-id read", ), ( - CommandError::Bootstrap(BootstrapError::InvalidProtocolTiming( - ProtocolTimingError::MarginNotLessThanMaxWait { - margin: sequencer_core::MAX_WAIT_BLOCKS, - max_wait: sequencer_core::MAX_WAIT_BLOCKS, - }, - )), - "invalid protocol timing", - ), - ( - CommandError::Bootstrap(BootstrapError::BatchOpenPastDanger { - max_batch_open_seconds: 9601, + CommandError::Bootstrap(BootstrapError::BatchOpenPastWindow { + max_batch_open_seconds: 13201, seconds_per_block: 12, - open_blocks: 801, - danger_threshold: 900, + open_blocks: 1101, + max_wait_blocks: 1200, margin_blocks: 100, }), - "a batch-open window past the danger budget", - ), - ( - CommandError::Bootstrap(BootstrapError::Identity(IdentityError::OrphanedState)), - "orphaned state", + "a batch-open window past the staleness window", ), ( CommandError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { @@ -1173,20 +1136,72 @@ mod tests { })), "a pinned-identity mismatch", ), - // `setup --recovery` operator-fixable failures are terminal (a - // restart re-runs the same bad inputs). + // Setup's refusals of its directory, base and rebuild are + // terminal (30), never the escape class (40): a fresh setup in + // another directory does not clear them. ( - CommandError::from(SetupRecoveryError::AlreadySetUp), - "setup --recovery over an already set-up directory", + CommandError::from(SetupError::DataDirNotEmpty { + path: "/data".into(), + entry: "sequencer.db".into(), + }), + "a setup in a data directory that holds anything", ), - // A checkpoint predating genesis is operator misconfig — terminal - // (30), not a recovery trigger (40). ( - CommandError::Bootstrap(BootstrapError::CheckpointBeforeAppDeployment { - checkpoint_block: 5, - app_deployment_block: 10, + CommandError::from(SetupError::BaseApplication { + path: "/machines/base".into(), + source: AppError::Internal { + reason: "count mismatch".into(), + }, }), - "a checkpoint predating genesis", + "an application that cannot be built from the base's state", + ), + ( + CommandError::from(SetupError::BaseAheadOfStop { + l1_input_count: 9, + inputs_through_stop: 8, + stop_block: 100, + }), + "a base past the stop block", + ), + ( + CommandError::from(SetupError::ExportDisagrees { + l1_input_count: 9, + clock: 40, + fridge_len: 2, + first_queued_input: 5, + admitted: 1, + first_admitted: Some(6), + }), + "an export that disagrees with L1", + ), + ( + CommandError::from(SetupError::ResumeRefused( + sequencer_core::scheduler::ResumeError::FirstQueuedNotADirect { index: 5 }, + )), + "an export whose queued range the scheduler refuses", + ), + ( + CommandError::from(SetupError::ClockPastStop { + clock: 101, + stop_block: 100, + }), + "a rebuilt clock past the stop block", + ), + ( + CommandError::from(SetupError::SyncBehindStopBlock { + synced_safe_block: 99, + stop_block: 100, + }), + "a sync that fell behind the flush's stop block", + ), + ( + CommandError::from(SetupError::HistoryExecution { + l1_input_index: 7, + position: 2, + transaction: "direct input 4".to_owned(), + source: AppError::Io(std::io::Error::other("disk unavailable")), + }), + "an application failure on the rebuilt history", ), // Reader-level chain-id mismatch (warm-boot backstop) is terminal, // like the boot-time BootstrapError::ChainIdMismatch. @@ -1197,31 +1212,32 @@ mod tests { }), "the reader's warm-boot chain-id backstop", ), - // The same mismatch surfacing during a startup-recovery safe-head - // sync (RecoveryError path) is terminal too — not the unclassified - // Recovery catch-all (which would loop on the wrong chain). + // The same mismatch surfacing during run's startup sync is + // terminal too, not a retry that would loop on the wrong chain. ( - recovery_refuse(RecoveryFailure::InputReader(InputReaderError::ChainIdMismatch { + startup_sync(InputReaderError::ChainIdMismatch { rpc: 1, expected: 31337, - })), + }), "the same mismatch during the startup sync", ), ( - recovery_refuse(RecoveryFailure::InputReader( - InputReaderError::StorageTaskPanicked { - operation: "startup sync", - }, - )), + startup_sync(InputReaderError::StorageTaskPanicked { + operation: "startup sync", + }), "a storage-task panic during the startup sync", ), ( - recovery_refuse(RecoveryFailure::OpenStorage(StorageOpenError::Sqlite(sqlite( + startup_sync(InputReaderError::OpenStorage(StorageOpenError::Sqlite(sqlite( rusqlite::ffi::ErrorCode::NotADatabase, 26, )))), "a persistent open failure during the startup sync", ), + ( + startup_sync(InputReaderError::Bootstrap("plaintext rpc url".into())), + "a malformed RPC URL at run's startup sync", + ), ( CommandError::from(FeeOracleError::Join("storage task panicked".into())), "a fee-oracle storage-task panic at bootstrap", @@ -1263,26 +1279,25 @@ mod tests { rows } - /// Class 40: sticky setup refusals. The operator must wipe the - /// uncompleted data dir and run `setup --recovery`, not plain-restart - /// (which would re-detect and re-refuse) — so they get a dedicated code, - /// distinct from the auto-recovery class (10). - fn setup_needs_recovery() -> Vec { + /// Class 40: the escape check. The old instance must be stopped before + /// a setup in a fresh directory can succeed, so it gets a dedicated code, + /// distinct from the terminal class (30). + fn setup_escape() -> Vec { vec![ ( - CommandError::from(SetupRefuse::WalletNonceUnsettled { + CommandError::from(SetupRefuse::WalletUnsettledAfterFlush { pending: 14, safe: 13, }), - "an unsettled wallet nonce past the checkpoint", + "a wallet still unsettled after the flush", ), ( - CommandError::from(SetupRefuse::BatchPastCheckpoint { - checkpoint_block: 100, + CommandError::from(SetupRefuse::SubmitterInputPastStop { + stop_block: 100, found_block: 250, safe_input_index: 7, }), - "a batch past the checkpoint", + "a submitter input past the stop block", ), ] } @@ -1364,16 +1379,15 @@ mod tests { } /// Every failure shape the suite pins, projected once onto the exit-code - /// contract. Each row asserts its class and that the black-box verdict - /// agrees with it (`is_terminal` exactly for class 30). The five classes + /// contract. Each row asserts its class and that the terminal verdict + /// agrees with it (`is_terminal` exactly for class 30). The four classes /// carry distinct codes (pinned below), so a code pin is a verdict pin. #[test] fn every_pinned_failure_projects_to_its_class() { let table = [ - (EXIT_RESTART_EXPECT_RECOVERY, expect_recovery()), (EXIT_RESTART_TRANSIENT, transient()), (EXIT_TERMINAL, terminal()), - (EXIT_SETUP_NEEDS_RECOVERY, setup_needs_recovery()), + (EXIT_SETUP_ESCAPE, setup_escape()), (EXIT_UNCLASSIFIED, unclassified()), ]; for (code, rows) in table { @@ -1389,42 +1403,59 @@ mod tests { } } - /// A detector exit aborts the run exactly when its exit code is terminal: - /// only canonical divergence. + /// Of every failure shape the suite pins, exactly the ones that state a + /// divergence or a void history are terminal facts, and each is terminal. #[test] - fn danger_exit_aborts_only_on_canonical_divergence() { - for status in [ - DangerStatus::ClosedBatchInDanger(1), - DangerStatus::TipInDanger(1), - DangerStatus::L1ViewStale, - DangerStatus::EstimatedBatchInDanger(1), - DangerStatus::CanonicalDivergence(1), - ] { - let terminal = matches!(status, DangerStatus::CanonicalDivergence(_)); - assert_eq!( - WorkerExit::DangerDetected { status }.is_terminal(), - terminal, - "{status:?}" - ); + fn only_a_divergence_or_a_void_history_is_a_terminal_fact() { + let pinned = [transient(), terminal(), setup_escape(), unclassified()]; + let facts: Vec<&str> = pinned + .iter() + .flatten() + .filter(|(error, _)| error.is_terminal_fact()) + .map(|(error, why)| { + assert!(error.failure_verdict().is_terminal(), "{why}: {error:?}"); + *why + }) + .collect(); + assert_eq!( + facts, + [ + "canonical divergence at runtime", + "a void history at runtime", + "canonical divergence at the lane's frontier read", + "canonical divergence at startup", + "a void history at startup", + "a void history the start batch's transaction refuses", + "divergence pages", + ] + ); + } + + /// Every terminal fact the landing watch reports aborts the run, with the + /// operator-facing text naming the remedy. + #[test] + fn terminal_fact_exits_abort_and_name_the_remedy() { + for fact in [divergence(), void_history()] { + assert!(WorkerExit::TerminalFact(fact).is_terminal(), "{fact:?}"); } + assert_eq!( + void_history().to_string(), + "the history is void: batch 3, opened at safe block 10, was not accepted by safe \ + block 1209, and no later block can accept it (max_wait_blocks 1200); recover with \ + setup --base in a new data directory" + ); } - /// The exhaustive match makes a sixth verdict a compile error until its + /// The exhaustive match makes a new verdict a compile error until its /// exit-line level is chosen here. #[test] fn exit_line_logs_at_error_only_for_terminal_and_unclassified() { use CommandFailureVerdict::*; let at_error = |verdict: CommandFailureVerdict| match verdict { Terminal | Unclassified => true, - ExpectedRecovery | Retryable | SetupRecoveryRequired => false, + Retryable | SetupEscape => false, }; - let all = [ - ExpectedRecovery, - Retryable, - Terminal, - SetupRecoveryRequired, - Unclassified, - ]; + let all = [Retryable, Terminal, SetupEscape, Unclassified]; for verdict in all { assert_eq!(verdict.logs_at_error(), at_error(verdict), "{verdict:?}"); } @@ -1438,27 +1469,20 @@ mod tests { /// The wire values a supervisor matches on, as integers: a renumbered /// constant must fail here, not in a runbook. They are pairwise distinct, /// which is what lets the table above pin verdicts through codes; the - /// exhaustive match makes a sixth verdict a compile error until it is - /// given its integer here. + /// exhaustive match makes a new verdict a compile error until it is + /// given its integer here. Code 10 is retired and never reused. #[test] fn exit_codes_are_the_documented_integers() { use CommandFailureVerdict::*; let documented = |verdict: CommandFailureVerdict| -> u8 { match verdict { - ExpectedRecovery => 10, Retryable => 20, Terminal => 30, - SetupRecoveryRequired => 40, + SetupEscape => 40, Unclassified => 1, } }; - let all = [ - ExpectedRecovery, - Retryable, - Terminal, - SetupRecoveryRequired, - Unclassified, - ]; + let all = [Retryable, Terminal, SetupEscape, Unclassified]; for verdict in all { assert_eq!(verdict.exit_code(), documented(verdict), "{verdict:?}"); } diff --git a/sequencer/src/commands/flush.rs b/sequencer/src/commands/flush.rs index 9f4d3c4d..7c26f3cb 100644 --- a/sequencer/src/commands/flush.rs +++ b/sequencer/src/commands/flush.rs @@ -2,14 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! The `flush-mempool` command: settle the batch-submitter wallet nonce on demand -//! with the same flush startup recovery and `setup --recovery` run. It needs the -//! signing key and a completed setup, and never syncs, cascades or launches -//! workers (README "Running"). +//! with the same flush that setup runs. It needs the signing key and a +//! completed setup, and never syncs or launches workers (README "Running"). use super::load_setup_identity; use crate::commands::config::FlushConfig; use crate::commands::error::CommandError; -use crate::recovery::MempoolFlusher; +use crate::l1::flusher::MempoolFlusher; use crate::storage::{self, LifecycleCommand}; pub async fn flush_mempool(config: FlushConfig) -> Result<(), CommandError> { @@ -42,7 +41,7 @@ async fn flush_mempool_admitted( // The durable flush anchor: every slot we ever broadcast // must resolve at safe depth, regardless of the local pool's memory. let watermark = { - let mut storage = storage::Storage::open_writer(&db_path)?; + let mut storage = storage::Storage::open_read_only(&db_path)?; storage.wallet_nonce_watermark()? }; diff --git a/sequencer/src/commands/mod.rs b/sequencer/src/commands/mod.rs index f969ac01..c8bd6d9e 100644 --- a/sequencer/src/commands/mod.rs +++ b/sequencer/src/commands/mod.rs @@ -3,7 +3,7 @@ //! Operator commands. Each child is one bracket: take the process lock, preflight //! the admission facts, run the body, and best-effort record a terminal cause. -//! Mechanisms live in `crate::recovery` and `crate::runtime`; this module holds +//! Mechanisms live in `crate::landing` and `crate::runtime`; this module holds //! the helpers shared by brackets, and [`crate::harness`] dispatches them. pub mod config; @@ -12,9 +12,7 @@ pub mod flush; pub mod run; pub mod setup; -pub use error::{ - BootstrapError, CommandError, IdentityError, SetupRecoveryError, SetupRefuse, WorkerExit, -}; +pub use error::{BootstrapError, CommandError, IdentityError, SetupError, SetupRefuse, WorkerExit}; use std::time::Duration; @@ -82,13 +80,28 @@ fn preflight_lifecycle_command( /// still carry it. Failures raised before a command's bracket (the /// admission preflight, identity, the key file) return before this write /// and leave only the exit code and logs. +/// +/// The black box keeps causes that would otherwise be lost. A terminal fact, +/// a recorded divergence or a void history, is never lost: the database holds +/// it and every `run` states it, so it is not copied, wherever the command +/// met it. pub(crate) fn record_terminal_fault_best_effort( db_path: &str, command: LifecycleCommand, result: &Result<(), CommandError>, ) { - let Err(error) = result else { return }; - if !error.failure_verdict().is_terminal() { + if let Err(error) = result { + record_terminal_error_best_effort(db_path, command, error); + } +} + +/// [`record_terminal_fault_best_effort`] for an error the caller still owns. +pub(crate) fn record_terminal_error_best_effort( + db_path: &str, + command: LifecycleCommand, + error: &CommandError, +) { + if !error.failure_verdict().is_terminal() || error.is_terminal_fact() { return; } let cause = error.to_string(); @@ -108,14 +121,6 @@ pub(crate) fn record_terminal_fault_best_effort( } } -/// A setup that completed after a fault proves the fault resolved; any fault -/// at or after the last completion is still news. The order is by wall clock, -/// so a backward step larger than the gap between completion and a later -/// fault demotes that fault to debug; the black box keeps it either way. -fn fault_is_news(fault: &storage::TerminalFault, setup_completed_at_ms: Option) -> bool { - setup_completed_at_ms.is_none_or(|completed| fault.recorded_at_ms >= completed) -} - /// Verdict-neutral startup read of the black box: if a previous command on /// this data directory died terminal, say so once at boot, so the cause is in /// this process's log even when the last process's final lines were lost. @@ -124,39 +129,26 @@ fn fault_is_news(fault: &storage::TerminalFault, setup_completed_at_ms: Option - { - tracing::warn!( - command = %fault.command, - cause = %fault.cause, - recorded_at_ms = fault.recorded_at_ms, - age_secs = crate::clock::unix_now_ms().saturating_sub(fault.recorded_at_ms) / 1000, - "a previous command on this data directory ended in a terminal fault" - ) - } - Ok((Some(fault), _)) => tracing::debug!( + Ok(Some(fault)) => tracing::warn!( command = %fault.command, cause = %fault.cause, recorded_at_ms = fault.recorded_at_ms, - "latest terminal fault predates the completed setup, which resolved it" + age_secs = crate::clock::unix_now_ms().saturating_sub(fault.recorded_at_ms) / 1000, + "a previous command on this data directory ended in a terminal fault" ), - Ok((None, _)) => {} + Ok(None) => {} Err(error) => tracing::debug!( error = %error, "no readable black box at startup (fresh data directory, or the \ @@ -183,7 +175,7 @@ pub(crate) fn batch_submitter_address_from_private_key( } /// Gate `run`/`flush` on a completed `setup` and return the pinned identity. -/// A missing completion fact — or a completion fact without an identity (a +/// A missing era — or an era without an identity (a /// corrupt/incomplete setup) — is a terminal `SetupNotComplete`: the operator must /// (re-)run `setup`, not retry `run`. pub(crate) fn load_setup_identity(data_dir: &str) -> Result { @@ -244,21 +236,6 @@ pub(crate) async fn validate_rpc_chain_id( } } -pub(crate) fn ensure_deployment_identity( - db_path: &str, - expected: DeploymentIdentity, -) -> Result<(), CommandError> { - let mut storage = storage::Storage::open(db_path)?; - if let Some(stored) = storage.deployment_identity()? { - return require_deployment_identity_match(stored, expected); - } - if storage.has_persisted_deployment_state()? { - return Err(IdentityError::OrphanedState.into()); - } - let stored = storage.load_or_insert_deployment_identity(expected)?; - require_deployment_identity_match(stored, expected) -} - fn require_deployment_identity_match( stored: DeploymentIdentity, expected: DeploymentIdentity, @@ -295,6 +272,25 @@ pub(crate) fn verify_submitter_key( Ok(key) } +/// `run`'s check that the binary's scheduler limits are the ones `setup` +/// pinned. Storage classifies directs by the pinned cap while the lane's +/// application executes under the binary's, and acceptance is judged by the +/// staleness window. A different value is an identity mismatch, so a new +/// limit needs a new `setup`. +pub(crate) fn require_pinned_scheduler_limits( + identity: &DeploymentIdentity, + binary: sequencer_core::application::SchedulerLimits, +) -> Result<(), CommandError> { + require_deployment_identity_match( + *identity, + DeploymentIdentity { + max_direct_payload_bytes: binary.max_direct_payload_bytes, + max_wait_blocks: binary.max_wait_blocks, + ..*identity + }, + ) +} + fn deployment_identity_mismatch_fields( stored: DeploymentIdentity, expected: DeploymentIdentity, @@ -315,6 +311,12 @@ fn deployment_identity_mismatch_fields( if stored.batch_submitter_address != expected.batch_submitter_address { fields.push("batch_submitter_address"); } + if stored.max_direct_payload_bytes != expected.max_direct_payload_bytes { + fields.push("max_direct_payload_bytes"); + } + if stored.max_wait_blocks != expected.max_wait_blocks { + fields.push("max_wait_blocks"); + } if stored.fee_oracle != expected.fee_oracle { fields.push("fee_oracle"); } @@ -325,21 +327,16 @@ fn deployment_identity_mismatch_fields( mod tests { use super::{ BootstrapError, CommandError, IdentityError, batch_submitter_address_from_private_key, - deployment_identity_mismatch_fields, ensure_deployment_identity, - preflight_lifecycle_command, require_data_dir, require_deployment_identity_match, + deployment_identity_mismatch_fields, preflight_lifecycle_command, require_data_dir, + require_deployment_identity_match, require_pinned_scheduler_limits, }; - use crate::recovery::{RecoveryError, RecoveryRetryReason}; - use crate::storage::test_helpers::{SENDER_A, default_protocol_timing}; use crate::storage::{DeploymentIdentity, LifecycleCommand, Storage}; use crate::test_support::temp_db; use alloy_primitives::Address; - use sequencer_core::protocol::ProtocolTimingError; - // Margin/stale-boundary validation is exercised directly in - // `sequencer-core/src/protocol.rs`. The runtime tests below only cover - // the typed `From` conversions into `CommandError` and the bootstrap-time - // identity guards. Worker `From` conversions live in - // `run::workers`. + // The runtime tests below cover the typed `From` conversions into + // `CommandError` and the bootstrap-time identity guards. Worker + // `From` conversions live in `run::workers`. #[test] fn previous_terminal_fault_read_is_verdict_neutral() { @@ -365,7 +362,9 @@ mod tests { #[test] fn settlement_write_records_terminal_verdicts_only() { use super::record_terminal_fault_best_effort; - use crate::storage::LifecycleCommand; + use crate::storage::{ + Divergence, LandingLag, LifecycleCommand, LifecycleError, TerminalFact, + }; let db = temp_db("settlement-write"); let storage = @@ -386,8 +385,37 @@ mod tests { None ); - // A terminal startup verdict lands as one row carrying the error's - // Display form. Runtime faults abort before settlement. + // A terminal fact is terminal and leaves the black box empty too: the + // database holds the fact itself. + let facts = [ + CommandError::from(TerminalFact::VoidHistory { + lag: LandingLag { + batch_nonce: 3, + first_frame_safe_block: 10, + safe_block: 1209, + }, + max_wait_blocks: 1200, + }), + CommandError::from(TerminalFact::Divergence(Divergence { + nonce: 0, + safe_input_index: 4, + })), + CommandError::Lifecycle(LifecycleError::CanonicalDivergence { nonce: 0 }), + ]; + for fact in facts { + assert_eq!(fact.exit_code(), crate::commands::error::EXIT_TERMINAL); + record_terminal_fault_best_effort(&db.path, LifecycleCommand::Run, &Err(fact)); + assert_eq!( + Storage::open_read_only(&db.path) + .expect("reopen") + .latest_terminal_fault() + .expect("read"), + None + ); + } + + // Any other terminal startup verdict lands as one row carrying the + // error's Display form. Runtime faults abort before settlement. let error = CommandError::Storage(rusqlite::Error::InvalidQuery); let expected_cause = error.to_string(); let terminal = Err(error); @@ -402,24 +430,20 @@ mod tests { } #[test] - fn invalid_protocol_config_propagates_through_run_error() { - let err: CommandError = ProtocolTimingError::MarginNotLessThanMaxWait { - margin: sequencer_core::MAX_WAIT_BLOCKS, - max_wait: sequencer_core::MAX_WAIT_BLOCKS, - } - .into(); - assert!(matches!( - err, - CommandError::Bootstrap(BootstrapError::InvalidProtocolTiming(_)) - )); - } - - #[test] - fn startup_recovery_error_preserves_recovery_category() { - let err: CommandError = RecoveryError::retry(RecoveryRetryReason::L1ViewStale).into(); + fn startup_terminal_fact_propagates_as_a_bootstrap_refusal() { + use crate::storage::{LandingLag, TerminalFact}; + let fact = TerminalFact::VoidHistory { + lag: LandingLag { + batch_nonce: 0, + first_frame_safe_block: 0, + safe_block: 1199, + }, + max_wait_blocks: 1200, + }; + let err: CommandError = fact.into(); assert!(matches!( err, - CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::Retry(_))) + CommandError::Bootstrap(BootstrapError::TerminalFact(inner)) if inner == fact )); } @@ -430,10 +454,50 @@ mod tests { input_box_address: Address::repeat_byte(0x22), app_deployment_block: 42, batch_submitter_address: Address::repeat_byte(0x33), + max_direct_payload_bytes: 1024, + max_wait_blocks: 1200, fee_oracle: crate::storage::FeeOracleIdentity::Fixed { log_gas_price: 0 }, } } + #[test] + fn run_refuses_scheduler_limits_other_than_the_pinned_ones() { + use sequencer_core::application::SchedulerLimits; + let pinned = identity(); + // The fridge bound is not pinned: only setup's rebuild reads it. + let binary = SchedulerLimits { + max_wait_blocks: 1200, + fridge_bound: 7, + max_direct_payload_bytes: 1024, + }; + require_pinned_scheduler_limits(&pinned, binary).expect("the pinned limits match"); + for (changed, field) in [ + ( + SchedulerLimits { + max_direct_payload_bytes: 2048, + ..binary + }, + "max_direct_payload_bytes", + ), + ( + SchedulerLimits { + max_wait_blocks: 7200, + ..binary + }, + "max_wait_blocks", + ), + ] { + let err = require_pinned_scheduler_limits(&pinned, changed) + .expect_err("a different limit refuses startup"); + assert_eq!(err.exit_code(), crate::commands::error::EXIT_TERMINAL); + assert!(matches!( + err, + CommandError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { fields, .. })) + if fields == field + )); + } + } + #[test] fn deployment_identity_match_accepts_same_identity() { let identity = identity(); @@ -463,24 +527,6 @@ mod tests { )); } - #[test] - fn deployment_identity_refuses_non_empty_unpinned_db() { - let db = temp_db("runtime-unpinned-deployment-state"); - { - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) - .expect("seed deployment-bound state"); - } - - let err = ensure_deployment_identity(db.path.as_str(), identity()) - .expect_err("non-empty unpinned DB must refuse"); - assert!(matches!( - err, - CommandError::Bootstrap(BootstrapError::Identity(IdentityError::OrphanedState)) - )); - } - #[test] fn invalid_private_key_is_terminal_misconfig_and_does_not_echo_key_material() { let secret = "0xabc123SECRET"; @@ -542,17 +588,4 @@ mod tests { assert!(matches!(err, CommandError::Io(_)), "{err:?}"); assert!(err.to_string().contains(&db_path), "{err}"); } - - #[test] - fn fault_is_news_only_at_or_after_the_last_completed_setup() { - let fault = crate::storage::TerminalFault { - command: LifecycleCommand::Setup, - cause: "prior terminal death".to_string(), - recorded_at_ms: 100, - }; - assert!(super::fault_is_news(&fault, None)); - assert!(!super::fault_is_news(&fault, Some(200))); - assert!(super::fault_is_news(&fault, Some(100))); - assert!(super::fault_is_news(&fault, Some(50))); - } } diff --git a/sequencer/src/commands/run/mod.rs b/sequencer/src/commands/run/mod.rs index e7b9ffd8..ece83a00 100644 --- a/sequencer/src/commands/run/mod.rs +++ b/sequencer/src/commands/run/mod.rs @@ -2,20 +2,25 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! The `run` command: refuse unless `setup` completed, read the pinned identity, -//! run startup recovery (`crate::recovery`), then prepare, admit and launch the -//! workers (`workers`). The `_admitted` suffix names the lifecycle gate, not the -//! runtime admission witness, which is minted after preparation. +//! then refuse on a terminal fact, sync L1 and open a batch if none is open +//! (`startup`), then prepare and launch the workers (`workers`). The +//! `_admitted` suffix names the lifecycle gate. +mod startup; mod startup_hygiene; mod workers; +#[cfg(test)] +pub(crate) use startup::start; + use std::time::Duration; use crate::commands::config::RunConfig; use crate::commands::error::CommandError; use crate::commands::{ INPUT_READER_POLL_INTERVAL, load_setup_identity, preflight_lifecycle_command, - record_terminal_fault_best_effort, verify_submitter_key, warn_on_previous_terminal_fault, + record_terminal_fault_best_effort, require_pinned_scheduler_limits, verify_submitter_key, + warn_on_previous_terminal_fault, }; use crate::l1::L1Config; use crate::l1::reader::{InputReader, InputReaderConfig}; @@ -42,7 +47,6 @@ where // settlement; runtime-owned tasks separately retain clones until they stop. let process_lock = process_lock::ProcessLock::acquire(&config.data_dir)?; let db_path = config.db_path(); - let timing = config.protocol_timing()?; // Local absorbing facts are inspected before identity/key checks and any // RPC: canonical divergence and the two-sided setup-completion rule. @@ -50,6 +54,10 @@ where warn_on_previous_terminal_fault(&db_path); preflight_lifecycle_command(&config.data_dir, LifecycleCommand::Run)?; let identity = load_setup_identity(&config.data_dir)?; + // Storage classifies directs by the pinned cap, and acceptance is judged + // by the pinned window; the binary must agree with both. + require_pinned_scheduler_limits(&identity, A::scheduler_limits())?; + config.check_batch_open_window(identity.max_wait_blocks)?; // `run` holds the signing key (it submits). The key's address must match // the pinned submitter address — running with the wrong key against a DB @@ -65,33 +73,23 @@ where allow_insecure_rpc: config.rpc.allow_insecure_rpc, }; - let result = run_admitted::(config, timing, l1_config, process_lock.clone()).await; - // The Ok-path divergence fact check (run's counterpart of the one - // `complete_baseline_setup` keeps): divergence persisted during this run must - // exit terminal even through a clean drain — exit 0 is the one code - // that breaks the supervisor's rediscovery chain (restart → preflight - // refusal), and the detector's poll cadence leaves a window where a - // clean shutdown outruns re-detection. - let result = result.and_then(|()| refuse_divergence_on_clean_exit(&db_path)); - // Verdict-neutral black-box settlement: a terminal failure records - // its cause best-effort — never changing the verdict — while a - // panic/cancellation/SIGKILL writes nothing; the next boot proceeds and - // re-derives everything from facts. + let result = run_admitted::(config, l1_config, process_lock.clone()).await; + // The Ok-path terminal-fact check: a divergence recorded or a history + // that became void during this run must exit terminal even through a + // clean drain. Exit 0 is the one code that breaks the supervisor's + // rediscovery chain (restart, then refusal), and the landing watch's poll + // cadence leaves a window where a clean shutdown outruns its detection. + let result = result.and_then(|()| startup::refuse_terminal_fact(&db_path)); + // Verdict-neutral black-box settlement: a terminal failure that is not a + // terminal fact records its cause best-effort — never changing the + // verdict — while a panic/cancellation/SIGKILL writes nothing; the next + // boot proceeds and re-derives everything from facts. record_terminal_fault_best_effort(&db_path, LifecycleCommand::Run, &result); result } -fn refuse_divergence_on_clean_exit(db_path: &str) -> Result<(), CommandError> { - let mut storage = storage::Storage::open_read_only(db_path)?; - if let Some((nonce, _)) = storage.canonical_divergence()? { - return Err(storage::LifecycleError::CanonicalDivergence { nonce }.into()); - } - Ok(()) -} - async fn run_admitted( config: RunConfig, - timing: sequencer_core::protocol::ProtocolTiming, l1_config: L1Config, process_lock: process_lock::ProcessLock, ) -> Result<(), CommandError> @@ -113,11 +111,9 @@ where l1_config.identity.input_box_address, l1_config.identity.app_deployment_block, db_path.clone(), - l1_config.identity.batch_submitter_address, - timing, - // Bootstrap syncs use nested blocking SQLite jobs. The reader takes - // its retained lock clone at construction—not only after worker - // admission—so cancellation of this async command cannot release + // The startup sync uses nested blocking SQLite jobs. The reader takes + // its retained lock clone at construction—not only at worker + // launch—so cancellation of this async command cannot release // exclusivity beneath an orphaned DB write. process_lock.clone(), ); @@ -132,11 +128,8 @@ where chain_id = l1_config.identity.chain_id, app_address = %l1_config.identity.app_address, batch_submitter_address = %l1_config.identity.batch_submitter_address, - max_wait_blocks = timing.max_wait_blocks(), - preemptive_margin_blocks = timing.preemptive_margin_blocks(), - danger_threshold = timing.danger_threshold(), - l1_read_stale_after_blocks = timing.l1_read_stale_after_blocks(), - seconds_per_block = timing.seconds_per_block(), + max_wait_blocks = l1_config.identity.max_wait_blocks, + seconds_per_block = config.timing.seconds_per_block, max_batch_open_seconds = config.max_batch_open_seconds, batch_submitter_confirmation_depth = config.batch_submitter_confirmation_depth, batch_submitter_idle_poll_interval_ms = config.batch_submitter_idle_poll_interval_ms, @@ -144,15 +137,24 @@ where "sequencer startup" ); - // ── Recovery ───────────────────────────────────────────── - // Local terminal facts are inspected before the first provider call. - crate::recovery::run_startup_recovery(&db_path, &mut input_reader, &l1_config, &timing).await?; + // ── Schema ─────────────────────────────────────────────── + // The run's one migrating open. Only the black-box warning, the lifecycle + // preflight and the identity load, which read setup's own facts, open the + // database before it; the startup, the lane's handle and the workers open + // without migrating. + drop(storage::Storage::open(&db_path)?); + + // ── Check, sync, open ──────────────────────────────────── + // A divergence or a void history is read from the database as it is, + // before any L1 call, so an unreachable provider cannot hide it and no + // batch is opened over it. + startup::start(&db_path, &mut input_reader).await?; // Setup persisted a real first price, so run performs no synchronous - // fee-source I/O and fee availability never precedes recovery's local - // terminal-fact inspection. The exhaustive identity match is the worker + // fee-source I/O and fee availability never precedes the terminal-fact + // check. The exhaustive identity match is the worker // launch decision: fixed mode has no task; Uniswap's supervised worker - // performs the first runtime quote after admission. + // performs the first runtime quote after launch. let fee_oracle = match l1_config.identity.fee_oracle { storage::FeeOracleIdentity::Fixed { .. } => None, storage::FeeOracleIdentity::Uniswap { @@ -187,22 +189,18 @@ where } }; - // ── Prepare → admit → launch ───────────────────────────── + // ── Prepare → launch ───────────────────────────────────── + // Under the process lock with no worker running, nothing changes the + // facts checked above before the launch. let prepared = PreparedRuntime::::prepare(WorkersConfig { run_config: config, l1_config, - timing, input_reader, fee_oracle, process_lock, }) .await?; - - // Preparation may take long enough for a clock/view refusal to arise. - // Recheck one consistent fact set; workers are - // never launched from an aged decision. - let admission = crate::recovery::admit_runtime(&db_path, &timing)?; - let mut workers = prepared.launch(admission); + let mut workers = prepared.launch(); let first_exit = workers.select_first_exit().await; workers.finish(first_exit).await @@ -226,7 +224,7 @@ mod tests { record_canonical_divergence(&mut storage, 7, 0); drop(storage); - let error = refuse_divergence_on_clean_exit(&db.path) + let error = startup::refuse_terminal_fact(&db.path) .expect_err("a clean drain over divergence must not exit 0"); assert_eq!( error.exit_code(), @@ -237,6 +235,6 @@ mod tests { let clean = temp_db("run-clean-exit-clean"); storage::Storage::initialize_for_command(&clean.path, LifecycleCommand::Setup) .expect("initialize clean"); - refuse_divergence_on_clean_exit(&clean.path).expect("no divergence passes clean"); + startup::refuse_terminal_fact(&clean.path).expect("no terminal fact passes clean"); } } diff --git a/sequencer/src/commands/run/startup.rs b/sequencer/src/commands/run/startup.rs new file mode 100644 index 00000000..9dc0e094 --- /dev/null +++ b/sequencer/src/commands/run/startup.rs @@ -0,0 +1,69 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! `run`'s startup, under the process lock with no worker running: check the +//! terminal facts on the database as it is, sync L1, and open a batch if none +//! is open. Nothing else is repaired, flushed or replaced, and +//! nothing can change between these steps and the launch. + +use crate::commands::error::{BootstrapError, CommandError}; +use crate::l1::reader::{InputReader, InputReaderError}; +use crate::storage::Storage; + +/// The one L1 operation startup performs. Tests substitute it and keep the +/// real storage reads and the real open transaction. +pub(crate) trait StartupSync { + async fn sync(&mut self) -> Result<(), InputReaderError>; +} + +impl StartupSync for InputReader { + async fn sync(&mut self) -> Result<(), InputReaderError> { + self.sync_to_current_safe_head().await + } +} + +/// Refuse while a fact that ends the era holds: a recorded divergence or a +/// void history. Reads only the database, so it answers with L1 unreachable. +pub(super) fn refuse_terminal_fact(db_path: &str) -> Result<(), CommandError> { + let mut storage = Storage::open_read_only(db_path)?; + match storage.terminal_fact()? { + Some(fact) => Err(fact.into()), + None => Ok(()), + } +} + +/// The whole startup: refuse on a terminal fact before any L1 call, sync L1, +/// then open a batch if none is open. The opening transaction checks the +/// terminal facts again, so one the sync revealed refuses there. +/// +/// Only a provider failure of the sync falls back to the stored view, which +/// setup always leaves, and only when a batch is already open: a stale view +/// does not refuse a restart. With no batch open (only the first `run` of an +/// era) the provider failure refuses as retryable instead: a batch opened now +/// would take its first frame's safe block from a stored view that may be +/// hours old, that block is fixed for good, and the batch could then be void +/// before it can land. +pub(crate) async fn start(db_path: &str, l1: &mut impl StartupSync) -> Result<(), CommandError> { + refuse_terminal_fact(db_path)?; + match l1.sync().await { + Ok(()) => tracing::info!("L1 safe head synced"), + Err(InputReaderError::Provider(error)) => { + if Storage::open_read_only(db_path)?.open_state()?.is_none() { + return Err(BootstrapError::StartupSync(InputReaderError::Provider(error)).into()); + } + tracing::warn!( + error = %error, + "L1 unreachable during the startup sync; starting on the stored L1 view" + ); + } + Err(error) => return Err(BootstrapError::StartupSync(error).into()), + } + let mut storage = Storage::open_writer(db_path)?; + if storage.open_batch_if_none()? { + tracing::info!("opened a batch at the safe head"); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/commands/run/startup/tests.rs b/sequencer/src/commands/run/startup/tests.rs new file mode 100644 index 00000000..1063d5f2 --- /dev/null +++ b/sequencer/src/commands/run/startup/tests.rs @@ -0,0 +1,319 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::collections::VecDeque; + +use super::*; +use crate::commands::error::{EXIT_RESTART_TRANSIENT, EXIT_TERMINAL}; +use crate::storage::test_helpers::{ + LandingFixture, SENDER_A, open_temp_storage, record_canonical_divergence, +}; +use crate::storage::{SafeInputRange, StoredSafeInput, TerminalFact}; +use crate::test_support::TestDb; + +/// The window the test identity pins (the wallet's). +const W: u64 = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; + +enum SyncStep { + /// L1 shows nothing new. + Keep, + Fail(InputReaderError), + /// L1's safe head moved to `block`, carrying `inputs`. + Advance { + block: u64, + inputs: Vec, + }, +} + +/// The substituted L1: each sync takes the next step, and the real storage +/// append records what it shows. +struct TestL1 { + db_path: String, + steps: VecDeque, + syncs: usize, +} + +impl TestL1 { + fn new(db: &TestDb, steps: Vec) -> Self { + Self { + db_path: db.path.clone(), + steps: steps.into(), + syncs: 0, + } + } +} + +impl StartupSync for TestL1 { + async fn sync(&mut self) -> Result<(), InputReaderError> { + self.syncs += 1; + match self.steps.pop_front().expect("unexpected L1 sync") { + SyncStep::Keep => Ok(()), + SyncStep::Fail(error) => Err(error), + SyncStep::Advance { block, inputs } => Storage::open_writer(&self.db_path) + .expect("open writer") + .append_safe_inputs(block, &inputs, SENDER_A) + .map_err(InputReaderError::Storage), + } + } +} + +/// What setup leaves: the era, the identity, a safe head at 50, no batch. +fn after_setup(name: &str) -> TestDb { + let (db, mut storage) = open_temp_storage(name); + storage.advance_safe_head(50); + db +} + +/// The batches as `(nonce, sealed)`, in nonce order. +fn batch_rows(db: &TestDb) -> Vec<(i64, bool)> { + Storage::open(&db.path) + .expect("open storage") + .raw_read(|tx| { + tx.prepare("SELECT nonce, sealed_at_ms IS NOT NULL FROM batches ORDER BY nonce")? + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect() + }) + .expect("read batches") +} + +#[tokio::test] +async fn the_first_start_of_an_era_opens_its_first_batch_once() { + let db = after_setup("startup-first-batch"); + let mut l1 = TestL1::new( + &db, + vec![SyncStep::Advance { + block: 60, + inputs: vec![], + }], + ); + start(&db.path, &mut l1).await.expect("start"); + assert_eq!(l1.syncs, 1); + assert_eq!(batch_rows(&db), vec![(0, false)]); + let head = Storage::open(&db.path) + .unwrap() + .open_state() + .unwrap() + .expect("the era's first batch"); + assert_eq!(head.safe_block(), 60, "opened at the synced safe head"); + + // A restart finds it open and opens nothing. + let mut l1 = TestL1::new(&db, vec![SyncStep::Keep]); + start(&db.path, &mut l1).await.expect("restart"); + assert_eq!(batch_rows(&db), vec![(0, false)]); +} + +/// A start inside the window repairs nothing: the same batches, the same +/// nonces, the same open batch. That holds with the provider down and the +/// stored view hours old. +#[tokio::test] +async fn a_start_inside_the_window_changes_nothing() { + let db = after_setup("startup-inside-window"); + { + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(50, SafeInputRange::empty_at(0)) + .unwrap(); + storage.close_batch_at(&mut head, 50).unwrap(); + storage.close_batch_at(&mut head, 60).unwrap(); + // The safe block is stamped at the epoch: a view decades old. + storage + .append_safe_inputs_with_timestamp(50 + W - 2, 0, &[], SENDER_A) + .unwrap(); + } + let before = batch_rows(&db); + assert_eq!(before, vec![(0, true), (1, true), (2, false)]); + + for step in [ + SyncStep::Keep, + SyncStep::Fail(InputReaderError::Provider("offline".into())), + ] { + let mut l1 = TestL1::new(&db, vec![step]); + start(&db.path, &mut l1) + .await + .expect("a start inside the window"); + assert_eq!(batch_rows(&db), before); + } +} + +/// The sync can show a void history the stored view did not: startup refuses +/// before opening or changing anything. +#[tokio::test] +async fn a_void_history_shown_by_the_sync_refuses() { + let db = after_setup("startup-void-after-sync"); + { + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(50, SafeInputRange::empty_at(0)) + .unwrap(); + storage.close_batch_at(&mut head, 50).unwrap(); + } + let before = batch_rows(&db); + + let mut l1 = TestL1::new( + &db, + vec![SyncStep::Advance { + block: 50 + W - 1, + inputs: vec![], + }], + ); + let error = start(&db.path, &mut l1) + .await + .expect_err("a void history refuses"); + assert!( + matches!( + &error, + CommandError::Bootstrap(BootstrapError::TerminalFact(TerminalFact::VoidHistory { + lag, + .. + })) if lag.first_frame_safe_block == 50 + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), EXIT_TERMINAL); + assert_eq!(batch_rows(&db), before); + // Every later start refuses before L1. + assert!(matches!( + refuse_terminal_fact(&db.path), + Err(CommandError::Bootstrap(BootstrapError::TerminalFact( + TerminalFact::VoidHistory { .. } + ))) + )); +} + +/// A foreign batch accepted at the era's first nonce, seen by the sync, +/// refuses before the era's first batch is opened. +#[tokio::test] +async fn a_divergence_shown_by_the_sync_refuses_before_opening() { + let db = after_setup("startup-divergence-after-sync"); + let foreign = StoredSafeInput { + sender: SENDER_A, + payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { + nonce: 0, + frames: vec![], + }), + block_number: 55, + }; + let mut l1 = TestL1::new( + &db, + vec![SyncStep::Advance { + block: 60, + inputs: vec![foreign], + }], + ); + let error = start(&db.path, &mut l1) + .await + .expect_err("a divergence refuses"); + assert!( + matches!( + &error, + CommandError::Bootstrap(BootstrapError::TerminalFact(TerminalFact::Divergence( + divergence + ))) if divergence.nonce == 0 + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), EXIT_TERMINAL); + assert!(batch_rows(&db).is_empty(), "no batch was opened"); +} + +#[test] +fn a_recorded_divergence_refuses_before_l1() { + let db = after_setup("startup-recorded-divergence"); + record_canonical_divergence(&mut Storage::open(&db.path).unwrap(), 7, 0); + let error = refuse_terminal_fact(&db.path).expect_err("a divergence refuses"); + assert!(matches!( + error, + CommandError::Bootstrap(BootstrapError::TerminalFact(TerminalFact::Divergence(_))) + )); + assert_eq!(error.exit_code(), EXIT_TERMINAL); +} + +/// A void history refuses before any L1 call: the stub has no scripted +/// sync and panics if called. +#[tokio::test] +async fn a_void_history_refuses_before_any_l1_call() { + let db = after_setup("startup-void-before-l1"); + { + let mut storage = Storage::open(&db.path).unwrap(); + storage + .initialize_open_state(50, SafeInputRange::empty_at(0)) + .unwrap(); + storage.advance_safe_head(50 + W - 1); + } + let before = batch_rows(&db); + let mut l1 = TestL1::new(&db, vec![]); + let error = start(&db.path, &mut l1) + .await + .expect_err("a void history refuses"); + assert!( + matches!( + error, + CommandError::Bootstrap(BootstrapError::TerminalFact( + TerminalFact::VoidHistory { .. } + )) + ), + "{error:?}" + ); + assert_eq!(l1.syncs, 0); + assert_eq!(batch_rows(&db), before); +} + +/// The first start of an era with the provider down opens no batch on the +/// stored view, whose safe block would fix the batch's first frame for good: +/// it refuses as retryable. +#[tokio::test] +async fn the_first_start_with_the_provider_down_opens_nothing() { + let db = after_setup("startup-first-start-provider-down"); + let mut l1 = TestL1::new( + &db, + vec![SyncStep::Fail(InputReaderError::Provider("offline".into()))], + ); + let error = start(&db.path, &mut l1) + .await + .expect_err("no batch is opened on the stored view"); + assert!( + matches!( + error, + CommandError::Bootstrap(BootstrapError::StartupSync(InputReaderError::Provider(_))) + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), EXIT_RESTART_TRANSIENT); + assert!(batch_rows(&db).is_empty()); +} + +/// Every sync failure other than an unreachable provider keeps its startup +/// class and opens nothing. +#[tokio::test] +async fn other_sync_failures_keep_their_startup_class() { + for (error, code) in [ + ( + InputReaderError::InconsistentL1Response("gap".into()), + EXIT_RESTART_TRANSIENT, + ), + ( + InputReaderError::ChainIdMismatch { + rpc: 1, + expected: 2, + }, + EXIT_TERMINAL, + ), + (InputReaderError::Bootstrap("bad url".into()), EXIT_TERMINAL), + ] { + let db = after_setup("startup-sync-failure"); + let description = error.to_string(); + let mut l1 = TestL1::new(&db, vec![SyncStep::Fail(error)]); + let refused = start(&db.path, &mut l1) + .await + .expect_err("a non-provider sync failure refuses"); + assert!( + matches!( + refused, + CommandError::Bootstrap(BootstrapError::StartupSync(_)) + ), + "{description}: {refused:?}" + ); + assert_eq!(refused.exit_code(), code, "{description}"); + assert!(batch_rows(&db).is_empty(), "{description}"); + } +} diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index 3f565f33..d15a890c 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -1,8 +1,9 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Startup clears stale leases, checks rollback checkpoint metadata, then collects -//! obsolete snapshots and orphan directories before workers are admitted. +//! Startup checks the accepted checkpoint's metadata, then collects obsolete +//! snapshots and orphan directories before the listeners are bound or the +//! workers launched. use std::os::unix::fs::MetadataExt; @@ -10,14 +11,15 @@ use crate::commands::error::CommandError; use crate::ingress::inclusion_lane::dump_info::{self, delete_dump_dir}; /// Repair interrupted artifact creation/collection before starting workers. +/// Nothing serves yet, so `leases` is empty. pub(super) fn run_snapshot_hygiene( storage: &mut crate::storage::Storage, + leases: &crate::storage::SnapshotLeases, data_dir: &str, dumps_dir: &std::path::Path, ) -> Result<(), CommandError> { - storage.reset_dump_leases()?; - require_rollback_snapshot(storage, data_dir)?; - let gc_removed = snapshot_gc_at_startup(storage)?; + require_accepted_snapshot(storage, data_dir)?; + let gc_removed = snapshot_gc_at_startup(storage, leases)?; let sweep_removed = sweep_orphan_dumps(storage, dumps_dir)?; tracing::debug!( gc_removed, @@ -27,18 +29,18 @@ pub(super) fn run_snapshot_hygiene( Ok(()) } -fn require_rollback_snapshot( +fn require_accepted_snapshot( storage: &mut crate::storage::Storage, data_dir: &str, ) -> Result<(), CommandError> { - let snapshot = storage.rollback_snapshot()?.ok_or_else(|| { + let snapshot = storage.accepted_snapshot()?.ok_or_else(|| { CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupNotComplete { data_dir: data_dir.to_string(), }) })?; - dump_info::read_info(&snapshot.dump.prefix).map_err(|source| { + dump_info::read_info(&snapshot.prefix).map_err(|source| { CommandError::ReferencedSnapshotArtifact { - path: snapshot.dump.prefix, + path: snapshot.prefix, source, } })?; @@ -46,13 +48,16 @@ fn require_rollback_snapshot( } /// Delete obsolete rows before their files; the sweep retries leftover files. -fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result { - let removed = storage.gc_unreferenced_dumps()?; - for row in &removed { - if let Err(err) = delete_dump_dir(&row.prefix) { +fn snapshot_gc_at_startup( + storage: &mut crate::storage::Storage, + leases: &crate::storage::SnapshotLeases, +) -> Result { + let removed = storage.collect_snapshots(leases)?; + for prefix in &removed { + if let Err(err) = delete_dump_dir(prefix) { tracing::warn!( error = %err, - prefix = ?row.prefix, + ?prefix, "startup GC: filesystem delete failed; orphan left for sweep", ); } @@ -60,8 +65,8 @@ fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result Result { let known = storage - .list_dump_rows()? + .list_snapshots()? .into_iter() - .map(|row| { - dump_identity(&row.prefix).map_err(|source| CommandError::ReferencedSnapshotArtifact { - path: row.prefix, - source, + .map(|snapshot| { + dump_identity(&snapshot.prefix).map_err(|source| { + CommandError::ReferencedSnapshotArtifact { + path: snapshot.prefix, + source, + } }) }) .collect::, _>>()?; @@ -133,6 +140,9 @@ fn dump_identity(path: &std::path::Path) -> std::io::Result<(u64, u64)> { mod tests { use super::*; use crate::storage::Storage; + use crate::storage::test_helpers::{ + SENDER_A, seed_safe_inputs_with_batch_nonces, set_up_storage, + }; use crate::test_support::create_structured_dump; use crate::test_support::temp_db; @@ -143,10 +153,10 @@ mod tests { let root = tempfile::tempdir().expect("snapshot parent"); let missing = root.path().join("missing"); storage - .insert_baseline_snapshot(&missing, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&missing, crate::storage::L2TxCount::ZERO) .expect("register missing fixture"); - let err = require_rollback_snapshot(&mut storage, "/data") + let err = require_accepted_snapshot(&mut storage, "/data") .expect_err("a durable DB reference cannot point at a missing artifact"); assert!(matches!( @@ -166,10 +176,10 @@ mod tests { std::fs::write(corrupt.join("info.toml"), "not = valid = toml") .expect("write corrupt metadata"); storage - .insert_baseline_snapshot(&corrupt, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&corrupt, crate::storage::L2TxCount::ZERO) .expect("register corrupt fixture"); - let err = require_rollback_snapshot(&mut storage, "/data") + let err = require_accepted_snapshot(&mut storage, "/data") .expect_err("corrupt durable metadata cannot be retried as operational I/O"); assert!(matches!( @@ -189,7 +199,7 @@ mod tests { let tracked = dumps_dir.path().join("tracked"); create_structured_dump(&tracked); storage - .insert_baseline_snapshot(&tracked, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&tracked, crate::storage::L2TxCount::ZERO) .expect("register tracked"); // Two orphans (NOT in SQLite). One is fully formed; the other @@ -244,19 +254,14 @@ mod tests { storage .insert_baseline_snapshot( &stored_dir.join("tracked"), - crate::storage::ExecutedInputCount::ZERO, + crate::storage::L2TxCount::ZERO, ) .unwrap(); // A literal match must not hide the other row's aliased spelling. let literal_match = sweep_dir.join("literal-match"); create_structured_dump(&literal_match); storage - .write(|tx| { - tx.execute( - "INSERT INTO dumps(prefix) VALUES (?1)", - [literal_match.to_str().unwrap()], - ) - }) + .insert_closed_batch_snapshot(&literal_match) .unwrap(); let orphan = absolute.join("orphan"); create_structured_dump(&orphan); @@ -270,7 +275,7 @@ mod tests { assert!(tracked.join("info.toml").is_file(), "{spelling}"); assert!(literal_match.join("info.toml").is_file(), "{spelling}"); assert!(!orphan.exists(), "{spelling}"); - assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + assert_eq!(storage.list_snapshots().unwrap().len(), 2); } } @@ -295,7 +300,7 @@ mod tests { tracked.clone() }; storage - .insert_baseline_snapshot(&stored, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&stored, crate::storage::L2TxCount::ZERO) .unwrap(); let orphan = dumps.join("orphan"); create_structured_dump(&orphan); @@ -327,7 +332,7 @@ mod tests { let tracked = dumps.join("tracked"); create_structured_dump(&tracked); storage - .insert_baseline_snapshot(&tracked, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&tracked, crate::storage::L2TxCount::ZERO) .unwrap(); // This alias survives realpath resolution, unlike an ordinary symlink. @@ -347,12 +352,7 @@ mod tests { let literal_match = alias.join("literal-match"); create_structured_dump(&literal_match); storage - .write(|tx| { - tx.execute( - "INSERT INTO dumps(prefix) VALUES (?1)", - [literal_match.to_str().unwrap()], - ) - }) + .insert_closed_batch_snapshot(&literal_match) .unwrap(); let orphan = dumps.join("orphan"); create_structured_dump(&orphan); @@ -366,7 +366,7 @@ mod tests { assert!(literal_match.join("info.toml").is_file()); assert!(!orphan.exists()); assert_eq!(removed, 1); - assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + assert_eq!(storage.list_snapshots().unwrap().len(), 2); } #[test] @@ -379,18 +379,11 @@ mod tests { storage .insert_baseline_snapshot( &dumps.path().join(".").join("tracked"), - crate::storage::ExecutedInputCount::ZERO, + crate::storage::L2TxCount::ZERO, ) .unwrap(); let missing = dumps.path().join("missing"); - storage - .write(|tx| { - tx.execute( - "INSERT INTO dumps(prefix) VALUES (?1)", - [missing.to_str().unwrap()], - ) - }) - .unwrap(); + storage.insert_closed_batch_snapshot(&missing).unwrap(); let orphan = dumps.path().join("orphan"); create_structured_dump(&orphan); @@ -407,7 +400,7 @@ mod tests { orphan.join("info.toml").is_file(), "resolution precedes deletion" ); - assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + assert_eq!(storage.list_snapshots().unwrap().len(), 2); } #[test] @@ -458,7 +451,7 @@ mod tests { let tracked = dumps.join("tracked"); create_structured_dump(&tracked); storage - .insert_baseline_snapshot(&tracked, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&tracked, crate::storage::L2TxCount::ZERO) .unwrap(); let outside = root.path().join("outside"); create_structured_dump(&outside); @@ -480,72 +473,28 @@ mod tests { .map(|entry| entry.unwrap().file_name()) .collect(); assert_eq!(remaining, vec![std::ffi::OsString::from("tracked")]); - assert_eq!(storage.list_dump_rows().unwrap().len(), 1); - } - - #[test] - fn startup_resets_persisted_crash_leases_before_collecting_artifacts() { - let db = temp_db("startup-crash-leases"); - let mut storage = Storage::open(&db.path).unwrap(); - let dumps = tempfile::tempdir().unwrap(); - let baseline = dumps.path().join("baseline"); - create_structured_dump(&baseline); - let baseline_id = storage - .insert_baseline_snapshot(&baseline, crate::storage::ExecutedInputCount::ZERO) - .unwrap(); - let obsolete = dumps.path().join("obsolete"); - create_structured_dump(&obsolete); - let obsolete_id = storage - .write(|tx| { - tx.execute( - "UPDATE dumps SET lease_count = 1 WHERE id = ?1", - [baseline_id], - )?; - tx.execute( - "INSERT INTO dumps(prefix, lease_count) VALUES (?1, 1)", - [obsolete.to_str().unwrap()], - )?; - Ok(tx.last_insert_rowid()) - }) - .unwrap(); - drop(storage); - - let mut storage = Storage::open(&db.path).unwrap(); - assert_eq!(storage.dump_lease_count(obsolete_id).unwrap(), Some(1)); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - assert!(obsolete.exists(), "the persisted lease blocks ordinary GC"); - - run_snapshot_hygiene(&mut storage, "/data", dumps.path()).unwrap(); - - assert_eq!(storage.dump_lease_count(baseline_id).unwrap(), Some(0)); - assert_eq!(storage.dump_lease_count(obsolete_id).unwrap(), None); - assert!( - baseline.exists(), - "the rollback baseline survives startup GC" - ); - assert!( - !obsolete.exists(), - "startup collects the abandoned leased artifact" - ); + assert_eq!(storage.list_snapshots().unwrap().len(), 1); } #[test] fn snapshot_gc_at_startup_removes_unreferenced_rows() { let db = temp_db("gc-startup"); - let mut storage = Storage::open(db.path.as_str()).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let dumps_dir = tempfile::tempdir().unwrap(); let orphan = dumps_dir.path().join("orphan"); create_structured_dump(&orphan); - storage - .write(|tx| { - tx.execute( - "INSERT INTO dumps(prefix) VALUES (?1)", - [orphan.to_str().unwrap()], - ) - }) - .unwrap(); - let removed = snapshot_gc_at_startup(&mut storage).unwrap(); - assert_eq!(removed, 1); + let accepted = dumps_dir.path().join("accepted"); + create_structured_dump(&accepted); + storage.insert_closed_batch_snapshot(&orphan).unwrap(); + storage.insert_closed_batch_snapshot(&accepted).unwrap(); + // With both batches accepted, the second's snapshot is the accepted + // checkpoint; the baseline and the first batch's are collectible. + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0, 1]); + let removed = + snapshot_gc_at_startup(&mut storage, &crate::storage::SnapshotLeases::default()) + .unwrap(); + assert_eq!(removed, 2); assert!(!orphan.exists()); + assert!(accepted.join("info.toml").is_file()); } } diff --git a/sequencer/src/commands/run/workers.rs b/sequencer/src/commands/run/workers.rs index 59971cf4..96b5a25e 100644 --- a/sequencer/src/commands/run/workers.rs +++ b/sequencer/src/commands/run/workers.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Runtime worker lifecycle: prepare -> admit -> launch -> stop. Each worker is one +//! Runtime worker lifecycle: prepare -> launch -> stop. Each worker is one //! field, one spawn, one select arm and one cleanup entry. The `Workers` literal in //! `launch` and the `let Self { .. }` destructures in `select_first_exit` and //! `finish` stay exhaustive (no `..`), so a missed site fails the build. @@ -31,16 +31,15 @@ use crate::l1::submitter::{ BatchPosterConfig, BatchSubmitter, BatchSubmitterConfig, BatchSubmitterError, EthereumBatchPoster, }; -use crate::recovery::{DangerDetector, DangerDetectorError, DetectorExit}; +use crate::landing::{LandingWatch, LandingWatchError, LandingWatchExit}; use crate::runtime::process_lock::ProcessLock; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; use sequencer_core::application::Application; -use sequencer_core::protocol::ProtocolTiming; const QUEUE_CAPACITY: usize = 8192; -/// Danger detector cadence. Cheap DB-only check; re-running quickly bounds the -/// lag on entering the danger zone. The preemptive margin absorbs bounded lag. -const DANGER_DETECTOR_POLL_INTERVAL: Duration = Duration::from_secs(2); +/// Landing watch cadence. Cheap DB-only check; re-running quickly bounds how +/// long the process serves after a divergence or a void history is recorded. +const LANDING_WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); /// Which event ended the `select!` race in [`Workers::select_first_exit`]. pub(super) enum FirstExit { @@ -51,9 +50,8 @@ pub(super) enum FirstExit { /// Inputs to [`PreparedRuntime::prepare`]. Consumed entirely; the caller has /// nothing further to do with these after the call. /// -/// Everything here is built by `run` because the recovery procedure consumes -/// it or must run after it; anything the workers alone need is derived -/// inside `prepare`. +/// Everything here is built by `run` because its startup consumes it or must +/// run after it; anything the workers alone need is derived inside `prepare`. /// /// No genesis app instance: `setup` already registered the finalized genesis /// snapshot, so the lane reloads via `A::from_dump`. No EIP-712 domain @@ -62,14 +60,13 @@ pub(super) enum FirstExit { pub(super) struct WorkersConfig { pub run_config: RunConfig, pub l1_config: L1Config, - pub timing: ProtocolTiming, pub input_reader: InputReader, /// Launch-ready without source I/O: setup supplied the persisted price, /// and a Uniswap worker quotes on its first supervised iteration. Fixed /// pricing has no worker (`None`). pub fee_oracle: Option, /// Exclusive data-directory ownership, acquired before bootstrap and - /// transferred into the runtime lifetime at worker admission. + /// transferred into the runtime lifetime at worker launch. pub process_lock: ProcessLock, } @@ -78,7 +75,7 @@ pub(super) struct WorkersConfig { /// Data-directory exclusivity does not depend on this guard: every worker /// holds a construction-required [`ProcessLock`] clone — the lane and /// server through their [`RuntimeScope`], the submitter both ways, the -/// reader, detector, and fee oracle directly — so the lock stays held until +/// reader, landing watch, and fee oracle directly — so the lock stays held until /// the workers finish even though Drop cannot join asynchronously. struct ShutdownOnDrop(RuntimeScope); @@ -102,7 +99,8 @@ pub(super) struct PreparedRuntime { storage: crate::storage::Storage, lane_config: InclusionLaneConfig, submitter: BatchSubmitter, - detector: DangerDetector, + landing_watch: LandingWatch, + landing_feed: crate::landing::LandingFeed, tx_feed: L2TxFeed, listeners: http::ApiListeners, public_addr: std::net::SocketAddr, @@ -124,7 +122,7 @@ pub(super) struct Workers { lane: JoinHandle>, reader: JoinHandle>, submitter: JoinHandle>, - detector: JoinHandle>, + landing_watch: JoinHandle>, fee_oracle: Option>>, shutdown: RuntimeScope, _shutdown_on_drop: ShutdownOnDrop, @@ -132,12 +130,11 @@ pub(super) struct Workers { impl PreparedRuntime { /// Prepare every fallible or awaited runtime dependency while launching - /// zero tasks. Durable admission remains the controller's responsibility. + /// zero tasks. pub(super) async fn prepare(cfg: WorkersConfig) -> Result { let WorkersConfig { run_config, l1_config, - timing, input_reader, fee_oracle, process_lock, @@ -162,14 +159,19 @@ impl PreparedRuntime { // panic unwind, or cancellation of the owning `run` future. let shutdown = RuntimeScope::new(process_lock); let shutdown_on_drop = ShutdownOnDrop(shutdown.clone()); - let mut storage = crate::storage::Storage::open(&db_path)?; + let mut storage = crate::storage::Storage::open_writer(&db_path)?; let dumps_dir = std::path::Path::new(&run_config.data_dir).join("dumps"); std::fs::create_dir_all(&dumps_dir) .map_err(|err| crate::commands::io_error_at("dumps dir", &dumps_dir, err))?; - // Validate rollback checkpoint metadata and collect obsolete snapshots before admission. + // One lease registry for the process: the lane's GC, startup hygiene + // and the egress handlers share it (`SnapshotLeases`). + let snapshot_leases = crate::storage::SnapshotLeases::default(); + + // Validate checkpoint metadata and collect obsolete snapshots before launch. super::startup_hygiene::run_snapshot_hygiene( &mut storage, + &snapshot_leases, &run_config.data_dir, &dumps_dir, )?; @@ -184,7 +186,7 @@ impl PreparedRuntime { batch_submitter_address: l1_config.identity.batch_submitter_address, start_block: l1_config.identity.app_deployment_block, confirmation_depth: run_config.batch_submitter_confirmation_depth, - seconds_per_block: timing.seconds_per_block(), + seconds_per_block: run_config.timing.seconds_per_block, expected_chain_id: l1_config.identity.chain_id, }; let providers = build_batch_submitter_providers(&l1_config)?; @@ -200,21 +202,21 @@ impl PreparedRuntime { ); submitter.preflight_storage()?; - let detector = DangerDetector::new( + let (landing_watch, landing_feed) = LandingWatch::new( db_path.clone(), - timing, - DANGER_DETECTOR_POLL_INTERVAL, + l1_config.identity.max_wait_blocks, + LANDING_WATCH_POLL_INTERVAL, shutdown.process_lock(), ); - detector.preflight_storage()?; + landing_watch.preflight_storage()?; let tx_feed = L2TxFeed::new(db_path.clone(), shutdown.clone(), L2TxFeedConfig::default()); // Configuration ends here: the remaining values are exactly what // `launch` hands to the workers, so the config structs never cross // the authority boundary. - let lane_config = - InclusionLaneConfig::new(dumps_dir).with_max_batch_open(run_config.max_batch_open()); + let lane_config = InclusionLaneConfig::new(dumps_dir, snapshot_leases.clone()) + .with_max_batch_open(run_config.max_batch_open()); let api_config = ApiConfig::new(domain, A::max_method_payload_bytes()); let bind = |what: &'static str, addr: &str| { let addr = addr.to_owned(); @@ -235,6 +237,7 @@ impl PreparedRuntime { // The DB row stores the dump *directory*; the app's state // file is located relative to the opaque application prefix. state_file_in_dump: |dump_dir| A::state_file_in_dump(&dump_info::app_prefix(dump_dir)), + leases: snapshot_leases, }; Ok(Self { @@ -244,7 +247,8 @@ impl PreparedRuntime { storage, lane_config, submitter, - detector, + landing_watch, + landing_feed, tx_feed, listeners, public_addr, @@ -257,10 +261,7 @@ impl PreparedRuntime { } /// Launch all workers in one infallible, non-async, non-yielding step. - /// Consuming the [`crate::recovery::RuntimeAdmission`] witness here is - /// what gates launching on the reducer's fresh clean decision — the - /// witness's sole constructor is `admit_runtime`, and launch uses it up. - pub(super) fn launch(self, _admission: crate::recovery::RuntimeAdmission) -> Workers { + pub(super) fn launch(self) -> Workers { let Self { input_reader, api_config, @@ -268,7 +269,8 @@ impl PreparedRuntime { storage, lane_config, submitter, - detector, + landing_watch, + landing_feed, tx_feed, listeners, public_addr, @@ -281,13 +283,13 @@ impl PreparedRuntime { let (tx, lane) = InclusionLane::::start(QUEUE_CAPACITY, shutdown.clone(), storage, lane_config); - // The reader, detector, and fee oracle only need to stop: they take + // The reader, landing watch, and fee oracle only need to stop: they take // the notification half and hold the process lock directly. The // lane, server, and submitter retain the complete scope, so they take // the scope. let reader = input_reader.start_preflighted(shutdown.signal()); let submitter = submitter.start_preflighted(shutdown.clone()); - let detector = detector.start_preflighted(shutdown.signal()); + let landing_watch = landing_watch.start_preflighted(shutdown.signal()); let fee_oracle = fee_oracle.map(|oracle| oracle.start(shutdown.signal())); // One task serves the public and internal listeners. let server = http::start_on_listeners( @@ -297,6 +299,7 @@ impl PreparedRuntime { tx_feed, api_config, snapshot_state, + landing_feed, ); tracing::info!(public = %public_addr, internal = %internal_addr, "listening"); @@ -305,7 +308,7 @@ impl PreparedRuntime { lane, reader, submitter, - detector, + landing_watch, fee_oracle, shutdown, _shutdown_on_drop: shutdown_on_drop, @@ -325,7 +328,7 @@ impl Workers { lane, reader, submitter, - detector, + landing_watch, fee_oracle, shutdown, _shutdown_on_drop: _, @@ -345,7 +348,8 @@ impl Workers { submitter_result = &mut *submitter => FirstExit::Worker(WorkerExit::BatchSubmitter( WorkerStop::from_select(submitter_result), )), - detector_result = &mut *detector => FirstExit::detector(detector_result), + landing_watch_result = &mut *landing_watch => + FirstExit::landing_watch(landing_watch_result), fee_oracle_result = async { match fee_oracle.as_mut() { Some(handle) => handle.await, @@ -374,7 +378,7 @@ impl Workers { lane, reader, submitter, - detector, + landing_watch, fee_oracle, shutdown: _shutdown, _shutdown_on_drop, @@ -391,8 +395,8 @@ impl Workers { Box::pin(wait_for_batch_submitter_shutdown(submitter)), ), ( - WorkerId::DangerDetector, - Box::pin(wait_for_danger_detector_shutdown(detector)), + WorkerId::LandingWatch, + Box::pin(wait_for_landing_watch_shutdown(landing_watch)), ), ]; if let Some(fee_oracle) = fee_oracle { @@ -491,7 +495,7 @@ enum WorkerId { Lane, InputReader, BatchSubmitter, - DangerDetector, + LandingWatch, FeeOracle, } @@ -503,7 +507,7 @@ impl WorkerId { WorkerId::Lane => "inclusion lane", WorkerId::InputReader => "input reader", WorkerId::BatchSubmitter => "batch submitter", - WorkerId::DangerDetector => "danger detector", + WorkerId::LandingWatch => "landing watch", WorkerId::FeeOracle => "fee oracle", } } @@ -517,9 +521,7 @@ impl WorkerExit { WorkerExit::Lane(_) => WorkerId::Lane, WorkerExit::InputReader(_) => WorkerId::InputReader, WorkerExit::BatchSubmitter(_) => WorkerId::BatchSubmitter, - WorkerExit::DangerDetector(_) | WorkerExit::DangerDetected { .. } => { - WorkerId::DangerDetector - } + WorkerExit::LandingWatch(_) | WorkerExit::TerminalFact(_) => WorkerId::LandingWatch, WorkerExit::FeeOracle(_) => WorkerId::FeeOracle, } } @@ -538,20 +540,20 @@ impl FirstExit { FirstExit::Signal(result.err().map(CommandError::from)) } - /// The detector's `RecoveryRequired` trip is a first-class exit, not an + /// The landing watch's terminal fact is a first-class exit, not an /// error; `Shutdown` outside a real shutdown means it stopped on its own. - fn detector( - result: Result, tokio::task::JoinError>, + fn landing_watch( + result: Result, tokio::task::JoinError>, ) -> Self { let stop = match result { - Ok(Ok(DetectorExit::RecoveryRequired { status })) => { - return FirstExit::Worker(WorkerExit::DangerDetected { status }); + Ok(Ok(LandingWatchExit::Terminal(fact))) => { + return FirstExit::Worker(WorkerExit::TerminalFact(fact)); } - Ok(Ok(DetectorExit::Shutdown)) => WorkerStop::StoppedUnexpectedly, + Ok(Ok(LandingWatchExit::Shutdown)) => WorkerStop::StoppedUnexpectedly, Ok(Err(source)) => WorkerStop::Source(source), Err(source) => WorkerStop::Join(source), }; - FirstExit::Worker(WorkerExit::DangerDetector(stop)) + FirstExit::Worker(WorkerExit::LandingWatch(stop)) } } @@ -614,18 +616,16 @@ async fn wait_for_batch_submitter_shutdown( WorkerStop::from_shutdown(handle.await).map_err(WorkerExit::BatchSubmitter) } -/// Detector `Shutdown` is the graceful drain; a `RecoveryRequired` trip -/// during drain still surfaces as the first-class danger exit. -async fn wait_for_danger_detector_shutdown( - handle: JoinHandle>, +/// The watch's `Shutdown` is the graceful drain; a terminal fact found +/// during drain still surfaces as the first-class terminal exit. +async fn wait_for_landing_watch_shutdown( + handle: JoinHandle>, ) -> Result<(), WorkerExit> { match handle.await { - Ok(Ok(DetectorExit::Shutdown)) => Ok(()), - Ok(Ok(DetectorExit::RecoveryRequired { status })) => { - Err(WorkerExit::DangerDetected { status }) - } - Ok(Err(source)) => Err(WorkerExit::DangerDetector(WorkerStop::Source(source))), - Err(source) => Err(WorkerExit::DangerDetector(WorkerStop::Join(source))), + Ok(Ok(LandingWatchExit::Shutdown)) => Ok(()), + Ok(Ok(LandingWatchExit::Terminal(fact))) => Err(WorkerExit::TerminalFact(fact)), + Ok(Err(source)) => Err(WorkerExit::LandingWatch(WorkerStop::Source(source))), + Err(source) => Err(WorkerExit::LandingWatch(WorkerStop::Join(source))), } } @@ -653,39 +653,45 @@ fn build_batch_submitter_providers(l1: &L1Config) -> Result` conversions ────────────────── // - // The detector arm is the interesting one (DangerDetected vs Shutdown vs - // Source vs Join). The other workers follow a uniform 3-way mapping - // covered by the type system. - - type DetectorJoinResult = - Result, tokio::task::JoinError>; + // The landing watch arm is the interesting one (TerminalFact vs + // Shutdown vs Source vs Join). The other workers follow a uniform 3-way + // mapping covered by the type system. + + type LandingWatchJoinResult = + Result, tokio::task::JoinError>; + + fn void_history() -> TerminalFact { + TerminalFact::VoidHistory { + lag: LandingLag { + batch_nonce: 7, + first_frame_safe_block: 10, + safe_block: 1209, + }, + max_wait_blocks: 1200, + } + } #[test] - fn detector_shutdown_in_select_maps_to_stopped_unexpectedly() { - let result: DetectorJoinResult = Ok(Ok(DetectorExit::Shutdown)); + fn landing_watch_shutdown_in_select_maps_to_stopped_unexpectedly() { + let result: LandingWatchJoinResult = Ok(Ok(LandingWatchExit::Shutdown)); assert!(matches!( - FirstExit::detector(result), - FirstExit::Worker(WorkerExit::DangerDetector(WorkerStop::StoppedUnexpectedly)) + FirstExit::landing_watch(result), + FirstExit::Worker(WorkerExit::LandingWatch(WorkerStop::StoppedUnexpectedly)) )); } #[test] - fn detector_recovery_required_maps_to_danger_detected() { - let result: DetectorJoinResult = Ok(Ok(DetectorExit::RecoveryRequired { - status: DangerStatus::ClosedBatchInDanger(7), - })); + fn landing_watch_terminal_fact_maps_to_the_terminal_exit() { + let result: LandingWatchJoinResult = Ok(Ok(LandingWatchExit::Terminal(void_history()))); assert!(matches!( - FirstExit::detector(result), - FirstExit::Worker(WorkerExit::DangerDetected { - status: DangerStatus::ClosedBatchInDanger(7) - }) + FirstExit::landing_watch(result), + FirstExit::Worker(WorkerExit::TerminalFact(fact)) if fact == void_history() )); } @@ -720,11 +726,11 @@ mod tests { Ok(()) } }); - let detector = tokio::spawn({ + let landing_watch = tokio::spawn({ let shutdown = shutdown.clone(); async move { shutdown.wait_for_shutdown().await; - Ok(DetectorExit::Shutdown) + Ok(LandingWatchExit::Shutdown) } }); ( @@ -733,7 +739,7 @@ mod tests { lane, reader, submitter, - detector, + landing_watch, fee_oracle: None, shutdown: shutdown.clone(), _shutdown_on_drop: ShutdownOnDrop(shutdown.clone()), @@ -819,7 +825,7 @@ mod tests { }; let run_config = *run_config; let db_path = run_config.db_path(); - let timing = run_config.protocol_timing().expect("valid timing"); + let window = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; let submitter_address = crate::commands::batch_submitter_address_from_private_key(KEY).expect("key address"); let dumps_dir = dir.path().join("dumps"); @@ -832,16 +838,10 @@ mod tests { ) .expect("open storage"); storage - .complete_baseline_setup( - &finalized, - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, - ) + .publish_era(&finalized, &crate::storage::NewEra::EMPTY_GENESIS) .expect("register finalized dump"); storage - .append_safe_inputs(0, &[], submitter_address, &timing) + .append_safe_inputs(0, &[], submitter_address) .expect("initialize safe head"); storage .initialize_open_state(0, crate::storage::SafeInputRange::empty_at(0)) @@ -860,6 +860,8 @@ mod tests { .expect("input box address"), app_deployment_block: 0, batch_submitter_address: submitter_address, + max_direct_payload_bytes: crate::storage::test_helpers::TEST_MAX_DIRECT_PAYLOAD_BYTES, + max_wait_blocks: window, fee_oracle: crate::storage::FeeOracleIdentity::Fixed { log_gas_price: 0 }, }; let input_reader = InputReader::from_parts( @@ -873,8 +875,6 @@ mod tests { identity.input_box_address, identity.app_deployment_block, db_path.clone(), - identity.batch_submitter_address, - timing, ProcessLock::test(), ); let l1_config = L1Config { @@ -892,7 +892,6 @@ mod tests { WorkersConfig { run_config, l1_config, - timing, input_reader, fee_oracle: None, process_lock, @@ -921,7 +920,7 @@ mod tests { assert!(err.to_string().contains("public listener"), "{err}"); // The boundary check: a worker spawned during preparation would keep // the process lock held — through its `RuntimeScope` clone or, for - // the reader, detector, and fee oracle, its own `ProcessLock` — so + // the reader, landing watch, and fee oracle, its own `ProcessLock` — so // this acquire succeeding proves zero workers launched. ProcessLock::acquire_released(&data_dir); } @@ -948,17 +947,14 @@ mod tests { } #[tokio::test] - async fn launch_requires_a_fresh_admission_after_preparation() { - let (_dir, data_dir, db_path, workers_config) = + async fn launched_workers_drain_cleanly_and_release_the_lock() { + let (_dir, data_dir, _db_path, workers_config) = startup_workers_config("127.0.0.1:0".to_string(), "127.0.0.1:0".to_string()); - let timing = workers_config.timing; let prepared = PreparedRuntime::::prepare(workers_config) .await .expect("runtime prepares"); - let admission = crate::recovery::admit_runtime(&db_path, &timing) - .expect("the reducer admits over clean facts and returns its witness"); - let workers = prepared.launch(admission); + let workers = prepared.launch(); workers .finish(FirstExit::Signal(None)) @@ -979,35 +975,6 @@ mod tests { .expect("nested runtime work releases process ownership after clean drain"); } - #[tokio::test] - async fn admission_refuses_when_facts_age_during_preparation() { - let (_dir, _data_dir, db_path, workers_config) = - startup_workers_config("127.0.0.1:0".to_string(), "127.0.0.1:0".to_string()); - let timing = workers_config.timing; - - let prepared = PreparedRuntime::::prepare(workers_config) - .await - .expect("runtime prepares"); - - // Simulate time moving behind the persisted local-progress baseline - // while preparation was in flight. The final reducer invocation must - // classify the new clock refusal before durable Live or launch. - let future_ms = i64::try_from(crate::clock::unix_now_ms()).expect("clock fits") - + i64::try_from(timing.seconds_per_block() * 2_000).expect("offset fits"); - crate::storage::Storage::open_connection(&db_path) - .expect("open raw test connection") - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [future_ms], - ) - .expect("move progress baseline into the future"); - - let error = crate::recovery::admit_runtime(&db_path, &timing) - .expect_err("aged clean decision must not admit"); - assert!(matches!(error, crate::recovery::RecoveryError::Retry(_))); - drop(prepared); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[cfg(unix)] async fn terminal_primary_worker_exit_aborts() { @@ -1019,9 +986,12 @@ mod tests { let shutdown = RuntimeScope::default(); let (workers, _dir) = waiting_workers(&shutdown); let _ = workers - .finish(FirstExit::Worker(WorkerExit::DangerDetected { - status: DangerStatus::CanonicalDivergence(7), - })) + .finish(FirstExit::Worker(WorkerExit::TerminalFact( + TerminalFact::Divergence(Divergence { + nonce: 7, + safe_input_index: 0, + }), + ))) .await; panic!("terminal runtime returned instead of aborting"); } @@ -1042,18 +1012,22 @@ mod tests { panic!("terminal runtime returned instead of aborting"); } - #[tokio::test] - async fn recovery_primary_uses_ordinary_shutdown() { + /// A void history found by the landing watch stops the process the way a + /// divergence does: the terminal abort. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg(unix)] + async fn void_history_primary_exit_aborts() { + if !crate::runtime::shutdown::abort_test_child( + "commands::run::workers::tests::void_history_primary_exit_aborts", + ) { + return; + } let shutdown = RuntimeScope::default(); let (workers, _dir) = waiting_workers(&shutdown); - - let err = workers - .finish(FirstExit::Worker(WorkerExit::DangerDetected { - status: DangerStatus::TipInDanger(3), - })) - .await - .expect_err("danger exit must fail run"); - assert!(matches!(err, CommandError::Worker(_))); + let _ = workers + .finish(FirstExit::Worker(WorkerExit::TerminalFact(void_history()))) + .await; + panic!("terminal runtime returned instead of aborting"); } #[tokio::test] @@ -1164,11 +1138,11 @@ mod tests { } #[test] - fn detector_inner_error_maps_to_source_variant() { - let result: DetectorJoinResult = Ok(Err(DangerDetectorError::Join("boom".into()))); + fn landing_watch_inner_error_maps_to_source_variant() { + let result: LandingWatchJoinResult = Ok(Err(LandingWatchError::Join("boom".into()))); assert!(matches!( - FirstExit::detector(result), - FirstExit::Worker(WorkerExit::DangerDetector(WorkerStop::Source(_))) + FirstExit::landing_watch(result), + FirstExit::Worker(WorkerExit::LandingWatch(WorkerStop::Source(_))) )); } } diff --git a/sequencer/src/commands/setup/base.rs b/sequencer/src/commands/setup/base.rs new file mode 100644 index 00000000..06417f9e --- /dev/null +++ b/sequencer/src/commands/setup/base.rs @@ -0,0 +1,254 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Step 1 of setup: the base it rebuilds from. A stored machine is read +//! through its export, and the sequencer's application is built from the +//! machine's state bytes; genesis is the empty scheduler and the host's +//! genesis application. + +use std::path::{Path, PathBuf}; + +use sequencer_core::application::{AppError, Application}; +use sequencer_core::history::L2TxCount; +use sequencer_core::scheduler::{Scheduler, SchedulerConfig, SchedulerExport}; + +use crate::commands::error::{CommandError, SetupError}; +use crate::machine::MachineExportError; + +/// How setup reads a stored machine: [`crate::machine::read_machine_export`] +/// in production. A test hands in an export and state bytes through it, so +/// setup's own code and tests never need an emulator. +pub(crate) type ReadMachineExport = + Box Result + Send>; + +pub(crate) fn read_stored_machine() -> ReadMachineExport { + Box::new(crate::machine::read_machine_export) +} + +/// The point setup rebuilds from: the machine's export, and the application +/// at the export's L2 tx count. +pub(crate) struct Base { + pub(crate) export: SchedulerExport, + pub(crate) app: A, +} + +/// Read the base: the machine at `machine` through `read_export`, its state +/// bytes written under `scratch`, or genesis without one. The genesis factory +/// runs only for a genesis base. +pub(crate) async fn load_base( + machine: Option<&Path>, + genesis_app: F, + read_export: ReadMachineExport, + scheduler_config: SchedulerConfig, + scratch: &Path, +) -> Result, CommandError> +where + A: Application + 'static, + F: FnOnce() -> Result, +{ + let Some(machine) = machine else { + let app = genesis_app()?; + if app.l2_tx_count() != L2TxCount::ZERO { + return Err(CommandError::AppBootstrap(AppError::Internal { + reason: format!( + "a genesis application must start at l2_tx_count = 0, got {}", + app.l2_tx_count().get() + ), + })); + } + let export = SchedulerExport::new( + Scheduler::genesis(scheduler_config).state(), + L2TxCount::ZERO, + ); + return Ok(Base { export, app }); + }; + + // The emulator blocks and its machine handle is not `Send`. + let machine_dir = machine.to_path_buf(); + let state_file = scratch.join("base-state"); + let export = { + let (dir, file) = (machine_dir.clone(), state_file.clone()); + tokio::task::spawn_blocking(move || read_export(&dir, &file)) + .await + .map_err(|join| { + if join.is_panic() { + std::panic::resume_unwind(join.into_panic()); + } + CommandError::Io(std::io::Error::other(format!( + "reading the base machine was cancelled: {join}" + ))) + })? + .map_err(|source| SetupError::BaseMachine { + path: machine_dir.clone(), + source, + })? + }; + let app = application_from_state::(&machine_dir, &state_file, &export)?; + Ok(Base { export, app }) +} + +/// The application at the export's count, held to it: an instance at another +/// count would number the rebuilt history wrongly. +fn application_from_state( + machine_dir: &Path, + state_file: &Path, + export: &SchedulerExport, +) -> Result { + let count = L2TxCount::new(export.l2_tx_count); + let base_application = |source| SetupError::BaseApplication { + path: PathBuf::from(machine_dir), + source, + }; + let app = A::from_canonical_state(state_file, count).map_err(base_application)?; + if app.l2_tx_count() != count { + return Err(base_application(AppError::Internal { + reason: format!( + "the application built from the machine's state is at L2 tx count {}, \ + but the machine reports {}", + app.l2_tx_count().get(), + export.l2_tx_count + ), + })); + } + Ok(app) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::Address; + use app_core::application::WalletApp; + use sequencer_core::application::{Admission, AppOutputs, CanonicalState, SchedulerLimits}; + use sequencer_core::l2_tx::{Direct, UserOpTx}; + use sequencer_core::user_op::UserOp; + + /// Builds itself from the count its state bytes hold, ignoring the one it + /// is handed: setup must hold it to the export's count itself. + struct IgnoresCount(L2TxCount); + + impl Application for IgnoresCount { + fn user_nonce(&self, _sender: Address) -> Result { + unreachable!("setup's base never executes") + } + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { + unreachable!("setup's base never executes") + } + fn apply_direct(&mut self, _direct: &Direct) -> Result { + unreachable!("setup's base never executes") + } + fn l2_tx_count(&self) -> L2TxCount { + self.0 + } + fn max_method_payload_bytes() -> usize { + 0 + } + fn scheduler_limits() -> SchedulerLimits { + app_core::application::WALLET_SCHEDULER_LIMITS + } + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + unreachable!("setup never admits") + } + fn from_dump(_prefix: &Path) -> Result { + unreachable!("setup's base is no dump") + } + fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { + unreachable!("setup's base writes no dump") + } + fn state_file_in_dump(prefix: &Path) -> PathBuf { + prefix.to_path_buf() + } + fn from_canonical_state(state_file: &Path, _count: L2TxCount) -> Result { + let bytes: [u8; 8] = + std::fs::read(state_file)? + .try_into() + .map_err(|_| AppError::Internal { + reason: "eight bytes".to_owned(), + })?; + Ok(Self(L2TxCount::new(u64::from_le_bytes(bytes)))) + } + } + + fn export_at(l2_tx_count: u64) -> SchedulerExport { + SchedulerExport { + l2_tx_count, + ..SchedulerExport::new( + Scheduler::genesis(SchedulerConfig::for_application::(Address::ZERO)) + .state(), + L2TxCount::ZERO, + ) + } + } + + /// A machine reader that writes `state` and answers `export`. + fn machine(state: Vec, export: SchedulerExport) -> ReadMachineExport { + Box::new(move |_, state_file| { + std::fs::write(state_file, &state).unwrap(); + Ok(export) + }) + } + + async fn refusal(read_export: ReadMachineExport) -> SetupError { + let scratch = tempfile::tempdir().unwrap(); + let error = load_base::( + Some(Path::new("/machines/base")), + || -> Result { unreachable!("a machine base needs no genesis") }, + read_export, + SchedulerConfig::for_application::(Address::ZERO), + scratch.path(), + ) + .await + .err() + .expect("the base is refused"); + match error { + CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error)) => *error, + other => panic!("expected a setup refusal, got {other:?}"), + } + } + + #[tokio::test] + async fn a_failed_machine_read_names_the_machine() { + let error = refusal::(Box::new(|_, state_file| { + Err(MachineExportError::Io { + path: state_file.to_path_buf(), + source: std::io::Error::other("disk full"), + }) + })) + .await; + assert!( + matches!( + &error, + SetupError::BaseMachine { path, source: MachineExportError::Io { .. } } + if path == Path::new("/machines/base") + ), + "{error:?}" + ); + } + + /// The count the built application reports must be the export's, whether + /// or not the application checks it itself. + #[tokio::test] + async fn an_application_at_another_count_than_the_export_is_refused() { + let error = + refusal::(machine(7_u64.to_le_bytes().to_vec(), export_at(8))).await; + assert!( + matches!( + &error, + SetupError::BaseApplication { path, source: AppError::Internal { reason } } + if path == Path::new("/machines/base") && reason.contains("machine reports 8") + ), + "{error:?}" + ); + + let state = WalletApp::default().canonical_snapshot_bytes().unwrap(); + let error = refusal::(machine(state, export_at(1))).await; + assert!( + matches!(error, SetupError::BaseApplication { .. }), + "{error:?}" + ); + } +} diff --git a/sequencer/src/commands/setup/chain_tests.rs b/sequencer/src/commands/setup/chain_tests.rs new file mode 100644 index 00000000..550ade02 --- /dev/null +++ b/sequencer/src/commands/setup/chain_tests.rs @@ -0,0 +1,995 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Setup end to end against Anvil, with a real InputBox and application and a +//! history already on chain: directs, the submitter's accepted and rejected +//! batches, a skipped op, and directs still queued at the stop block. The +//! result must equal what the canonical runner executes over the same inputs. + +use std::path::Path; + +use alloy::network::EthereumWallet; +use alloy::node_bindings::{Anvil, AnvilInstance}; +use alloy::providers::{DynProvider, Provider, ProviderBuilder}; +use alloy::rpc::types::BlockNumberOrTag; +use alloy::signers::SignerSync; +use alloy::signers::local::PrivateKeySigner; +use alloy_primitives::{Address, B256}; +use alloy_sol_types::SolStruct; +use app_core::application::{WalletApp, WalletConfig}; +use cartesi_rollups_contracts::application_factory::ApplicationFactory; +use cartesi_rollups_contracts::input_box::InputBox; +use sequencer_core::application::{AppError, Application, CanonicalState}; +use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::history::{HistoryClaim, L2TxCount}; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::{CanonicalRunner, SchedulerConfig, SchedulerExport, TxFate}; +use ssz::Encode; + +use super::*; +use crate::ingress::inclusion_lane::dump_info; +use crate::storage::L2TxContext; + +/// Anvil's dev accounts 0 (the submitter) and 1 (a depositor of directs). +const SUBMITTER_KEY: &str = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; +const DEPOSITOR_KEY: &str = "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"; +const CHAIN_ID: u64 = 31337; + +struct Chain { + _anvil: AnvilInstance, + url: String, + provider: DynProvider, + input_box: Address, + app: Address, + app_deployment_block: u64, + submitter: Address, + depositor: Address, + /// Each posted input's inclusion block, by index. + blocks: Vec, +} + +impl Chain { + /// A chain with the InputBox and the application deployed, and nothing + /// posted yet. + async fn deploy() -> Self { + let anvil = Anvil::default() + .args(["--slots-in-an-epoch", "1"]) + .timeout(30_000) + .spawn(); + let submitter_key: PrivateKeySigner = SUBMITTER_KEY.parse().unwrap(); + let depositor_key: PrivateKeySigner = DEPOSITOR_KEY.parse().unwrap(); + let (submitter, depositor) = (submitter_key.address(), depositor_key.address()); + let mut wallet = EthereumWallet::from(submitter_key); + wallet.register_signer(depositor_key); + let provider = ProviderBuilder::new() + .wallet(wallet) + .connect_http(anvil.endpoint_url()) + .erased(); + + let input_box = *InputBox::deploy(provider.clone()).await.unwrap().address(); + let factory = ApplicationFactory::deploy(provider.clone(), Address::repeat_byte(0x55)) + .await + .unwrap(); + let new_application = factory.newApplication_0( + submitter, + submitter, + B256::repeat_byte(1), + input_box, + ApplicationFactory::WithdrawalConfig { + guardian: Address::ZERO, + log2LeavesPerAccount: 0, + log2MaxNumOfAccounts: 0, + accountsDriveStartIndex: 0, + withdrawalOutputBuilder: Address::ZERO, + }, + ); + let app = new_application.call().await.unwrap(); + let receipt = new_application + .send() + .await + .unwrap() + .get_receipt() + .await + .unwrap(); + assert!(receipt.status()); + Self { + url: anvil.endpoint_url().to_string(), + _anvil: anvil, + provider, + input_box, + app, + app_deployment_block: receipt.block_number.unwrap(), + submitter, + depositor, + blocks: Vec::new(), + } + } + + /// The deployed chain with the shared history posted and safe: + /// + /// | index | input | + /// |---|---| + /// | 0, 1 | directs | + /// | 2 | batch 0, a frame at input 0's block: drains 0, one op | + /// | 3 | a batch with a wrong nonce: rejected | + /// | 4 | a direct | + /// | 5 | batch 1, a frame at input 1's block: drains 1, one op, one skipped | + /// | 6 | a direct | + /// + /// Inputs 4 and 6 are still queued at the stop block. After the first + /// five, inputs 1 and 4 are queued. + async fn with_history() -> Self { + let mut chain = Self::deploy().await; + chain.direct(vec![1]).await; + chain.direct(vec![2]).await; + let frame = chain.blocks[0]; + chain.batch(0, frame, &[0]).await; + let frame = chain.latest().await; + chain.batch(5, frame, &[1]).await; + chain.direct(vec![3]).await; + let frame = chain.blocks[1]; + chain.batch(1, frame, &[1, 7]).await; + chain.direct(vec![4]).await; + chain.mine_until_safe(*chain.blocks.last().unwrap()).await; + chain + } + + async fn post(&mut self, sender: Address, payload: Vec) { + let receipt = InputBox::new(self.input_box, &self.provider) + .addInput(self.app, payload.into()) + .from(sender) + .send() + .await + .unwrap() + .get_receipt() + .await + .unwrap(); + assert!(receipt.status()); + self.blocks.push(receipt.block_number.unwrap()); + } + + async fn direct(&mut self, payload: Vec) { + self.post(self.depositor, payload).await; + } + + /// A batch with one frame at `safe_block` holding one op of the user per + /// nonce in `op_nonces`. + async fn batch(&mut self, nonce: u64, safe_block: u64, op_nonces: &[u32]) { + let user: PrivateKeySigner = format!("{:064x}", 7).parse().unwrap(); + let domain = sequencer_core::build_input_domain(CHAIN_ID, self.app); + let user_ops = op_nonces + .iter() + .map(|&nonce| { + let op = sequencer_core::user_op::UserOp { + nonce, + max_fee: 0, + data: vec![0xde, 0xad].into(), + }; + let signature = user + .sign_hash_sync(&op.eip712_signing_hash(&domain)) + .unwrap(); + WireUserOp { + nonce, + max_fee: 0, + data: op.data.to_vec(), + signature: signature.as_bytes().to_vec(), + } + }) + .collect(); + let payload = Batch { + nonce, + frames: vec![Frame { + safe_block, + fee_price: 0, + user_ops, + }], + } + .as_ssz_bytes(); + self.post(self.submitter, payload).await; + } + + async fn latest(&self) -> u64 { + self.provider.get_block_number().await.unwrap() + } + + async fn safe(&self) -> u64 { + self.provider + .get_block_by_number(BlockNumberOrTag::Safe) + .await + .unwrap() + .unwrap() + .header + .number + } + + async fn mine_until_safe(&self, block: u64) { + while self.safe().await < block { + self.provider + .raw_request::<_, serde_json::Value>("evm_mine".into(), ()) + .await + .unwrap(); + } + } + + fn setup_config(&self, data_dir: &Path, extra: &[&str]) -> SetupConfig { + use crate::harness::{Cli, Command}; + use clap::Parser; + let app = self.app.to_string(); + let submitter = self.submitter.to_string(); + let mut args = vec![ + "sequencer", + "setup", + "--data-dir", + data_dir.to_str().unwrap(), + "--eth-rpc-url", + &self.url, + "--chain-id", + "31337", + "--app-address", + &app, + "--batch-submitter-address", + &submitter, + "--seconds-per-block", + "1", + ]; + if !extra.contains(&"--fixed-log-gas-price") { + args.extend_from_slice(&["--fixed-log-gas-price", "0"]); + } + args.extend_from_slice(extra); + let Command::Setup(config) = Cli::try_parse_from(args).unwrap().command else { + panic!("expected setup subcommand"); + }; + *config + } + + fn wallet(&self) -> WalletApp { + WalletApp::new(WalletConfig { + sequencer_address: self.submitter, + ..WalletConfig::default() + }) + } + + fn scheduler_config(&self) -> SchedulerConfig { + SchedulerConfig::for_application::(self.submitter) + } + + /// The canonical runner over the first `count` inputs of the synced table. + fn reference( + &self, + storage: &mut Storage, + count: u64, + ) -> (CanonicalRunner, Vec) { + let mut runner = CanonicalRunner::genesis(self.wallet(), self.scheduler_config()); + let mut rows = Vec::new(); + for input in storage.l1_inputs_page(0, count, usize::MAX).unwrap() { + for record in runner.process_input(input).unwrap().txs { + if let TxFate::Done { index } = record.fate { + rows.push(( + index.get(), + record.scheduled.tx, + record.scheduled.batch_nonce, + )); + } + } + } + (runner, rows) + } +} + +/// One executed row: its offset, the transaction and its batch nonce. +type Row = (u64, L2Tx, u64); + +fn with_key(extra: &[&'static str]) -> Vec<&'static str> { + let mut args = vec!["--batch-submitter-private-key", SUBMITTER_KEY]; + args.extend_from_slice(extra); + args +} + +/// The rows the feed serves from `from` on. +fn feed(storage: &mut Storage, from: u64) -> Vec { + let era_id = storage.era().unwrap().id; + let page = storage + .canonical_history_page( + HistoryClaim { + era_id, + next_input: L2TxCount::new(from), + }, + 1024, + ) + .unwrap(); + page.rows + .into_iter() + .map(|row| match row.context { + L2TxContext::UserOp { tx, batch_nonce } => { + (row.offset.get(), L2Tx::UserOp(tx), batch_nonce) + } + L2TxContext::Direct { + tx, batch_nonce, .. + } => (row.offset.get(), L2Tx::Direct(tx), batch_nonce), + }) + .collect() +} + +/// How many rows the rebuilt history holds. +fn rebuilt_rows(storage: &mut Storage) -> u64 { + storage + .raw_read(|tx| tx.query_row("SELECT COUNT(*) FROM rebuilt_history", [], |row| row.get(0))) + .map(|count: i64| u64::try_from(count).unwrap()) + .unwrap() +} + +fn genesis_wallet(chain: &Chain) -> impl FnOnce() -> Result { + let app = chain.wallet(); + move || Ok(app) +} + +/// A machine reader that answers with `runner`'s export and state, as the +/// stored machine of the same history would. +fn machine_like(runner: &CanonicalRunner, export: SchedulerExport) -> ReadMachineExport { + let state = runner.app().canonical_snapshot_bytes().unwrap(); + Box::new(move |_machine, state_file| { + std::fs::write(state_file, &state).unwrap(); + Ok(export) + }) +} + +fn no_machine() -> ReadMachineExport { + Box::new(|_, _| unreachable!("a genesis setup reads no machine")) +} + +fn setup_refusal(error: CommandError) -> SetupError { + match error { + CommandError::Bootstrap(BootstrapError::Setup(error)) => *error, + other => panic!("expected a setup refusal, got {other:?}"), + } +} + +/// Assert the published era is the reference's at the stop block, row for +/// row, with the baseline's bytes. +fn assert_matches_reference( + chain: &Chain, + db_path: &str, + base_count: u64, + summary: &SetupSummary, +) -> CanonicalRunner { + let mut storage = Storage::open(db_path).unwrap(); + let era = summary.era.expect("setup publishes an era"); + let history = storage.era().unwrap(); + let stop_block = history.stop_block; + assert_eq!(era.stop_block, stop_block); + let through_stop = storage.safe_input_count_through_block(stop_block).unwrap(); + assert_eq!( + through_stop, 7, + "every posted input lies at or below the stop block" + ); + let (runner, reference) = chain.reference(&mut storage, through_stop); + let state = runner.scheduler().state(); + + assert_eq!(history.base.get(), base_count); + assert_eq!(history.rebuilt_end, runner.app().l2_tx_count()); + assert_eq!(history.first_queued_input, state.first_queued_input); + assert_eq!((state.first_queued_input, state.fridge_len), (4, 2)); + assert_eq!(history.anchor_batch_nonce, state.next_batch_nonce); + assert_eq!(state.next_batch_nonce, 2); + assert_eq!( + era, + NewEra { + base: L2TxCount::new(base_count), + rebuilt_end: runner.app().l2_tx_count(), + stop_block, + first_queued_input: 4, + anchor_batch_nonce: 2, + } + ); + + // The feed serves, from the base count, what the reference executed. + let expected: Vec = reference + .into_iter() + .filter(|(offset, _, _)| *offset >= base_count) + .collect(); + assert!(!expected.is_empty()); + assert_eq!(feed(&mut storage, base_count), expected); + + // The baseline is the reference's state at the end of the stop block, and + // the era's first comparison checkpoint. + let baseline = storage.latest_snapshot().unwrap().expect("baseline"); + let prefix = dump_info::app_prefix(&baseline.prefix); + assert_eq!( + std::fs::read(WalletApp::state_file_in_dump(&prefix)).unwrap(), + runner.app().canonical_snapshot_bytes().unwrap() + ); + let checkpoint = storage.finalized_dump().unwrap().expect("era baseline"); + assert_eq!(checkpoint.inclusion_block, stop_block); + assert_eq!(checkpoint.snapshot.l2_tx_count, runner.app().l2_tx_count()); + assert_eq!(checkpoint.next_batch_nonce, 2); + assert!( + storage.open_state().unwrap().is_none(), + "setup opens no root" + ); + + assert_eq!(summary.through_block, stop_block); + assert_eq!( + summary.submitter_inputs_past_base, + if base_count == 0 { 3 } else { 1 } + ); + assert_eq!(summary.unsettled_wallet_txs, 0); + runner +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn setup_from_genesis_rebuilds_a_chain_with_batches() { + let chain = Chain::with_history().await; + let data_dir = tempfile::tempdir().unwrap(); + + let summary = run_setup( + chain.setup_config(data_dir.path(), &with_key(&[])), + genesis_wallet(&chain), + no_machine(), + ) + .await + .unwrap(); + + assert_eq!( + (summary.base_l1_input_count, summary.base_l2_tx_count), + (0, 0) + ); + assert_eq!(summary.inputs_to_replay, 7); + let db_path = crate::commands::config::db_path_in(data_dir.path().to_str().unwrap()); + let runner = assert_matches_reference(&chain, &db_path, 0, &summary); + assert!( + feed(&mut Storage::open(&db_path).unwrap(), 0) + .iter() + .any(|(_, tx, _)| matches!(tx, L2Tx::UserOp(_))), + "the rebuilt history holds the accepted batches' ops" + ); + let skipped = { + let mut storage = Storage::open(&db_path).unwrap(); + let mut reference = CanonicalRunner::genesis(chain.wallet(), chain.scheduler_config()); + storage + .l1_inputs_page(0, 7, usize::MAX) + .unwrap() + .into_iter() + .flat_map(|input| reference.process_input(input).unwrap().txs) + .filter(|record| matches!(record.fate, TxFate::Skipped(_))) + .count() + }; + assert_eq!(skipped, 1, "the canonical run of this chain skips one op"); + assert_eq!( + rebuilt_rows(&mut Storage::open(&db_path).unwrap()), + runner.app().l2_tx_count().get(), + "only executed transactions are stored: the skipped op has no row" + ); + let left: Vec = std::fs::read_dir(data_dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!( + left.iter() + .all(|name| !name.starts_with("sequencer-setup-scratch")), + "the scratch directory is removed: {left:?}" + ); +} + +/// From a machine after five inputs, with two directs queued; then `run` +/// starts on the era and its root frame drains the directs still queued at +/// the stop block. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn setup_from_a_base_with_queued_directs_then_run_drains_them() { + let chain = Chain::with_history().await; + let data_dir = tempfile::tempdir().unwrap(); + let db_path = crate::commands::config::db_path_in(data_dir.path().to_str().unwrap()); + + // The base: the canonical runner over the first five inputs, read from a + // scratch sync of the same chain. + let scratch = crate::test_support::temp_db("setup-base-source"); + let base = { + let lock = ProcessLock::test(); + let mut reader = discover( + &chain.setup_config(data_dir.path(), &with_key(&[])), + &scratch.path, + lock, + ) + .await + .unwrap(); + let mut storage = + Storage::initialize_for_command(&scratch.path, LifecycleCommand::Setup).unwrap(); + storage + .pin_deployment_identity(DeploymentIdentity { + chain_id: CHAIN_ID, + app_address: chain.app, + input_box_address: chain.input_box, + app_deployment_block: chain.app_deployment_block, + batch_submitter_address: chain.submitter, + max_direct_payload_bytes: WalletApp::scheduler_limits().max_direct_payload_bytes, + max_wait_blocks: WalletApp::scheduler_limits().max_wait_blocks, + fee_oracle: FeeOracleIdentity::Fixed { log_gas_price: 0 }, + }) + .unwrap(); + reader.sync_to_current_safe_head().await.unwrap(); + chain.reference(&mut storage, 5).0 + }; + let export = base.export(); + assert_eq!( + ( + export.l1_input_count, + export.first_queued_input, + export.fridge_len + ), + (5, 1, 2) + ); + + let summary = run_setup( + chain.setup_config( + data_dir.path(), + &with_key(&["--base", "/machines/after-five"]), + ), + || -> Result { unreachable!("a machine base needs no genesis") }, + machine_like(&base, export), + ) + .await + .unwrap(); + assert_eq!( + (summary.base_l1_input_count, summary.base_l2_tx_count), + (5, export.l2_tx_count) + ); + assert_eq!(summary.inputs_to_replay, 2); + let runner = assert_matches_reference(&chain, &db_path, export.l2_tx_count, &summary); + let rebuilt_end = runner.app().l2_tx_count().get(); + + // `run`'s startup: sync with the frontier on, then the root frame at its + // own safe head. + let lock = ProcessLock::acquire(data_dir.path().to_str().unwrap()).unwrap(); + let mut reader = InputReader::new( + db_path.clone(), + crate::l1::reader::InputReaderConfig { + rpc_url: chain.url.clone(), + allow_insecure_rpc: false, + app_address: chain.app, + poll_interval: std::time::Duration::from_secs(1), + expected_chain_id: CHAIN_ID, + }, + lock, + ) + .await + .unwrap(); + crate::commands::run::start(&db_path, &mut reader) + .await + .unwrap(); + + let mut storage = Storage::open(&db_path).unwrap(); + let root = storage.open_state().unwrap().expect("run opened the root"); + assert!(root.safe_block() >= summary.through_block); + let drained: Vec<(u64, u64, u64)> = feed(&mut storage, rebuilt_end) + .into_iter() + .map(|(offset, tx, batch_nonce)| match tx { + L2Tx::Direct(direct) => (offset, direct.input_index, batch_nonce), + other => panic!("the root frame drains only directs, got {other:?}"), + }) + .collect(); + assert_eq!( + drained, + vec![(rebuilt_end, 4, 2), (rebuilt_end + 1, 6, 2)], + "the root frame drains the directs queued at the stop block, in the anchor batch" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn setup_refuses_an_export_that_disagrees_with_l1_before_publishing() { + let chain = Chain::with_history().await; + let data_dir = tempfile::tempdir().unwrap(); + let db_path = crate::commands::config::db_path_in(data_dir.path().to_str().unwrap()); + // A machine at genesis that claims a queued direct L1 does not hold. + let runner = CanonicalRunner::genesis(chain.wallet(), chain.scheduler_config()); + let export = SchedulerExport { + fridge_len: 1, + ..runner.export() + }; + + let error = run_setup( + chain.setup_config(data_dir.path(), &with_key(&["--base", "/machines/bad"])), + || -> Result { unreachable!("a machine base needs no genesis") }, + machine_like(&runner, export), + ) + .await + .expect_err("an export that disagrees with L1 is refused"); + let refusal = setup_refusal(error); + assert!( + matches!( + refusal, + SetupError::ExportDisagrees { + fridge_len: 1, + admitted: 0, + .. + } + ), + "{refusal:?}" + ); + let mut storage = Storage::open_read_only(&db_path).unwrap(); + assert!( + !storage.is_setup_complete().unwrap(), + "nothing is published" + ); + assert_eq!(rebuilt_rows(&mut storage), 0); + assert!( + storage + .latest_terminal_fault() + .unwrap() + .is_some_and(|fault| fault.cause.contains("disagrees with L1")), + "the terminal cause is in the black box" + ); +} + +/// A setup before the deployment block is safe exits 20 and leaves the +/// directory empty, so the retry may run in it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_deployment_not_yet_safe_exits_20_and_leaves_the_directory_empty() { + let chain = Chain::deploy().await; + assert!(chain.safe().await < chain.app_deployment_block); + let data_dir = tempfile::tempdir().unwrap(); + + let error = run_setup( + chain.setup_config(data_dir.path(), &with_key(&[])), + genesis_wallet(&chain), + no_machine(), + ) + .await + .expect_err("the deployment is not safe yet"); + assert!( + matches!( + error, + CommandError::Bootstrap(BootstrapError::Identity( + IdentityError::DeploymentNotYetSafe { .. } + )) + ), + "{error:?}" + ); + assert_eq!( + error.exit_code(), + crate::commands::error::EXIT_RESTART_TRANSIENT + ); + let left: Vec = std::fs::read_dir(data_dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(left, vec![LOCK_FILE_NAME.to_owned()]); + + // Once it is safe, the retry in the same directory completes. + chain.mine_until_safe(chain.app_deployment_block).await; + let summary = run_setup( + chain.setup_config(data_dir.path(), &with_key(&[])), + genesis_wallet(&chain), + no_machine(), + ) + .await + .unwrap(); + assert_eq!(summary.era.unwrap().rebuilt_end, L2TxCount::ZERO); +} + +/// The dry run needs no key, reports the base and the old instance's +/// activity past it, and never touches the data directory. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_dry_run_reports_and_changes_nothing() { + let chain = Chain::with_history().await; + let parent = tempfile::tempdir().unwrap(); + let absent = parent.path().join("absent"); + let occupied = parent.path().join("occupied"); + std::fs::create_dir(&occupied).unwrap(); + std::fs::write(occupied.join("sequencer.db"), b"someone else's").unwrap(); + + for data_dir in [&absent, &occupied] { + let summary = run_setup( + chain.setup_config(data_dir, &["--dry-run"]), + genesis_wallet(&chain), + no_machine(), + ) + .await + .unwrap(); + assert_eq!( + summary, + SetupSummary { + base_l1_input_count: 0, + base_l2_tx_count: 0, + submitter_inputs_past_base: 3, + unsettled_wallet_txs: 0, + inputs_to_replay: 7, + through_block: chain.safe().await, + era: None, + } + ); + assert!(summary.to_string().contains("re-judges")); + } + assert!( + !absent.exists(), + "the dry run never creates the data directory" + ); + let entries: Vec<_> = std::fs::read_dir(&occupied).unwrap().collect(); + assert_eq!( + entries.len(), + 1, + "the dry run never touches the data directory" + ); + assert_eq!( + std::fs::read(occupied.join("sequencer.db")).unwrap(), + b"someone else's" + ); +} + +/// The escape check against a table synced from the chain: a submitter +/// input past the stop block refuses, and so does a wallet left unsettled. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_escape_check_refuses_activity_past_the_stop_block() { + let chain = Chain::with_history().await; + let scratch = crate::test_support::temp_db("setup-escape"); + let data_dir = tempfile::tempdir().unwrap(); + let mut reader = discover( + &chain.setup_config(data_dir.path(), &with_key(&[])), + &scratch.path, + ProcessLock::test(), + ) + .await + .unwrap(); + let mut storage = + Storage::initialize_for_command(&scratch.path, LifecycleCommand::Setup).unwrap(); + storage + .pin_deployment_identity(DeploymentIdentity { + chain_id: CHAIN_ID, + app_address: chain.app, + input_box_address: chain.input_box, + app_deployment_block: chain.app_deployment_block, + batch_submitter_address: chain.submitter, + max_direct_payload_bytes: WalletApp::scheduler_limits().max_direct_payload_bytes, + max_wait_blocks: WalletApp::scheduler_limits().max_wait_blocks, + fee_oracle: FeeOracleIdentity::Fixed { log_gas_price: 0 }, + }) + .unwrap(); + reader.sync_to_current_safe_head().await.unwrap(); + let safe_block = storage.current_safe_block().unwrap().unwrap(); + let settled = read_submitter_wallet_nonces(&chain.url, chain.submitter, safe_block, false) + .await + .unwrap(); + assert_eq!(settled.0, settled.1); + + // The batch at input 5 lies past a stop block just below it. + let stop_block = chain.blocks[5] - 1; + match check_escape(&mut storage, chain.submitter, stop_block, settled) { + Err(CommandError::Bootstrap(BootstrapError::SetupRefuse( + SetupRefuse::SubmitterInputPastStop { + found_block, + safe_input_index: 5, + .. + }, + ))) => assert_eq!(found_block, chain.blocks[5]), + other => panic!("expected SubmitterInputPastStop, got {other:?}"), + } + check_escape(&mut storage, chain.submitter, chain.blocks[6], settled).unwrap(); + + // The old instance broadcasts a batch that is not mined yet. + chain + .provider + .raw_request::<_, serde_json::Value>("evm_setAutomine".into(), (false,)) + .await + .unwrap(); + let _pending = InputBox::new(chain.input_box, &chain.provider) + .addInput(chain.app, vec![9].into()) + .from(chain.submitter) + .send() + .await + .unwrap(); + let unsettled = read_submitter_wallet_nonces(&chain.url, chain.submitter, safe_block, false) + .await + .unwrap(); + let error = check_escape(&mut storage, chain.submitter, chain.blocks[6], unsettled) + .expect_err("a wallet left unsettled refuses"); + assert!( + matches!( + error, + CommandError::Bootstrap(BootstrapError::SetupRefuse( + SetupRefuse::WalletUnsettledAfterFlush { .. } + )) + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), crate::commands::error::EXIT_SETUP_ESCAPE); +} + +fn entries(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +/// A key that does not sign for the pinned submitter, or a fixed price below +/// the fee floor, is refused with exit 30 before anything is written: no +/// base is read, no transaction is sent, and the directory holds only the lock. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_wrong_key_or_a_price_below_the_floor_is_refused_before_any_write() { + let chain = Chain::with_history().await; + let nonce_before = chain + .provider + .get_transaction_count(chain.submitter) + .await + .unwrap(); + + let data_dir = tempfile::tempdir().unwrap(); + let error = run_setup( + chain.setup_config( + data_dir.path(), + &["--batch-submitter-private-key", DEPOSITOR_KEY], + ), + || -> Result { unreachable!("a refused key reads no base") }, + Box::new(|_, _| unreachable!("a refused key reads no machine")), + ) + .await + .expect_err("the key signs for another address"); + assert!( + matches!( + &error, + CommandError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { fields, .. })) + if fields == "batch_submitter_address" + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), crate::commands::error::EXIT_TERMINAL); + assert_eq!(entries(data_dir.path()), vec![LOCK_FILE_NAME.to_owned()]); + assert_eq!( + chain + .provider + .get_transaction_count(chain.submitter) + .await + .unwrap(), + nonce_before, + "nothing was sent from either account" + ); + + let data_dir = tempfile::tempdir().unwrap(); + let error = run_setup( + chain.setup_config( + data_dir.path(), + &with_key(&["--fixed-log-gas-price", "-1357"]), + ), + || -> Result { unreachable!("a refused price reads no base") }, + Box::new(|_, _| unreachable!("a refused price reads no machine")), + ) + .await + .expect_err("the price is below the floor"); + assert!( + matches!( + error, + CommandError::Bootstrap(BootstrapError::FeeOracleMisconfig { .. }) + ), + "{error:?}" + ); + assert_eq!(error.exit_code(), crate::commands::error::EXIT_TERMINAL); + assert_eq!(entries(data_dir.path()), vec![LOCK_FILE_NAME.to_owned()]); +} + +/// The dry run runs the scheduler's own resume checks too, so it refuses a +/// base that only setup's rebuild would otherwise find wrong. This export +/// queues nothing, so the table check passes, but it names a first queued +/// input past its own input count. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_dry_run_refuses_a_base_that_resume_refuses() { + let chain = Chain::with_history().await; + let data_dir = tempfile::tempdir().unwrap(); + let runner = CanonicalRunner::genesis(chain.wallet(), chain.scheduler_config()); + let export = SchedulerExport { + first_queued_input: 2, + ..runner.export() + }; + + let error = run_setup( + chain.setup_config(data_dir.path(), &["--dry-run", "--base", "/machines/bad"]), + || -> Result { unreachable!("a machine base needs no genesis") }, + machine_like(&runner, export), + ) + .await + .expect_err("the dry run refuses what setup refuses"); + let refusal = setup_refusal(error); + assert!( + matches!( + refusal, + SetupError::ResumeRefused( + sequencer_core::scheduler::ResumeError::FirstQueuedAfterCount { + first_queued_input: 2, + l1_input_count: 0, + } + ) + ), + "{refusal:?}" + ); + assert!(!data_dir.path().join(LOCK_FILE_NAME).exists()); +} + +/// Mines a block every `interval` until dropped. +struct Miner(tokio::task::JoinHandle<()>); + +impl Miner { + fn start(provider: DynProvider, interval: std::time::Duration) -> Self { + Self(tokio::spawn(async move { + loop { + let _ = provider + .raw_request::<_, serde_json::Value>("evm_mine".into(), ()) + .await; + tokio::time::sleep(interval).await; + } + })) + } +} + +impl Drop for Miner { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// The old instance left a submitter transaction in the mempool. Setup's +/// flush settles that wallet slot, by the original or a no-op, and pins the +/// stop block at or after it; the era is then published. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn setup_flushes_a_pending_submitter_transaction_before_the_stop_block() { + let chain = Chain::with_history().await; + let data_dir = tempfile::tempdir().unwrap(); + let settled = chain + .provider + .get_transaction_count(chain.submitter) + .await + .unwrap(); + + chain + .provider + .raw_request::<_, serde_json::Value>("evm_setAutomine".into(), (false,)) + .await + .unwrap(); + let _pending = InputBox::new(chain.input_box, &chain.provider) + .addInput(chain.app, vec![9].into()) + .from(chain.submitter) + .send() + .await + .unwrap(); + let pending = chain + .provider + .get_transaction_count(chain.submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .unwrap(); + assert_eq!( + pending, + settled + 1, + "the old instance's transaction is pending" + ); + let _miner = Miner::start( + chain.provider.clone(), + std::time::Duration::from_millis(100), + ); + + let summary = run_setup( + chain.setup_config(data_dir.path(), &with_key(&[])), + genesis_wallet(&chain), + no_machine(), + ) + .await + .unwrap(); + let era = summary.era.expect("setup publishes an era"); + assert_eq!(summary.unsettled_wallet_txs, 0); + let at_stop = chain + .provider + .get_transaction_count(chain.submitter) + .block_id(BlockNumberOrTag::Number(era.stop_block).into()) + .await + .unwrap(); + assert!( + at_stop >= pending, + "the pending slot is consumed at or below the stop block {}: nonce {at_stop}", + era.stop_block + ); + let db_path = crate::commands::config::db_path_in(data_dir.path().to_str().unwrap()); + assert!( + Storage::open(&db_path) + .unwrap() + .is_setup_complete() + .unwrap() + ); +} diff --git a/sequencer/src/commands/setup/checkpoint_tests.rs b/sequencer/src/commands/setup/checkpoint_tests.rs deleted file mode 100644 index c1d43c86..00000000 --- a/sequencer/src/commands/setup/checkpoint_tests.rs +++ /dev/null @@ -1,225 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -use std::path::{Path, PathBuf}; - -use alloy_primitives::{Address, U256}; -use app_core::application::{WalletApp, WalletConfig}; -use sequencer_core::application::{ - AppError, AppOutputs, Application, ApplicationProgress, ValidationOutcome, execute_direct_input, -}; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; -use sequencer_core::user_op::UserOp; - -use super::{Checkpoint, rebuild_from_checkpoint}; -use crate::commands::error::{BootstrapError, CommandError, EXIT_RESTART_TRANSIENT}; -use crate::ingress::inclusion_lane::dump_info; -use crate::recovery::{RecoveryError, RecoveryFailure, RecoveryRetryReason}; -use crate::storage::test_helpers::{SENDER_A, default_protocol_timing}; -use crate::storage::{FrontierMode, LifecycleCommand, Storage, StoredSafeInput}; -use crate::test_support::{pin_test_deployment_identity, temp_db}; - -struct ReplayForbiddenApp(ApplicationProgress); - -impl Application for ReplayForbiddenApp { - fn max_method_payload_bytes() -> usize { - 0 - } - - fn validate_user_op( - &self, - _: Address, - _: &UserOp, - _: u16, - ) -> Result { - panic!("an uncovered checkpoint must refuse before application execution") - } - - fn apply_valid_user_op(&mut self, _: &ValidUserOp, _: u64) -> Result { - panic!("an uncovered checkpoint must refuse before application execution") - } - - fn apply_direct_input(&mut self, _: &DirectInput) -> Result { - panic!("an uncovered checkpoint must refuse before application execution") - } - - fn progress(&self) -> ApplicationProgress { - self.0 - } - - fn from_dump(_: &Path) -> Result { - unreachable!("the checkpoint is supplied by the fixture") - } - - fn create_dump(&mut self, _: &Path) -> Result<(), AppError> { - panic!("an uncovered checkpoint must refuse before artifact creation") - } - - fn state_file_in_dump(_: &Path) -> PathBuf { - unreachable!("the refused checkpoint has no new artifact") - } -} - -#[test] -fn recovery_refuses_stop_before_checkpoint_without_replay_or_publication() { - // The last case has H1 >= B: resync reaching the checkpoint cannot replace - // the fixed fold boundary C. The earlier cases model an honestly lagging node. - for (application_clock, checkpoint_block, stop_block, resynced_head) in [ - (900, 901, 899, 899), - (900, 964, 930, 930), - (900, 964, 930, 964), - ] { - let db = temp_db("recovery-uncovered-checkpoint"); - let mut storage = - Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); - let identity = storage.deployment_identity().unwrap().unwrap(); - let inputs = if resynced_head > application_clock { - vec![StoredSafeInput { - sender: Address::repeat_byte(0x22), - payload: vec![1], - block_number: application_clock + 1, - }] - } else { - vec![] - }; - storage - .append_safe_inputs_with_timestamp( - resynced_head, - resynced_head, - &inputs, - SENDER_A, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, - ) - .unwrap(); - let checkpoint = Checkpoint { - app: ReplayForbiddenApp( - ApplicationProgress::try_new(ExecutedInputCount::new(1), application_clock) - .unwrap(), - ), - executed_safe_block: application_clock, - checkpoint_nonce: 1, - checkpoint_block, - }; - let dumps = tempfile::tempdir().unwrap(); - - let error = rebuild_from_checkpoint( - checkpoint, - &identity, - stop_block, - &mut storage, - dumps.path(), - ) - .expect_err("the RPC stopping block must cover the trusted checkpoint"); - - assert_eq!(error.exit_code(), EXIT_RESTART_TRANSIENT); - assert!(matches!( - error, - CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::Retry(ref failure))) - if matches!(failure.as_ref(), RecoveryFailure::PolicyRetry( - RecoveryRetryReason::CheckpointAheadOfStop { - checkpoint_block: found_checkpoint, - stop_block: found_stop, - }) if *found_checkpoint == checkpoint_block && *found_stop == stop_block) - )); - assert!(!storage.is_setup_complete().unwrap()); - assert!(matches!( - storage.history_state(), - Err(rusqlite::Error::QueryReturnedNoRows) - )); - assert!(storage.open_state().unwrap().is_none()); - assert!(storage.latest_snapshot().unwrap().is_none()); - assert_eq!(std::fs::read_dir(dumps.path()).unwrap().count(), 0); - assert_eq!(storage.current_safe_block().unwrap(), Some(resynced_head)); - assert_eq!( - storage.safe_input_end_exclusive().unwrap(), - inputs.len() as u64 - ); - } -} - -#[test] -fn recovery_publishes_at_checkpoint_or_later_including_genesis() { - let owner = Address::repeat_byte(0x77); - for (checkpoint_block, stop_block, expected_count, expected_balance) in - [(10, 10, 2, 120_u64), (10, 15, 3, 150), (0, 0, 0, 0)] - { - let db = temp_db("recovery-covered-checkpoint"); - let mut storage = - Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); - let identity = storage.deployment_identity().unwrap().unwrap(); - let config = WalletConfig::devnet(); - let deposit = |block, amount: u64| DirectInput { - sender: config.erc20_portal_address, - block_number: block, - payload: [ - config.supported_erc20_token.as_slice(), - owner.as_slice(), - U256::from(amount).to_be_bytes::<32>().as_slice(), - ] - .concat(), - }; - let mut app = WalletApp::new(config); - if checkpoint_block != 0 { - execute_direct_input(&mut app, &deposit(5, 100)).unwrap(); - } - let checkpoint = Checkpoint { - executed_safe_block: app.last_executed_safe_block(), - app, - checkpoint_nonce: u64::from(checkpoint_block != 0), - checkpoint_block, - }; - let resynced_head = stop_block + 5; - let inputs = [(5, 100), (7, 20), (12, 30), (16, 40)] - .into_iter() - .filter(|(block, _)| *block <= resynced_head) - .map(|(block, amount)| { - let input = deposit(block, amount); - StoredSafeInput { - sender: input.sender, - payload: input.payload, - block_number: block, - } - }) - .collect::>(); - storage - .append_safe_inputs_with_timestamp( - resynced_head, - resynced_head, - &inputs, - SENDER_A, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, - ) - .unwrap(); - let dumps = tempfile::tempdir().unwrap(); - - rebuild_from_checkpoint( - checkpoint, - &identity, - stop_block, - &mut storage, - dumps.path(), - ) - .unwrap(); - - assert!(storage.is_setup_complete().unwrap()); - let history = storage.history_state().unwrap(); - assert_eq!(history.base_safe_block, stop_block); - assert_eq!(history.base_executed_input_count, expected_count); - assert_eq!( - storage.open_state().unwrap().unwrap().safe_block, - stop_block - ); - let snapshot = storage.latest_snapshot().unwrap().unwrap(); - let app = WalletApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)).unwrap(); - assert_eq!(app.executed_input_count().get(), expected_count); - assert_eq!( - app.current_user_balance(owner), - U256::from(expected_balance) - ); - } -} diff --git a/sequencer/src/commands/setup/fill.rs b/sequencer/src/commands/setup/fill.rs index a141e8ed..7b308102 100644 --- a/sequencer/src/commands/setup/fill.rs +++ b/sequencer/src/commands/setup/fill.rs @@ -1,59 +1,50 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! File-first baseline creation for setup. The complete baseline, optional -//! recovery root, and setup-complete fact become visible in one transaction. +//! Step 7 of setup: write the baseline dump of the application at the end of +//! the stop block, then publish the era with it in one transaction. Setup +//! opens no root frame; `run`'s startup does. -use crate::commands::error::{CommandError, SetupRecoveryError}; +use std::path::{Path, PathBuf}; + +use sequencer_core::application::Application; +use sequencer_core::history::L2TxCount; +use sequencer_core::scheduler::SchedulerState; + +use crate::commands::error::CommandError; use crate::ingress::inclusion_lane::dump_info::{ self, CreateDumpDirError, create_dump_dir_with_info, }; -use sequencer_core::application::Application; +use crate::storage::{NewEra, Storage}; -pub(crate) fn register_genesis_baseline( - mut initial_app: A, - storage: &mut crate::storage::Storage, - dumps_dir: &std::path::Path, -) -> Result<(), CommandError> { - let count = initial_app.executed_input_count(); - if count.get() != 0 { - return Err(CommandError::AppBootstrap( - sequencer_core::application::AppError::Internal { - reason: format!( - "a genesis application must start at executed_input_count = 0, got {}", - count.get() - ), - }, - )); - } - let prefix = write_baseline_dump(&mut initial_app, 0, dumps_dir)?; - storage.complete_baseline_setup(&prefix, count, 0, 0, false)?; - Ok(()) -} - -/// The terminal fold has already applied every direct through C. Its dump -/// is a local resume baseline; it is not a canonical comparison checkpoint. -pub(crate) fn fill_recovery_state( - mut recovered_app: A, - resume_nonce: u64, +/// Publish the era: `app` has executed the whole rebuilt history, and +/// `scheduler` is the scheduler at the end of the stop block. Publishing +/// seals the rebuilt rows. +pub(crate) fn publish( + app: &mut A, + storage: &mut Storage, + dumps_dir: &Path, + base_count: L2TxCount, stop_block: u64, - storage: &mut crate::storage::Storage, - dumps_dir: &std::path::Path, -) -> Result<(), CommandError> { - if storage.is_setup_complete()? { - return Err(SetupRecoveryError::AlreadySetUp.into()); - } - let count = recovered_app.executed_input_count(); - let prefix = write_baseline_dump(&mut recovered_app, resume_nonce, dumps_dir)?; - storage.complete_baseline_setup(&prefix, count, stop_block, resume_nonce, true)?; - Ok(()) + scheduler: SchedulerState, +) -> Result { + let era = NewEra { + base: base_count, + rebuilt_end: app.l2_tx_count(), + stop_block, + first_queued_input: scheduler.first_queued_input, + anchor_batch_nonce: scheduler.next_batch_nonce, + }; + let prefix = write_baseline_dump(app, era.anchor_batch_nonce, dumps_dir)?; + storage.publish_era(&prefix, &era)?; + Ok(era) } fn write_baseline_dump( app: &mut A, next_batch_nonce: u64, - dumps_dir: &std::path::Path, -) -> Result { + dumps_dir: &Path, +) -> Result { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_err(std::io::Error::other)? @@ -79,105 +70,105 @@ mod tests { use super::*; use crate::ingress::inclusion_lane::{InclusionLane, InclusionLaneConfig}; use crate::runtime::shutdown::RuntimeScope; - use crate::storage::{LifecycleCommand, Storage}; + use crate::storage::{LifecycleCommand, StoredSafeInput}; use crate::test_support::{pin_test_deployment_identity, temp_db}; use crate::test_support::{shutdown_lane, wait_until}; use alloy_primitives::{Address, U256}; use app_core::application::{WalletApp, WalletConfig}; - use sequencer_core::application::{ - AppError, AppOutputs, ApplicationProgress, ValidationOutcome, - }; - use sequencer_core::history::ExecutedInputCount; - use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; + use sequencer_core::application::{Admission, AppError, AppOutputs, SchedulerLimits}; + use sequencer_core::history::HistoryClaim; + use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; - use std::path::Path; use std::time::Duration; + /// Counts the directs it executes and persists only that count. #[derive(Clone)] - struct CountedSweepTestApp(ApplicationProgress); + struct CountedSweepTestApp(L2TxCount); impl CountedSweepTestApp { - fn new(executed_input_count: u64) -> Self { - Self( - ApplicationProgress::try_new(ExecutedInputCount::new(executed_input_count), 0) - .expect("coherent progress"), - ) + fn new(l2_tx_count: u64) -> Self { + Self(L2TxCount::new(l2_tx_count)) } } impl Application for CountedSweepTestApp { - fn max_method_payload_bytes() -> usize { - 0 - } - - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result { - Ok(ValidationOutcome::Accept) + fn user_nonce(&self, _sender: Address) -> Result { + unreachable!("not used by setup-fill tests") } - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - _safe_block: u64, - ) -> Result { + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { unreachable!("not used by setup-fill tests") } - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { - self.0.advance(input.block_number); + fn apply_direct(&mut self, _direct: &Direct) -> Result { + self.0.advance(); Ok(Vec::new()) } - fn progress(&self) -> ApplicationProgress { + fn l2_tx_count(&self) -> L2TxCount { self.0 } + fn max_method_payload_bytes() -> usize { + 0 + } + + fn scheduler_limits() -> SchedulerLimits { + app_core::application::WALLET_SCHEDULER_LIMITS + } + + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + unreachable!("not used by setup-fill tests") + } + fn from_dump(prefix: &Path) -> Result { let bytes = std::fs::read(prefix.join("state"))?; - let bytes: [u8; 16] = bytes.try_into().map_err(|_| AppError::Internal { + let bytes: [u8; 8] = bytes.try_into().map_err(|_| AppError::Internal { reason: "invalid counted test dump".to_string(), })?; - let count = u64::from_le_bytes(bytes[..8].try_into().expect("eight-byte count")); - let safe_block = - u64::from_le_bytes(bytes[8..].try_into().expect("eight-byte safe block")); - Ok(Self( - ApplicationProgress::try_new(ExecutedInputCount::new(count), safe_block) - .expect("coherent progress"), - )) + Ok(Self::new(u64::from_le_bytes(bytes))) } fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { std::fs::create_dir(prefix)?; - let mut bytes = Vec::with_capacity(16); - bytes.extend_from_slice(&self.0.executed_input_count().get().to_le_bytes()); - bytes.extend_from_slice(&self.0.last_executed_safe_block().to_le_bytes()); - std::fs::write(prefix.join("state"), bytes)?; + std::fs::write(prefix.join("state"), self.0.get().to_le_bytes())?; Ok(()) } - fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { + fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } + + fn from_canonical_state( + _state_file: &Path, + _l2_tx_count: L2TxCount, + ) -> Result { + unreachable!("not used by setup-fill tests") + } } - #[test] - fn recovery_publishes_complete_baseline_without_application_padding() { - use crate::storage::test_helpers::default_protocol_timing; - use crate::storage::{FrontierMode, StoredSafeInput}; - let db = temp_db("complete-recovery-baseline"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps = tempfile::tempdir().expect("dumps"); - let submitter = Address::repeat_byte(0x99); - pin_test_deployment_identity(&mut storage, submitter); + const SUBMITTER: Address = Address::repeat_byte(0x99); + + /// A setup database holding a direct at block 100 and one at `later`, + /// synced through `safe_block`. + fn synced_setup( + name: &str, + later: u64, + safe_block: u64, + ) -> (crate::test_support::TestDb, Storage) { + let db = temp_db(name); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); storage .append_safe_inputs_with_timestamp( - 150, - 150, + safe_block, + crate::clock::unix_now_ms() / 1000, &[ StoredSafeInput { sender: Address::repeat_byte(0x22), @@ -187,356 +178,209 @@ mod tests { StoredSafeInput { sender: Address::repeat_byte(0x22), payload: vec![2], - block_number: 120, + block_number: later, }, ], - submitter, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, + SUBMITTER, ) - .expect("ingest prefix and later direct"); - fill_recovery_state( - CountedSweepTestApp::new(41), - 3, - 100, + .unwrap(); + (db, storage) + } + + /// The scheduler at stop block 100: it has consumed only input 0, and + /// the later direct (input 1) lies past the stop block, so the lane drains + /// from input 1. + fn queued_at_stop(next_batch_nonce: u64) -> SchedulerState { + SchedulerState { + l1_input_count: 1, + next_batch_nonce, + clock: 100, + first_queued_input: 1, + fridge_len: 0, + } + } + + #[test] + fn publish_records_the_era_and_its_baseline_and_opens_no_tip() { + let (_db, mut storage) = synced_setup("publish-era", 120, 150); + let dumps = tempfile::tempdir().unwrap(); + let mut app = CountedSweepTestApp::new(41); + + let era = publish( + &mut app, &mut storage, dumps.path(), + L2TxCount::new(41), + 100, + queued_at_stop(3), ) - .expect("complete baseline"); - assert!(storage.is_setup_complete().expect("completion")); - assert_eq!(storage.batch_tree_anchor().expect("anchor"), 3); - let history = storage.history_state().expect("history"); - assert_eq!(history.base_executed_input_count, 41); - assert_eq!(history.base_safe_block, 100); - let root = storage.open_state().expect("root").expect("open root"); - assert_eq!(root.safe_block, 100); + .unwrap(); + assert_eq!( - storage.next_executed_input_count().expect("head"), - ExecutedInputCount::new(41) + era, + NewEra { + base: L2TxCount::new(41), + rebuilt_end: L2TxCount::new(41), + stop_block: 100, + first_queued_input: 1, + anchor_batch_nonce: 3, + } ); + assert!(storage.is_setup_complete().unwrap()); + let history = storage.era().unwrap(); + assert_eq!(history.anchor_batch_nonce, 3); + assert_eq!(history.base, L2TxCount::new(41)); + assert_eq!(history.rebuilt_end, L2TxCount::new(41)); + assert_eq!(history.stop_block, 100); + assert_eq!(history.first_queued_input, 1); assert!( - storage - .finalized_dump() - .expect("accepted checkpoint") - .is_none() + storage.open_state().unwrap().is_none(), + "run opens the root" ); - let snapshot = storage - .latest_snapshot() - .expect("baseline") - .expect("baseline"); - assert_eq!(snapshot.executed_input_count, ExecutedInputCount::new(41)); + // The baseline is canonical at the stop block: the first checkpoint. + let checkpoint = storage.finalized_dump().unwrap().expect("era baseline"); + assert_eq!(checkpoint.inclusion_block, 100); + assert_eq!(checkpoint.snapshot.l2_tx_count, L2TxCount::new(41)); + assert_eq!(checkpoint.next_batch_nonce, 3); + let snapshot = storage.latest_snapshot().unwrap().expect("baseline"); let restored = - CountedSweepTestApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)) - .expect("restore baseline"); - assert_eq!( - restored.executed_input_count(), - snapshot.executed_input_count - ); - assert_eq!( - storage - .read(|tx| tx - .query_row("SELECT COUNT(*) FROM application_inputs", [], |row| row - .get::<_, i64>(0))) - .expect("application rows"), - 0 - ); - let err = fill_recovery_state( - CountedSweepTestApp::new(42), - 4, - 150, + CountedSweepTestApp::from_dump(&dump_info::app_prefix(&snapshot.prefix)).unwrap(); + assert_eq!(restored.l2_tx_count(), L2TxCount::new(41)); + + let err = publish( + &mut CountedSweepTestApp::new(41), &mut storage, dumps.path(), + L2TxCount::new(41), + 150, + queued_at_stop(4), ) - .expect_err("completed rebuild is one-shot"); + .expect_err("an era is published once"); assert!(matches!( err, - CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupRecovery( - SetupRecoveryError::AlreadySetUp - )) + CommandError::Lifecycle(crate::storage::LifecycleError::NotAdmissible { .. }) )); - assert_eq!( - storage.history_state().expect("unchanged baseline"), - history - ); + assert_eq!(storage.era().unwrap(), history); } + /// Run's root frame drains from the era's first queued input. #[test] - fn failed_baseline_registration_rolls_back_root_history_and_completion() { - let db = temp_db("baseline-registration-rollback"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps = tempfile::tempdir().expect("dumps"); - storage - .write(|tx| { - tx.execute_batch( - "CREATE TRIGGER fail_baseline BEFORE INSERT ON snapshots - BEGIN SELECT RAISE(ABORT, 'injected baseline failure'); END;", - ) - }) - .expect("inject failure"); - let err = fill_recovery_state( - CountedSweepTestApp::new(41), - 3, - 100, - &mut storage, - dumps.path(), - ) - .expect_err("registration must fail"); - assert!(err.to_string().contains("injected baseline failure")); - assert!(!storage.is_setup_complete().expect("completion absent")); - assert!(storage.open_state().expect("root absent").is_none()); - assert_eq!(storage.batch_tree_anchor().expect("unchanged anchor"), 0); - assert_eq!( - storage - .read( - |tx| tx.query_row("SELECT COUNT(*) FROM history_state", [], |row| row - .get::<_, i64>(0)) - ) - .expect("no history"), - 0 - ); - assert_eq!( - std::fs::read_dir(dumps.path()) - .expect("orphan dump") - .count(), - 1 - ); - storage - .write(|tx| tx.execute_batch("DROP TRIGGER fail_baseline")) - .expect("remove failure"); - fill_recovery_state( - CountedSweepTestApp::new(42), - 4, - 150, + fn the_root_drains_from_the_first_queued_input() { + let (_db, mut storage) = synced_setup("root-drains-queued", 1400, 1500); + let dumps = tempfile::tempdir().unwrap(); + publish( + &mut CountedSweepTestApp::new(41), &mut storage, dumps.path(), + L2TxCount::new(41), + 100, + queued_at_stop(3), ) - .expect("fresh retry after atomic rollback"); - assert_eq!( - storage - .history_state() - .expect("history") - .base_executed_input_count, - 42 - ); - let root = storage.open_state().expect("open state").expect("root tip"); - assert_eq!( - storage.batch_nonce(root.batch_index).expect("root nonce"), - 4 - ); - } - - #[test] - fn plain_setup_refuses_a_nonzero_genesis_application_boundary() { - let db = temp_db("nonzero-genesis"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup) - .expect("initialize setup"); - let dumps = tempfile::tempdir().expect("dumps"); - assert!( - register_genesis_baseline(CountedSweepTestApp::new(1), &mut storage, dumps.path()) - .is_err() - ); - assert!(!storage.is_setup_complete().expect("completion absent")); - assert!( - storage - .latest_snapshot() - .expect("snapshot absent") - .is_none() - ); - assert_eq!( - std::fs::read_dir(dumps.path()) - .expect("no artifacts") - .count(), - 0 - ); - } + .unwrap(); + assert!(storage.open_batch_if_none().unwrap(), "run opens the root"); + assert_eq!(storage.open_state().unwrap().unwrap().safe_block(), 1500); - #[test] - fn root_invalidation_keeps_baseline_floor_and_replays_only_post_baseline_directs() { - use crate::storage::test_helpers::default_protocol_timing; - use crate::storage::{FrontierMode, StoredSafeInput}; - use sequencer_core::history::HistoryClaim; - let db = temp_db("baseline-survives-root-invalidation"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps = tempfile::tempdir().expect("dumps"); - let submitter = Address::repeat_byte(0x99); - pin_test_deployment_identity(&mut storage, submitter); - storage - .append_safe_inputs_with_timestamp( - 1500, - 1500, - &[ - StoredSafeInput { - sender: Address::repeat_byte(0x22), - payload: vec![1], - block_number: 100, + let drained = |storage: &mut Storage| { + let era_id = storage.era().unwrap().id; + let page = storage + .canonical_history_page( + HistoryClaim { + era_id, + next_input: L2TxCount::new(41), }, - StoredSafeInput { - sender: Address::repeat_byte(0x22), - payload: vec![2], - block_number: 1400, - }, - ], - submitter, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, - ) - .expect("ingest inputs"); - fill_recovery_state( - CountedSweepTestApp::new(41), - 3, - 100, - &mut storage, - dumps.path(), - ) - .expect("complete baseline"); - let history = storage.history_state().expect("history"); - assert_eq!( - storage.recover_aging_tip(1200).expect("recover root"), - vec![0] - ); - let current = storage.history_state().expect("current history"); - assert_eq!(current.base_safe_block, 100); - assert_eq!(current.base_executed_input_count, 41); - assert_eq!( - current.version.recovery_generation.get(), - history.version.recovery_generation.get() + 1 - ); - let snapshot = storage - .latest_snapshot() - .expect("baseline") - .expect("baseline survives"); - let mut app = CountedSweepTestApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)) - .expect("restore"); - let page = storage - .canonical_history_page( - HistoryClaim { - version: current.version, - next_input: snapshot.executed_input_count, - }, - 16, - ) - .expect("replacement history"); - assert_eq!(page.rows.len(), 1); - assert_eq!(page.rows[0].offset, ExecutedInputCount::new(41)); - match &page.rows[0].context { - crate::storage::L2TxContext::DirectInput { tx, .. } => { - assert_eq!(tx.block_number, 1400); - sequencer_core::application::execute_direct_input(&mut app, tx) - .expect("replay direct"); - } - _ => panic!("expected post-baseline direct"), - } - assert_eq!(app.executed_input_count(), ExecutedInputCount::new(42)); - assert!( - storage - .recover_aging_tip(1200) - .expect("fresh replacement") - .is_empty() - ); - assert_eq!( - storage.next_executed_input_count().expect("head"), - ExecutedInputCount::new(42) - ); + 16, + ) + .unwrap(); + page.rows + .iter() + .map(|row| match &row.context { + crate::storage::L2TxContext::Direct { tx, .. } => { + (row.offset.get(), tx.input_index) + } + other => panic!("expected a drained direct, got {other:?}"), + }) + .collect::>() + }; + assert_eq!(drained(&mut storage), vec![(41, 1)]); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn recovery_then_lane_credits_post_c_deposit_exactly_once() { - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("recovery-credited-once"); + async fn run_after_setup_credits_a_queued_deposit_exactly_once() { + let db = temp_db("setup-then-lane-credits-once"); let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let timing = default_protocol_timing(); - // The recovered checkpoint state S' is the real `WalletApp` (only it - // exposes a queryable balance). Its config round-trips through the SSZ - // dump, so the lane reloads a devnet-configured app. + // The real `WalletApp`: only it exposes a queryable balance. Its config + // round-trips through the SSZ dump, so the lane reloads it. let cfg = WalletConfig::devnet(); let portal = cfg.erc20_portal_address; let token = cfg.supported_erc20_token; - let recovered = WalletApp::new(cfg); - let submitter = Address::repeat_byte(0x99); // ingest classifier (not the portal) - let pre_c_direct = Address::repeat_byte(0x22); // non-portal => decode None => inert - let nested_sender = Address::repeat_byte(0x77); // fresh account, zero balance in S' + let mut base = WalletApp::new(cfg); + let other = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x77); let deposit_value = U256::from(2_000_000_u64); + let mut deposit_payload = Vec::with_capacity(20 + 20 + 32); + deposit_payload.extend_from_slice(token.as_slice()); + deposit_payload.extend_from_slice(recipient.as_slice()); + deposit_payload.extend_from_slice(deposit_value.to_be_bytes::<32>().as_slice()); - // C = 100; the resync reached H1 = 150. Three `<= C` directs (already - // folded into S'), plus ONE portal USDC deposit at block 120 in the - // (C, H1] window. Deposit payload mirrors `encode_erc20_deposit_payload`: - // token || nested_sender || value_be32. - let deposit_payload = { - let mut p = Vec::with_capacity(20 + 20 + 32); - p.extend_from_slice(token.as_slice()); - p.extend_from_slice(nested_sender.as_slice()); - p.extend_from_slice(deposit_value.to_be_bytes::<32>().as_slice()); - p - }; - let inputs = vec![ - StoredSafeInput { - sender: pre_c_direct, - payload: vec![0x01], - block_number: 10, - }, - StoredSafeInput { - sender: pre_c_direct, - payload: vec![0x02], - block_number: 20, - }, - StoredSafeInput { - sender: pre_c_direct, - payload: vec![0x03], - block_number: 30, - }, - StoredSafeInput { + // Three directs the base already executed, then a deposit at block + // 120, past stop block 100: the era's first queued input. + let inputs: Vec = [10, 20, 30] + .into_iter() + .map(|block_number| StoredSafeInput { + sender: other, + payload: vec![1], + block_number, + }) + .chain([StoredSafeInput { sender: portal, payload: deposit_payload, block_number: 120, - }, - ]; + }]) + .collect(); + for _ in 0..3 { + base.apply_direct(&Direct { + input_index: 0, + sender: other, + block: 10, + timestamp: 0, + payload: vec![1], + }) + .unwrap(); + } { let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - pin_test_deployment_identity(&mut storage, submitter); + Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0x99)); storage - .append_safe_inputs(150, &inputs, submitter, &timing) - .expect("sync to H1 = 150"); - // Fill at N' = 3, C = 100: drains only the three `<= C` directs into - // the recovery root frame; the deposit (index 3) stays undrained. - fill_recovery_state(recovered, 3, 100, &mut storage, dumps_dir.path()) - .expect("recovery fill"); - - // Preconditions (the (C, H1] cap — covered by the cap tests; pinned - // here so a regression that drained the deposit fails early). - assert_eq!( - storage.next_undrained_safe_input_index().expect("cursor"), - 3, - "the (C, H1] deposit must be left UNDRAINED at index 3" - ); - let finalized = storage - .latest_snapshot() - .expect("read") - .expect("baseline exists"); - let s_prime = WalletApp::from_dump(&dump_info::app_prefix(&finalized.dump.prefix)) - .expect("load S'"); - assert_eq!( - s_prime.current_user_balance(nested_sender), - U256::ZERO, - "S' itself must NOT have credited the deposit" - ); + .append_safe_inputs(150, &inputs, Address::repeat_byte(0x99)) + .unwrap(); + publish( + &mut base, + &mut storage, + dumps_dir.path(), + L2TxCount::new(3), + 100, + SchedulerState { + l1_input_count: 3, + next_batch_nonce: 0, + clock: 30, + first_queued_input: 3, + fridge_len: 0, + }, + ) + .unwrap(); + let baseline = storage.latest_snapshot().unwrap().expect("baseline"); + let at_stop = WalletApp::from_dump(&dump_info::app_prefix(&baseline.prefix)).unwrap(); + assert_eq!(at_stop.current_user_balance(recipient), U256::ZERO); + // Run's startup opens the root frame at its own safe head. + assert!(storage.open_batch_if_none().unwrap()); } - // Drive the REAL run-side inclusion lane against the recovered DB. The - // frontier advance `C -> H1` is already staged (l1_safe_head = 150, the - // recovery tip frame's safe block = 100), so the lane's first iteration - // leads the deposit at index 3 and executes it once. `batch_submitter` - // differs from the portal, so the deposit is not skipped as an own-batch - // input; the short `max_batch_open` forces a batch-close snapshot. let storage = Storage::open(db.path.as_str()).expect("reopen for lane"); let config = InclusionLaneConfig { dumps_dir: dumps_dir.path().to_path_buf(), + snapshot_leases: crate::storage::SnapshotLeases::default(), max_user_ops_per_chunk: 16, safe_input_buffer_capacity: 16, max_batch_open: Duration::from_millis(10), @@ -547,48 +391,32 @@ mod tests { let (_tx, handle) = InclusionLane::::start(128, shutdown.clone(), storage, config); - // Observe via a SECOND storage handle (WAL); the lane owns its own. The - // executed-deposit state is externalized only through the pending dump - // the lane writes at batch close. Wait for it to reflect the credit. + // The lane's state shows only through the dump it writes at batch close. let credited = wait_until(Duration::from_secs(5), || { let mut s = Storage::open(db.path.as_str()).expect("open observer"); match s.latest_snapshot().expect("read pending") { Some(p) => { - WalletApp::from_dump(&dump_info::app_prefix(&p.dump.prefix)) + WalletApp::from_dump(&dump_info::app_prefix(&p.prefix)) .expect("load lane snapshot") - .current_user_balance(nested_sender) + .current_user_balance(recipient) == deposit_value } None => false, } }) .await; - assert!( - credited, - "the lane must lead + execute the (C, H1] deposit, crediting it once" - ); - - // Exactly once: the drain cursor advanced to 4 (one past the deposit), so - // a restart would not re-lead it; and the balance is the deposit value - // (== once; 0 would be lost, 2x would be double-credited). + assert!(credited, "the root frame executes the queued deposit"); { let mut s = Storage::open(db.path.as_str()).expect("open observer"); + assert_eq!(s.next_undrained_safe_input_index().unwrap(), 4); + let dump = s.latest_snapshot().unwrap().expect("a post-deposit dump"); + let app = WalletApp::from_dump(&dump_info::app_prefix(&dump.prefix)).unwrap(); assert_eq!( - s.next_undrained_safe_input_index().expect("cursor"), - 4, - "the deposit's drain cursor must advance exactly one past it" - ); - let dump = s - .latest_snapshot() - .expect("pending") - .expect("a post-deposit pending dump"); - let app = - WalletApp::from_dump(&dump_info::app_prefix(&dump.dump.prefix)).expect("load"); - assert_eq!( - app.current_user_balance(nested_sender), + app.current_user_balance(recipient), deposit_value, - "credited EXACTLY once" + "credited once" ); + assert_eq!(app.l2_tx_count(), L2TxCount::new(4)); } shutdown_lane(&shutdown, handle).await; diff --git a/sequencer/src/commands/setup/mod.rs b/sequencer/src/commands/setup/mod.rs index 1af58695..871bfe1c 100644 --- a/sequencer/src/commands/setup/mod.rs +++ b/sequencer/src/commands/setup/mod.rs @@ -1,87 +1,483 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! The `setup` command: pin the deployment identity and fee source, sync L1 to -//! the safe head, and register the genesis baseline (or the `--recovery` rebuild), -//! committing `setup_complete` last as its single completion point. Storage phases -//! are re-entry safe. Plain `setup` reads L1 only and never signs. +//! The `setup` command: start a new era in an empty data directory from a +//! base, genesis or a stored canonical machine. It reads the base, flushes the +//! submitter's wallet to pin the stop block, syncs L1 through it, checks for +//! an escape and checks the base's export, rebuilds and executes the history, +//! and publishes the era (`docs/recovery/cockroach.md`, "What setup does"). +//! Every L1 check that can fail retryably runs before anything is written, so +//! a retry finds the directory empty. + +use std::path::{Path, PathBuf}; use alloy_primitives::Address; use sequencer_core::application::{AppError, Application}; -use sequencer_core::scheduler::{FoldInput, SchedulerConfig, fold_replay}; +use sequencer_core::history::L2TxCount; +use sequencer_core::scheduler::SchedulerConfig; +pub(crate) mod base; #[cfg(test)] -mod checkpoint_tests; +mod chain_tests; pub(crate) mod fill; +pub(crate) mod rebuild; -use super::{ensure_deployment_identity, validate_rpc_chain_id}; +use super::validate_rpc_chain_id; use crate::commands::config::{FeeOracleMode, SetupConfig}; use crate::commands::error::{ - BootstrapError, CommandError, IdentityError, SetupRecoveryError, SetupRefuse, WorkerExit, + BootstrapError, CommandError, IdentityError, SetupError, SetupRefuse, }; -use crate::ingress::inclusion_lane::dump_info; +use crate::l1::flusher::MempoolFlusher; use crate::l1::reader::{InputReader, InputReaderConfig, InputReaderError, sample_safe_block}; -use crate::recovery::{ - MempoolFlusher, RecoveryError, RecoveryRetryReason, assert_resync_caught_up, -}; -use crate::storage::{self, DeploymentIdentity, FeeOracleIdentity}; +use crate::runtime::process_lock::ProcessLock; +use crate::storage::{DeploymentIdentity, FeeOracleIdentity, LifecycleCommand, NewEra, Storage}; +use base::{Base, ReadMachineExport}; + +/// The process lock's anchor file. It survives every command, so a directory +/// holding only it is still empty: an exit 20 before any write leaves it. +const LOCK_FILE_NAME: &str = "sequencer.lock"; pub async fn setup(config: SetupConfig, genesis_app: F) -> Result<(), CommandError> where A: Application + 'static, F: FnOnce() -> Result, { - // Cross-field config validation (recovery vs the recovery-only args). A - // misconfig is operator error — terminal, before any filesystem touch. - config - .validate() - .map_err(|message| SetupRecoveryError::InvalidConfig { message })?; + let summary = run_setup(config, genesis_app, base::read_stored_machine()).await?; + println!("{summary}"); + Ok(()) +} +/// [`setup`] with the machine reader as a parameter, returning what it +/// prints. Tests supply the base's export and state bytes through it. +pub(crate) async fn run_setup( + config: SetupConfig, + genesis_app: F, + read_export: ReadMachineExport, +) -> Result +where + A: Application + 'static, + F: FnOnce() -> Result, +{ + if config.dry_run { + return dry_run(&config, genesis_app, read_export).await; + } + let key = config.resolve_private_key()?; + + // Refuse a directory that holds anything before touching it, then once + // more under the lock, which another setup may have raced to. + require_empty_data_dir(&config.data_dir)?; std::fs::create_dir_all(&config.data_dir) .map_err(|err| super::io_error_at("data dir", &config.data_dir, err))?; - // Exclusive process ownership before any read or mutation: setup rewrites - // deployment state and must never run beside a live sequencer (or a - // second setup) on the same data dir. - let process_lock = crate::runtime::process_lock::ProcessLock::acquire(&config.data_dir)?; + let process_lock = ProcessLock::acquire(&config.data_dir)?; + require_empty_data_dir(&config.data_dir)?; let db_path = config.db_path(); - config.timing.protocol_timing()?; - let command = if config.recovery { - storage::LifecycleCommand::Rebuild - } else { - storage::LifecycleCommand::Setup - }; - if let SetupAdmission::AlreadyComplete = - admit_setup_lifecycle(&db_path, command, config.recovery)? + let result = setup_locked(&config, key, genesis_app, read_export, process_lock).await; + // A failure before the first write leaves no database to record into, + // and creating one would make the directory non-empty. + if let Err(error) = &result + && Path::new(&db_path).exists() { - tracing::info!(data_dir = %config.data_dir, "setup already complete — nothing to do"); - return Ok(()); + crate::commands::record_terminal_error_best_effort( + &db_path, + LifecycleCommand::Setup, + error, + ); } - - let result = setup_admitted(config, genesis_app, process_lock.clone()).await; - settle_setup_lifecycle(&db_path, command, &result)?; result } -async fn setup_admitted( - config: SetupConfig, +async fn setup_locked( + config: &SetupConfig, + key: crate::l1::SubmitterKey, genesis_app: F, - process_lock: crate::runtime::process_lock::ProcessLock, -) -> Result<(), CommandError> + read_export: ReadMachineExport, + process_lock: ProcessLock, +) -> Result where A: Application + 'static, F: FnOnce() -> Result, { let db_path = config.db_path(); - let timing = config.timing.protocol_timing()?; - // Do not create auxiliary data-dir entries before durable lifecycle - // admission. The lock file itself is the unavoidable ownership anchor. - let dumps_dir = std::path::Path::new(&config.data_dir).join("dumps"); + // ── Before anything is written: every L1 check that can fail retryably. + let mut input_reader = discover(config, &db_path, process_lock.clone()).await?; + let fee_oracle = prepare_fee_oracle(config).await?; + let identity = DeploymentIdentity { + chain_id: config.chain_id, + app_address: config.app_address, + input_box_address: input_reader.input_box_address(), + app_deployment_block: input_reader.app_deployment_block(), + batch_submitter_address: config.batch_submitter_address, + max_direct_payload_bytes: A::scheduler_limits().max_direct_payload_bytes, + max_wait_blocks: A::scheduler_limits().max_wait_blocks, + fee_oracle: fee_oracle.identity(), + }; + // The flush settles the pinned account, so the key must sign for it. + let key = super::verify_submitter_key(key, &identity)?; + let providers = crate::l1::provider::create_verified_keyed_providers( + &config.rpc.eth_rpc_url, + key.expose_secret(), + config.chain_id, + config.rpc.allow_insecure_rpc, + ) + .await?; + let submitter = identity.batch_submitter_address; + let scheduler_config = SchedulerConfig::for_application::(submitter); + + // ── Step 1: the base. Its state bytes live in a scratch directory outside + // the data directory, so a setup killed while it loads a large machine + // leaves the data directory empty for the retry. + let scratch = ScratchDir::create(&std::env::temp_dir())?; + let base = base::load_base( + config.base.as_deref(), + genesis_app, + read_export, + scheduler_config, + scratch.path(), + ) + .await?; + tracing::info!( + base = ?config.base, + l1_input_count = base.export.l1_input_count, + l2_tx_count = base.export.l2_tx_count, + "setup read its base" + ); + + // ── Writes begin: from here a failure leaves a directory to discard. + let mut storage = Storage::initialize_for_command(&db_path, LifecycleCommand::Setup)?; + storage.pin_deployment_identity(identity)?; + match fee_oracle { + PreparedFeeOracle::Fixed { log_gas_price } => storage.set_log_gas_price(log_gas_price)?, + PreparedFeeOracle::Uniswap { + provider, token, .. + } => { + crate::l1::fee_oracle::persist_first_price( + db_path.clone(), + provider, + token, + process_lock.clone(), + ) + .await? + } + } + + // ── Step 2: flush, then pin the stop block at a safe block after it. + let watermark = storage.wallet_nonce_watermark()?; + let stop_block = MempoolFlusher::flush_to_safe( + providers, + submitter, + config.timing.seconds_per_block, + db_path.clone(), + watermark, + ) + .await?; + tracing::info!(stop_block, "setup flushed the submitter's wallet"); + + // ── Step 3: sync through the stop block. Before the era exists there + // is no stop block or anchor to judge batches by, so storage leaves the + // acceptance frontier empty; `run`'s first sync populates it. + input_reader + .sync_to_current_safe_head() + .await + .map_err(BootstrapError::SetupInputReader)?; + let synced_safe_block = storage + .current_safe_block()? + .expect("a completed sync persists its safe head"); + let wallet = read_submitter_wallet_nonces( + &config.rpc.eth_rpc_url, + submitter, + synced_safe_block, + config.rpc.allow_insecure_rpc, + ) + .await?; + let summary = finish_setup( + &mut storage, + base, + scheduler_config, + Synced { + stop_block, + synced_safe_block, + wallet, + }, + Path::new(&config.data_dir), + )?; + tracing::info!( + data_dir = %config.data_dir, + chain_id = identity.chain_id, + app_address = %identity.app_address, + input_box_address = %identity.input_box_address, + app_deployment_block = identity.app_deployment_block, + batch_submitter_address = %identity.batch_submitter_address, + "setup complete" + ); + drop(scratch); + Ok(summary) +} + +/// Where step 3 left setup: the stop block the flush pinned, the safe head +/// the sync reached, and the submitter's `(pending, safe)` wallet nonces read +/// at that head. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Synced { + pub(crate) stop_block: u64, + pub(crate) synced_safe_block: u64, + pub(crate) wallet: (u64, u64), +} + +/// The rest of step 3 and steps 4 to 7, after the sync: the sync must reach +/// the stop block and the escape check must pass before the history is +/// rebuilt, executed and published with its dumps under `data_dir`. +pub(crate) fn finish_setup( + storage: &mut Storage, + base: Base, + scheduler_config: SchedulerConfig, + synced: Synced, + data_dir: &Path, +) -> Result { + let Synced { + stop_block, + synced_safe_block, + wallet, + } = synced; + if synced_safe_block < stop_block { + return Err(SetupError::SyncBehindStopBlock { + synced_safe_block, + stop_block, + } + .into()); + } + check_escape( + storage, + scheduler_config.sequencer_address, + stop_block, + wallet, + )?; + + let dumps_dir = data_dir.join("dumps"); std::fs::create_dir_all(&dumps_dir) .map_err(|err| super::io_error_at("dumps dir", &dumps_dir, err))?; + rebuild_and_publish( + storage, + base, + scheduler_config, + stop_block, + &dumps_dir, + wallet, + ) +} + +/// Steps 4 to 7 over an input table synced through `stop_block`: check the +/// export, rebuild and execute the history, and publish the era. +pub(crate) fn rebuild_and_publish( + storage: &mut Storage, + base: Base, + scheduler_config: SchedulerConfig, + stop_block: u64, + dumps_dir: &Path, + (pending_wallet_nonce, safe_wallet_nonce): (u64, u64), +) -> Result { + let Base { export, mut app } = base; + let inputs_through_stop = storage.safe_input_count_through_block(stop_block)?; + rebuild::check_export(storage, &export, inputs_through_stop, stop_block)?; + let submitter_inputs_past_base = rebuild::count_submitter_inputs( + storage, + scheduler_config.sequencer_address, + export.l1_input_count, + inputs_through_stop, + )?; + let scheduler = rebuild::rebuild_history( + storage, + &mut app, + scheduler_config, + &export, + inputs_through_stop, + stop_block, + )?; + // The rebuild fed exactly the inputs through the stop block, so the era's + // first queued input and anchor are the scheduler's there. + assert_eq!( + scheduler.l1_input_count, inputs_through_stop, + "the rebuilt scheduler consumed every input through the stop block, and no other" + ); + let era = fill::publish( + &mut app, + storage, + dumps_dir, + L2TxCount::new(export.l2_tx_count), + stop_block, + scheduler, + )?; + Ok(SetupSummary { + base_l1_input_count: export.l1_input_count, + base_l2_tx_count: export.l2_tx_count, + submitter_inputs_past_base, + unsettled_wallet_txs: pending_wallet_nonce.saturating_sub(safe_wallet_nonce), + inputs_to_replay: inputs_through_stop - export.l1_input_count, + through_block: stop_block, + era: Some(era), + }) +} + +/// The dry run: read the base, sync into a scratch directory with its own +/// lock, and check the export there. No flush and no key; the data directory +/// is never touched. The escape facts are findings here, not refusals. +async fn dry_run( + config: &SetupConfig, + genesis_app: F, + read_export: ReadMachineExport, +) -> Result +where + A: Application + 'static, + F: FnOnce() -> Result, +{ + let scratch = ScratchDir::create(&std::env::temp_dir())?; + let scratch_dir = scratch.path().to_string_lossy().into_owned(); + let process_lock = ProcessLock::acquire(&scratch_dir)?; + let db_path = crate::commands::config::db_path_in(&scratch_dir); - // ── L1 discovery (required) ────────────────────────────── + let mut input_reader = discover(config, &db_path, process_lock).await?; + let fee_oracle = resolve_fee_oracle(config)?; + let identity = DeploymentIdentity { + chain_id: config.chain_id, + app_address: config.app_address, + input_box_address: input_reader.input_box_address(), + app_deployment_block: input_reader.app_deployment_block(), + batch_submitter_address: config.batch_submitter_address, + max_direct_payload_bytes: A::scheduler_limits().max_direct_payload_bytes, + max_wait_blocks: A::scheduler_limits().max_wait_blocks, + fee_oracle: fee_oracle_identity(&fee_oracle), + }; + let submitter = identity.batch_submitter_address; + let scheduler_config = SchedulerConfig::for_application::(submitter); + let base = base::load_base( + config.base.as_deref(), + genesis_app, + read_export, + scheduler_config, + scratch.path(), + ) + .await?; + + let mut storage = Storage::initialize_for_command(&db_path, LifecycleCommand::Setup)?; + storage.pin_deployment_identity(identity)?; + input_reader + .sync_to_current_safe_head() + .await + .map_err(BootstrapError::SetupInputReader)?; + let safe_block = storage + .current_safe_block()? + .expect("a completed sync persists its safe head"); + let (pending, safe) = read_submitter_wallet_nonces( + &config.rpc.eth_rpc_url, + submitter, + safe_block, + config.rpc.allow_insecure_rpc, + ) + .await?; + let synced_inputs = storage.safe_input_count_through_block(safe_block)?; + rebuild::check_export(&mut storage, &base.export, synced_inputs, safe_block)?; + // The scheduler's own resume checks are part of the export check. Feeding + // never moves the clock past the block of the input that names it, so a + // resumed clock past the safe head is one setup would refuse too. + let resumed = rebuild::resume_from_export(&mut storage, scheduler_config, &base.export)?; + rebuild::check_clock(&resumed.state(), safe_block)?; + let submitter_inputs_past_base = rebuild::count_submitter_inputs( + &mut storage, + submitter, + base.export.l1_input_count, + synced_inputs, + )?; + drop(storage); + drop(scratch); + Ok(SetupSummary { + base_l1_input_count: base.export.l1_input_count, + base_l2_tx_count: base.export.l2_tx_count, + submitter_inputs_past_base, + unsettled_wallet_txs: pending.saturating_sub(safe), + inputs_to_replay: synced_inputs - base.export.l1_input_count, + through_block: safe_block, + era: None, + }) +} + +/// What setup and its dry run report. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SetupSummary { + pub(crate) base_l1_input_count: u64, + pub(crate) base_l2_tx_count: u64, + /// The submitter's inputs past the base: the old instance's batches. + pub(crate) submitter_inputs_past_base: u64, + /// The submitter's transactions not yet safe. + pub(crate) unsettled_wallet_txs: u64, + /// How many inputs past the base the rebuild feeds the scheduler. + pub(crate) inputs_to_replay: u64, + /// The stop block; for a dry run, the safe head it synced to. + pub(crate) through_block: u64, + /// The published era; `None` for a dry run. + pub(crate) era: Option, +} + +impl SetupSummary { + /// Recovery is needed exactly when L1 shows the old submitter's activity + /// past the base; otherwise the rebuild only moves directs. + fn needs_recovery(&self) -> bool { + self.submitter_inputs_past_base > 0 || self.unsettled_wallet_txs > 0 + } +} + +impl std::fmt::Display for SetupSummary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "base: {} L1 inputs, L2 tx count {}", + self.base_l1_input_count, self.base_l2_tx_count + )?; + writeln!( + f, + "submitter inputs past the base: {}", + self.submitter_inputs_past_base + )?; + writeln!( + f, + "unsettled submitter transactions: {}", + self.unsettled_wallet_txs + )?; + writeln!( + f, + "inputs the rebuild replays: {} (through block {})", + self.inputs_to_replay, self.through_block + )?; + if self.needs_recovery() { + write!( + f, + "the rebuild re-judges the old instance's batches past the base" + )?; + } else { + write!( + f, + "no submitter activity past the base: the rebuild only moves directs" + )?; + } + if let Some(era) = &self.era { + write!( + f, + "\nstop block: {}\nL2 tx count at its end: {}", + era.stop_block, + era.rebuilt_end.get() + )?; + } + Ok(()) + } +} + +/// Setup's L1 discovery, which writes nothing: the chain id, then a safe head +/// sampled before the deployment block is read, then that block safe. +async fn discover( + config: &SetupConfig, + db_path: &str, + process_lock: ProcessLock, +) -> Result { // The chain check comes first, so a wrong-chain endpoint is reported as // such rather than as a missing contract. validate_rpc_chain_id( @@ -90,7 +486,6 @@ where config.rpc.allow_insecure_rpc, ) .await?; - let input_reader_config = InputReaderConfig { rpc_url: config.rpc.eth_rpc_url.clone(), allow_insecure_rpc: config.rpc.allow_insecure_rpc, @@ -105,55 +500,79 @@ where let discovered = async { let safe_block = sample_safe_block(&config.rpc.eth_rpc_url, config.rpc.allow_insecure_rpc).await?; - let reader = InputReader::new( - db_path.clone(), - input_reader_config, - config.batch_submitter_address, - timing, - process_lock.clone(), - ) - .await?; + let reader = + InputReader::new(db_path.to_owned(), input_reader_config, process_lock).await?; Ok::<_, InputReaderError>((safe_block, reader)) } .await; - let (safe_before_discovery, mut input_reader) = match discovered { + let (safe_before_discovery, input_reader) = match discovered { Ok(discovered) => discovered, Err(InputReaderError::Provider(e)) => { - // First boot needs L1 to discover + pin identity. Retryable: the - // operator brings L1 up and re-runs `setup`. + // Retryable: the operator brings L1 up and re-runs `setup`. tracing::warn!(error = %e, "L1 unreachable during setup — cannot discover identity"); return Err(IdentityError::FirstBootRequiresL1.into()); } Err(source) => return Err(BootstrapError::SetupInputReader(source).into()), }; require_deployment_safe(input_reader.app_deployment_block(), safe_before_discovery)?; + Ok(input_reader) +} + +/// The fee source, resolved and, for Uniswap, connected and validated, and a +/// fixed price held to the fee floor, before anything is written. `run` later +/// reads only the pinned tuple. +enum PreparedFeeOracle { + Fixed { + log_gas_price: i16, + }, + Uniswap { + identity: FeeOracleIdentity, + provider: alloy::providers::DynProvider, + token: crate::l1::fee_oracle::UniswapV3PriceSource, + }, +} - // Resolve and validate the complete fee source while setup still owns all - // address-bearing configuration. `run` later reads only this pinned tuple. - // Both modes write the first `log_gas_price` here so `run` can always assume - // a real price exists (Uniswap Tip never samples the migration default). - let fee_oracle = config +impl PreparedFeeOracle { + fn identity(&self) -> FeeOracleIdentity { + match self { + Self::Fixed { log_gas_price } => FeeOracleIdentity::Fixed { + log_gas_price: *log_gas_price, + }, + Self::Uniswap { identity, .. } => *identity, + } + } +} + +fn resolve_fee_oracle(config: &SetupConfig) -> Result { + config .fee_oracle .resolve(config.chain_id) - .map_err(|message| BootstrapError::FeeOracleMisconfig { message })?; - enum PreparedFeeOracle { - Fixed { - log_gas_price: i16, + .map_err(|message| BootstrapError::FeeOracleMisconfig { message }.into()) +} + +fn fee_oracle_identity(mode: &FeeOracleMode) -> FeeOracleIdentity { + match mode { + FeeOracleMode::Fixed { log_gas_price } => FeeOracleIdentity::Fixed { + log_gas_price: *log_gas_price, }, - Uniswap { - identity: FeeOracleIdentity, - provider: alloy::providers::DynProvider, - token: crate::l1::fee_oracle::UniswapV3PriceSource, + FeeOracleMode::Uniswap(uniswap) => FeeOracleIdentity::Uniswap { + weth: uniswap.weth, + fee_token: uniswap.fee_token, + pool: uniswap.pool, + twap_window_secs: uniswap.twap_window_secs, }, } - let prepared_fee_oracle = match fee_oracle { +} + +async fn prepare_fee_oracle(config: &SetupConfig) -> Result { + let mode = resolve_fee_oracle(config)?; + let identity = fee_oracle_identity(&mode); + Ok(match mode { FeeOracleMode::Fixed { log_gas_price } => { - check_fixed_log_gas_price(&db_path, log_gas_price)?; + check_fixed_log_gas_price(log_gas_price)?; PreparedFeeOracle::Fixed { log_gas_price } } FeeOracleMode::Uniswap(uniswap) => { - // Setup requires L1: transient and misconfig both abort before - // anything pins. let (provider, token) = crate::l1::fee_oracle::connect_uniswap( &config.rpc.eth_rpc_url, config.rpc.allow_insecure_rpc, @@ -161,178 +580,19 @@ where ) .await?; PreparedFeeOracle::Uniswap { - identity: FeeOracleIdentity::Uniswap { - weth: uniswap.weth, - fee_token: uniswap.fee_token, - pool: uniswap.pool, - twap_window_secs: uniswap.twap_window_secs, - }, + identity, provider, token, } } - }; - let pinned_fee_oracle = match &prepared_fee_oracle { - PreparedFeeOracle::Fixed { log_gas_price } => FeeOracleIdentity::Fixed { - log_gas_price: *log_gas_price, - }, - PreparedFeeOracle::Uniswap { identity, .. } => *identity, - }; - - // ── Pin identity ───────────────────────────────────────── - // INVARIANT: identity is pinned (this step) BEFORE the initial sync - // (next step). The sync is the first writer of `safe_inputs` / - // `l1_safe_head`, which `has_persisted_deployment_state` keys on. Pinning - // first means a crash mid-sync re-runs cleanly; reordering would make a - // crashed setup look like `OrphanedState` (DB has state, no identity) on - // the retry. See `ensure_deployment_identity` + docs/invariants.md. - let identity = DeploymentIdentity { - chain_id: config.chain_id, - app_address: config.app_address, - input_box_address: input_reader.input_box_address(), - app_deployment_block: input_reader.app_deployment_block(), - batch_submitter_address: config.batch_submitter_address, - fee_oracle: pinned_fee_oracle, - }; - ensure_deployment_identity(&db_path, identity)?; - - // Persist the first price under setup's hard L1 requirement. Fixed writes - // the configured exponent; Uniswap quotes once so Tip never samples empty. - match prepared_fee_oracle { - PreparedFeeOracle::Fixed { log_gas_price } => { - storage::Storage::open(&db_path)?.set_log_gas_price(log_gas_price)?; - } - PreparedFeeOracle::Uniswap { - provider, token, .. - } => { - crate::l1::fee_oracle::persist_first_price( - db_path.clone(), - provider, - token, - process_lock.clone(), - ) - .await?; - } - } - - // ── Detection gate, step 0: checkpoint sanity ──────────── - // A checkpoint promotion cannot predate the application's deployment - // block (the scan genesis — no input exists before it). - // `B = 0` is the genesis bootstrap (no checkpoint) and is always valid. - // (plain setup detects only; loading a non-genesis checkpoint machine and - // the checkpoint clock check are `setup --recovery`'s job.) - if config.checkpoint_block != 0 && config.checkpoint_block < input_reader.app_deployment_block() - { - return Err(BootstrapError::CheckpointBeforeAppDeployment { - checkpoint_block: config.checkpoint_block, - app_deployment_block: input_reader.app_deployment_block(), - } - .into()); - } - - // ── Initial L1 sync ────────────────────────────────────── - // One pass reads every direct input up to the current safe head into - // `safe_inputs` + `safe_accepted_batches` and persists `l1_safe_head`. - // Re-entry-safe: a retry resumes from the persisted safe head. A - // transient sync failure leaves setup incomplete; the operator simply - // re-runs `setup`. - // - // Recovery rebuilds the batch tree from the checkpoint *after* this sync, so - // the local tree is empty here. Disable frontier population for recovery's - // syncs (this one and the post-flush re-sync): otherwise the - // content-identity check would see every L1 batch as "foreign" and falsely - // freeze the frontier, poisoning the rebuilt DB. `run`'s first sync - // populates it correctly once the anchor = N' is set (I16). - if config.recovery { - input_reader.set_frontier_mode(storage::FrontierMode::DeferUntilAnchorSet); - } - input_reader - .sync_to_current_safe_head() - .await - .map_err(BootstrapError::SetupInputReader)?; - - // ── Branch: recovery rebuild vs the genesis-style detect-and-refuse ── - let mut storage = storage::Storage::open(&db_path)?; - - if config.recovery { - // ── Recovery: flush → fold → fill. Replaces the - // detection gate + genesis snapshot; the operator is acting *on* the - // refusal the gate would otherwise raise. - recover::( - &config, - &identity, - timing, - &mut input_reader, - &mut storage, - &dumps_dir, - ) - .await?; - } else { - // ── Detection gate, steps 1–2: refuse if a previous instance left work ── - // Read-only: no key, no L1 write. Runs *after* - // the sync so step 2 reads a `safe_inputs` table populated to the safe - // head, and *before* the genesis snapshot / completion transaction so - // a refusing setup cannot complete. A retry while still incomplete - // re-detects identically. Once setup is complete, - // plain `setup` is a no-op; detecting a dirty chain is the job of a - // fresh setup/rebuild. - // - // Coherence: read the submitter's `safe` wallet nonce at the - // **persisted** safe block from the sync, not the live `Safe` tag. The head can advance - // between the sync and this read; pinning step 1 to the same block step 2 - // scans (`safe_inputs` up to that block) closes the window where a batch - // landing in between would be counted as settled by a live-tag read yet - // be missing from the not-yet-resynced scan. `pending` stays live, so any - // submitter activity past the synced head still trips `pending > safe`. - // - // Step 1 reads the LOCAL provider's pool view — a zombie tx dropped - // from this pool but alive elsewhere evades it, bounded at runtime by the - // content-identity check (CanonicalDivergence → cockroach recovery). - let synced_safe_block = storage.current_safe_block()?; - let wallet_nonces = read_submitter_wallet_nonces( - &config.rpc.eth_rpc_url, - config.batch_submitter_address, - synced_safe_block, - config.rpc.allow_insecure_rpc, - ) - .await?; - run_detection_gate( - &mut storage, - config.batch_submitter_address, - config.checkpoint_block, - wallet_nonces, - )?; - - // ── Genesis snapshot ───────────────────────────────────── - // Construct only after the admission facts and every - // detect-and-refuse gate. Factory errors follow normal setup settlement; - // completed no-ops and recovery never construct genesis state. - let genesis_app = genesis_app()?; - fill::register_genesis_baseline::(genesis_app, &mut storage, &dumps_dir)?; - } - - // Baseline registration committed setup_complete with the restore point. - tracing::info!( - data_dir = %config.data_dir, - chain_id = identity.chain_id, - app_address = %identity.app_address, - input_box_address = %identity.input_box_address, - app_deployment_block = identity.app_deployment_block, - batch_submitter_address = %identity.batch_submitter_address, - "setup complete" - ); - Ok(()) -} - -enum SetupAdmission { - Proceed, - AlreadyComplete, + }) } /// Refuse a fixed price the fee-floor CHECK would reject, before it pins an -/// identity whose first price write could never succeed. -fn check_fixed_log_gas_price(db_path: &str, log_gas_price: i16) -> Result<(), CommandError> { - let floor = storage::Storage::open(db_path)?.log_gas_price_floor()?; +/// identity whose first price write could never succeed. The floor is read +/// from the schema alone, so the data directory is still untouched. +fn check_fixed_log_gas_price(log_gas_price: i16) -> Result<(), CommandError> { + let floor = Storage::schema_only()?.log_gas_price_floor()?; if log_gas_price < floor { return Err(BootstrapError::FeeOracleMisconfig { message: format!( @@ -345,445 +605,121 @@ fn check_fixed_log_gas_price(db_path: &str, log_gas_price: i16) -> Result<(), Co Ok(()) } -fn admit_setup_lifecycle( - db_path: &str, - command: storage::LifecycleCommand, - recovery: bool, -) -> Result { - let exists = std::path::Path::new(db_path) - .try_exists() - .map_err(|err| super::io_error_at("database", db_path, err))?; - let populated = exists && !existing_sqlite_schema_is_empty(db_path)?; - if !populated { - storage::Storage::initialize_for_command(db_path, command)?; - return Ok(SetupAdmission::Proceed); - } - - // Admission facts only: divergence is absorbing; a completed setup - // is once-per-database (already-complete plain setup is a no-op, an - // already-complete rebuild is an error); a crashed prior attempt left - // only idempotent residue behind — the retry proceeds fresh over it. - let mut storage = storage::Storage::open_read_only(db_path)?; - if let Some((nonce, _)) = storage.canonical_divergence()? { - return Err(storage::LifecycleError::CanonicalDivergence { nonce }.into()); - } - if storage.is_setup_complete()? { - if recovery { - return Err(SetupRecoveryError::AlreadySetUp.into()); - } - return Ok(SetupAdmission::AlreadyComplete); - } - Ok(SetupAdmission::Proceed) -} - -fn settle_setup_lifecycle( - db_path: &str, - command: storage::LifecycleCommand, - result: &Result<(), CommandError>, -) -> Result<(), CommandError> { - match result { - Ok(()) => { - let storage = storage::Storage::open_read_only(db_path)?; - if !storage.is_setup_complete()? { - return Err(storage::LifecycleError::Malformed( - "setup returned without its complete baseline".to_string(), - ) - .into()); +/// Setup runs only in an empty data directory: absent, empty, or holding only +/// the lock file. Nothing else there can be trusted to fit this run. +fn require_empty_data_dir(data_dir: &str) -> Result<(), CommandError> { + let entries = match std::fs::read_dir(data_dir) { + Ok(entries) => entries, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(super::io_error_at("data dir", data_dir, err)), + }; + for entry in entries { + let entry = entry.map_err(|err| super::io_error_at("data dir", data_dir, err))?; + if entry.file_name() != LOCK_FILE_NAME { + return Err(SetupError::DataDirNotEmpty { + path: data_dir.to_owned(), + entry: entry.file_name().to_string_lossy().into_owned(), } - Ok(()) - } - Err(_) => { - // Verdict-neutral black-box settlement: the write never - // replaces the command's own error. - crate::commands::record_terminal_fault_best_effort(db_path, command, result); - Ok(()) + .into()); } } + Ok(()) } -/// The trusted checkpoint a `setup --recovery` folds from — the machine state -/// `S` at block `B`, plus the two scalars the fold needs that the bare-metal app -/// cannot recompute: `A` (`S`'s last-executed safe block — the fridge is the -/// `(A, B]` directs) and `N` (the resume batch nonce at `B`, which rides in the -/// dump's `info.toml`). See the data dictionary in `docs/recovery/cockroach.md`. -struct Checkpoint { - /// `S` — the checkpoint app state at block `B`. - app: A, - /// `A` — `S`'s last-executed safe block. - executed_safe_block: u64, - /// `N` — the resume batch nonce at `B` (checkpoint metadata). Named to - /// contrast with the fold's output `N'` (`resume_nonce` in `recover`). - checkpoint_nonce: u64, - /// `B` — the checkpoint's L1 inclusion block (`--checkpoint-block`). - checkpoint_block: u64, -} - -impl Checkpoint { - /// Load `S` from the dump dir, derive `A` and `N`, and enforce the load-time - /// precondition `A < B`, except for the known empty genesis checkpoint. - /// Equality otherwise hides pending directs later in the same block. - /// The exporter must separately verify no canonical direct at or below A - /// remains queued; this bundle carries no queue evidence to check here. - fn load(dir: &std::path::Path, checkpoint_block: u64) -> Result { - let load_err = |message: String| SetupRecoveryError::CheckpointLoad { - path: dir.display().to_string(), - message, - }; - let info = dump_info::read_info(dir).map_err(|e| load_err(e.to_string()))?; - let checkpoint = - dump_info::read_checkpoint_info(dir).map_err(|e| load_err(e.to_string()))?; - if checkpoint.inclusion_block != checkpoint_block { - return Err(load_err(format!( - "checkpoint receipt block {} differs from configured block {checkpoint_block}", - checkpoint.inclusion_block, - ))); - } - if checkpoint.next_batch_nonce != info.next_batch_nonce { - return Err(load_err( - "checkpoint receipt nonce differs from immutable dump metadata".into(), - )); - } - let app = A::from_dump(&dump_info::app_prefix(dir)).map_err(|e| load_err(e.to_string()))?; - let executed_safe_block = app.last_executed_safe_block(); - let is_genesis = checkpoint_block == 0 - && info.next_batch_nonce == 0 - && app.executed_input_count() == sequencer_core::history::ExecutedInputCount::ZERO; - if executed_safe_block >= checkpoint_block && !is_genesis { - return Err(SetupRecoveryError::CheckpointNotBeforeBlock { - executed_safe_block, - checkpoint_block, - }); - } - Ok(Self { - app, - executed_safe_block, - checkpoint_nonce: info.next_batch_nonce, - checkpoint_block, - }) +/// A directory removed when dropped, for the base's state bytes and the dry +/// run's database. +struct ScratchDir(PathBuf); + +impl ScratchDir { + fn create(parent: &Path) -> Result { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(std::io::Error::other)? + .as_nanos(); + let path = parent.join(format!( + "sequencer-setup-scratch-{}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&path) + .map_err(|err| super::io_error_at("setup scratch dir", &path, err))?; + Ok(Self(path)) } -} - -/// Source the fold inputs from the synced `safe_inputs`: the -/// `(A, B]` direct **seeds** and the full `(B, C]` **replay** stream. -/// -/// Seeds must be directs only — the fold enqueues them straight into the fridge -/// (`enqueue_direct`, no classification). The `sender != submitter` filter IS -/// the scheduler's own sender-based classification: the fold's `process_input` -/// routes `sender == sequencer_address` to the batch path and everything else to -/// a direct, so a dropped `sender == submitter` row is never a direct-with- -/// effect — it is a batch already folded into `S` (every valid batch with -/// inclusion `≤ B` is in `S` by the operator's checkpoint, the same trust -/// boundary as the resume nonce `N`) or an undecodable scheduler no-op. -/// -/// Replay is the full stream; the fold classifies each input itself, so no -/// pre-filter. -fn source_fold_inputs( - storage: &mut storage::Storage, - checkpoint: &Checkpoint, - stop_block: u64, - submitter: Address, -) -> Result<(Vec, Vec), CommandError> { - let seeds = storage - .safe_inputs_in_block_range(checkpoint.executed_safe_block, checkpoint.checkpoint_block)? - .into_iter() - .filter(|input| input.sender != submitter) - .map(to_fold_input) - .collect(); - let replay = storage - .safe_inputs_in_block_range(checkpoint.checkpoint_block, stop_block)? - .into_iter() - .map(to_fold_input) - .collect(); - Ok((seeds, replay)) -} -/// The `setup --recovery` procedure: rebuild a freshly-wiped DB from -/// a trusted checkpoint instead of refusing. Runs after the shared prefix -/// (identity pinned, initial sync done); replaces the detection gate + genesis -/// snapshot. Invalid checkpoint/configuration failures are terminal; transient -/// L1 failures leave setup incomplete for a fresh attempt. -/// -/// The `flush → fold → fill` steps are enumerated authoritatively in -/// **[`docs/recovery/cockroach.md`](../../../../docs/recovery/cockroach.md)** (spec, -/// data dictionary `A`/`B`/`C`/`N`/`N'`, and code map) and anchored inline below -/// (`// 1.`…`// 6.`). Read the doc before editing this function. -/// -/// `N` (and the whole checkpoint tuple `S`/`A`/`B`/`N`) is **trusted** metadata -/// from the sequencer-produced finalized dump; recovery does not independently -/// re-verify it (there is no local oracle without replaying the scheduler from -/// genesis). See cockroach.md "Data dictionary" for the trust boundary. -async fn recover( - config: &SetupConfig, - identity: &DeploymentIdentity, - timing: sequencer_core::protocol::ProtocolTiming, - input_reader: &mut InputReader, - storage: &mut storage::Storage, - dumps_dir: &std::path::Path, -) -> Result<(), CommandError> -where - A: Application + 'static, -{ - // 1. Load the trusted checkpoint (S, A, N, B); require A < B or empty genesis. - let checkpoint_dir = std::path::Path::new( - config - .checkpoint_dump_dir - .as_deref() - .expect("validated: recovery requires a checkpoint dump dir"), - ); - let checkpoint = Checkpoint::::load(checkpoint_dir, config.checkpoint_block)?; - - // 2. Flush the previous instance's batch txs → C, the post-flush safe head. - let stop_block = - flush_wallet_nonce(config, identity, timing.seconds_per_block(), storage).await?; - - // 3. Re-sync through C, then verify the resynced safe block reaches the - // flush observation before cascade. Frontier population stays OFF (the - // caller disabled it before the initial sync): the tree is rebuilt in - // step 6, so a frontier built here against an empty tree would falsely - // diverge. `run`'s first sync populates it correctly once anchor = N'. - tracing::info!("re-syncing L1 safe head after flush"); - input_reader - .sync_to_current_safe_head() - .await - .map_err(|e| { - CommandError::Worker(WorkerExit::InputReader( - crate::commands::error::WorkerStop::Source(e), - )) - })?; - let resynced_safe_block = require_resynced_safe_block(storage.current_safe_block()?)?; - assert_resync_caught_up(resynced_safe_block, stop_block)?; - - rebuild_from_checkpoint(checkpoint, identity, stop_block, storage, dumps_dir) -} - -fn rebuild_from_checkpoint( - checkpoint: Checkpoint, - identity: &DeploymentIdentity, - stop_block: u64, - storage: &mut storage::Storage, - dumps_dir: &std::path::Path, -) -> Result<(), CommandError> { - // A valid checkpoint can outpace an honestly lagging RPC node. Labelling - // its state with an earlier C would let run execute its directs again. - if stop_block < checkpoint.checkpoint_block { - return Err( - RecoveryError::retry(RecoveryRetryReason::CheckpointAheadOfStop { - checkpoint_block: checkpoint.checkpoint_block, - stop_block, - }) - .into(), - ); + fn path(&self) -> &Path { + &self.0 } - - // 4. Source the (A, B] direct seeds + the (B, C] replay stream. - let submitter = identity.batch_submitter_address; - let (seeds, replay) = source_fold_inputs(storage, &checkpoint, stop_block, submitter)?; - - // 5. Fold (S, N) over the inputs → (S', N'). - let Checkpoint { - app, - executed_safe_block, - checkpoint_nonce, - checkpoint_block, - } = checkpoint; - let domain = sequencer_core::build_input_domain(identity.chain_id, identity.app_address); - let scheduler_config = SchedulerConfig::new(submitter); - let (recovered_app, resume_nonce) = fold_replay( - app, - checkpoint_nonce, - scheduler_config, - domain, - seeds, - replay, - stop_block, - )?; - - // 6. Publish the local resume baseline and root atomically. The first run - // scans acceptance only after C, beginning with expected nonce N'. - fill::fill_recovery_state(recovered_app, resume_nonce, stop_block, storage, dumps_dir)?; - - tracing::info!( - executed_safe_block, - checkpoint_block, - stop_block, - resume_nonce, - "recovery complete — DB rebuilt from checkpoint" - ); - Ok(()) -} - -fn require_resynced_safe_block(observed: Option) -> Result { - observed.ok_or(SetupRecoveryError::MissingResyncedSafeHead) -} - -/// Recovery step 2: flush the previous instance's stranded batch txs and wait -/// for the wallet nonce to settle, returning `C` — the post-flush safe head at -/// which every prior batch is resolved at safe depth. This is `setup`'s only -/// L1-signing action (it otherwise takes the submitter *address*, not the key). -/// -/// The flush-acquire core (flusher + watermark sink + `flush_and_wait`) is -/// shared with the runtime danger path and `flush-mempool` via -/// [`MempoolFlusher::flush_to_safe`]; this site supplies the recovery key, the -/// pinned submitter address, and the watermark from the open `storage`. -async fn flush_wallet_nonce( - config: &SetupConfig, - identity: &DeploymentIdentity, - seconds_per_block: u64, - storage: &mut storage::Storage, -) -> Result { - // The recovery key must match the pinned batch-submitter address — flushing - // under a different account's key would settle the wrong wallet's nonce (and - // burn its gas) while recovery never settles `identity.batch_submitter_address`. - // Gate before signing, exactly as `flush-mempool` does. - let key = super::verify_submitter_key(config.resolve_recovery_key()?, identity)?; - // Keyed-write chain-id gate: `setup --recovery`'s flush signs L1 no-op txs, - // so re-confirm the RPC serves the pinned chain right before building the - // signer (the earlier `validate_rpc_chain_id` may have gone stale). Same - // guarded constructor the runtime flush and `flush-mempool` use. - let providers = crate::l1::provider::create_verified_keyed_providers( - &config.rpc.eth_rpc_url, - key.expose_secret(), - config.chain_id, - config.rpc.allow_insecure_rpc, - ) - .await - .map_err(CommandError::from)?; - let watermark = storage.wallet_nonce_watermark()?; - let stop_block = MempoolFlusher::flush_to_safe( - providers, - identity.batch_submitter_address, - seconds_per_block, - config.db_path(), - watermark, - ) - .await?; - Ok(stop_block) } -/// Map a synced `safe_inputs` row to a fold input. The EIP-712 domain is a -/// deployment-wide constant the engine clones onto each, so it is not stored -/// per row. -fn to_fold_input(input: crate::storage::StoredSafeInput) -> FoldInput { - FoldInput { - sender: input.sender, - inclusion_block: input.block_number, - payload: input.payload, +impl Drop for ScratchDir { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_dir_all(&self.0) { + tracing::warn!(path = %self.0.display(), %error, "could not remove setup's scratch directory"); + } } } -/// The read-only detection gate, separated -/// from the L1/RPC I/O so its logic is unit-testable against a seeded DB. -/// -/// `wallet_nonces` is the submitter's `(pending, safe)` wallet nonce read from -/// L1; `storage` holds the safe inputs synced to the safe head. The two steps -/// are ordered on purpose: step 1 (wallet nonce settled?) gates step 2 (scan). -/// An unsettled wallet nonce refuses *without* scanning — unsafe batches are not yet in -/// `safe_inputs`, so the scan would be incomplete and could false-negative. -fn run_detection_gate( - storage: &mut storage::Storage, +/// Setup's escape check, after the flush and the sync through the stop +/// block. An unsettled wallet refuses before the scan: the old instance's +/// unsafe batches are not in the synced table yet. +fn check_escape( + storage: &mut Storage, batch_submitter: Address, - checkpoint_block: u64, + stop_block: u64, (pending_wallet_nonce, safe_wallet_nonce): (u64, u64), ) -> Result<(), CommandError> { - // Step 1 — `pending > safe` means a previous instance left in-flight - // (pending or mined-but-unsafe) batch txs. if pending_wallet_nonce > safe_wallet_nonce { - return Err(SetupRefuse::WalletNonceUnsettled { + return Err(SetupRefuse::WalletUnsettledAfterFlush { pending: pending_wallet_nonce, safe: safe_wallet_nonce, } .into()); } - // Step 2 — settled ⟹ nothing of ours sits unsafe, so the synced - // `safe_inputs` already reflect every previous batch (scanning them is - // equivalent to scanning `(B, safe]`). Refuse if any batch-submitter tx - // landed strictly past the checkpoint block. The query reads the - // reader-synced table, inheriting the reader's range-completeness - // hardening — see `first_batch_submitter_input_after_block`. - if let Some((safe_input_index, found_block)) = - storage.first_batch_submitter_input_after_block(batch_submitter, checkpoint_block)? + // The scan reads the reader-synced table, which carries the reader's + // completeness witness; see `first_batch_submitter_input_after_block`. + if let Some(found) = + storage.first_batch_submitter_input_after_block(batch_submitter, stop_block)? { - return Err(SetupRefuse::BatchPastCheckpoint { - checkpoint_block, - found_block, - safe_input_index, + return Err(SetupRefuse::SubmitterInputPastStop { + stop_block, + found_block: found.block_number, + safe_input_index: found.safe_input_index, } .into()); } Ok(()) } -/// Read the batch-submitter wallet nonce at the pending tag and at `safe_block` -/// (read-only): returns `(pending, safe)`. `pending > safe` means a previous -/// instance left batch txs past the synced safe head — the step-1 signal. -/// -/// `safe_block` is the safe head the preceding sync persisted; the `safe` nonce -/// is read at exactly that block (`Number(safe_block)`) so step 1 is coherent -/// with step 2's scan of `safe_inputs` synced to the same block — not the live -/// `Safe` tag, which can advance between the sync and this read and reintroduce -/// a TOCTOU gap. `None` (no safe head persisted — not expected after a -/// successful sync) falls back to the `Safe` tag. -/// -/// Builds a provider from the RPC URL via the same `create_read_provider` the input -/// reader uses (the reader holds no long-lived provider — it builds one per -/// pass), so no second persistent connection is introduced. Both reads are the -/// batch-submitter's `eth_getTransactionCount`; an RPC failure here is -/// transient ([`BootstrapError::DetectionWalletNonceRead`]) — the prior sync already -/// proved L1 reachable. +/// The submitter's wallet nonce at the pending tag and at `safe_block`, the +/// safe head the sync persisted, so the settled check agrees with the table +/// the escape scan reads. Returns `(pending, safe)`. async fn read_submitter_wallet_nonces( eth_rpc_url: &str, batch_submitter: Address, - safe_block: Option, + safe_block: u64, allow_insecure: bool, ) -> Result<(u64, u64), BootstrapError> { use alloy::providers::Provider; use alloy::rpc::types::BlockNumberOrTag; let provider = crate::l1::provider::create_read_provider(eth_rpc_url, allow_insecure) - .map_err(|message| BootstrapError::DetectionWalletNonceRead { message })?; - let pending = provider - .get_transaction_count(batch_submitter) - .block_id(BlockNumberOrTag::Pending.into()) - .await - .map_err(|e| BootstrapError::DetectionWalletNonceRead { - message: crate::l1::provider::error_chain(&e), - })?; - let safe_tag = match safe_block { - Some(block) => BlockNumberOrTag::Number(block), - None => BlockNumberOrTag::Safe, + .map_err(|message| BootstrapError::WalletNonceRead { message })?; + let read = |tag: BlockNumberOrTag| { + let provider = &provider; + async move { + provider + .get_transaction_count(batch_submitter) + .block_id(tag.into()) + .await + .map_err(|e| BootstrapError::WalletNonceRead { + message: crate::l1::provider::error_chain(&e), + }) + } }; - let safe = provider - .get_transaction_count(batch_submitter) - .block_id(safe_tag.into()) - .await - .map_err(|e| BootstrapError::DetectionWalletNonceRead { - message: crate::l1::provider::error_chain(&e), - })?; + let pending = read(BlockNumberOrTag::Pending).await?; + let safe = read(BlockNumberOrTag::Number(safe_block)).await?; Ok((pending, safe)) } -/// A first `Connection::open` can create the DB file before the transactional -/// baseline migration begins. That exact empty-schema crash state contains no -/// possible DB verdict and may resume setup. Any object at all makes the DB -/// non-empty; malformed/populated databases then take the strict read-only -/// lifecycle gate and are never migrated speculatively. -fn existing_sqlite_schema_is_empty( - db_path: &str, -) -> Result { - let metadata = - std::fs::metadata(db_path).map_err(|err| super::io_error_at("database", db_path, err))?; - if metadata.len() == 0 { - return Ok(true); - } - let connection = rusqlite::Connection::open_with_flags( - db_path, - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - let object_count: i64 = - connection.query_row("SELECT count(*) FROM sqlite_master", [], |row| row.get(0))?; - Ok(object_count == 0) -} - /// Setup pins the application's deployment block, the reader's scan floor, only /// once it is at or below a safe head sampled before the block was read. Until /// then a reorg could move the deployment earlier, and the reader would advance @@ -807,11 +743,47 @@ fn require_deployment_safe( #[cfg(test)] mod tests { use super::*; - use crate::storage::Storage; use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::{SENDER_A, default_protocol_timing}; + use crate::storage::test_helpers::SENDER_A; use crate::test_support::temp_db; + const ANVIL_KEY: &str = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + + pub(super) fn setup_config(data_dir: &str, rpc_url: &str, extra: &[&str]) -> SetupConfig { + use crate::harness::{Cli, Command}; + use clap::Parser; + let mut args = vec![ + "sequencer", + "setup", + "--data-dir", + data_dir, + "--eth-rpc-url", + rpc_url, + "--chain-id", + "31337", + "--app-address", + "0x1111111111111111111111111111111111111111", + "--batch-submitter-address", + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "--batch-submitter-private-key", + ANVIL_KEY, + ]; + args.extend_from_slice(extra); + let Command::Setup(config) = Cli::try_parse_from(args).expect("parse setup").command else { + panic!("expected setup subcommand"); + }; + *config + } + + fn entries(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .expect("read dir") + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + #[test] fn setup_pins_only_a_safe_deployment() { assert!(matches!( @@ -825,195 +797,86 @@ mod tests { require_deployment_safe(10, 11).expect("a deployment below the safe head is final"); } + /// The emptiness rule skips the lock file by name, so the name must be + /// the one the lock creates. #[test] - fn checkpoint_load_requires_an_export_receipt_matching_the_artifact_and_block() { - use crate::test_support::StubApp; - let dir = tempfile::tempdir().expect("checkpoint parent"); - let prefix = dir.path().join("checkpoint"); - let mut app = StubApp::default(); - dump_info::create_dump_dir_with_info( - &mut app, - &prefix, - &dump_info::DumpInfo::at_baseline(3), - ) - .expect("create artifact"); - assert!(matches!( - Checkpoint::::load(&prefix, 10), - Err(SetupRecoveryError::CheckpointLoad { .. }) - )); - let write_receipt = |nonce| { - let receipt = dump_info::CheckpointInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: nonce, - inclusion_block: 10, - }; - std::fs::write( - prefix.join("checkpoint.toml"), - toml::to_string(&receipt).unwrap(), - ) - .expect("write receipt"); - }; - write_receipt(4); - assert!(matches!( - Checkpoint::::load(&prefix, 10), - Err(SetupRecoveryError::CheckpointLoad { .. }) - )); - write_receipt(3); - assert!(matches!( - Checkpoint::::load(&prefix, 11), - Err(SetupRecoveryError::CheckpointLoad { .. }) - )); - let checkpoint = Checkpoint::::load(&prefix, 10).unwrap(); - assert_eq!( - (checkpoint.checkpoint_nonce, checkpoint.checkpoint_block), - (3, 10) + fn only_the_lock_file_counts_as_empty() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap(); + require_empty_data_dir(&dir.path().join("absent").to_string_lossy()).unwrap(); + require_empty_data_dir(path).unwrap(); + drop(ProcessLock::acquire(path).unwrap()); + assert_eq!(entries(dir.path()), vec![LOCK_FILE_NAME.to_owned()]); + require_empty_data_dir(path).unwrap(); + std::fs::create_dir(dir.path().join("dumps")).unwrap(); + let err = require_empty_data_dir(path).expect_err("a directory with a dump is not empty"); + assert!( + matches!( + &err, + CommandError::Bootstrap(BootstrapError::Setup(error)) + if matches!(&**error, SetupError::DataDirNotEmpty { entry, .. } if entry == "dumps") + ), + "{err:?}" ); + assert_eq!(err.exit_code(), crate::commands::error::EXIT_TERMINAL); } - #[test] - fn genesis_checkpoint_loads_and_has_an_empty_seed_interval() { - use crate::test_support::StubApp; - let dir = tempfile::tempdir().expect("checkpoint parent"); - let prefix = dir.path().join("genesis"); - let mut app = StubApp::default(); - dump_info::create_dump_dir_with_info( - &mut app, - &prefix, - &dump_info::DumpInfo::at_baseline(0), - ) - .expect("genesis artifact"); - std::fs::write( - prefix.join("checkpoint.toml"), - toml::to_string(&dump_info::CheckpointInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: 0, - inclusion_block: 0, - }) - .unwrap(), - ) - .expect("genesis receipt"); - let checkpoint = Checkpoint::::load(&prefix, 0).expect("load genesis export"); - assert_eq!(checkpoint.executed_safe_block, checkpoint.checkpoint_block); - let db = temp_db("genesis-checkpoint-seed"); - let mut storage = Storage::open(&db.path).expect("storage"); - storage - .append_safe_inputs( - 10, - &[StoredSafeInput { - sender: Address::repeat_byte(0x22), - payload: vec![1], - block_number: 5, - }], - SENDER_A, - &default_protocol_timing(), - ) - .expect("post-genesis direct"); - let (seeds, replay) = source_fold_inputs(&mut storage, &checkpoint, 10, SENDER_A) - .expect("source empty seed and replay"); - assert!(seeds.is_empty()); - assert_eq!(replay.len(), 1); - assert_eq!(replay[0].inclusion_block, 5); - } - - #[test] - fn non_genesis_checkpoint_rejects_equal_execution_and_inclusion_blocks() { - use app_core::application::{WalletApp, WalletConfig}; - use sequencer_core::application::execute_direct_input; - use sequencer_core::l2_tx::DirectInput; - - let dir = tempfile::tempdir().expect("checkpoint parent"); - let prefix = dir.path().join("checkpoint"); - let mut app = WalletApp::new(WalletConfig::devnet()); - execute_direct_input( - &mut app, - &DirectInput { - sender: Address::repeat_byte(0x22), - payload: vec![], - block_number: 10, - }, - ) - .expect("execute direct at checkpoint block"); - dump_info::create_dump_dir_with_info( - &mut app, - &prefix, - &dump_info::DumpInfo::at_batch_close(0), - ) - .expect("checkpoint artifact"); - std::fs::write( - prefix.join("checkpoint.toml"), - toml::to_string(&dump_info::CheckpointInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: 1, - inclusion_block: 10, - }) - .unwrap(), - ) - .expect("checkpoint receipt"); - - // A direct arriving after this batch in the same L1 block is still - // pending, but the (A, B] seed interval would omit the whole block. - let error = Checkpoint::::load(&prefix, 10).err().unwrap(); - assert!(matches!( - error, - SetupRecoveryError::CheckpointNotBeforeBlock { - executed_safe_block: 10, - checkpoint_block: 10, - } - )); - assert!(error.to_string().contains("pending same-block directs")); - } - - #[test] - fn completed_plain_setup_is_a_noop_and_writes_nothing() { - let db = temp_db("setup-noop-preserves-recovery"); + /// A completed (or failed) setup's directory is refused with exit 30 + /// before L1, the key's account or the genesis factory are touched, and + /// it is left exactly as it was. + #[tokio::test] + async fn a_non_empty_directory_is_refused_untouched() { + let data_dir = tempfile::tempdir().unwrap(); + let db = data_dir.path().join("sequencer.db"); let mut storage = - Storage::initialize_for_command(db.path.as_str(), storage::LifecycleCommand::Setup) - .expect("initialize"); - storage - .complete_baseline_setup( - &db._dir.path().join("baseline"), - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, - ) - .expect("complete setup"); + Storage::initialize_for_command(db.to_str().unwrap(), LifecycleCommand::Setup).unwrap(); storage - .record_terminal_fault(storage::LifecycleCommand::Run, "prior terminal death") - .expect("record prior fault"); - let before = storage - .latest_terminal_fault() - .expect("read") - .expect("recorded fault"); + .publish_era(&data_dir.path().join("baseline"), &NewEra::EMPTY_GENESIS) + .unwrap(); drop(storage); + let before = entries(data_dir.path()); + let modified = std::fs::metadata(&db).unwrap().modified().unwrap(); - assert!(matches!( - admit_setup_lifecycle(db.path.as_str(), storage::LifecycleCommand::Setup, false) - .expect("plain setup no-op"), - SetupAdmission::AlreadyComplete - )); - let after = Storage::open_read_only(db.path.as_str()) - .expect("reopen") - .latest_terminal_fault() - .expect("read") - .expect("still recorded"); - assert_eq!(after, before, "plain setup must write nothing"); - } - - #[test] - fn recovery_resync_requires_a_persisted_safe_head() { - assert_eq!(require_resynced_safe_block(Some(7)).unwrap(), 7); - assert!(matches!( - require_resynced_safe_block(None), - Err(SetupRecoveryError::MissingResyncedSafeHead) - )); + let err = run_setup( + setup_config(data_dir.path().to_str().unwrap(), "http://127.0.0.1:1", &[]), + || -> Result { + unreachable!("a refused directory never builds the genesis app") + }, + Box::new(|_, _| unreachable!("a refused directory never reads a machine")), + ) + .await + .expect_err("a non-empty directory is refused"); + assert!( + matches!( + &err, + CommandError::Bootstrap(BootstrapError::Setup(error)) + if matches!(**error, SetupError::DataDirNotEmpty { .. }) + ), + "{err:?}" + ); + assert_eq!(err.exit_code(), crate::commands::error::EXIT_TERMINAL); + assert_eq!( + entries(data_dir.path()), + before, + "not even a lock file appears" + ); + assert_eq!( + std::fs::metadata(&db).unwrap().modified().unwrap(), + modified + ); + assert!( + Storage::open_read_only(db.to_str().unwrap()) + .unwrap() + .latest_terminal_fault() + .unwrap() + .is_none(), + "the refusal records nothing in another setup's black box" + ); } /// Seed `safe_inputs` with one batch-submitter (`SENDER_A`) input per block - /// in `blocks`, synced up to the max block. Payloads are junk (scheduler - /// no-ops) — detection scans raw rows, not the accepted frontier. + /// in `blocks`, synced up to the max block. fn seed_submitter_inputs(storage: &mut Storage, blocks: &[u64]) { - let protocol = default_protocol_timing(); let inputs: Vec = blocks .iter() .map(|&block_number| StoredSafeInput { @@ -1024,71 +887,63 @@ mod tests { .collect(); let safe_block = blocks.iter().copied().max().unwrap_or(0); storage - .append_safe_inputs(safe_block, inputs.as_slice(), SENDER_A, &protocol) + .append_safe_inputs(safe_block, &inputs, SENDER_A) .expect("seed safe inputs"); } #[test] - fn detection_passes_when_settled_and_no_batch_past_checkpoint() { - let db = temp_db("detect-pass"); + fn escape_check_passes_a_settled_wallet_with_nothing_past_the_stop_block() { + let db = temp_db("escape-pass"); let mut storage = Storage::open(db.path.as_str()).expect("open"); - // Settled (pending == safe) and no submitter inputs at all. - assert!(run_detection_gate(&mut storage, SENDER_A, 0, (5, 5)).is_ok()); + seed_submitter_inputs(&mut storage, &[18]); + // A batch at the stop block is part of the rebuilt history. + check_escape(&mut storage, SENDER_A, 18, (5, 5)).unwrap(); } #[test] - fn detection_refuses_unsettled_wallet_nonce_before_scanning() { - let db = temp_db("detect-unsettled"); + fn escape_check_refuses_an_unsettled_wallet_before_scanning() { + let db = temp_db("escape-unsettled"); let mut storage = Storage::open(db.path.as_str()).expect("open"); - // Step 1 gates step 2: `pending > safe` refuses regardless of inputs - // (and the DB is empty here, proving step 2 never ran). - match run_detection_gate(&mut storage, SENDER_A, 0, (14, 13)) { - Err(CommandError::Bootstrap(BootstrapError::SetupRefuse( - SetupRefuse::WalletNonceUnsettled { pending, safe }, - ))) => assert_eq!((pending, safe), (14, 13)), - other => panic!("expected WalletNonceUnsettled, got {other:?}"), + // The table is empty, so only the wallet can refuse. + match check_escape(&mut storage, SENDER_A, 0, (14, 13)) { + Err( + error @ CommandError::Bootstrap(BootstrapError::SetupRefuse( + SetupRefuse::WalletUnsettledAfterFlush { + pending: 14, + safe: 13, + }, + )), + ) => { + assert_eq!(error.exit_code(), crate::commands::error::EXIT_SETUP_ESCAPE); + assert!(error.to_string().contains("stop the old instance")); + } + other => panic!("expected WalletUnsettledAfterFlush, got {other:?}"), } } #[test] - fn detection_refuses_batch_past_checkpoint_when_settled() { - let db = temp_db("detect-batch-past"); + fn escape_check_refuses_a_submitter_input_past_the_stop_block() { + let db = temp_db("escape-past-stop"); let mut storage = Storage::open(db.path.as_str()).expect("open"); - seed_submitter_inputs(&mut storage, &[18]); - match run_detection_gate(&mut storage, SENDER_A, 0, (1, 1)) { + seed_submitter_inputs(&mut storage, &[10, 18]); + match check_escape(&mut storage, SENDER_A, 17, (1, 1)) { Err(CommandError::Bootstrap(BootstrapError::SetupRefuse( - SetupRefuse::BatchPastCheckpoint { - checkpoint_block, - found_block, - .. + SetupRefuse::SubmitterInputPastStop { + stop_block: 17, + found_block: 18, + safe_input_index: 1, }, - ))) => { - assert_eq!(checkpoint_block, 0); - assert_eq!(found_block, 18); - } - other => panic!("expected BatchPastCheckpoint, got {other:?}"), + ))) => {} + other => panic!("expected SubmitterInputPastStop, got {other:?}"), } } - #[test] - fn detection_passes_when_batch_at_or_below_checkpoint() { - let db = temp_db("detect-batch-below"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - seed_submitter_inputs(&mut storage, &[18]); - // A batch at block 18 is part of `S` for a checkpoint at 18 (not - // strictly past it), so a settled deployment proceeds. - assert!(run_detection_gate(&mut storage, SENDER_A, 18, (1, 1)).is_ok()); - } - /// A fixed price below the fee floor would pin an identity whose first /// price write the schema refuses; setup refuses it first. #[test] fn fixed_price_below_the_floor_is_refused() { - let db = crate::test_support::temp_db("fixed-price-floor"); - drop(storage::Storage::open(db.path.as_str()).expect("initialize schema")); - - let err = check_fixed_log_gas_price(&db.path, -1357) - .expect_err("one below the floor must be refused"); + let err = + check_fixed_log_gas_price(-1357).expect_err("one below the floor must be refused"); assert!( matches!( err, @@ -1096,51 +951,58 @@ mod tests { ), "{err:?}" ); - check_fixed_log_gas_price(&db.path, -1356).expect("the floor itself is accepted"); + check_fixed_log_gas_price(-1356).expect("the floor itself is accepted"); + } + + #[test] + fn summary_says_which_case_it_found() { + let mut summary = SetupSummary { + base_l1_input_count: 8, + base_l2_tx_count: 7, + submitter_inputs_past_base: 0, + unsettled_wallet_txs: 0, + inputs_to_replay: 3, + through_block: 120, + era: None, + }; + assert!(summary.to_string().contains("only moves directs")); + summary.unsettled_wallet_txs = 1; + assert!(summary.to_string().contains("re-judges")); + summary.era = Some(NewEra { + stop_block: 130, + rebuilt_end: L2TxCount::new(9), + ..NewEra::EMPTY_GENESIS + }); + let text = summary.to_string(); + assert!(text.contains("stop block: 130"), "{text}"); + assert!(text.contains("L2 tx count at its end: 9"), "{text}"); } /// Setup's L1 discovery uses the startup polarity: a URL the client refuses /// re-fails identically, so it is terminal, and so is an endpoint that /// redirects the chain-id read; an unreachable RPC fails at the chain-id - /// read (`ChainIdRpc`) and is retried. The fee-source connect follows the - /// runtime classification with setup's messages. + /// read (`ChainIdRpc`) and is retried, leaving the directory empty for the + /// retry. The fee-source connect follows the runtime classification with + /// setup's messages. #[tokio::test] async fn setup_l1_failures_exit_by_startup_phase() { use crate::commands::error::{EXIT_RESTART_TRANSIENT, EXIT_TERMINAL}; - use crate::harness::{Cli, Command}; - use clap::Parser; - async fn setup_with(rpc_url: &str) -> CommandError { + async fn setup_with(rpc_url: &str) -> (CommandError, Vec) { let data_dir = tempfile::tempdir().expect("data dir"); - let cli = Cli::try_parse_from([ - "sequencer", - "setup", - "--data-dir", - data_dir.path().to_str().expect("utf8 path"), - "--eth-rpc-url", - rpc_url, - "--chain-id", - "31337", - "--app-address", - "0x1111111111111111111111111111111111111111", - "--batch-submitter-address", - "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - ]) - .expect("parse setup"); - let Command::Setup(config) = cli.command else { - panic!("expected setup subcommand"); - }; - setup( - *config, + let error = run_setup( + setup_config(data_dir.path().to_str().expect("utf8 path"), rpc_url, &[]), || -> Result { unreachable!("setup fails before building the genesis app") }, + base::read_stored_machine(), ) .await - .expect_err("setup cannot discover L1") + .expect_err("setup cannot discover L1"); + (error, entries(data_dir.path())) } - let malformed = setup_with("not a url").await; + let (malformed, _) = setup_with("not a url").await; assert!( matches!( malformed, @@ -1152,12 +1014,17 @@ mod tests { ); assert_eq!(malformed.exit_code(), EXIT_TERMINAL); - let unreachable = setup_with("http://127.0.0.1:1").await; + let (unreachable, left) = setup_with("http://127.0.0.1:1").await; assert_eq!( unreachable.exit_code(), EXIT_RESTART_TRANSIENT, "{unreachable:?}" ); + assert_eq!( + left, + vec![LOCK_FILE_NAME.to_owned()], + "an exit 20 leaves the directory empty for the retry" + ); // Called directly: clap would read CARTESI_SEQUENCER_ALLOW_INSECURE_RPC // from the environment. @@ -1263,168 +1130,23 @@ mod tests { assert_eq!(unavailable.exit_code(), EXIT_RESTART_TRANSIENT); } - /// `setup --recovery`'s flush signs L1 no-ops with the recovery key, so it - /// must match the pinned batch-submitter — otherwise it would settle some - /// other account's nonce while recovery never settles the intended wallet. - /// The guard is pure and runs first, so a wrong key is refused before any L1 - /// contact (the RPC here is unreachable) or DB write. #[tokio::test] - async fn flush_wallet_nonce_refuses_mismatched_recovery_key() { - use crate::harness::{Cli, Command}; - use crate::storage::DeploymentIdentity; - use clap::Parser; - - // A recovery key whose address is NOT the pinned submitter below. - const OTHER_KEY: &str = - "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - let pinned_submitter: Address = "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" - .parse() - .unwrap(); - - let cli = Cli::try_parse_from([ - "sequencer", - "setup", - "--eth-rpc-url", - "http://127.0.0.1:1", // unreachable — the verify fails before any contact - "--chain-id", - "31337", - "--app-address", - "0x1111111111111111111111111111111111111111", - "--batch-submitter-address", - "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "--recovery", - "--checkpoint-block", - "10", - "--checkpoint-dump-dir", - "/nonexistent/checkpoint", // never read — refusal precedes the load - "--batch-submitter-private-key", - OTHER_KEY, - ]) - .expect("parse setup --recovery"); - let config = match cli.command { - Command::Setup(c) => *c, - other => panic!("expected setup subcommand, got {other:?}"), - }; - - let identity = DeploymentIdentity { - chain_id: 31337, - app_address: "0x1111111111111111111111111111111111111111" - .parse() - .unwrap(), - input_box_address: "0x2222222222222222222222222222222222222222" - .parse() - .unwrap(), - app_deployment_block: 0, - batch_submitter_address: pinned_submitter, - fee_oracle: FeeOracleIdentity::Fixed { log_gas_price: 0 }, - }; - let db = temp_db("recovery-wrong-key"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let result = flush_wallet_nonce(&config, &identity, 12, &mut storage).await; - match result { - Err(CommandError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { - fields, - .. - }))) => assert_eq!(fields, "batch_submitter_address"), - other => panic!("expected batch_submitter_address mismatch, got: {other:?}"), - } - } - - #[test] - fn source_fold_inputs_splits_seeds_and_replay_dropping_submitter_batches() { - use crate::storage::test_helpers::SENDER_B; - let db = temp_db("source-fold-inputs"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let protocol = default_protocol_timing(); - let submitter = SENDER_A; - let direct = SENDER_B; - - // A=5, B=20, C=40. (A,B] directs are seeds (submitter batches dropped); - // (B,C] is the full replay stream. - let mk = |sender, block, marker| StoredSafeInput { - sender, - payload: vec![marker], - block_number: block, - }; - let inputs = vec![ - mk(direct, 10, 0x10), // (A,B] direct → seed - mk(submitter, 15, 0x11), // (A,B] batch → dropped (already in S) - mk(direct, 20, 0x12), // (A,B] direct@B → seed - mk(submitter, 25, 0x13), // (B,C] batch → replay - mk(direct, 30, 0x14), // (B,C] direct → replay - ]; - storage - .append_safe_inputs(40, &inputs, submitter, &protocol) - .expect("seed"); - - let checkpoint = Checkpoint::<()> { - app: (), - executed_safe_block: 5, - checkpoint_nonce: 0, - checkpoint_block: 20, - }; - let (seeds, replay) = - source_fold_inputs(&mut storage, &checkpoint, 40, submitter).expect("source"); - - assert_eq!( - seeds.iter().map(|s| s.inclusion_block).collect::>(), - vec![10, 20], - "seeds are the (A,B] directs; the submitter batch @15 is dropped" - ); - assert_eq!( - replay.iter().map(|r| r.inclusion_block).collect::>(), - vec![25, 30], - "replay is the full (B,C] stream (batch + direct)" - ); - } - - #[test] - fn source_fold_inputs_replay_upper_bound_stays_at_c_when_resync_overshoots() { - // The recovery resync runs to the live head H1 > C, so safe_inputs holds - // directs past C. The fold's replay range must stay (B, C]: a direct in - // (C, H1] must NOT leak into replay (it would be folded into S' AND then - // left undrained for run = double execution — the (C, H1] bug class). Unit - // twin of the recovery e2e, pinning the boundary the bug lived at. - use crate::storage::test_helpers::SENDER_B; - let db = temp_db("source-fold-overshoot"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let protocol = default_protocol_timing(); - let submitter = SENDER_A; - let direct = SENDER_B; - - let mk = |block, marker| StoredSafeInput { - sender: direct, - payload: vec![marker], - block_number: block, - }; - // A=5, B=20, C=40, H1=50: a direct at block 50 is in the (C, H1] overshoot. - let inputs = vec![ - mk(10, 0x10), // (A,B] → seed - mk(30, 0x14), // (B,C] → replay - mk(50, 0x20), // (C,H1] → must NOT appear in replay - ]; - storage - .append_safe_inputs(50, &inputs, submitter, &protocol) - .expect("seed"); - - let checkpoint = Checkpoint::<()> { - app: (), - executed_safe_block: 5, - checkpoint_nonce: 0, - checkpoint_block: 20, - }; - let (seeds, replay) = - source_fold_inputs(&mut storage, &checkpoint, 40, submitter).expect("source"); - - assert_eq!( - seeds.iter().map(|s| s.inclusion_block).collect::>(), - vec![10] - ); - assert_eq!( - replay.iter().map(|r| r.inclusion_block).collect::>(), - vec![30], - "the (C, H1] direct @50 must be excluded from replay (upper bound is C, not H1)" + async fn a_genesis_application_must_start_at_zero() { + let scratch = tempfile::tempdir().unwrap(); + let error = base::load_base( + None, + || Ok(crate::test_support::StubApp::with_progress(1)), + base::read_stored_machine(), + SchedulerConfig::for_application::(SENDER_A), + scratch.path(), + ) + .await + .err() + .expect("a nonzero genesis is refused"); + assert!( + matches!(error, CommandError::AppBootstrap(AppError::Internal { .. })), + "{error:?}" ); + assert!(entries(scratch.path()).is_empty()); } } diff --git a/sequencer/src/commands/setup/rebuild.rs b/sequencer/src/commands/setup/rebuild.rs new file mode 100644 index 00000000..28e93d10 --- /dev/null +++ b/sequencer/src/commands/setup/rebuild.rs @@ -0,0 +1,1331 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Steps 4 to 6 of setup, over an input table synced through the stop block: +//! check the base's export against L1, then rebuild the history with the +//! scheduler, executing each transaction as it is handed out. Inputs and rows +//! move in pages, so memory stays bounded however long the range is. + +use alloy_primitives::Address; +use sequencer_core::application::{Application, Executed, execute}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::{ + L1Input, Scheduled, Scheduler, SchedulerConfig, SchedulerExport, SchedulerState, +}; + +use crate::commands::error::{CommandError, SetupError}; +use crate::storage::{SafeInputRange, Storage}; + +/// Inputs and rows per page. Tests use a tiny page, so their few rows +/// already cross page boundaries inside one input and inside the resume range. +#[cfg(not(test))] +const PAGE: usize = 1024; +#[cfg(test)] +const PAGE: usize = 2; + +/// Step 4: among the inputs the base consumed, the admitted directs above its +/// clock must be exactly the ones its export queues, and the base must lie at +/// or below the stop block. `inputs_through_stop` is the L1 input count +/// through the stop block. +pub(crate) fn check_export( + storage: &mut Storage, + export: &SchedulerExport, + inputs_through_stop: u64, + stop_block: u64, +) -> Result<(), CommandError> { + if export.l1_input_count > inputs_through_stop { + return Err(SetupError::BaseAheadOfStop { + l1_input_count: export.l1_input_count, + inputs_through_stop, + stop_block, + } + .into()); + } + let admitted = storage.admitted_directs_above_clock(export.l1_input_count, export.clock)?; + let starts_at_first_queued = admitted + .first_index + .is_none_or(|index| index == export.first_queued_input); + if admitted.count != export.fridge_len || !starts_at_first_queued { + return Err(SetupError::ExportDisagrees { + l1_input_count: export.l1_input_count, + clock: export.clock, + fridge_len: export.fridge_len, + first_queued_input: export.first_queued_input, + admitted: admitted.count, + first_admitted: admitted.first_index, + } + .into()); + } + Ok(()) +} + +/// The rest of step 4: resume the scheduler from the export over the inputs +/// it still had queued. Its refusals are the export check's, so the dry run +/// runs this too. Reads only the queued range and writes nothing. +pub(crate) fn resume_from_export( + storage: &mut Storage, + config: SchedulerConfig, + export: &SchedulerExport, +) -> Result { + let state = export.scheduler_state(); + let mut queued = InputPages::new(storage, state.first_queued_input, state.l1_input_count); + let resumed = Scheduler::resume(config, state, &mut queued); + // A failed read ends the range early; report it, not the short range. + if let Some(error) = queued.error { + return Err(error.into()); + } + Ok(resumed.map_err(SetupError::ResumeRefused)?) +} + +/// The scheduler's clock is a safe block a frame named, so it never lies past +/// the block the scheduler has consumed through. +pub(crate) fn check_clock(state: &SchedulerState, stop_block: u64) -> Result<(), CommandError> { + if state.clock > stop_block { + return Err(SetupError::ClockPastStop { + clock: state.clock, + stop_block, + } + .into()); + } + Ok(()) +} + +/// Steps 5 and 6: resume the scheduler from the export with the inputs it +/// still had queued, feed it every input it has not consumed through the stop +/// block, and execute each transaction it hands out through the shared executor. +/// An executed one is stored with its count; a skipped user op is not. An +/// application error stops setup at that transaction. Returns the +/// scheduler's state at the end of the stop block. +pub(crate) fn rebuild_history( + storage: &mut Storage, + app: &mut A, + config: SchedulerConfig, + export: &SchedulerExport, + inputs_through_stop: u64, + stop_block: u64, +) -> Result { + let mut scheduler = resume_from_export(storage, config, export)?; + check_clock(&scheduler.state(), stop_block)?; + + let mut rows: Vec<(L2TxCount, Scheduled)> = Vec::with_capacity(PAGE); + let mut next = export.l1_input_count; + while next < inputs_through_stop { + let page = storage.l1_inputs_page(next, inputs_through_stop, PAGE)?; + assert!( + !page.is_empty(), + "the input table ends at {next}, below the {inputs_through_stop} inputs it counted \ + through the stop block" + ); + for input in page { + let l1_input_index = input.index; + scheduler.feed(input); + // Every transaction the input released, skipped ones included. + let mut position: u64 = 0; + while let Some(scheduled) = scheduler.next_tx() { + let executed = execute(app, &scheduled.tx); + match (executed, &scheduled.tx) { + (Ok(Executed::Done { index, .. }), _) => { + rows.push((index, scheduled)); + if rows.len() == PAGE { + storage.append_rebuilt_history(&rows)?; + rows.clear(); + } + } + (Ok(Executed::Skipped(_)), L2Tx::UserOp(_)) => {} + (Ok(Executed::Skipped(_)), L2Tx::Direct(_)) => { + unreachable!("a direct never skips") + } + (Err(source), tx) => { + return Err(SetupError::HistoryExecution { + l1_input_index, + position, + transaction: describe(tx), + source, + } + .into()); + } + } + position = position + .checked_add(1) + .expect("an input's transaction count fits u64"); + } + next = l1_input_index + .checked_add(1) + .expect("an input index has a successor"); + } + } + storage.append_rebuilt_history(&rows)?; + + let end = scheduler.state(); + check_clock(&end, stop_block)?; + assert_drained_directs_stored(storage, export.first_queued_input, &end)?; + Ok(end) +} + +/// The stored rows hold only what executed, so the directs are checked against +/// the input table. The queue is FIFO and holds every admitted direct from the +/// base's first queued input on, so the drained ones are exactly the admitted +/// inputs before the final first queued input, and a direct always executes: +/// the stored directs must be those inputs, each one and no other. +fn assert_drained_directs_stored( + storage: &mut Storage, + base_first_queued_input: u64, + end: &SchedulerState, +) -> Result<(), CommandError> { + let drained = SafeInputRange::new(base_first_queued_input, end.first_queued_input); + let admitted = storage.admitted_directs_in(drained)?; + let rebuilt = storage.rebuilt_directs(drained)?; + assert!( + rebuilt.stored == admitted && rebuilt.admitted_in_range == admitted, + "the rebuild stored {} directs, {} of them admitted inputs in [{base_first_queued_input}, {}), \ + where {admitted} admitted inputs lie", + rebuilt.stored, + rebuilt.admitted_in_range, + end.first_queued_input + ); + Ok(()) +} + +/// The failing transaction for an error: it has no row to point at. +fn describe(tx: &L2Tx) -> String { + match tx { + L2Tx::Direct(direct) => format!("direct input {}", direct.input_index), + L2Tx::UserOp(op) => format!("user op of {} at nonce {}", op.sender, op.nonce), + } +} + +/// How many inputs in `[start, end)` came from `submitter`: the old +/// instance's activity past a base. +pub(crate) fn count_submitter_inputs( + storage: &mut Storage, + submitter: Address, + start: u64, + end: u64, +) -> Result { + let mut pages = InputPages::new(storage, start, end); + let count = pages + .by_ref() + .filter(|input| input.sender == submitter) + .count(); + if let Some(error) = pages.error { + return Err(error.into()); + } + Ok(u64::try_from(count).expect("a count of inputs fits u64")) +} + +/// The inputs `[start, end)` from storage, a page at a time. A read failure +/// ends the iteration and is kept for the caller. +struct InputPages<'a> { + storage: &'a mut Storage, + next: u64, + end: u64, + page: std::vec::IntoIter, + error: Option, +} + +impl<'a> InputPages<'a> { + fn new(storage: &'a mut Storage, start: u64, end: u64) -> Self { + Self { + storage, + next: start, + end, + page: Vec::new().into_iter(), + error: None, + } + } +} + +impl Iterator for InputPages<'_> { + type Item = L1Input; + + fn next(&mut self) -> Option { + if let Some(input) = self.page.next() { + return Some(input); + } + if self.error.is_some() || self.next >= self.end { + return None; + } + match self.storage.l1_inputs_page(self.next, self.end, PAGE) { + Ok(page) => { + let last = page.last()?.index; + self.next = last.checked_add(1).expect("an input index has a successor"); + self.page = page.into_iter(); + self.page.next() + } + Err(error) => { + self.error = Some(error); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{IngestedSafeInput, LifecycleCommand}; + use crate::test_support::{TestDb, pin_test_deployment_identity, temp_db}; + use alloy_primitives::B256; + use app_core::application::{WalletApp, WalletConfig}; + use sequencer_core::application::{AppError, AppOutputs, CanonicalState}; + use sequencer_core::batch::{Batch, Frame, WireUserOp}; + use sequencer_core::history::L2TxCount; + use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; + use sequencer_core::scheduler::{CanonicalRunner, TxFate}; + use ssz::Encode; + + const SUBMITTER: Address = Address::repeat_byte(0x99); + const DEPOSITOR: Address = Address::repeat_byte(0x22); + /// The chain and application `pin_test_deployment_identity` pins. + const CHAIN_ID: u64 = 1; + const APP: Address = Address::repeat_byte(0x11); + + fn wallet() -> WalletApp { + WalletApp::new(WalletConfig { + sequencer_address: SUBMITTER, + ..WalletConfig::default() + }) + } + + fn config() -> SchedulerConfig { + SchedulerConfig::for_application::(SUBMITTER) + } + + fn cap() -> u64 { + WalletApp::scheduler_limits().max_direct_payload_bytes + } + + fn direct(block: u64, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender: DEPOSITOR, + payload, + block_number: block, + block_timestamp: 1_700_000_000 + block, + transaction_hash: B256::repeat_byte(u8::try_from(block % 251).unwrap()), + } + } + + /// A batch from the submitter with one frame at `safe_block` and one + /// signed op of `signer` at `op_nonce`. + fn batch(nonce: u64, block: u64, safe_block: u64, op_nonce: u32) -> IngestedSafeInput { + use alloy::signers::{SignerSync, local::PrivateKeySigner}; + use alloy_sol_types::SolStruct; + let signer: PrivateKeySigner = format!("{:064x}", 7).parse().unwrap(); + let op = sequencer_core::user_op::UserOp { + nonce: op_nonce, + max_fee: 0, + data: vec![0xde, 0xad].into(), + }; + let domain = sequencer_core::build_input_domain(CHAIN_ID, APP); + let signature = signer + .sign_hash_sync(&op.eip712_signing_hash(&domain)) + .unwrap(); + IngestedSafeInput { + sender: SUBMITTER, + payload: Batch { + nonce, + frames: vec![Frame { + safe_block, + fee_price: 0, + user_ops: vec![WireUserOp { + nonce: op_nonce, + max_fee: 0, + data: op.data.to_vec(), + signature: signature.as_bytes().to_vec(), + }], + }], + } + .as_ssz_bytes(), + block_number: block, + block_timestamp: 1_700_000_000 + block, + transaction_hash: B256::repeat_byte(0xbb), + } + } + + fn l1_input(index: u64, input: &IngestedSafeInput) -> L1Input { + L1Input { + index, + sender: input.sender, + block: input.block_number, + timestamp: input.block_timestamp, + chain_id: CHAIN_ID, + app_contract: APP, + payload: input.payload.clone(), + } + } + + /// A setup database with `inputs` synced through `safe_block`. + fn synced(name: &str, inputs: &[IngestedSafeInput], safe_block: u64) -> (TestDb, Storage) { + let db = temp_db(name); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .append_ingested_safe_inputs_with_timestamp( + safe_block, + 1_700_000_000 + safe_block, + inputs, + ) + .unwrap(); + (db, storage) + } + + /// The chain the tests share: directs, an accepted batch that drains the + /// first, an over-cap direct, a stale batch, a direct left queued, and a + /// batch whose op is skipped for its nonce. + fn chain() -> Vec { + vec![ + direct(10, vec![1]), + direct(11, vec![2]), + batch(0, 12, 10, 0), + direct(13, vec![0; usize::try_from(cap()).unwrap() + 1]), + direct(14, vec![3]), + batch(0, 15, 13, 1), + direct(16, vec![4]), + batch(1, 17, 14, 5), + direct(30, vec![5]), + ] + } + + /// Run the reference over `inputs[..count]`. + fn reference(inputs: &[IngestedSafeInput], count: usize) -> CanonicalRunner { + let mut runner = CanonicalRunner::genesis(wallet(), config()); + for (index, input) in (0_u64..).zip(&inputs[..count]) { + runner.process_input(l1_input(index, input)).unwrap(); + } + runner + } + + /// Every stored row with its count, in count order. A direct reads its + /// fields back from the input table. + fn stored(storage: &mut Storage) -> Vec<(L2TxCount, Scheduled)> { + let to_u64 = |value: i64| u64::try_from(value).unwrap(); + storage + .raw_read(|tx| { + let mut statement = tx.prepare( + "SELECT r.l2_tx_count, r.batch_nonce, r.safe_input_index, r.sender, \ + r.nonce, r.fee, r.safe_block, r.data, \ + d.sender, d.block_number, d.block_timestamp, d.payload \ + FROM rebuilt_history r \ + LEFT JOIN safe_inputs d ON d.safe_input_index = r.safe_input_index \ + ORDER BY r.l2_tx_count", + )?; + statement + .query_map([], |row| { + let tx = match row.get::<_, Option>(2)? { + Some(input_index) => L2Tx::Direct(Direct { + input_index: to_u64(input_index), + sender: Address::from_slice(&row.get::<_, Vec>(8)?), + block: to_u64(row.get(9)?), + timestamp: to_u64(row.get(10)?), + payload: row.get(11)?, + }), + None => L2Tx::UserOp(UserOpTx { + sender: Address::from_slice(&row.get::<_, Vec>(3)?), + nonce: u32::try_from(row.get::<_, i64>(4)?).unwrap(), + fee: u16::try_from(row.get::<_, i64>(5)?).unwrap(), + block: to_u64(row.get(6)?), + data: row.get(7)?, + }), + }; + Ok(( + L2TxCount::new(to_u64(row.get(0)?)), + Scheduled { + batch_nonce: to_u64(row.get(1)?), + tx, + }, + )) + })? + .collect::>>() + }) + .unwrap() + } + + /// What the reference does with `inputs[from..to]`: the transactions it + /// executes, each with its count, and the user ops it skips. + struct Expected { + executed: Vec<(L2TxCount, Scheduled)>, + skipped: Vec, + } + + fn expected( + runner: &mut CanonicalRunner, + inputs: &[IngestedSafeInput], + from: usize, + to: usize, + ) -> Expected { + let mut expected = Expected { + executed: Vec::new(), + skipped: Vec::new(), + }; + for (index, input) in (u64::try_from(from).unwrap()..).zip(&inputs[from..to]) { + let advanced = runner.process_input(l1_input(index, input)).unwrap(); + for record in advanced.txs { + match record.fate { + TxFate::Done { index } => expected.executed.push((index, record.scheduled)), + TxFate::Skipped(_) => expected.skipped.push(record.scheduled), + } + } + } + expected + } + + /// The stored rows are the executed ones, in order, with their counts, + /// batch nonces and contents; no skipped op is stored. + fn assert_stored(storage: &mut Storage, expected: &Expected) { + let stored = stored(storage); + assert_eq!(stored, expected.executed); + for skipped in &expected.skipped { + assert!( + stored.iter().all(|(_, row)| row != skipped), + "a skipped op is stored: {skipped:?}" + ); + } + } + + #[test] + fn genesis_rebuild_matches_the_canonical_runner_row_for_row() { + let inputs = chain(); + let (_db, mut storage) = synced("rebuild-genesis", &inputs, 30); + let through_stop = storage.safe_input_count_through_block(30).unwrap(); + assert_eq!(through_stop, 9); + let mut runner = reference(&inputs, 0); + let export = runner.export(); + + check_export(&mut storage, &export, through_stop, 30).unwrap(); + let mut app = wallet(); + let state = + rebuild_history(&mut storage, &mut app, config(), &export, through_stop, 30).unwrap(); + + let expected = expected(&mut runner, &inputs, 0, 9); + assert_stored(&mut storage, &expected); + assert!( + !expected.skipped.is_empty(), + "the fixture holds a skipped op" + ); + assert_eq!(state, runner.scheduler().state()); + assert_eq!( + (state.first_queued_input, state.fridge_len), + (6, 2), + "the directs after the last frame stay queued" + ); + assert_eq!(app.l2_tx_count(), runner.app().l2_tx_count()); + assert_eq!( + app.canonical_snapshot_bytes().unwrap(), + runner.app().canonical_snapshot_bytes().unwrap() + ); + } + + /// The worked boundary over the wallet's window `W`. A malformed batch + /// from the submitter consumes no batch nonce, yet the backstop runs + /// before decoding and executes the overdue directs; the last frame + /// drains the direct at its own safe block, so nothing stays queued. + /// + /// | L1 input | Block | Input | Executes | + /// |---:|---:|---|---| + /// | 0 | 5 | `D0` | queued | + /// | 1 | 12 | `D1` | queued | + /// | 2 | 20 | batch 0, safe block 10, `U0` | `D0`, `U0` | + /// | 3 | 20 | `D2`, after the batch | queued | + /// | 4 | 24 | `D3` | queued | + /// | 5 | `W + 30` | malformed batch | the backstop: `D1`, `D2`, `D3` | + /// | 6 | `W + 31` | `D4` | queued | + /// | 7 | `W + 32` | batch 1, safe block `W + 31`, `U1` | `D4`, `U1` | + fn worked_boundary() -> Vec { + let window = config().max_wait_blocks; + vec![ + direct(5, vec![1]), + direct(12, vec![2]), + batch(0, 20, 10, 0), + direct(20, vec![3]), + direct(24, vec![4]), + IngestedSafeInput { + sender: SUBMITTER, + ..direct(window + 30, vec![0]) + }, + direct(window + 31, vec![5]), + batch(1, window + 32, window + 31, 1), + ] + } + + /// Each stored row: its L2 tx index, and the direct's input index (`None` + /// for a user op). + fn stored_rows(storage: &mut Storage) -> Vec<(u64, Option)> { + stored(storage) + .into_iter() + .map(|(count, scheduled)| { + let direct = match scheduled.tx { + L2Tx::Direct(direct) => Some(direct.input_index), + L2Tx::UserOp(_) => None, + }; + (count.get(), direct) + }) + .collect() + } + + /// The [worked boundary](worked_boundary) through stop block `W + 40`: + /// the era has 7 transactions, next batch nonce 2 and first queued input + /// 8, the input count through the stop block. + #[test] + fn the_worked_boundary_rebuilds_seven_transactions_and_queues_nothing() { + let stop = config().max_wait_blocks + 40; + let inputs = worked_boundary(); + let (_db, mut storage) = synced("rebuild-worked-boundary", &inputs, stop); + let data_dir = tempfile::tempdir().unwrap(); + let summary = finish(&mut storage, stop, stop, (3, 3), data_dir.path()).unwrap(); + + // Every transaction of this trace executes. + assert_eq!( + stored_rows(&mut storage), + vec![ + (0, Some(0)), + (1, None), + (2, Some(1)), + (3, Some(3)), + (4, Some(4)), + (5, Some(6)), + (6, None), + ] + ); + assert_eq!(summary.inputs_to_replay, 8); + assert_eq!( + summary.era, + Some(crate::storage::NewEra { + base: L2TxCount::ZERO, + rebuilt_end: L2TxCount::new(7), + stop_block: stop, + first_queued_input: 8, + anchor_batch_nonce: 2, + }) + ); + assert_eq!(storage.era().unwrap().anchor_batch_nonce, 2); + // The first live frame drains from the first input past the stop block. + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 8); + + let mut runner = reference(&inputs, 0); + let expected = expected(&mut runner, &inputs, 0, 8); + assert!(expected.skipped.is_empty()); + assert_stored(&mut storage, &expected); + } + + /// The [worked boundary](worked_boundary) stopped right after the + /// malformed batch, at `W + 30`: only the backstop can have executed + /// `D1`, `D2` and `D3`, since no later frame drains them. Without it they + /// would still be queued. + #[test] + fn the_worked_boundary_backstop_drains_at_the_malformed_batch() { + let stop = config().max_wait_blocks + 30; + let inputs = &worked_boundary()[..6]; + let (_db, mut storage) = synced("rebuild-worked-boundary-backstop", inputs, stop); + let data_dir = tempfile::tempdir().unwrap(); + let summary = finish(&mut storage, stop, stop, (3, 3), data_dir.path()).unwrap(); + + // `D0`, `U0`, then the backstop's `D1`, `D2`, `D3`. + assert_eq!( + stored_rows(&mut storage), + vec![ + (0, Some(0)), + (1, None), + (2, Some(1)), + (3, Some(3)), + (4, Some(4)), + ] + ); + assert_eq!(summary.inputs_to_replay, 6); + assert_eq!( + summary.era, + Some(crate::storage::NewEra { + base: L2TxCount::ZERO, + rebuilt_end: L2TxCount::new(5), + stop_block: stop, + first_queued_input: 6, + anchor_batch_nonce: 1, + }) + ); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 6); + + let mut runner = reference(inputs, 0); + assert_stored(&mut storage, &expected(&mut runner, inputs, 0, 6)); + } + + /// A base in the middle with two directs queued: the rebuild resumes with + /// them and lands where the reference over the whole range does. + #[test] + fn rebuild_from_a_base_with_queued_directs_matches_the_whole_range() { + let inputs = chain(); + let (_db, mut storage) = synced("rebuild-middle", &inputs, 30); + let mut runner = reference(&inputs, 5); + let export = runner.export(); + assert_eq!((export.fridge_len, export.first_queued_input), (2, 1)); + + check_export(&mut storage, &export, 9, 30).unwrap(); + let mut app = runner.app().clone(); + let state = rebuild_history(&mut storage, &mut app, config(), &export, 9, 30).unwrap(); + + let expected = expected(&mut runner, &inputs, 5, 9); + assert_stored(&mut storage, &expected); + assert_eq!(state, runner.scheduler().state()); + assert_eq!( + app.canonical_snapshot_bytes().unwrap(), + runner.app().canonical_snapshot_bytes().unwrap() + ); + } + + /// Inputs past the stop block are synced but never fed. + #[test] + fn rebuild_stops_at_the_stop_block_not_the_synced_head() { + let inputs = chain(); + let (_db, mut storage) = synced("rebuild-stop", &inputs, 30); + let through_stop = storage.safe_input_count_through_block(17).unwrap(); + assert_eq!(through_stop, 8); + let mut runner = reference(&inputs, 0); + let export = runner.export(); + + let mut app = wallet(); + let state = + rebuild_history(&mut storage, &mut app, config(), &export, through_stop, 17).unwrap(); + + assert_stored(&mut storage, &expected(&mut runner, &inputs, 0, 8)); + assert_eq!(state, runner.scheduler().state()); + assert_eq!(app.l2_tx_count(), runner.app().l2_tx_count()); + assert_eq!(state.l1_input_count, 8); + } + + #[test] + fn export_check_refuses_a_disagreeing_queue() { + let inputs = chain(); + let (_db, mut storage) = synced("export-check", &inputs, 30); + let honest = reference(&inputs, 5).export(); + check_export(&mut storage, &honest, 9, 30).unwrap(); + + let refused = |storage: &mut Storage, export: SchedulerExport| match check_export( + storage, &export, 9, 30, + ) { + Err(CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error))) => *error, + other => panic!("expected a setup refusal, got {other:?}"), + }; + // One queued direct too few. + assert!(matches!( + refused( + &mut storage, + SchedulerExport { + fridge_len: 1, + ..honest + } + ), + SetupError::ExportDisagrees { + admitted: 2, + fridge_len: 1, + first_admitted: Some(1), + .. + } + )); + // The right number, starting at the wrong input. + assert!(matches!( + refused( + &mut storage, + SchedulerExport { + first_queued_input: 4, + ..honest + } + ), + SetupError::ExportDisagrees { + first_queued_input: 4, + first_admitted: Some(1), + .. + } + )); + // A clock past a queued direct: one fewer lies above it. + assert!(matches!( + refused( + &mut storage, + SchedulerExport { + clock: 11, + ..honest + } + ), + SetupError::ExportDisagrees { + admitted: 1, + first_admitted: Some(4), + .. + } + )); + // Nothing queued while a direct lies above the clock. + assert!(matches!( + refused( + &mut storage, + SchedulerExport { + fridge_len: 0, + first_queued_input: 5, + ..honest + } + ), + SetupError::ExportDisagrees { admitted: 2, .. } + )); + // A base past the stop block. + assert!(matches!( + match check_export(&mut storage, &honest, 4, 13) { + Err(CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error))) => + *error, + other => panic!("expected a setup refusal, got {other:?}"), + }, + SetupError::BaseAheadOfStop { + l1_input_count: 5, + inputs_through_stop: 4, + stop_block: 13 + } + )); + } + + /// The scheduler's own resume checks count as the export check. This + /// export queues nothing and no admitted direct lies above its clock, so + /// the table check passes; but it names the over-cap direct as its first + /// queued input, which resume refuses. + #[test] + fn resume_refusal_is_an_export_refusal_and_stores_nothing() { + let inputs = chain(); + let (_db, mut storage) = synced("resume-refused", &inputs, 30); + let export = SchedulerExport { + clock: 14, + first_queued_input: 3, + fridge_len: 0, + ..reference(&inputs, 5).export() + }; + check_export(&mut storage, &export, 9, 30).unwrap(); + let mut app = reference(&inputs, 5).app().clone(); + let error = rebuild_history(&mut storage, &mut app, config(), &export, 9, 30) + .expect_err("a range that does not match the state is refused"); + assert!( + matches!( + &error, + CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error)) + if matches!( + **error, + SetupError::ResumeRefused( + sequencer_core::scheduler::ResumeError::FirstQueuedNotADirect { + index: 3 + } + ) + ) + ), + "{error:?}" + ); + assert!(stored(&mut storage).is_empty()); + } + + /// An application that fails on its direct number `fail_at` (from 0). + struct FailingApp { + count: L2TxCount, + nonce: u32, + directs: u32, + fail_at: u32, + } + + impl Application for FailingApp { + fn user_nonce(&self, _sender: Address) -> Result { + Ok(self.nonce) + } + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { + self.nonce += 1; + self.count.advance(); + Ok(Vec::new()) + } + fn apply_direct(&mut self, _direct: &Direct) -> Result { + if self.directs == self.fail_at { + return Err(AppError::Internal { + reason: "injected".into(), + }); + } + self.directs += 1; + self.count.advance(); + Ok(Vec::new()) + } + fn l2_tx_count(&self) -> L2TxCount { + self.count + } + fn max_method_payload_bytes() -> usize { + 0 + } + fn scheduler_limits() -> sequencer_core::application::SchedulerLimits { + WalletApp::scheduler_limits() + } + fn admit_user_op( + &self, + _sender: Address, + _op: &sequencer_core::user_op::UserOp, + _fee: u16, + ) -> Result { + unreachable!("setup never admits") + } + fn from_dump(_prefix: &std::path::Path) -> Result { + unreachable!("setup never loads a dump") + } + fn create_dump(&mut self, _prefix: &std::path::Path) -> Result<(), AppError> { + unreachable!("a failed setup writes no dump") + } + fn state_file_in_dump(prefix: &std::path::Path) -> std::path::PathBuf { + prefix.to_path_buf() + } + fn from_canonical_state( + _state_file: &std::path::Path, + _count: L2TxCount, + ) -> Result { + unreachable!("built directly") + } + } + + /// An error stops setup at the transaction, named by its input, its + /// position among the transactions that input released, and what it is, + /// since it has no row. Pages flushed before it stay. + #[test] + fn an_application_error_names_the_input_position_and_transaction() { + let inputs = chain(); + let (_db, mut storage) = synced("execute-fails", &inputs, 30); + let export = reference(&inputs, 0).export(); + let mut app = FailingApp { + count: L2TxCount::ZERO, + nonce: 0, + directs: 0, + fail_at: 2, + }; + let error = rebuild_history(&mut storage, &mut app, config(), &export, 9, 30) + .expect_err("the application fails"); + // The third direct is input 4, the second drained by the batch at + // input 7, after input 1. + let CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error)) = error else { + panic!("expected a setup refusal, got {error:?}"); + }; + assert!( + matches!( + &*error, + SetupError::HistoryExecution { + l1_input_index: 7, + position: 1, + transaction, + .. + } if transaction == "direct input 4" + ), + "{error:?}" + ); + assert!( + error + .to_string() + .contains("at input 7, position 1 (direct input 4): "), + "{error}" + ); + let counts: Vec = stored(&mut storage) + .into_iter() + .map(|(count, _)| count.get()) + .collect(); + assert_eq!( + counts, + vec![0, 1], + "the first page is stored, the second not" + ); + + let op = L2Tx::UserOp(UserOpTx { + sender: DEPOSITOR, + nonce: 3, + fee: 0, + block: 0, + data: Vec::new(), + }); + assert_eq!(describe(&op), format!("user op of {DEPOSITOR} at nonce 3")); + } + + /// The cross-check reads the stored rows: a drained direct with no row + /// leaves the stored directs short of the input table's. No input makes the + /// loop drop a direct, so this drives the check on a tampered table; every + /// rebuild test above passes through the same check untampered. + #[test] + #[should_panic( + expected = "the rebuild stored 2 directs, 2 of them admitted inputs in [0, 6), \ + where 3 admitted inputs lie" + )] + fn a_drained_direct_without_a_row_trips_the_cross_check() { + let inputs = chain(); + let (_db, mut storage) = synced("cross-check", &inputs, 30); + let export = reference(&inputs, 0).export(); + let mut app = wallet(); + let end = rebuild_history(&mut storage, &mut app, config(), &export, 9, 30).unwrap(); + // Inputs 0, 1 and 4 drained; 2 is the submitter's and 3 is over the cap. + assert_eq!(end.first_queued_input, 6); + assert_drained_directs_stored(&mut storage, 0, &end).unwrap(); + storage + .raw_write(|tx| { + tx.execute_batch( + "DROP TRIGGER trg_rebuilt_history_not_deletable; + DELETE FROM rebuilt_history WHERE safe_input_index = 4;", + ) + }) + .unwrap(); + let _ = assert_drained_directs_stored(&mut storage, 0, &end); + } + + /// A fridge at its bound drains whole blocks on arrival, and those + /// directs execute too: the rebuild still matches the reference, and the + /// cross-check counts them. + #[test] + fn an_overflowing_fridge_rebuilds_like_the_reference() { + let inputs = chain(); + let (_db, mut storage) = synced("rebuild-overflow", &inputs, 30); + let bounded = SchedulerConfig { + fridge_bound: 1, + ..config() + }; + let mut runner = CanonicalRunner::genesis(wallet(), bounded); + let export = runner.export(); + let mut app = wallet(); + let state = rebuild_history(&mut storage, &mut app, bounded, &export, 9, 30).unwrap(); + + let expected = expected(&mut runner, &inputs, 0, 9); + assert_stored(&mut storage, &expected); + assert_eq!(state, runner.scheduler().state()); + assert_eq!( + (state.first_queued_input, state.fridge_len), + (8, 1), + "every direct but the last drained" + ); + } + + /// An export whose clock lies past the stop block passes the table check + /// and resumes, but no scheduler at the stop block can hold that clock. + #[test] + fn an_export_clock_past_the_stop_block_is_refused() { + let inputs = chain(); + let (_db, mut storage) = synced("clock-past-stop", &inputs, 30); + let past = SchedulerExport { + clock: 31, + first_queued_input: 5, + fridge_len: 0, + ..reference(&inputs, 5).export() + }; + check_export(&mut storage, &past, 9, 30).unwrap(); + let mut app = reference(&inputs, 5).app().clone(); + let error = rebuild_history(&mut storage, &mut app, config(), &past, 9, 30) + .expect_err("the clock is past the stop block"); + assert!( + matches!( + &error, + CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error)) + if matches!(**error, SetupError::ClockPastStop { clock: 31, stop_block: 30 }) + ), + "{error:?}" + ); + + // At the stop block itself the clock is admissible. + let (_db, mut storage) = synced("clock-at-stop", &inputs, 30); + let at = SchedulerExport { clock: 30, ..past }; + let mut app = reference(&inputs, 5).app().clone(); + rebuild_history(&mut storage, &mut app, config(), &at, 9, 30).unwrap(); + } + + /// A batch whose frame drains every queued direct, then one more direct. + fn drained_chain() -> Vec { + vec![ + direct(10, vec![1]), + batch(0, 12, 11, 0), + direct(13, vec![2]), + ] + } + + /// A base with nothing queued resumes over an empty range, and one at the + /// stop block feeds nothing: the era rebuilds no row and ends at the base. + #[test] + fn a_base_with_an_empty_fridge_or_at_the_stop_block_rebuilds_nothing() { + let inputs = drained_chain(); + let base = reference(&inputs, 2); + let export = base.export(); + assert_eq!( + ( + export.l1_input_count, + export.first_queued_input, + export.fridge_len + ), + (2, 2, 0) + ); + + // Past the base, a direct that only queues. + let (_db, mut storage) = synced("empty-fridge-base", &inputs, 13); + check_export(&mut storage, &export, 3, 13).unwrap(); + let mut app = base.app().clone(); + let state = rebuild_history(&mut storage, &mut app, config(), &export, 3, 13).unwrap(); + assert!(stored(&mut storage).is_empty()); + assert_eq!(state, reference(&inputs, 3).scheduler().state()); + + // The base is the stop block's own state. + let (_db, mut storage) = synced("base-at-stop", &inputs, 13); + let dumps = tempfile::tempdir().unwrap(); + let summary = crate::commands::setup::finish_setup( + &mut storage, + crate::commands::setup::base::Base { + export, + app: base.app().clone(), + }, + config(), + crate::commands::setup::Synced { + stop_block: 12, + synced_safe_block: 13, + wallet: (1, 1), + }, + dumps.path(), + ) + .unwrap(); + assert_eq!(summary.inputs_to_replay, 0); + assert!(stored(&mut storage).is_empty()); + assert_eq!( + summary.era, + Some(crate::storage::NewEra { + base: L2TxCount::new(export.l2_tx_count), + rebuilt_end: L2TxCount::new(export.l2_tx_count), + stop_block: 12, + first_queued_input: 2, + anchor_batch_nonce: 1, + }) + ); + } + + fn genesis_base() -> crate::commands::setup::base::Base { + crate::commands::setup::base::Base { + export: reference(&[], 0).export(), + app: wallet(), + } + } + + fn finish( + storage: &mut Storage, + stop_block: u64, + synced_safe_block: u64, + wallet: (u64, u64), + data_dir: &std::path::Path, + ) -> Result { + crate::commands::setup::finish_setup( + storage, + genesis_base(), + config(), + crate::commands::setup::Synced { + stop_block, + synced_safe_block, + wallet, + }, + data_dir, + ) + } + + /// The sync overshoots the stop block; the era is the scheduler's at the + /// stop block, never at the synced head. + #[test] + fn setup_publishes_the_era_at_the_stop_block_not_the_synced_head() { + let inputs = chain(); + let (_db, mut storage) = synced("finish-overshoot", &inputs, 30); + let data_dir = tempfile::tempdir().unwrap(); + let summary = finish(&mut storage, 17, 30, (3, 3), data_dir.path()).unwrap(); + + let at_stop = reference(&inputs, 8); + let state = at_stop.scheduler().state(); + assert_eq!( + summary.era, + Some(crate::storage::NewEra { + base: L2TxCount::ZERO, + rebuilt_end: at_stop.app().l2_tx_count(), + stop_block: 17, + first_queued_input: state.first_queued_input, + anchor_batch_nonce: state.next_batch_nonce, + }) + ); + assert_eq!(summary.inputs_to_replay, 8); + let published = storage.era().unwrap(); + assert_eq!(published.stop_block, 17); + assert_eq!(published.anchor_batch_nonce, state.next_batch_nonce); + // No row comes from an input past the stop block. + assert_stored( + &mut storage, + &expected(&mut reference(&inputs, 0), &inputs, 0, 8), + ); + } + + /// Every refusal after the sync comes before the rebuild writes a row. + #[test] + fn setup_refuses_after_the_sync_before_rebuilding() { + let inputs = chain(); + type Refusal = (&'static str, u64, (u64, u64), fn(&CommandError) -> bool); + let refusals: [Refusal; 3] = [ + // The sync stopped below the stop block. + ("finish-sync-behind", 31, (3, 3), |error| { + matches!( + error, + CommandError::Bootstrap(crate::commands::BootstrapError::Setup(error)) + if matches!( + **error, + SetupError::SyncBehindStopBlock { + synced_safe_block: 30, + stop_block: 31 + } + ) + ) + }), + // The batch at input 7 (block 17) lies past stop block 15. + ("finish-escape-input", 15, (3, 3), |error| { + matches!( + error, + CommandError::Bootstrap(crate::commands::BootstrapError::SetupRefuse( + crate::commands::error::SetupRefuse::SubmitterInputPastStop { + stop_block: 15, + found_block: 17, + safe_input_index: 7, + } + )) + ) + }), + ("finish-escape-wallet", 30, (4, 3), |error| { + matches!( + error, + CommandError::Bootstrap(crate::commands::BootstrapError::SetupRefuse( + crate::commands::error::SetupRefuse::WalletUnsettledAfterFlush { + pending: 4, + safe: 3, + } + )) + ) + }), + ]; + for (name, stop_block, wallet, expected) in refusals { + let (_db, mut storage) = synced(name, &inputs, 30); + let data_dir = tempfile::tempdir().unwrap(); + let error = + finish(&mut storage, stop_block, 30, wallet, data_dir.path()).expect_err(name); + assert!(expected(&error), "{name}: {error:?}"); + if !name.starts_with("finish-sync") { + assert_eq!( + error.exit_code(), + crate::commands::error::EXIT_SETUP_ESCAPE, + "{name}" + ); + } + assert!(!storage.is_setup_complete().unwrap(), "{name}"); + assert!(stored(&mut storage).is_empty(), "{name}"); + assert!(!data_dir.path().join("dumps").exists(), "{name}"); + } + } + + /// The era's root frame drains what the canonical scheduler drains with + /// the era's first frame: the directs queued at the stop block, skipping + /// the submitter's input and the over-cap direct among them, then the + /// directs up to the root frame's safe block, all in the anchor batch. + #[test] + fn the_root_frame_drains_what_the_scheduler_drains_with_the_first_frame() { + let inputs = vec![ + direct(10, vec![1]), + direct(11, vec![2]), + batch(0, 12, 10, 0), + direct(13, vec![0; usize::try_from(cap()).unwrap() + 1]), + direct(14, vec![3]), + direct(20, vec![4]), + ]; + // Inside the window, so run's terminal-fact check lets the root open. + let db = temp_db("root-drain"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + let now_ms = crate::clock::unix_now_ms(); + storage + .append_ingested_safe_inputs_with_timestamp(30, now_ms / 1000, &inputs) + .unwrap(); + let data_dir = tempfile::tempdir().unwrap(); + let era = finish(&mut storage, 14, 30, (1, 1), data_dir.path()) + .unwrap() + .era + .unwrap(); + assert_eq!((era.first_queued_input, era.anchor_batch_nonce), (1, 1)); + + // Run's startup opens the root at its own safe head. + assert!(storage.open_batch_if_none().unwrap()); + let root = storage.open_state().unwrap().expect("the root is open"); + assert_eq!(root.safe_block(), 30); + let era_id = storage.era().unwrap().id; + let drained: Vec<(u64, u64, u64)> = storage + .canonical_history_page( + sequencer_core::history::HistoryClaim { + era_id, + next_input: era.rebuilt_end, + }, + 16, + ) + .unwrap() + .rows + .into_iter() + .map(|row| match row.context { + crate::storage::L2TxContext::Direct { + tx, batch_nonce, .. + } => (row.offset.get(), tx.input_index, batch_nonce), + other => panic!("the root frame drains only directs, got {other:?}"), + }) + .collect(); + + // The canonical scheduler at the stop block, then the inputs past it, + // then the root's batch landing with its one frame. + let mut runner = reference(&inputs, 5); + let mut handed_out = Vec::new(); + let root_batch = IngestedSafeInput { + sender: SUBMITTER, + payload: Batch { + nonce: era.anchor_batch_nonce, + frames: vec![Frame { + safe_block: root.safe_block(), + fee_price: 0, + user_ops: Vec::new(), + }], + } + .as_ssz_bytes(), + block_number: 31, + block_timestamp: 1_700_000_031, + transaction_hash: B256::repeat_byte(0xcc), + }; + for (index, input) in (5_u64..).zip(inputs[5..].iter().chain([&root_batch])) { + for record in runner.process_input(l1_input(index, input)).unwrap().txs { + let TxFate::Done { index: l2_tx_index } = record.fate else { + panic!("a direct never skips"); + }; + let L2Tx::Direct(direct) = record.scheduled.tx else { + panic!("the frame holds no op"); + }; + handed_out.push(( + l2_tx_index.get(), + direct.input_index, + record.scheduled.batch_nonce, + )); + } + } + assert_eq!(drained, handed_out); + assert_eq!( + drained + .iter() + .map(|(_, input_index, _)| *input_index) + .collect::>(), + vec![1, 4, 5] + ); + } + + #[test] + fn counts_the_submitters_inputs_in_a_range() { + let inputs = chain(); + let (_db, mut storage) = synced("count-submitter", &inputs, 30); + assert_eq!( + count_submitter_inputs(&mut storage, SUBMITTER, 0, 9).unwrap(), + 3 + ); + assert_eq!( + count_submitter_inputs(&mut storage, SUBMITTER, 3, 9).unwrap(), + 2 + ); + assert_eq!( + count_submitter_inputs(&mut storage, SUBMITTER, 8, 9).unwrap(), + 0 + ); + assert_eq!( + count_submitter_inputs(&mut storage, DEPOSITOR, 9, 9).unwrap(), + 0 + ); + } +} diff --git a/sequencer/src/egress/api/health.rs b/sequencer/src/egress/api/health.rs index 83e65559..a3a0129a 100644 --- a/sequencer/src/egress/api/health.rs +++ b/sequencer/src/egress/api/health.rs @@ -3,7 +3,8 @@ //! Health probes `/livez`, `/readyz` and `/healthz` (contract: README "Health //! probes"). They live on the internal listener because operators and kubelet -//! probe from inside the cluster. +//! probe from inside the cluster. `/healthz` also carries the landing watch's +//! latest reading for monitoring; it never changes the status code. use std::sync::Arc; @@ -15,6 +16,7 @@ use serde::Serialize; use tokio::sync::mpsc; use crate::ingress::inclusion_lane::PendingUserOp; +use crate::landing::LandingFeed; use crate::runtime::shutdown::RuntimeScope; /// Narrow health-check state. Holds only the signals the probes inspect; the @@ -24,12 +26,24 @@ use crate::runtime::shutdown::RuntimeScope; pub(crate) struct HealthState { pub tx_sender: mpsc::Sender, pub shutdown: RuntimeScope, + /// The landing watch's latest reading. + pub landing: LandingFeed, } #[derive(Serialize)] struct HealthStatus { status: &'static str, inclusion_lane: &'static str, + /// The oldest unaccepted batch's age, in blocks, at the earliest block it + /// can still land in (`H + 1 - s`); the history is void once it reaches + /// `max_wait_blocks`. `null` when every batch is accepted or before the + /// first reading. + landing_lag_blocks: Option, + /// The application's staleness window, pinned at setup. + max_wait_blocks: u64, + /// Wall clock minus the safe block's L1 timestamp, saturating at 0; + /// `null` before the first reading. + safe_head_age_seconds: Option, } pub(crate) async fn livez() -> StatusCode { @@ -49,9 +63,13 @@ pub(crate) async fn healthz(State(state): State>) -> impl IntoR let shutting_down = state.shutdown.is_shutdown_requested(); let all_ok = lane_ok && !shutting_down; + let reading = *state.landing.borrow(); let body = HealthStatus { status: if all_ok { "ok" } else { "degraded" }, inclusion_lane: if lane_ok { "ok" } else { "stopped" }, + landing_lag_blocks: reading.landing_lag_blocks(), + max_wait_blocks: reading.max_wait_blocks, + safe_head_age_seconds: reading.safe_head_age_seconds(crate::clock::unix_now_ms()), }; let status = if all_ok { @@ -65,16 +83,26 @@ pub(crate) async fn healthz(State(state): State>) -> impl IntoR #[cfg(test)] mod tests { use super::*; + use crate::landing::LandingReading; + use crate::storage::LandingLag; fn fresh_state() -> (Arc, mpsc::Receiver) { let (tx_sender, rx) = mpsc::channel::(1); let state = Arc::new(HealthState { tx_sender, shutdown: RuntimeScope::default(), + landing: crate::landing::test_feed(1200), }); (state, rx) } + async fn body_of(response: axum::response::Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read health body"); + serde_json::from_slice(&bytes).expect("decode health body") + } + #[tokio::test] async fn livez_is_always_ok() { assert_eq!(livez().await, StatusCode::OK); @@ -107,4 +135,68 @@ mod tests { let response = healthz(State(state)).await.into_response(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } + + /// Before the first reading only the window is known. + #[tokio::test] + async fn healthz_reports_the_window_and_no_lag_before_the_first_reading() { + let (state, _rx) = fresh_state(); + let response = healthz(State(state)).await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body_of(response).await, + serde_json::json!({ + "status": "ok", + "inclusion_lane": "ok", + "landing_lag_blocks": null, + "max_wait_blocks": 1200, + "safe_head_age_seconds": null, + }) + ); + } + + /// The published reading reaches the body, and a lag past the window does + /// not change the status code: the landing watch, not the probe, stops + /// the process. + #[tokio::test] + async fn healthz_reports_the_published_landing_reading() { + let (tx_sender, _rx) = mpsc::channel::(1); + let (publish, landing) = tokio::sync::watch::channel(LandingReading { + lag: None, + max_wait_blocks: 1200, + safe_block_timestamp: None, + }); + let state = Arc::new(HealthState { + tx_sender, + shutdown: RuntimeScope::default(), + landing, + }); + let now_secs = crate::clock::unix_now_ms() / 1000; + publish.send_replace(LandingReading { + lag: Some(LandingLag { + batch_nonce: 4, + first_frame_safe_block: 100, + safe_block: 1400, + }), + max_wait_blocks: 1200, + // An hour old, so the age is not at the 0 floor. + safe_block_timestamp: Some(now_secs - 3600), + }); + let response = healthz(State(state.clone())).await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + let body = body_of(response).await; + assert_eq!(body["landing_lag_blocks"], 1301); + assert_eq!(body["max_wait_blocks"], 1200); + let age = body["safe_head_age_seconds"].as_u64().expect("an age"); + assert!((3600..3700).contains(&age), "age {age}"); + + // A safe block stamped ahead of the local clock reads as age 0. + publish.send_replace(LandingReading { + lag: None, + max_wait_blocks: 1200, + safe_block_timestamp: Some(now_secs + 600), + }); + let body = body_of(healthz(State(state)).await.into_response()).await; + assert_eq!(body["landing_lag_blocks"], serde_json::Value::Null); + assert_eq!(body["safe_head_age_seconds"], 0); + } } diff --git a/sequencer/src/egress/api/history.rs b/sequencer/src/egress/api/history.rs index caaad0e8..afc583e6 100644 --- a/sequencer/src/egress/api/history.rs +++ b/sequencer/src/egress/api/history.rs @@ -1,76 +1,38 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Coherent history metadata and bounded pages of the immutable era L1 prefix. +//! Coherent history metadata. -use std::io::Cursor; -use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; -use axum::body::Body; use axum::extract::rejection::QueryRejection; use axum::extract::{Query, State}; -use axum::http::{StatusCode, header}; +use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::routing::get; use axum::{Json, Router}; -use sequencer_core::history::{EraId, RecoveryGeneration}; -use sequencer_core::history_api::{HISTORICAL_INPUT_MAX_ITEMS, HistoricalL1InputStart}; +use sequencer_core::history::EraId; use serde::Deserialize; -use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; -use tokio_util::io::ReaderStream; use crate::http::{ApiError, StorageTaskError, storage_task}; use crate::runtime::shutdown::RuntimeScope; use crate::storage::{HistoricalReadError, Storage}; -const MAX_HISTORICAL_RESPONSES: usize = 8; - struct HistoryState { db_path: String, shutdown: RuntimeScope, - responses: Arc, } pub(super) fn router(db_path: String, shutdown: RuntimeScope) -> Router { Router::new() .route("/history", get(history)) - .route("/historical-l1-inputs", get(historical_l1_inputs)) - .with_state(Arc::new(HistoryState { - db_path, - shutdown, - responses: Arc::new(Semaphore::new(MAX_HISTORICAL_RESPONSES)), - })) + .with_state(Arc::new(HistoryState { db_path, shutdown })) } #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct HistoryQuery { era_id: Option, - from_generation: Option, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct HistoricalInputsQuery { - era_id: EraId, - next_input_index: Option, - after_block: Option, - limit: Option, -} - -impl HistoricalInputsQuery { - fn start(&self) -> Result { - match (self.next_input_index, self.after_block) { - (Some(index), None) => Ok(HistoricalL1InputStart::NextInputIndex(index)), - (None, Some(block)) => Ok(HistoricalL1InputStart::AfterBlock(block)), - _ => Err(ApiError::bad_request( - "provide exactly one of next_input_index or after_block", - )), - } - } } async fn history( @@ -81,15 +43,12 @@ async fn history( Ok(query) => query, Err(error) => return ApiError::bad_request(error.body_text()).into_response(), }; - if query.from_generation.is_some() && query.era_id.is_none() { - return ApiError::bad_request("from_generation requires era_id").into_response(); - } if let Err(error) = crate::http::reject_if_shutting_down(&state.shutdown) { return error.into_response(); } let db_path = state.db_path.clone(); match storage_task(state.shutdown.clone(), "read history metadata", move |_| { - Ok(Storage::open_read_only(&db_path)?.history_info(query.era_id, query.from_generation)?) + Ok(Storage::open_read_only(&db_path)?.history_info(query.era_id)?) }) .await { @@ -98,83 +57,14 @@ async fn history( } } -async fn historical_l1_inputs( - State(state): State>, - query: Result, QueryRejection>, -) -> Response { - let Query(query) = match query { - Ok(query) => query, - Err(error) => return ApiError::bad_request(error.body_text()).into_response(), - }; - let start = match query.start() { - Ok(start) => start, - Err(error) => return error.into_response(), - }; - if let Err(error) = crate::http::reject_if_shutting_down(&state.shutdown) { - return error.into_response(); - } - let permit = match state.responses.clone().try_acquire_owned() { - Ok(permit) => permit, - Err(_) => return ApiError::overloaded("historical response limit reached").into_response(), - }; - let db_path = state.db_path.clone(); - let result = storage_task( - state.shutdown.clone(), - "read historical L1 inputs", - move |_| { - // The blocking task owns admission even if its HTTP request is cancelled. - let page = Storage::open_read_only(&db_path)?.historical_l1_inputs( - query.era_id, - start, - query.limit.unwrap_or(HISTORICAL_INPUT_MAX_ITEMS), - )?; - let bytes = serde_json::to_vec(&page)?; - Ok((bytes, permit)) - }, - ) - .await; - match result { - Ok((bytes, permit)) => ( - [(header::CONTENT_TYPE, "application/json")], - Body::from_stream(ReaderStream::new(HistoricalResponseBody { - bytes: Cursor::new(bytes), - _permit: permit, - })), - ) - .into_response(), - Err(error) => read_error(error), - } -} - fn read_error(error: StorageTaskError) -> Response { - match error.downcast_ref::() { - Some(HistoricalReadError::Policy(policy)) => { - return (StatusCode::CONFLICT, Json(*policy)).into_response(); - } - Some(HistoricalReadError::BadRequest(message)) => { - return ApiError::bad_request(message.clone()).into_response(); - } - _ => {} + if let Some(HistoricalReadError::Policy(policy)) = error.downcast_ref::() { + return (StatusCode::CONFLICT, Json(*policy)).into_response(); } tracing::warn!(%error, "history read unavailable"); ApiError::unavailable("history read unavailable").into_response() } -struct HistoricalResponseBody { - bytes: Cursor>, - _permit: OwnedSemaphorePermit, -} - -impl AsyncRead for HistoricalResponseBody { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.bytes).poll_read(cx, buf) - } -} - #[cfg(test)] mod tests { use super::*; @@ -185,33 +75,15 @@ mod tests { Arc::new(HistoryState { db_path: "unused: invalid queries do not access storage".to_owned(), shutdown: RuntimeScope::default(), - responses: Arc::new(Semaphore::new(1)), }) } #[tokio::test] async fn malformed_queries_return_bad_request_json_before_storage() { for query in [ - "", - "?era_id=bad&next_input_index=0", - &format!("?era_id={ERA}"), - &format!("?era_id={ERA}&next_input_index=0&after_block=0"), - &format!("?era_id={ERA}&next_input_index=-1"), - &format!("?era_id={ERA}&next_input_index=0&unrecognized=1"), - ] { - let uri = format!("/historical-l1-inputs{query}").parse().unwrap(); - let response = historical_l1_inputs(State(state()), Query::try_from_uri(&uri)).await; - assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); - let bytes = axum::body::to_bytes(response.into_body(), 4096) - .await - .unwrap(); - let error: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(error["code"], "BAD_REQUEST"); - } - for query in [ + "?era_id=invalid", "?from_generation=0", - &format!("?era_id={ERA}&from_generation=-1"), - &format!("?era_id={ERA}&from_generation=invalid"), + &format!("?era_id={ERA}&from_generation=0"), &format!("?era_id={ERA}&unrecognized=1"), ] { let uri = format!("/history{query}").parse().unwrap(); @@ -226,23 +98,10 @@ mod tests { } #[tokio::test] - async fn shutdown_and_overload_refuse_before_opening_storage() { + async fn shutdown_refuses_before_opening_storage() { let state = state(); - let uri = format!("/historical-l1-inputs?era_id={ERA}&next_input_index=0") - .parse() - .unwrap(); - let permit = state.responses.clone().try_acquire_owned().unwrap(); - let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; - assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); - drop(permit); state.shutdown.request_shutdown(); - let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - for query in [ - "".to_owned(), - format!("?era_id={ERA}"), - format!("?era_id={ERA}&from_generation=0"), - ] { + for query in ["".to_owned(), format!("?era_id={ERA}")] { let uri = format!("/history{query}").parse().unwrap(); let response = history(State(state.clone()), Query::try_from_uri(&uri)).await; assert_eq!( @@ -254,23 +113,93 @@ mod tests { } #[tokio::test] - async fn response_body_holds_admission_until_consumed_or_dropped() { - let permits = Arc::new(Semaphore::new(1)); - for consume in [false, true] { - let body = Body::from_stream(ReaderStream::new(HistoricalResponseBody { - bytes: Cursor::new(vec![1; 100_000]), - _permit: permits.clone().try_acquire_owned().unwrap(), - })); - assert!(permits.clone().try_acquire_owned().is_err()); - if consume { - assert_eq!( - axum::body::to_bytes(body, 100_000).await.unwrap().len(), - 100_000 - ); - } else { - drop(body); - } - assert_eq!(permits.available_permits(), 1); - } + async fn history_reports_a_rebuilt_eras_base_and_rebuilt_end() { + use crate::egress::test_era; + + let db = crate::test_support::temp_db("history-rebuilt-era"); + let mut storage = + test_era::seed_rebuilt_era(&db.path, std::path::Path::new("/unused/baseline")); + test_era::append_live_user_op(&mut storage, 2); + let era_id = storage.era().unwrap().id; + drop(storage); + let state = Arc::new(HistoryState { + db_path: db.path.clone(), + shutdown: RuntimeScope::default(), + }); + let response = history( + State(state.clone()), + Query::try_from_uri(&format!("/history?era_id={era_id}").parse().unwrap()), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let info: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let keys: Vec<&str> = info + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + keys, + ["accepted_checkpoint", "baseline", "deployment", "history"] + ); + // The era starts at the base count; the head is past the queued + // direct and the live op. + assert_eq!( + info["history"], + serde_json::json!({ + "era_id": era_id.to_string(), + "available_from": test_era::BASE, + "head": test_era::REBUILT_END + 2, + }) + ); + assert_eq!( + info["baseline"], + serde_json::json!({ + "l1_stop_block": test_era::STOP_BLOCK, + "l1_end_input_index": test_era::FIRST_QUEUED_INPUT + 1, + "next_batch_nonce": test_era::NEXT_BATCH_NONCE, + "rebuilt_history_end": test_era::REBUILT_END, + }) + ); + // With no accepted batch yet, the era's baseline is the checkpoint. + assert_eq!( + info["accepted_checkpoint"], + serde_json::json!({ + "inclusion_block": test_era::STOP_BLOCK, + "executed_input_count": test_era::REBUILT_END, + "next_batch_nonce": test_era::NEXT_BATCH_NONCE, + }) + ); + } + + #[tokio::test] + async fn another_era_is_refused_with_the_current_era() { + use crate::egress::test_era; + + let db = crate::test_support::temp_db("history-era-changed"); + let mut storage = + test_era::seed_rebuilt_era(&db.path, std::path::Path::new("/unused/baseline")); + let era_id = storage.era().unwrap().id; + drop(storage); + assert_ne!(era_id.to_string(), ERA); + let state = Arc::new(HistoryState { + db_path: db.path.clone(), + shutdown: RuntimeScope::default(), + }); + let uri = format!("/history?era_id={ERA}").parse().unwrap(); + let response = history(State(state), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body, + serde_json::json!({ "code": "ERA_CHANGED", "era_id": era_id.to_string() }) + ); } } diff --git a/sequencer/src/egress/api/mod.rs b/sequencer/src/egress/api/mod.rs index 1ff749ab..a7851ca7 100644 --- a/sequencer/src/egress/api/mod.rs +++ b/sequencer/src/egress/api/mod.rs @@ -15,7 +15,6 @@ use axum::Router; use axum::routing::get; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::ReleaseScheduler; pub(crate) use health::HealthState; pub use snapshot::SnapshotState; @@ -28,7 +27,6 @@ pub(crate) fn router( health_state: Arc, snapshot_state: SnapshotState, shutdown: RuntimeScope, - snapshot_release_scheduler: ReleaseScheduler, ) -> Router { let history_router = history::router(snapshot_state.db_path.clone(), shutdown.clone()); let subscribe_router = Router::new() @@ -44,9 +42,5 @@ pub(crate) fn router( subscribe_router .merge(health_router) .merge(history_router) - .merge(snapshot::router( - snapshot_state, - shutdown, - snapshot_release_scheduler, - )) + .merge(snapshot::router(snapshot_state, shutdown)) } diff --git a/sequencer/src/egress/api/snapshot.rs b/sequencer/src/egress/api/snapshot.rs index edc8d7b3..70a5c654 100644 --- a/sequencer/src/egress/api/snapshot.rs +++ b/sequencer/src/egress/api/snapshot.rs @@ -1,8 +1,8 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Operator-only comparison files and complete restore/recovery archives. -//! Snapshot selection, history metadata, and leases are one SQLite transaction. +//! Operator-only comparison files and complete restore archives. +//! Snapshot selection and history metadata are one read, taken under the lease. use std::future::Future; use std::path::{Path, PathBuf}; @@ -27,37 +27,29 @@ use tokio_util::io::{ReaderStream, SyncIoBridge}; use crate::http::{ApiError, StorageTaskError, storage_task}; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; use crate::storage::{ - FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, ReleaseScheduler, Storage, + FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, SnapshotLeases, Storage, }; type BoxError = StorageTaskError; -/// Wiring for the snapshot endpoints: where the DB is, and how to find the -/// canonical state file inside a dump. `state_file_in_dump` is threaded as a -/// fn pointer (`A::state_file_in_dump`) so this layer stays free of an `A` -/// type parameter. +/// Wiring for the snapshot endpoints: where the DB is, how to find the +/// canonical state file inside a dump, and the lease registry the lane's GC +/// shares. `state_file_in_dump` is threaded as a fn pointer +/// (`A::state_file_in_dump`) so this layer stays free of an `A` type parameter. #[derive(Clone)] pub struct SnapshotState { pub db_path: String, pub state_file_in_dump: fn(&Path) -> PathBuf, + pub leases: SnapshotLeases, } struct SnapshotApiState { snapshot: SnapshotState, shutdown: RuntimeScope, - release_scheduler: ReleaseScheduler, } -pub(crate) fn router( - snapshot: SnapshotState, - shutdown: RuntimeScope, - release_scheduler: ReleaseScheduler, -) -> Router { - let state = Arc::new(SnapshotApiState { - snapshot, - shutdown, - release_scheduler, - }); +pub(crate) fn router(snapshot: SnapshotState, shutdown: RuntimeScope) -> Router { + let state = Arc::new(SnapshotApiState { snapshot, shutdown }); Router::new() .route("/finalized_state", get(finalized_state)) .route( @@ -89,7 +81,7 @@ async fn finalized_inclusion_block(State(state): State>) - match result { Ok(Some(finalized)) => Json(InclusionBlockResponse { inclusion_block: finalized.inclusion_block, - executed_input_count: finalized.executed_input_count.get(), + executed_input_count: finalized.snapshot.l2_tx_count.get(), }) .into_response(), Ok(None) => StatusCode::NOT_FOUND.into_response(), @@ -117,7 +109,7 @@ async fn finalized_state_digest(State(state): State>) -> R Err(err) => return finalized_error("acquire finalized lease", err), }; let path = state_file_path(&state.snapshot, &leased.prefix); - let executed_input_count = leased.executed_input_count.get(); + let l2_tx_count = leased.l2_tx_count.get(); let LeasedDump { guard, .. } = leased; let scope = state.shutdown.clone(); let hashed = tokio::task::spawn_blocking(move || { @@ -129,7 +121,7 @@ async fn finalized_state_digest(State(state): State>) -> R match hashed { Ok(Ok(digest)) => Json(DigestResponse { inclusion_block, - executed_input_count, + executed_input_count: l2_tx_count, sha256: alloy_primitives::hex::encode(digest), }) .into_response(), @@ -169,8 +161,8 @@ async fn finalized_state( } let path = state_file_path(&state.snapshot, &leased.prefix); - let executed_input_count = leased.executed_input_count.get(); - let history = leased.history_version; + let l2_tx_count = leased.l2_tx_count.get(); + let era_id = leased.era_id; let LeasedDump { guard, .. } = leased; match comparison_io(File::open(&path).await, &path) { @@ -179,12 +171,8 @@ async fn finalized_state( .header(header::CONTENT_TYPE, "application/octet-stream") .header(header::ETAG, etag) .header("X-Inclusion-Block", inclusion_block.to_string()) - .header("X-Executed-Input-Count", executed_input_count.to_string()) - .header("X-History-Era", history.era_id.to_string()) - .header( - "X-Recovery-Generation", - history.recovery_generation.get().to_string(), - ) + .header("X-Executed-Input-Count", l2_tx_count.to_string()) + .header("X-History-Era", era_id.to_string()) .body(stream_body(file, guard, path)) .expect("snapshot response headers are well-formed"), // `guard` is a local here; on this error path it drops → lease released. @@ -231,8 +219,8 @@ fn archive_response( ) -> Response { let LeasedDump { prefix, - executed_input_count, - history_version, + l2_tx_count, + era_id, next_batch_nonce, guard, } = leased; @@ -276,15 +264,8 @@ fn archive_response( let mut response = Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/x-tar") - .header("X-History-Era", history_version.era_id.to_string()) - .header( - "X-Recovery-Generation", - history_version.recovery_generation.get().to_string(), - ) - .header( - "X-Executed-Input-Count", - executed_input_count.get().to_string(), - ); + .header("X-History-Era", era_id.to_string()) + .header("X-Executed-Input-Count", l2_tx_count.get().to_string()); if let Some(block) = inclusion_block { response = response.header("X-Inclusion-Block", block.to_string()); } @@ -387,22 +368,18 @@ impl AsyncRead for ArchiveReader { } } -// ── Lease acquisition (storage returns the dump bundled with its release) ── +// ── Lease acquisition (storage returns the dump bundled with its guard) ── async fn acquire_finalized(state: &SnapshotApiState) -> Result, BoxError> { let db_path = state.snapshot.db_path.clone(); - let release_scheduler = state.release_scheduler.clone(); + let leases = state.snapshot.leases.clone(); storage_task( state.shutdown.clone(), "acquire finalized snapshot lease", - move |scope| { - let report_persistent_failure: crate::storage::PersistentReleaseFailureReporter = - Arc::new(move |cause: &str| { - let _runtime_lifetime = &scope; - abort_terminal(cause) - }); - let mut storage = Storage::open_writer(&db_path)?; - Ok(storage.acquire_finalized_lease(release_scheduler, report_persistent_failure)?) + move |_scope| { + // Opened before the registry lock: only the selection runs under it. + let mut storage = Storage::open_read_only(&db_path)?; + Ok(storage.acquire_finalized_lease(&leases)?) }, ) .await @@ -410,19 +387,13 @@ async fn acquire_finalized(state: &SnapshotApiState) -> Result Result, BoxError> { let db_path = state.snapshot.db_path.clone(); - let release_scheduler = state.release_scheduler.clone(); + let leases = state.snapshot.leases.clone(); storage_task( state.shutdown.clone(), "acquire latest snapshot lease", - move |scope| { - let report_persistent_failure: crate::storage::PersistentReleaseFailureReporter = - Arc::new(move |cause: &str| { - let _runtime_lifetime = &scope; - abort_terminal(cause) - }); - let mut storage = Storage::open_writer(&db_path)?; - Ok(storage - .acquire_latest_snapshot_lease(release_scheduler, report_persistent_failure)?) + move |_scope| { + let mut storage = Storage::open_read_only(&db_path)?; + Ok(storage.acquire_latest_snapshot_lease(&leases)?) }, ) .await @@ -481,15 +452,15 @@ mod tests { let db = temp_db("corrupt-comparison-path"); let mut storage = Storage::open(&db.path).unwrap(); storage - .insert_baseline_snapshot(path, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(path, crate::storage::L2TxCount::ZERO) .unwrap(); let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: Path::to_path_buf, + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }); let response = finalized_state(State(state), HeaderMap::new()).await; // Unix can open a directory successfully: the structural error first @@ -558,6 +529,7 @@ mod tests { _db: crate::test_support::TestDb, _root: tempfile::TempDir, state: Arc, + snapshot_id: i64, } fn baseline_state(name: &str, contents: &[u8]) -> BaselineFixture { @@ -567,22 +539,23 @@ mod tests { std::fs::create_dir(&dump_dir).expect("create dump directory"); std::fs::write(dump_dir.join("state"), contents).expect("write comparison file"); let mut storage = Storage::open(&db.path).expect("open storage"); - storage - .insert_baseline_snapshot(&dump_dir, crate::storage::ExecutedInputCount::ZERO) + let snapshot_id = storage + .insert_baseline_snapshot(&dump_dir, crate::storage::L2TxCount::ZERO) .expect("insert baseline snapshot"); drop(storage); let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { db_path: db.path.clone(), state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }); BaselineFixture { _db: db, _root: root, state, + snapshot_id, } } @@ -600,6 +573,8 @@ mod tests { let digest: serde_json::Value = serde_json::from_slice(&body).expect("digest JSON"); assert_eq!(digest["inclusion_block"], 0); assert_eq!(digest["executed_input_count"], 0); + // The hashing task held the lease and dropped it before responding. + assert_eq!(state.snapshot.leases.count(fixture.snapshot_id), Some(0)); let streamed = finalized_state(State(state), HeaderMap::new()).await; let bytes = axum::body::to_bytes(streamed.into_body(), contents.len() + 1) @@ -620,9 +595,9 @@ mod tests { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }); let response = finalized_state_digest(State(state)).await; assert_eq!(response.status(), StatusCode::NOT_FOUND); @@ -641,7 +616,7 @@ mod tests { storage .insert_baseline_snapshot( Path::new("/tmp/panicking-snapshot-path"), - crate::storage::ExecutedInputCount::ZERO, + crate::storage::L2TxCount::ZERO, ) .expect("insert baseline snapshot"); drop(storage); @@ -649,9 +624,9 @@ mod tests { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |_| panic!("application returned no state path"), + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }); // Exercise the request-task boundary that would swallow this panic. @@ -673,16 +648,16 @@ mod tests { .expect("write immutable dump metadata"); let mut storage = Storage::open(&db.path).expect("open storage"); storage - .insert_baseline_snapshot(&dump_dir, crate::storage::ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&dump_dir, crate::storage::L2TxCount::ZERO) .expect("insert baseline snapshot"); drop(storage); let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |_| panic!("archive must not ask for comparison bytes"), + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }); let response = latest_snapshot(State(state)).await; assert_eq!(response.status(), StatusCode::OK); @@ -716,7 +691,7 @@ mod tests { storage .insert_baseline_snapshot( Path::new("/tmp/corrupt-finalized"), - crate::storage::ExecutedInputCount::ZERO, + crate::storage::L2TxCount::ZERO, ) .expect("insert finalized snapshot"); drop(storage); @@ -727,7 +702,7 @@ mod tests { conn.execute_batch("DROP TRIGGER trg_snapshot_immutable") .unwrap(); conn.execute( - "UPDATE snapshots SET executed_input_count = -1 WHERE batch_index IS NULL", + "UPDATE snapshots SET l2_tx_count = -1 WHERE batch_nonce IS NULL", [], ) .expect("corrupt finalized cursor"); @@ -738,9 +713,9 @@ mod tests { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, shutdown: shutdown.clone(), - release_scheduler: Arc::new(|release| release()), }); let response = finalized_inclusion_block(State(state)).await; @@ -752,74 +727,93 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn graceful_shutdown_keeps_snapshot_reads_available() { - let db = temp_db("snapshot-gate-predicate"); + #[cfg(unix)] + async fn accepted_batch_without_its_snapshot_trips_terminal_storage_fault() { + use crate::storage::test_helpers::{SENDER_A, seed_safe_inputs_with_batch_nonces}; + if !crate::runtime::shutdown::abort_test_child( + "egress::api::snapshot::tests::accepted_batch_without_its_snapshot_trips_terminal_storage_fault", + ) { + return; + } + let db = temp_db("missing-accepted-snapshot-endpoint"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + crate::test_support::pin_test_deployment_identity(&mut storage, SENDER_A); storage .insert_baseline_snapshot( - Path::new("/tmp/gate-finalized"), - crate::storage::ExecutedInputCount::ZERO, + Path::new("/tmp/missing-accepted/baseline"), + crate::storage::L2TxCount::ZERO, ) - .expect("insert finalized snapshot"); + .expect("insert baseline snapshot"); + let mut head = storage + .initialize_open_state(0, crate::storage::SafeInputRange::empty_at(0)) + .expect("open the genesis tip"); + let batch_nonce = head.batch_nonce(); + let count = storage.next_l2_tx_count().expect("next count"); + storage + .close_frame_and_batch_with_snapshot( + &mut head, + Path::new("/tmp/missing-accepted/batch"), + count, + ) + .expect("close the batch with its snapshot"); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); drop(storage); - let shutdown = RuntimeScope::default(); + // The accepted batch's snapshot is the only comparison checkpoint: + // without its row the endpoint has nothing older to fall back to. + let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); + let deleted = conn + .execute( + "DELETE FROM snapshots WHERE batch_nonce = ?1", + [batch_nonce as i64], + ) + .expect("remove the accepted batch's snapshot row"); + assert_eq!(deleted, 1); + drop(conn); + let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, - shutdown: shutdown.clone(), - release_scheduler: Arc::new(|release| release()), + shutdown: RuntimeScope::default(), }); - // Immutable operator reads are not authority-bearing (ADR): an - // ordinary graceful drain keeps serving the watchdog's poll. - shutdown.request_shutdown(); - let response = finalized_inclusion_block(State(state.clone())).await; - assert_eq!(response.status(), StatusCode::OK); + let response = finalized_inclusion_block(State(state)).await; + panic!( + "terminal snapshot fault returned HTTP {}", + response.status() + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(unix)] - async fn dangling_finalized_snapshot_row_trips_terminal_storage_fault() { - if !crate::runtime::shutdown::abort_test_child( - "egress::api::snapshot::tests::dangling_finalized_snapshot_row_trips_terminal_storage_fault", - ) { - return; - } - let db = temp_db("dangling-finalized-endpoint"); + async fn graceful_shutdown_keeps_snapshot_reads_available() { + let db = temp_db("snapshot-gate-predicate"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage .insert_baseline_snapshot( - Path::new("/tmp/dangling-finalized"), - crate::storage::ExecutedInputCount::ZERO, + Path::new("/tmp/gate-finalized"), + crate::storage::L2TxCount::ZERO, ) .expect("insert finalized snapshot"); drop(storage); - let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); - conn.pragma_update(None, "foreign_keys", "OFF") - .expect("disable foreign keys for corruption fixture"); - conn.execute("DELETE FROM dumps", []) - .expect("remove referenced dump row"); - drop(conn); - let shutdown = RuntimeScope::default(); let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { db_path: db.path, state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, shutdown: shutdown.clone(), - release_scheduler: Arc::new(|release| release()), }); - let response = finalized_inclusion_block(State(state)).await; - panic!( - "terminal snapshot fault returned HTTP {}", - response.status() - ); + // Immutable operator reads are not authority-bearing (ADR): an + // ordinary graceful drain keeps serving the watchdog's poll. + shutdown.request_shutdown(); + let response = finalized_inclusion_block(State(state.clone())).await; + assert_eq!(response.status(), StatusCode::OK); } #[test] @@ -835,45 +829,6 @@ mod tests { assert!(!crate::http::persistent_storage_error(&error)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(unix)] - async fn missing_lease_row_aborts_through_runtime_reporter() { - if !crate::runtime::shutdown::abort_test_child( - "egress::api::snapshot::tests::missing_lease_row_aborts_through_runtime_reporter", - ) { - return; - } - let db = temp_db("missing-lease-row"); - let mut storage = Storage::open(&db.path).expect("open storage"); - storage - .insert_baseline_snapshot( - Path::new("/tmp/lease-probe"), - crate::storage::ExecutedInputCount::ZERO, - ) - .expect("register finalized dump"); - drop(storage); - let state = SnapshotApiState { - snapshot: SnapshotState { - db_path: db.path.clone(), - state_file_in_dump: |prefix| prefix.join("state"), - }, - shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), - }; - let lease = acquire_finalized(&state) - .await - .expect("acquire lease") - .expect("finalized dump"); - let conn = Storage::open_connection(&db.path).expect("raw connection"); - conn.pragma_update(None, "foreign_keys", "OFF") - .expect("corruption fixture"); - conn.execute("DELETE FROM dumps", []) - .expect("remove leased row"); - drop(conn); - drop(lease); - panic!("persistent release failure returned instead of aborting"); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[cfg(unix)] async fn cancelled_snapshot_request_cannot_discard_started_storage_fault() { @@ -886,9 +841,9 @@ mod tests { snapshot: SnapshotState { db_path: String::new(), state_file_in_dump: |prefix| prefix.join("state"), + leases: SnapshotLeases::default(), }, shutdown: RuntimeScope::default(), - release_scheduler: Arc::new(|release| release()), }; let (started_tx, started_rx) = tokio::sync::oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel(); diff --git a/sequencer/src/egress/api/subscribe.rs b/sequencer/src/egress/api/subscribe.rs index ca0debf8..9ea01270 100644 --- a/sequencer/src/egress/api/subscribe.rs +++ b/sequencer/src/egress/api/subscribe.rs @@ -18,20 +18,20 @@ use crate::egress::l2_tx_feed::Subscription; use crate::egress::l2_tx_feed::{BroadcastTxMessage, L2TxFeed, SubscribeError}; use crate::http::ApiError; use axum::{Json, http::StatusCode}; -use sequencer_core::history::{ - EraId, ExecutedInputCount, HistoryClaim, HistoryVersion, RecoveryGeneration, -}; +use sequencer_core::history::{EraId, HistoryClaim, L2TxCount}; use super::SubscribeState; const MAX_INBOUND_WS_MESSAGE_SIZE: usize = 8 * 1024; const MAX_INBOUND_WS_FRAME_SIZE: usize = 8 * 1024; +/// The claim, field for field. An unknown field is a `400`, so a client +/// that sends coordinates the server does not check fails loudly. #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub(crate) struct SubscribeQuery { era_id: EraId, - recovery_generation: RecoveryGeneration, - next_input: ExecutedInputCount, + next_input: L2TxCount, } pub(crate) async fn subscribe_l2_txs( @@ -44,10 +44,7 @@ pub(crate) async fn subscribe_l2_txs( } let claim = HistoryClaim { - version: HistoryVersion { - era_id: query.era_id, - recovery_generation: query.recovery_generation, - }, + era_id: query.era_id, next_input: query.next_input, }; let permit = match state.try_acquire_ws_subscriber_permit() { diff --git a/sequencer/src/egress/l2_tx_feed/error.rs b/sequencer/src/egress/l2_tx_feed/error.rs index 6de76e40..41dd6bde 100644 --- a/sequencer/src/egress/l2_tx_feed/error.rs +++ b/sequencer/src/egress/l2_tx_feed/error.rs @@ -74,7 +74,11 @@ pub enum SubscriptionError { impl SubscriptionError { pub(super) fn is_persistent_storage_invariant(&self) -> bool { match self { - Self::History(_) => false, + // The claim was admitted when the subscription was prepared, and + // inside one database the era is immutable and the history only + // appends. A page that refuses the claim it continues from means + // the history changed under the reader. + Self::History(_) => true, Self::OpenStorage { source } => open_error_is_persistent(source), Self::LoadReplay { source, .. } => is_persistent_storage_error(source), Self::Join { source } => source.is_panic(), diff --git a/sequencer/src/egress/l2_tx_feed/mod.rs b/sequencer/src/egress/l2_tx_feed/mod.rs index 3bfb3ee3..98e9d7a7 100644 --- a/sequencer/src/egress/l2_tx_feed/mod.rs +++ b/sequencer/src/egress/l2_tx_feed/mod.rs @@ -219,29 +219,15 @@ fn run_subscription( } let offset = row.offset.get(); - let event = match row.context { - L2TxContext::UserOp { - tx, - nonce, - safe_block, - batch_nonce, - .. - } => BroadcastTxMessage::from_user_op(offset, tx, nonce, safe_block, batch_nonce), - L2TxContext::DirectInput { - tx, - input_index, - batch_nonce, - block_timestamp, - transaction_hash, - .. - } => BroadcastTxMessage::from_direct_input( - offset, + let event = match &row.context { + L2TxContext::UserOp { tx, batch_nonce } => { + BroadcastTxMessage::from_user_op(offset, tx, *batch_nonce) + } + L2TxContext::Direct { tx, - input_index, batch_nonce, - block_timestamp, transaction_hash, - ), + } => BroadcastTxMessage::from_direct(offset, tx, *batch_nonce, *transaction_hash), }; if events_tx.blocking_send(event).is_err() { return Ok(()); diff --git a/sequencer/src/egress/l2_tx_feed/tests.rs b/sequencer/src/egress/l2_tx_feed/tests.rs index eae227e1..eb3c06fe 100644 --- a/sequencer/src/egress/l2_tx_feed/tests.rs +++ b/sequencer/src/egress/l2_tx_feed/tests.rs @@ -1,34 +1,35 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +use std::path::Path; use std::time::{Duration, SystemTime}; use alloy_primitives::{Address, B256, Signature}; use tokio::sync::oneshot; use super::{BroadcastTxMessage, L2TxFeed, L2TxFeedConfig, SubscribeError}; +use crate::egress::test_era; use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp, SequencerError}; use crate::runtime::process_lock::{ProcessLock, ProcessLockError}; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::{FrontierMode, IngestedSafeInput, SafeInputRange, Storage}; +use crate::storage::test_helpers::{SENDER_A, set_up_storage}; +use crate::storage::{IngestedSafeInput, SafeInputRange, Storage}; use crate::test_support::{pin_test_deployment_identity, temp_db}; -use sequencer_core::history::{ - ExecutedInputCount, HistoryClaim, HistoryPolicyError, RecoveryGeneration, -}; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::history::{EraId, HistoryClaim, HistoryPolicyError, L2TxCount}; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; #[test] fn broadcast_user_op_serializes_with_hex_data() { let msg = BroadcastTxMessage::from_user_op( 7, - ValidUserOp { + &UserOpTx { sender: Address::from_slice(&[0x11; 20]), + nonce: 11, fee: 3, + block: 1_234, data: vec![0xaa, 0xbb], }, - 11, - 1_234, 5, ); let json = serde_json::to_string(&msg).expect("serialize"); @@ -43,16 +44,16 @@ fn broadcast_user_op_serializes_with_hex_data() { #[test] fn broadcast_direct_input_serializes_with_hex_payload() { - let msg = BroadcastTxMessage::from_direct_input( + let msg = BroadcastTxMessage::from_direct( 9, - DirectInput { + &Direct { + input_index: 3, sender: Address::ZERO, - block_number: 42, + block: 42, + timestamp: 1_700_000_000, payload: vec![0xcc, 0xdd], }, - 3, 5, - 1_700_000_000, B256::repeat_byte(0xab), ); let json = serde_json::to_string(&msg).expect("serialize"); @@ -68,26 +69,26 @@ fn broadcast_direct_input_serializes_with_hex_payload() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn subscription_refuses_ahead_and_stale_claims() { +async fn subscription_refuses_ahead_and_other_era_claims() { let db = temp_db("subscription-claims"); seed_ordered_txs(&db.path); let feed = test_feed(&db.path, RuntimeScope::default()); assert!(matches!(feed.subscribe_from(claim(&db.path, 3)).await, Err(SubscribeError::History(HistoryPolicyError::AheadOfHead { head })) if head.get() == 2)); - let mut stale = claim(&db.path, 0); - stale.version.recovery_generation = RecoveryGeneration::new(1); + let current = claim(&db.path, 0); + let other_era: EraId = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert_ne!(other_era, current.era_id); assert!(matches!( - feed.subscribe_from(stale).await, - Err(SubscribeError::History( - HistoryPolicyError::StaleGeneration { .. } - )) + feed.subscribe_from(HistoryClaim { era_id: other_era, ..current }).await, + Err(SubscribeError::History(HistoryPolicyError::EraChanged { era_id })) + if era_id == current.era_id )); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn subscription_accepts_more_than_fifty_thousand_inputs_without_a_total_cap() { let db = temp_db("subscription-deep-history"); - let mut storage = Storage::open(&db.path).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); @@ -108,12 +109,12 @@ async fn subscription_accepts_more_than_fifty_thousand_inputs_without_a_total_ca respond_to, received_at: SystemTime::now(), }, - executed_input_offset: ExecutedInputCount::new(u64::from(offset)), + l2_tx_index: L2TxCount::new(u64::from(offset)), } }) .collect(); storage - .append_executed_user_ops_chunk(&mut head, &inputs) + .append_included_user_ops(&mut head, &inputs) .unwrap(); drop(storage); let feed = L2TxFeed::new( @@ -135,6 +136,75 @@ async fn subscription_accepts_more_than_fifty_thousand_inputs_without_a_total_ca subscription.finish().await.unwrap(); } +/// Subscribe from `from` and read every row through `end`, one feed per call +/// so each page size starts a fresh cursor. +async fn read_feed(db_path: &str, from: u64, end: u64, page_size: usize) -> Vec { + let feed = L2TxFeed::new( + db_path.to_string(), + RuntimeScope::default(), + L2TxFeedConfig { + idle_poll_interval: Duration::from_millis(2), + page_size, + }, + ); + let mut subscription = feed.subscribe_from(claim(db_path, from)).await.unwrap(); + let mut messages = Vec::new(); + for _ in from..end { + let message = tokio::time::timeout(Duration::from_secs(2), subscription.recv()) + .await + .expect("wait for a feed row") + .expect("the feed is open"); + messages.push(serde_json::to_value(message).unwrap()); + } + subscription.finish().await.unwrap(); + messages +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn feed_serves_the_rebuilt_history_then_the_live_rows_across_the_seam() { + let db = temp_db("feed-rebuilt-era"); + let mut storage = test_era::seed_rebuilt_era(&db.path, Path::new("/unused/baseline")); + let mut expected: Vec<_> = (test_era::BASE..=test_era::REBUILT_END) + .map(test_era::expected_feed_message) + .collect(); + expected.push(test_era::append_live_user_op(&mut storage, 2)); + expected.push(test_era::append_live_user_op(&mut storage, 3)); + drop(storage); + let head = test_era::BASE + expected.len() as u64; + assert_eq!(head, 48); + + // From the base, the middle of the rebuilt range, its last row, its end + // (the first live row) and past it, with page sizes that split the seam. + for from in [41, 43, 44, 45, 46] { + for page_size in [1, 2, 3, 64] { + let messages = read_feed(&db.path, from, head, page_size).await; + let skip = usize::try_from(from - test_era::BASE).unwrap(); + assert_eq!( + messages, + expected[skip..], + "from {from}, page size {page_size}" + ); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn feed_refuses_a_claim_below_the_rebuilt_eras_base() { + let db = temp_db("feed-rebuilt-era-below-base"); + drop(test_era::seed_rebuilt_era( + &db.path, + Path::new("/unused/baseline"), + )); + let feed = test_feed(&db.path, RuntimeScope::default()); + for next in [0, test_era::BASE - 1] { + assert!(matches!( + feed.subscribe_from(claim(&db.path, next)).await, + Err(SubscribeError::History(HistoryPolicyError::HistoryUnavailable { available_from })) + if available_from.get() == test_era::BASE + )); + } +} + #[test] fn cancelled_subscription_prepare_retains_process_lock_until_blocking_read_finishes() { let db = temp_db("cancelled-subscription-prepare-lock"); @@ -283,7 +353,15 @@ async fn corrupt_feed_head_trips_terminal_storage_fault() { seed_ordered_txs(db.path.as_str()); let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); let start = claim(&db.path, 0); - conn.execute_batch("PRAGMA ignore_check_constraints = ON; DROP TRIGGER trg_history_generation_monotonic; UPDATE history_state SET recovery_generation = 'broken';").expect("corrupt generation"); + // An era id of the wrong length: the head read decodes it and fails with + // a storage error, not a panic. The era is immutable and its CHECK bounds + // the length, so both are lifted for the corruption. + conn.execute_batch( + "PRAGMA ignore_check_constraints = ON; \ + DROP TRIGGER trg_era_immutable; \ + UPDATE era SET era_id = X'0102';", + ) + .expect("corrupt the era id"); drop(conn); let shutdown = RuntimeScope::default(); @@ -304,8 +382,12 @@ async fn corrupt_feed_page_trips_terminal_storage_fault() { let db = temp_db("corrupt-feed-page"); seed_ordered_txs(db.path.as_str()); let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); - conn.execute("UPDATE frames SET safe_block = 'not-an-integer'", []) - .expect("corrupt safe-block storage type"); + // A missing frame makes the page's LEFT JOIN read NULL frame columns (fee, + // safe block), which the decoder refuses as a wrongly typed stored value. + conn.execute_batch( + "PRAGMA foreign_keys = OFF; DROP TRIGGER trg_frames_not_deletable; DELETE FROM frames;", + ) + .expect("drop the frames under stored rows"); drop(conn); let shutdown = RuntimeScope::default(); @@ -341,7 +423,7 @@ fn seed_ordered_txs(db_path: &str) { .initialize_open_state(123, SafeInputRange::empty_at(0)) .expect("initialize open state"); storage - .close_frame_and_batch(&mut head, 123) + .close_frame_and_batch(&mut head) .expect("advance to batch nonce 1"); let (respond_to, _recv) = oneshot::channel::>(); @@ -360,11 +442,11 @@ fn seed_ordered_txs(db_path: &str) { }; storage - .append_executed_user_ops_chunk( + .append_included_user_ops( &mut head, &[IncludedUserOp { pending, - executed_input_offset: ExecutedInputCount::ZERO, + l2_tx_index: L2TxCount::ZERO, }], ) .expect("append user-op chunk"); @@ -379,9 +461,6 @@ fn seed_ordered_txs(db_path: &str) { block_timestamp: 1_700_000_000, transaction_hash: B256::repeat_byte(0xcd), }], - NO_OWN_BATCHES, - &crate::storage::test_helpers::default_protocol_timing(), - FrontierMode::Populate, ) .expect("append direct input"); storage @@ -390,9 +469,9 @@ fn seed_ordered_txs(db_path: &str) { } fn claim(db_path: &str, next: u64) -> HistoryClaim { - let storage = Storage::open_read_only(db_path).unwrap(); + let mut storage = Storage::open_read_only(db_path).unwrap(); HistoryClaim { - version: storage.history_state().unwrap().version, - next_input: ExecutedInputCount::new(next), + era_id: storage.era().unwrap().id, + next_input: L2TxCount::new(next), } } diff --git a/sequencer/src/egress/mod.rs b/sequencer/src/egress/mod.rs index 9a945f6f..2f63d0f6 100644 --- a/sequencer/src/egress/mod.rs +++ b/sequencer/src/egress/mod.rs @@ -7,3 +7,5 @@ pub mod api; pub mod l2_tx_feed; +#[cfg(test)] +pub(crate) mod test_era; diff --git a/sequencer/src/egress/test_era.rs b/sequencer/src/egress/test_era.rs new file mode 100644 index 00000000..86d965b1 --- /dev/null +++ b/sequencer/src/egress/test_era.rs @@ -0,0 +1,194 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! A recovered era for tests of the feed, `/history` and the lane: setup +//! rebuilt counts 41..45 from a base at count 41 through stop block 10, one +//! rebuilt op was skipped, and one direct was still queued at the stop +//! block, so the root frame drains it as the first live row. + +use std::path::Path; +use std::time::SystemTime; + +use alloy_primitives::{Address, B256, Signature}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; +use sequencer_core::scheduler::Scheduled; +use sequencer_core::user_op::{SignedUserOp, UserOp}; +use tokio::sync::oneshot; + +use crate::egress::l2_tx_feed::BroadcastTxMessage; +use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; +use crate::storage::{IngestedSafeInput, LifecycleCommand, NewEra, Storage}; +use crate::test_support::pin_test_deployment_identity; + +pub(crate) const BASE: u64 = 41; +pub(crate) const REBUILT_END: u64 = 45; +pub(crate) const STOP_BLOCK: u64 = 10; +pub(crate) const FIRST_QUEUED_INPUT: u64 = 3; +pub(crate) const NEXT_BATCH_NONCE: u64 = 5; +pub(crate) const SUBMITTER: Address = Address::repeat_byte(0x5b); +pub(crate) const DIRECT_SENDER: Address = Address::repeat_byte(0xd1); +pub(crate) const OP_SENDER: Address = Address::repeat_byte(0x0b); + +/// The L1 input at `index`, as the input table stores it: the timestamp and +/// transaction hash a rebuilt direct must carry on the feed. +pub(crate) fn safe_input(index: u64) -> IngestedSafeInput { + let block = [5, 6, 8, STOP_BLOCK][usize::try_from(index).expect("fixture index")]; + let tag = 0xa0 + u8::try_from(index).expect("fixture index"); + IngestedSafeInput { + sender: DIRECT_SENDER, + payload: vec![tag], + block_number: block, + block_timestamp: 1_000 + block, + transaction_hash: B256::repeat_byte(tag), + } +} + +/// A user op that setup rebuilt from input 1's batch. +pub(crate) fn rebuilt_op(nonce: u32) -> UserOpTx { + UserOpTx { + sender: OP_SENDER, + nonce, + fee: 9, + block: 6, + data: vec![0xd0, u8::try_from(nonce).expect("fixture nonce")], + } +} + +/// [`publish_rebuilt_era`], then open the root frame as `run`'s startup does. +pub(crate) fn seed_rebuilt_era(db_path: &str, baseline_prefix: &Path) -> Storage { + let mut storage = publish_rebuilt_era(db_path, baseline_prefix); + storage.ensure_open_tip().expect("open the root frame"); + storage +} + +/// Create the database as setup leaves it, with `baseline_prefix` as the +/// era's baseline dump. Inputs 0 and 2 are directs a batch drained (batch +/// nonces 3 and 4); input 1 stands in for a batch of three ops whose second +/// (nonce 5) the executor skipped, but it is not the submitter's, so no +/// acceptance scan parses it; input 3 is a direct still queued at the stop +/// block. +pub(crate) fn publish_rebuilt_era(db_path: &str, baseline_prefix: &Path) -> Storage { + let mut storage = Storage::initialize_for_command(db_path, LifecycleCommand::Setup) + .expect("create the setup schema"); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .append_ingested_safe_inputs_with_timestamp( + STOP_BLOCK, + 1_000 + STOP_BLOCK, + &(0..4).map(safe_input).collect::>(), + ) + .expect("sync through the stop block"); + let direct = |index: u64, batch_nonce| Scheduled { + batch_nonce, + tx: L2Tx::Direct(Direct { + input_index: index, + sender: DIRECT_SENDER, + block: safe_input(index).block_number, + timestamp: safe_input(index).block_timestamp, + payload: safe_input(index).payload, + }), + }; + let op = |nonce| Scheduled { + batch_nonce: 3, + tx: L2Tx::UserOp(rebuilt_op(nonce)), + }; + // The op at nonce 5 is skipped, so it has no row. + storage + .append_rebuilt_history(&[ + (L2TxCount::new(41), direct(0, 3)), + (L2TxCount::new(42), op(0)), + (L2TxCount::new(43), op(1)), + (L2TxCount::new(44), direct(2, 4)), + ]) + .expect("store the rebuilt history"); + storage + .publish_era( + baseline_prefix, + &NewEra { + base: L2TxCount::new(BASE), + rebuilt_end: L2TxCount::new(REBUILT_END), + stop_block: STOP_BLOCK, + first_queued_input: FIRST_QUEUED_INPUT, + anchor_batch_nonce: NEXT_BATCH_NONCE, + }, + ) + .expect("publish the era"); + storage +} + +/// The feed message for each count from the base through the root frame's +/// queued direct, as JSON: a direct carries its input's block, timestamp and +/// transaction hash with its own row's batch nonce, and the skipped op has +/// no row, so it never appears. +pub(crate) fn expected_feed_message(offset: u64) -> serde_json::Value { + let direct = |index: u64, batch_nonce| { + let input = safe_input(index); + BroadcastTxMessage::from_direct( + offset, + &Direct { + input_index: index, + sender: input.sender, + block: input.block_number, + timestamp: input.block_timestamp, + payload: input.payload, + }, + batch_nonce, + input.transaction_hash, + ) + }; + let message = match offset { + 41 => direct(0, 3), + 42 => BroadcastTxMessage::from_user_op(offset, &rebuilt_op(0), 3), + 43 => BroadcastTxMessage::from_user_op(offset, &rebuilt_op(1), 3), + 44 => direct(2, 4), + // The first live row: the root batch drains the queued direct. + 45 => direct(FIRST_QUEUED_INPUT, NEXT_BATCH_NONCE), + _ => panic!("the fixture fixes no message at count {offset}"), + }; + serde_json::to_value(message).expect("serialize a feed message") +} + +/// Append one live user op from `OP_SENDER` at the history head, as the lane +/// would after admitting it; returns the feed message expected for it. +pub(crate) fn append_live_user_op(storage: &mut Storage, nonce: u32) -> serde_json::Value { + let mut head = storage + .open_state() + .expect("read the open frame") + .expect("the root frame is open"); + let offset = storage.next_l2_tx_count().expect("read the history head"); + let data = vec![0xe0, u8::try_from(nonce).expect("fixture nonce")]; + let (respond_to, _response) = oneshot::channel(); + let included = IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: OP_SENDER, + signature: Signature::test_signature(), + user_op: UserOp { + nonce, + max_fee: u16::MAX, + data: data.clone().into(), + }, + }, + respond_to, + received_at: SystemTime::now(), + }, + l2_tx_index: offset, + }; + storage + .append_included_user_ops(&mut head, &[included]) + .expect("append a live user op"); + let op = UserOpTx { + sender: OP_SENDER, + nonce, + fee: head.frame_fee(), + block: head.safe_block(), + data, + }; + serde_json::to_value(BroadcastTxMessage::from_user_op( + offset.get(), + &op, + NEXT_BATCH_NONCE, + )) + .expect("serialize a feed message") +} diff --git a/sequencer/src/harness.rs b/sequencer/src/harness.rs index 1791f179..daa78d40 100644 --- a/sequencer/src/harness.rs +++ b/sequencer/src/harness.rs @@ -17,8 +17,9 @@ use crate::commands::config::{FlushConfig, RunConfig, SetupConfig}; name = "sequencer", version, about = "App-specific rollup sequencer.\n\n\ - Subcommands: `setup` (pin identity + initial sync, run once), \ - `run` (boot the sequencer from a set-up DB), and `flush-mempool` \ + Subcommands: `setup` (start an era in an empty data directory, from \ + genesis or a canonical machine), `run` (boot the sequencer from a \ + set-up DB), and `flush-mempool` \ (settle the batch-submitter wallet nonce on demand).\n\n\ All options can also be set via environment variables (shown in brackets)." )] @@ -31,10 +32,10 @@ pub struct Cli { /// `large_enum_variant`); `RunConfig` is the largest. #[derive(Debug, Subcommand)] pub enum Command { - /// Establish the deployment's timeless state: pin identity, do the initial - /// L1 sync, register the genesis snapshot, and mark setup complete. Run - /// once before `run`. L1-read-only (takes the submitter address, not the - /// key). + /// Start a new era in an empty data directory: pin identity, flush the + /// submitter's wallet with its key, sync L1, rebuild the history from the + /// base (genesis, or `--base `) and publish the era. Refuses a + /// directory that holds anything. `--dry-run` only reports. Setup(Box), /// Boot the sequencer from an already-set-up DB. Refuses unless `setup` /// completed. Reads identity from the DB; keeps the signing key. @@ -54,7 +55,7 @@ where } /// Run a parsed command with the same panic and exit-code handling as [`run_main`]. -/// The fallible genesis factory runs only when plain setup needs its initial snapshot. +/// The fallible genesis factory runs only when setup starts from genesis. pub async fn run_command(command: Command, genesis_app: F) -> std::process::ExitCode where A: Application + 'static, @@ -94,8 +95,8 @@ fn project_dispatch_join( /// `CommandError` maps through `CommandError::exit_code`. /// Terminal runtime faults abort the process directly; see [`crate::run`]. /// -/// `genesis_app` is called at most once — only when plain `setup` needs to -/// register the genesis snapshot. File-backed applications return load errors through +/// `genesis_app` is called at most once — only when `setup` starts from +/// genesis. File-backed applications return load errors through /// `AppError`; ordinary I/O failures are not invariant panics. pub async fn dispatch(command: Command, genesis_app: F) -> std::process::ExitCode where @@ -129,8 +130,10 @@ where mod tests { use super::*; + /// A completed setup's directory is refused with exit 30 before the + /// genesis factory runs; setup runs once per directory. #[tokio::test] - async fn completed_setup_does_not_construct_genesis_app() { + async fn setup_on_a_completed_data_dir_exits_30_without_genesis() { use crate::storage::{LifecycleCommand, Storage}; use crate::test_support::StubApp; use std::sync::Arc; @@ -151,6 +154,8 @@ mod tests { "0x1111111111111111111111111111111111111111", "--batch-submitter-address", "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "--batch-submitter-private-key", + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", ]) .expect("parse setup"); let db_path = match &cli.command { @@ -161,12 +166,9 @@ mod tests { let mut storage = Storage::initialize_for_command(&db_path, LifecycleCommand::Setup) .expect("initialize setup"); storage - .complete_baseline_setup( + .publish_era( &std::path::Path::new(&data_dir).join("baseline"), - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, + &crate::storage::NewEra::EMPTY_GENESIS, ) .expect("complete setup"); drop(storage); @@ -175,14 +177,11 @@ mod tests { let observed = Arc::clone(&constructions); let exit = run_command::(cli.command, move || { observed.fetch_add(1, Ordering::SeqCst); - Err(AppError::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - "genesis file no longer exists", - ))) + Ok(StubApp::default()) }) .await; - assert_eq!(exit, std::process::ExitCode::SUCCESS); + assert_eq!(exit, std::process::ExitCode::from(30)); assert_eq!(constructions.load(Ordering::SeqCst), 0); } @@ -191,9 +190,8 @@ mod tests { /// `setup` never completed is deterministic operator error, class 30. /// The per-variant table in `commands::error` pins every shape against /// the constants; this is the in-process assertion of a wire value - /// through the real command bracket, beside the e2e suite's 10 (the - /// aging-tip scenario) and 0 (`ManagedSequencer::stop_expecting_clean_exit`) - /// from a real process. Offline by construction: the admission preflight + /// through the real command bracket, beside the e2e suite's 0 + /// (`ManagedSequencer::stop_expecting_clean_exit`) from a real process. Offline by construction: the admission preflight /// refuses before the key is read or any provider is built. #[tokio::test] async fn run_on_a_never_set_up_data_dir_exits_30() { diff --git a/sequencer/src/http.rs b/sequencer/src/http.rs index 7776719b..6bd9cecd 100644 --- a/sequencer/src/http.rs +++ b/sequencer/src/http.rs @@ -17,7 +17,7 @@ use axum::response::{IntoResponse, Response}; use serde::Serialize; use thiserror::Error; use tokio::sync::mpsc; -use tokio::task::{JoinHandle, JoinSet}; +use tokio::task::JoinHandle; use tower_http::trace::TraceLayer; pub use crate::egress::api::SnapshotState; @@ -26,7 +26,6 @@ use crate::egress::l2_tx_feed::L2TxFeed; use crate::ingress::api::{ReadState, SubmitState}; use crate::ingress::inclusion_lane::{PendingUserOp, SequencerError}; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; -use crate::storage::ReleaseScheduler; use sequencer_core::api::{DomainResponse, TxRequest, TxRequestError}; #[derive(Debug, Error, Clone)] @@ -142,69 +141,6 @@ const DEFAULT_MAX_BODY_BYTES: usize = TxRequest::MAX_JSON_BYTES_RECOMMENDED; pub type ApiServerTask = JoinHandle>; -type SnapshotReleaseTask = Box; - -/// Joins every blocking snapshot-lease release before the HTTP worker exits. -/// -/// The scheduler passed into each lease guard owns a channel producer. The -/// receiver therefore cannot close while a guard can still submit a release, -/// including a guard dropping concurrently with graceful HTTP shutdown. -struct SnapshotReleaseDrain { - supervisor: JoinHandle<()>, -} - -impl SnapshotReleaseDrain { - async fn finish(self) { - if let Err(join) = self.supervisor.await { - abort_terminal(format_args!( - "snapshot lease release supervisor task failed: {join}" - )); - } - } -} - -fn supervise_snapshot_releases(shutdown: RuntimeScope) -> (ReleaseScheduler, SnapshotReleaseDrain) { - let (sender, receiver) = mpsc::unbounded_channel::(); - let schedule_shutdown = shutdown.clone(); - let scheduler: ReleaseScheduler = Arc::new(move |release| { - let _runtime_lifetime = &schedule_shutdown; - if sender.send(release).is_err() { - abort_terminal("snapshot lease release supervisor is unavailable"); - } - }); - let supervisor_shutdown = shutdown.clone(); - let supervisor = tokio::spawn(async move { - run_snapshot_release_supervisor(receiver, supervisor_shutdown).await; - }); - (scheduler, SnapshotReleaseDrain { supervisor }) -} - -async fn run_snapshot_release_supervisor( - mut receiver: mpsc::UnboundedReceiver, - _shutdown: RuntimeScope, -) { - let mut releases = JoinSet::new(); - let mut accepting = true; - - while accepting || !releases.is_empty() { - tokio::select! { - task = receiver.recv(), if accepting => { - match task { - Some(release) => { - releases.spawn_blocking(release); - } - None => accepting = false, - } - } - result = releases.join_next(), if !releases.is_empty() => { - if let Some(Err(join)) = result { - abort_terminal(format_args!("snapshot lease release task panicked: {join}")); - } - } - } - } -} - /// The `503` a handler returns when it refuses new work after shutdown is /// requested. pub(crate) fn reject_if_shutting_down(shutdown: &RuntimeScope) -> Result<(), ApiError> { @@ -312,12 +248,12 @@ pub(crate) fn start_on_listeners( tx_feed: L2TxFeed, config: ApiConfig, snapshot_state: SnapshotState, + landing: crate::landing::LandingFeed, ) -> ApiServerTask { - let (snapshot_release_scheduler, snapshot_release_drain) = - supervise_snapshot_releases(shutdown.clone()); let health_state = Arc::new(crate::egress::api::HealthState { tx_sender: tx_sender.clone(), shutdown: shutdown.clone(), + landing, }); // Serve exactly the domain `/tx` verifies against. Production builds it // with `build_input_domain`, which always sets all four fields. @@ -348,7 +284,6 @@ pub(crate) fn start_on_listeners( health_state, snapshot_state, shutdown.clone(), - snapshot_release_scheduler, ) .layer(TraceLayer::new_for_http()); @@ -359,8 +294,6 @@ pub(crate) fn start_on_listeners( serve(listeners.public, public, shutdown.clone()), serve(listeners.internal, internal, shutdown), ); - // Finish every scheduled release before ordinary HTTP shutdown ends. - snapshot_release_drain.finish().await; public.and(internal) }) } @@ -377,56 +310,23 @@ async fn serve( #[cfg(test)] mod tests { - use std::time::Duration; - - use tokio::sync::oneshot; - use super::*; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn snapshot_release_drain_waits_for_late_release() { - let shutdown = RuntimeScope::default(); - let (schedule, drain) = supervise_snapshot_releases(shutdown.clone()); - let (started_tx, started_rx) = oneshot::channel(); - let (unblock_tx, unblock_rx) = std::sync::mpsc::channel(); - let mut drain_task = tokio::spawn(drain.finish()); - - // Start draining first. The scheduler's producer token must keep the - // queue open so this release, submitted concurrently with shutdown, - // cannot be missed. - tokio::task::yield_now().await; - assert!(!drain_task.is_finished()); - - schedule(Box::new(move || { - let _ = started_tx.send(()); - unblock_rx.recv().expect("release test gate"); - })); - started_rx.await.expect("release task started"); - drop(schedule); - - assert!( - tokio::time::timeout(Duration::from_millis(25), &mut drain_task) - .await - .is_err(), - "drain returned before the in-flight release completed" + /// An application's policy refusal is a client error like any other lane + /// refusal: 422, and its text reaches the client unchanged. + #[tokio::test] + async fn lane_refusal_is_a_422_carrying_the_refusal_text() { + let text = "sender is frozen by policy"; + let response = ApiError::from(SequencerError::invalid(text)).into_response(); + + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read error body"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("decode error body"); + assert_eq!( + body, + serde_json::json!({ "ok": false, "code": "EXECUTION_REJECTED", "message": text }) ); - - unblock_tx.send(()).expect("unblock release"); - drain_task.await.expect("join release drain"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(unix)] - async fn snapshot_release_task_panic_is_terminal_before_drain_returns() { - if !crate::runtime::shutdown::abort_test_child( - "http::tests::snapshot_release_task_panic_is_terminal_before_drain_returns", - ) { - return; - } - let (schedule, drain) = supervise_snapshot_releases(RuntimeScope::default()); - schedule(Box::new(|| panic!("simulated lease release panic"))); - drop(schedule); - drain.finish().await; - panic!("terminal runtime returned instead of aborting"); } } diff --git a/sequencer/src/ingress/api.rs b/sequencer/src/ingress/api.rs index bd616849..4251589e 100644 --- a/sequencer/src/ingress/api.rs +++ b/sequencer/src/ingress/api.rs @@ -21,7 +21,7 @@ use tracing::debug; use crate::http::{ApiError, reject_if_shutting_down, storage_task}; use crate::ingress::inclusion_lane::PendingUserOp; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::Storage; +use crate::storage::{FeeQuote, Storage}; use sequencer_core::api::{ DomainResponse, FeeResponse, NonceResponse, TxRequest, TxResponse, parse_sender_address, }; @@ -94,8 +94,9 @@ pub(crate) fn router(submit: Arc, read: Arc) -> Router { ) } -/// Admission checks only size and signature. Nonce and `max_fee` are left to the -/// lane, where a stale op costs an in-memory comparison and no transaction; a +/// Admission checks only size and signature. The fee cap, the nonce and the +/// application's admission are left to the lane, whose refusal comes back as the +/// 422; there a stale op costs an in-memory comparison and no transaction; a /// pre-check here would add a SQLite read to every honest submit, and a future /// nonce or fresh address bypasses it. Revisit if persistent `429`s trace back to /// stale-nonce floods. @@ -135,9 +136,13 @@ async fn get_fee(State(state): State>) -> Result { - Ok(no_store(Json(FeeResponse::quote(fee, recommended_fee)))) - } + Ok(Some(FeeQuote { + frame_fee, + recommended_fee, + })) => Ok(no_store(Json(FeeResponse::quote( + frame_fee, + recommended_fee, + )))), Ok(None) => Err(ApiError::unavailable("no open frame")), Err(err) => { tracing::warn!(error = %err, "GET /fee failed"); diff --git a/sequencer/src/ingress/inclusion_lane/catch_up.rs b/sequencer/src/ingress/inclusion_lane/catch_up.rs index 76fe269d..d71c97ee 100644 --- a/sequencer/src/ingress/inclusion_lane/catch_up.rs +++ b/sequencer/src/ingress/inclusion_lane/catch_up.rs @@ -4,8 +4,9 @@ //! Replay committed application history after launch, before processing new user ops. use crate::storage::{HistoryReadError, L2TxContext, Snapshot, Storage}; -use sequencer_core::application::{Application, execute_direct_input, execute_valid_user_op}; -use sequencer_core::history::{ExecutedInputCount, HistoryClaim}; +use sequencer_core::application::{Application, Executed, execute}; +use sequencer_core::history::{HistoryClaim, L2TxCount}; +use sequencer_core::l2_tx::L2Tx; use super::error::CatchUpError; @@ -21,7 +22,7 @@ pub(super) fn catch_up_snapshot(storage: &mut Storage) -> Result Result<(), CatchUpError> { catch_up_application_paged(app, storage, start, DEFAULT_CATCH_UP_PAGE_SIZE) } @@ -29,17 +30,17 @@ pub(super) fn catch_up_application( pub(super) fn catch_up_application_paged( app: &mut impl Application, storage: &mut Storage, - start: ExecutedInputCount, + start: L2TxCount, page_size: usize, ) -> Result<(), CatchUpError> { - if app.executed_input_count() != start { + if app.l2_tx_count() != start { return Err(CatchUpError::SnapshotExecutionCountMismatch { - application: app.executed_input_count().get(), + application: app.l2_tx_count().get(), storage: start.get(), }); } tracing::debug!( - executed_input_count = start.get(), + l2_tx_count = start.get(), "inclusion lane resuming from snapshot" ); let bounds = storage @@ -49,7 +50,7 @@ pub(super) fn catch_up_application_paged( source, })?; let mut claim = HistoryClaim { - version: bounds.version, + era_id: bounds.era_id, next_input: start, }; loop { @@ -68,19 +69,24 @@ pub(super) fn catch_up_application_paged( claim = page.next_claim(); for row in page.rows { assert_eq!( - app.executed_input_count(), + app.l2_tx_count(), row.offset, "application replay offset differs from stored history" ); - match row.context { - L2TxContext::UserOp { tx, safe_block, .. } => { - execute_valid_user_op(app, &tx, safe_block) - .map_err(|source| CatchUpError::ReplayUserOp { source })?; - } - L2TxContext::DirectInput { tx, .. } => { - execute_direct_input(app, &tx) - .map_err(|source| CatchUpError::ReplayDirectInput { source })?; - } + // Replay runs the executor's own checks, so a stored op the + // restored application would skip stops the lane instead of + // silently shifting every later offset. + let executed = match row.context { + L2TxContext::UserOp { tx, .. } => execute(app, &L2Tx::UserOp(tx)) + .map_err(|source| CatchUpError::ReplayUserOp { source })?, + L2TxContext::Direct { tx, .. } => execute(app, &L2Tx::Direct(tx)) + .map_err(|source| CatchUpError::ReplayDirectInput { source })?, + }; + if let Executed::Skipped(reason) = executed { + return Err(CatchUpError::ReplaySkipped { + offset: row.offset.get(), + reason, + }); } } } diff --git a/sequencer/src/ingress/inclusion_lane/config.rs b/sequencer/src/ingress/inclusion_lane/config.rs index 505768fc..dbdc81df 100644 --- a/sequencer/src/ingress/inclusion_lane/config.rs +++ b/sequencer/src/ingress/inclusion_lane/config.rs @@ -7,6 +7,8 @@ use std::path::PathBuf; use std::time::Duration; +use crate::storage::SnapshotLeases; + const DEFAULT_MAX_USER_OPS_PER_CHUNK: usize = 64; const DEFAULT_SAFE_INPUT_BUFFER_CAPACITY: usize = 2048; const DEFAULT_MAX_BATCH_OPEN: Duration = Duration::from_secs(2 * 60 * 60); @@ -22,6 +24,9 @@ pub struct InclusionLaneConfig { /// lives in its own unique subdirectory of this path. The runtime /// sets this to `{data_dir}/dumps/`. pub dumps_dir: PathBuf, + /// The download leases `run` shares with egress; the lane's GC skips + /// leased snapshots. + pub snapshot_leases: SnapshotLeases, /// Cap on user ops dequeued per chunk. Bounds per-chunk SQL transaction /// size and (more importantly) ack latency for the first op in each chunk. pub max_user_ops_per_chunk: usize, @@ -39,9 +44,10 @@ pub struct InclusionLaneConfig { } impl InclusionLaneConfig { - pub fn new(dumps_dir: PathBuf) -> Self { + pub fn new(dumps_dir: PathBuf, snapshot_leases: SnapshotLeases) -> Self { Self { dumps_dir, + snapshot_leases, max_user_ops_per_chunk: DEFAULT_MAX_USER_OPS_PER_CHUNK, safe_input_buffer_capacity: DEFAULT_SAFE_INPUT_BUFFER_CAPACITY, max_batch_open: DEFAULT_MAX_BATCH_OPEN, diff --git a/sequencer/src/ingress/inclusion_lane/dump_info.rs b/sequencer/src/ingress/inclusion_lane/dump_info.rs index 17fe4425..e3cb1c37 100644 --- a/sequencer/src/ingress/inclusion_lane/dump_info.rs +++ b/sequencer/src/ingress/inclusion_lane/dump_info.rs @@ -206,14 +206,13 @@ pub fn read_info(dump_dir: &Path) -> io::Result { /// Explain why `dump_dir` is not a usable sequencer checkpoint dump. /// -/// Called when `info.toml` is missing. Distinguishes the common operator -/// mistake of pointing `--checkpoint-dump-dir` at a **watchdog** CM checkpoint -/// (`manifest.json` + `snapshot/`) from a merely empty/wrong path. +/// Called when `info.toml` is missing. Distinguishes a **watchdog** CM +/// checkpoint (`manifest.json` + `snapshot/`), an easy operator mix-up, from a +/// merely empty/wrong path. pub fn diagnose_missing_dump(dump_dir: &Path) -> String { let expected = format!( "expected a sequencer dump at {} with `info.toml` and a `state` app artifact. \ - Recovery additionally requires `checkpoint.toml` from `/finalized_snapshot`. \ - See docs/snapshots/lifecycle.md and docs/recovery/cockroach.md.", + See docs/snapshots/lifecycle.md.", dump_dir.display() ); @@ -235,9 +234,8 @@ pub fn diagnose_missing_dump(dump_dir: &Path) -> String { }; return format!( "this looks like a watchdog Cartesi Machine checkpoint (found {found}), \ - not a sequencer dump. `setup --recovery --checkpoint-dump-dir` cannot \ - use watchdog state under .../checkpoints// — those are CM \ - snapshots for the watchdog compare loop. {expected}" + not a sequencer dump. Watchdog state under .../checkpoints// \ + holds CM snapshots for the watchdog compare loop. {expected}" ); } @@ -255,61 +253,61 @@ pub fn diagnose_missing_dump(dump_dir: &Path) -> String { mod tests { use super::*; use alloy_primitives::Address; - use sequencer_core::application::{ - AppOutputs, ApplicationProgress, ValidationOutcome, execute_direct_input, - }; - use sequencer_core::history::ExecutedInputCount; - use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; + use sequencer_core::application::{Admission, AppOutputs, SchedulerLimits, execute}; + use sequencer_core::history::L2TxCount; + use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use sequencer_core::user_op::UserOp; + /// Persists only its count, as a single file or inside a directory. #[derive(Default)] struct PrefixDumpApp { - progress: ApplicationProgress, + count: L2TxCount, flushes: usize, } impl Application for PrefixDumpApp { - fn max_method_payload_bytes() -> usize { - 0 + fn user_nonce(&self, _sender: Address) -> Result { + unreachable!("dump-prefix tests execute only directs") } - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _fee: u16, - ) -> Result { - Ok(ValidationOutcome::Accept) + fn apply_user_op(&mut self, _op: &UserOpTx) -> Result { + unreachable!("dump-prefix tests execute only directs") } - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - self.progress.advance(safe_block); + fn apply_direct(&mut self, _direct: &Direct) -> Result { + self.count.advance(); Ok(Vec::new()) } - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { - self.progress.advance(input.block_number); - Ok(Vec::new()) + fn l2_tx_count(&self) -> L2TxCount { + self.count } - fn progress(&self) -> ApplicationProgress { - self.progress + fn max_method_payload_bytes() -> usize { + 0 + } + + fn scheduler_limits() -> SchedulerLimits { + app_core::application::WALLET_SCHEDULER_LIMITS + } + + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + Ok(Admission::Admit) } fn from_dump(prefix: &Path) -> Result { let bytes = std::fs::read(Self::state_file_in_dump(prefix))?; - let count = u64::from_le_bytes(bytes[..8].try_into().unwrap()); - let block = u64::from_le_bytes(bytes[8..].try_into().unwrap()); + let count = u64::from_le_bytes(bytes.as_slice().try_into().unwrap()); if DIRECTORY { assert_eq!(std::fs::read(prefix.join("manifest"))?, b"complete"); } Ok(Self { - progress: ApplicationProgress::try_new(ExecutedInputCount::new(count), block) - .expect("coherent dumped progress"), + count: L2TxCount::new(count), flushes: 0, }) } @@ -321,13 +319,8 @@ mod tests { std::fs::write(prefix.join("manifest"), b"complete")?; std::fs::File::open(prefix.join("manifest"))?.sync_all()?; } - let bytes = [ - self.progress.executed_input_count().get().to_le_bytes(), - self.progress.last_executed_safe_block().to_le_bytes(), - ] - .concat(); let state = Self::state_file_in_dump(prefix); - std::fs::write(&state, bytes)?; + std::fs::write(&state, self.count.get().to_le_bytes())?; std::fs::File::open(state)?.sync_all()?; if DIRECTORY { std::fs::File::open(prefix)?.sync_all()?; @@ -343,26 +336,35 @@ mod tests { prefix.to_path_buf() } } + + fn from_canonical_state( + _state_file: &Path, + _l2_tx_count: L2TxCount, + ) -> Result { + unreachable!("dump-prefix tests never adopt a canonical state") + } } fn assert_dump_prefix_round_trip() { let root = tempfile::tempdir().unwrap(); let dump = root.path().join("original"); - let input = DirectInput { + let input = L2Tx::Direct(Direct { + input_index: 0, sender: Address::ZERO, - block_number: 7, + block: 7, + timestamp: 0, payload: vec![], - }; + }); let mut app = PrefixDumpApp::::default(); - execute_direct_input(&mut app, &input).unwrap(); - let checkpoint = app.progress(); + execute(&mut app, &input).unwrap(); + let checkpoint = app.l2_tx_count(); create_dump_dir_with_info(&mut app, &dump, &sample()).unwrap(); assert_eq!( app.flushes, 1, "dump creation can flush mutable runtime state" ); assert_eq!( - app.progress(), + app.l2_tx_count(), checkpoint, "dumping preserves logical state" ); @@ -378,14 +380,14 @@ mod tests { assert_eq!( PrefixDumpApp::::from_dump(&app_prefix(&sibling)) .unwrap() - .progress(), + .l2_tx_count(), checkpoint, "other checkpoints survive deletion" ); - execute_direct_input(&mut first, &input).unwrap(); - assert_eq!(app.progress(), checkpoint); + execute(&mut first, &input).unwrap(); + assert_eq!(app.l2_tx_count(), checkpoint); assert_eq!( - second.progress(), + second.l2_tx_count(), checkpoint, "restored instances own independent state" ); @@ -393,8 +395,8 @@ mod tests { create_dump_dir_with_info(&mut first, &successor, &sample()).unwrap(); let reloaded = PrefixDumpApp::::from_dump(&app_prefix(&successor)).unwrap(); assert_eq!( - reloaded.progress(), - first.progress(), + reloaded.l2_tx_count(), + first.l2_tx_count(), "restored state survives source deletion" ); delete_dump_dir(&successor).unwrap(); @@ -435,7 +437,7 @@ mod tests { assert_eq!(read_checkpoint_info(&destination).unwrap(), receipt); let restored = PrefixDumpApp::::from_dump(&app_prefix(&destination)).unwrap(); - assert_eq!(restored.progress(), app.progress()); + assert_eq!(restored.l2_tx_count(), app.l2_tx_count()); } check::(); check::(); @@ -526,8 +528,7 @@ mod tests { "got: {msg}" ); assert!(msg.contains("manifest.json and snapshot/"), "got: {msg}"); - assert!(msg.contains("/finalized_snapshot"), "got: {msg}"); - assert!(msg.contains("checkpoint.toml"), "got: {msg}"); + assert!(msg.contains("with `info.toml` and a `state`"), "got: {msg}"); let err = read_info(dir.path()).unwrap_err(); assert_eq!(err.kind(), io::ErrorKind::NotFound); diff --git a/sequencer/src/ingress/inclusion_lane/error.rs b/sequencer/src/ingress/inclusion_lane/error.rs index 79c50724..0509a6b9 100644 --- a/sequencer/src/ingress/inclusion_lane/error.rs +++ b/sequencer/src/ingress/inclusion_lane/error.rs @@ -4,7 +4,7 @@ //! Lane-level error types. Returned from the lane's join handle; the runtime //! logs them and may shut down depending on severity. -use sequencer_core::application::AppError; +use sequencer_core::application::{AppError, SkipReason}; use thiserror::Error; use super::dump_info::CreateDumpDirError; @@ -36,6 +36,10 @@ pub enum InclusionLaneError { #[source] source: AppError, }, + /// The executor skipped an op the lane's pre-check had just found at its + /// sender's next nonce: the application's nonces are not deterministic. + #[error("executor skipped a user op the lane admitted: {reason}")] + AdmittedOpSkipped { reason: SkipReason }, #[error("snapshot at batch close failed: {0}")] Snapshot(#[from] TakeDumpError), #[error("loading Application from finalized snapshot failed: {0}")] @@ -43,8 +47,8 @@ pub enum InclusionLaneError { #[error("snapshot garbage collection failed: {0}")] Gc(#[from] GcError), #[error( - "no open Tip at lane startup; the runtime must establish it via \ - guarded startup recovery before starting the lane" + "no open Tip at lane startup; run must open one at startup before \ + starting the lane" )] NoOpenTip, } @@ -62,7 +66,9 @@ impl InclusionLaneError { Self::Gc(GcError::Storage(source)) => { crate::storage::is_persistent_storage_error(source) } - Self::CanonicalDivergence { .. } | Self::NoOpenTip => true, + Self::CanonicalDivergence { .. } | Self::NoOpenTip | Self::AdmittedOpSkipped { .. } => { + true + } Self::ChannelClosed => false, } } @@ -112,7 +118,11 @@ pub enum CatchUpError { #[source] source: AppError, }, - #[error("snapshot executed-input count mismatch: application={application}, storage={storage}")] + /// A stored user op that the executor skips on replay: the restored + /// application disagrees with the history it once executed. + #[error("replay skipped the stored user op at offset {offset}: {reason}")] + ReplaySkipped { offset: u64, reason: SkipReason }, + #[error("snapshot L2 tx count mismatch: application={application}, storage={storage}")] SnapshotExecutionCountMismatch { application: u64, storage: u64 }, #[error( "no snapshot registered before lane catch-up; \ @@ -130,9 +140,10 @@ impl CatchUpError { Self::ReplayUserOp { source } | Self::ReplayDirectInput { source } => { app_error_is_terminal(source) } - Self::History(_) | Self::NoSnapshot | Self::SnapshotExecutionCountMismatch { .. } => { - true - } + Self::History(_) + | Self::NoSnapshot + | Self::SnapshotExecutionCountMismatch { .. } + | Self::ReplaySkipped { .. } => true, } } } diff --git a/sequencer/src/ingress/inclusion_lane/mod.rs b/sequencer/src/ingress/inclusion_lane/mod.rs index dc1a7799..cca2c3ee 100644 --- a/sequencer/src/ingress/inclusion_lane/mod.rs +++ b/sequencer/src/ingress/inclusion_lane/mod.rs @@ -28,11 +28,10 @@ use tokio::sync::mpsc; use tokio::task::JoinHandle; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::{SafeFrontierState, SafeInputRange, Storage, StoredDirectInput, WriteHead}; -use sequencer_core::application::{ - Application, ExecutionOutcome, execute_direct_input, validate_and_execute_user_op, -}; -use sequencer_core::history::ExecutedInputCount; +use crate::storage::{SafeFrontierState, SafeInputRange, Storage, WriteHead}; +use sequencer_core::application::{Application, Executed, check_live_user_op, execute}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use sequencer_core::user_op::SignedUserOp; use catch_up::{catch_up_application, catch_up_snapshot}; @@ -48,14 +47,15 @@ pub struct InclusionLane { } impl InclusionLane { - /// Spawn the lane on a blocking thread. Startup recovery establishes the - /// open Tip, so the lane loads its resume state and never initializes a Tip. + /// Spawn the lane on a blocking thread. `run`'s startup opens the Tip if + /// none is open, so the lane loads its resume state and never initializes + /// a Tip. /// It fails loudly with /// [`InclusionLaneError::NoOpenTip`] if the invariant was somehow violated. /// - /// Restore the newest surviving snapshot and replay from its exact - /// application count. Startup preserves a rollback-safe checkpoint before - /// admitting any runtime work. + /// Restore the newest snapshot and replay from its exact application + /// count. Setup leaves one, the era's baseline, before `run` launches any + /// worker. /// /// Returns the input MPSC sender (for the API to enqueue user /// ops) and the join handle (for the runtime to observe lane @@ -79,7 +79,7 @@ impl InclusionLane { // can never drift apart. let checkpoint = catch_up_snapshot(&mut storage) .map_err(|source| InclusionLaneError::CatchUp { source })?; - let app = A::from_dump(&dump_info::app_prefix(&checkpoint.dump.prefix)) + let app = A::from_dump(&dump_info::app_prefix(&checkpoint.prefix)) .map_err(InclusionLaneError::LoadFromDump)?; let mut lane = Self { rx, @@ -88,20 +88,20 @@ impl InclusionLane { storage, config, }; - lane.run_forever(checkpoint.executed_input_count) + lane.run_forever(checkpoint.l2_tx_count) }); (tx, handle) } - fn run_forever(&mut self, catch_up_from: ExecutedInputCount) -> Result<(), InclusionLaneError> { + fn run_forever(&mut self, catch_up_from: L2TxCount) -> Result<(), InclusionLaneError> { self.run_catch_up(catch_up_from)?; let mut included = Vec::with_capacity(self.config.max_user_ops_per_chunk.max(1)); let mut safe_inputs = Vec::with_capacity(self.config.safe_input_buffer_capacity.max(1)); - // Startup recovery established the Tip before runtime admission. + // `run`'s startup opened the Tip before launching the workers. // Read the open frame (fail-loud if absent) and the drain // cursor together from storage, so both come from the same place. Any - // leading range already sequenced into the Tip's frames (genesis or a - // recovery batch) was replayed into the app by `run_catch_up` above, + // leading range already sequenced into the Tip's frames (an era's + // first batch included) was replayed into the app by `run_catch_up` above, // so there is no cold-start drain here. let head = self .storage @@ -129,7 +129,6 @@ impl InclusionLane { if turn.hit_batch_target() || should_close_batch_by_time(&lane_state.head, &self.config) { - let next_safe_block = lane_state.head.safe_block; // Atomic close: dump the app state, then seal the batch // and register its snapshot in one transaction. // A create_dump failure leaves the batch open for retry; @@ -139,7 +138,6 @@ impl InclusionLane { &mut self.app, &mut self.storage, &mut lane_state.head, - next_safe_block, &self.config.dumps_dir, ) .map_err(InclusionLaneError::Snapshot)?; @@ -150,7 +148,7 @@ impl InclusionLane { } } - fn run_catch_up(&mut self, start_offset: ExecutedInputCount) -> Result<(), InclusionLaneError> { + fn run_catch_up(&mut self, start_offset: L2TxCount) -> Result<(), InclusionLaneError> { catch_up_application(&mut self.app, &mut self.storage, start_offset) .map_err(|source| InclusionLaneError::CatchUp { source }) } @@ -189,7 +187,7 @@ impl InclusionLane { fn maybe_advance_safe_frontier( &mut self, lane_state: &mut LaneState, - safe_inputs: &mut Vec, + safe_inputs: &mut Vec, ) -> Result<(), InclusionLaneError> { if !lane_state.frontier_check_due(self.config.frontier_min_interval) { return Ok(()); @@ -216,13 +214,13 @@ impl InclusionLane { lane_state.next_safe_input_index ); assert!( - frontier.safe_block >= lane_state.head.safe_block, + frontier.safe_block >= lane_state.head.safe_block(), "safe-block frontier regressed: observed={}, frame={}", frontier.safe_block, - lane_state.head.safe_block, + lane_state.head.safe_block(), ); - if frontier.safe_block - lane_state.head.safe_block - < sequencer_core::protocol::ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS + if frontier.safe_block - lane_state.head.safe_block() + < sequencer_core::protocol::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS { return Ok(()); } @@ -240,27 +238,37 @@ impl InclusionLane { // Acceptance can advance independently of application inputs. A bounded // reconciliation cadence reclaims retired artifacts even under load. - let removed = snapshot::run_gc(&mut self.storage)?; + let removed = snapshot::run_gc(&mut self.storage, &self.config.snapshot_leases)?; if removed > 0 { tracing::debug!(removed, "snapshot garbage collection removed artifacts"); } Ok(()) } + /// Execute the directs `range` drains. Storage hands out only those that + /// execute: no batch-submitter input and none over the pinned payload + /// cap, which therefore get no execution row. fn execute_direct_range( &mut self, range: SafeInputRange, - chunk: &mut Vec, + chunk: &mut Vec, ) -> Result, InclusionLaneError> { let mut executions = Vec::new(); for chunk_range in range.chunks(self.config.safe_input_buffer_capacity.max(1) as u64) { self.storage.fill_direct_inputs(chunk_range, chunk)?; - for input in chunk.iter() { - let receipt = execute_direct_input(&mut self.app, &input.input) - .map_err(|source| InclusionLaneError::ExecuteDirectInput { source })?; + for direct in chunk.drain(..) { + let safe_input_index = direct.input_index; + let l2_tx_index = match execute(&mut self.app, &L2Tx::Direct(direct)) + .map_err(|source| InclusionLaneError::ExecuteDirectInput { source })? + { + Executed::Done { index, .. } => index, + Executed::Skipped(reason) => { + unreachable!("the executor never skips a direct, got {reason}") + } + }; executions.push(crate::storage::DirectInputExecution { - safe_input_index: input.safe_input_index, - executed_input_offset: receipt.offset, + safe_input_index, + l2_tx_index, }); } } @@ -294,7 +302,7 @@ fn persist_included_user_ops( included: &mut Vec, ) -> Result<(), InclusionLaneError> { storage - .append_executed_user_ops_chunk(head, included.as_slice()) + .append_executed_user_ops_chunk(head, included.iter().map(IncludedUserOp::executed)) .map_err(|err| { for item in included.drain(..) { let _ = item.pending.respond_to.send(Err(SequencerError::internal( @@ -336,14 +344,19 @@ impl TurnOutcome { fn should_close_batch_by_time(head: &WriteHead, config: &InclusionLaneConfig) -> bool { // A backwards clock step makes `duration_since` err; `unwrap_or_default` // then reads as age 0, silently stalling the time-based close trigger - // until the clock catches up. Acceptable: the size trigger - // is unaffected, and a wedge here is liveness-only, never correctness. + // until the clock catches up. The size trigger is unaffected, but a batch + // that is never sealed keeps aging toward the staleness window, and once + // it cannot land the history is void and the era ends. A backwards step + // longer than the window is therefore not only a liveness problem. let age = SystemTime::now() - .duration_since(head.batch_created_at) + .duration_since(head.batch_created_at()) .unwrap_or_default(); age >= config.max_batch_open } +/// Refuse the op with its 422 text, or execute it in the open frame at the +/// frame's price and safe block. The pre-check runs the executor's own nonce +/// rule, so an admitted op that `execute` skips is a terminal lane fault. fn execute_user_op( app: &mut impl Application, item: PendingUserOp, @@ -351,43 +364,67 @@ fn execute_user_op( frame_safe_block: u64, included: &mut Vec, ) -> Result<(), InclusionLaneError> { - match validate_and_execute_user_op( - app, - item.signed.sender, - &item.signed.user_op, + let sender = item.signed.sender; + let op = &item.signed.user_op; + match check_live_user_op(app, sender, op, current_frame_fee) { + Ok(None) => {} + Ok(Some(refusal)) => { + let _ = item + .respond_to + .send(Err(SequencerError::invalid(refusal.to_string()))); + return Ok(()); + } + Err(source) => { + return Err(fail_user_op( + item, + InclusionLaneError::ExecuteUserOp { source }, + )); + } + } + let tx = L2Tx::UserOp(UserOpTx::new( + sender, + op, current_frame_fee, frame_safe_block, - ) { - Ok(ExecutionOutcome::Included(receipt)) => included.push(IncludedUserOp { + )); + match execute(app, &tx) { + Ok(Executed::Done { index, .. }) => included.push(IncludedUserOp { pending: item, - executed_input_offset: receipt.offset, + l2_tx_index: index, }), - Ok(ExecutionOutcome::Invalid(reason)) => { - let _ = item - .respond_to - .send(Err(SequencerError::invalid(reason.to_string()))); + Ok(Executed::Skipped(reason)) => { + return Err(fail_user_op( + item, + InclusionLaneError::AdmittedOpSkipped { reason }, + )); } - // An application error has no canonical successor. Discard the - // application-owned state; this op is never persisted or acknowledged. - Err(err) => { - // The client gets a fixed message: the application's reason and - // any I/O detail travel on the lane error and the log, never - // into the public 500 body. - let _ = item.respond_to.send(Err(SequencerError::internal( - "application internal error".to_string(), - ))); - return Err(InclusionLaneError::ExecuteUserOp { source: err }); + Err(source) => { + return Err(fail_user_op( + item, + InclusionLaneError::ExecuteUserOp { source }, + )); } } Ok(()) } +/// An op whose execution failed has no canonical successor, so the lane stops +/// and the op is never persisted or acknowledged. The client gets a fixed +/// message: the cause travels on the lane error and the log, never into the +/// public 500 body. +fn fail_user_op(item: PendingUserOp, error: InclusionLaneError) -> InclusionLaneError { + let _ = item.respond_to.send(Err(SequencerError::internal( + "application internal error".to_string(), + ))); + error +} + /// Dequeue and execute up to `max_chunk` user ops, stopping early if the batch /// would cross its size target. Returns the outcome that drove the stop. /// -/// `head.batch_user_op_count` reflects already-persisted ops; `included.len()` +/// `head.batch_user_op_count()` reflects already-persisted ops; `included.len()` /// is the count we'd add by persisting now. When their sum's bytes equal or -/// exceed `head.max_batch_user_op_bytes`, we stop and the caller closes the +/// exceed `head.max_batch_user_op_bytes()`, we stop and the caller closes the /// batch. fn dequeue_and_execute_user_op_chunk( rx: &mut mpsc::Receiver, @@ -401,16 +438,16 @@ fn dequeue_and_execute_user_op_chunk( while attempted < max_chunk { match rx.try_recv() { Ok(item) => { - execute_user_op(app, item, head.frame_fee, head.safe_block, included)?; + execute_user_op(app, item, head.frame_fee(), head.safe_block(), included)?; attempted += 1; let included_count = u64::try_from(included.len()).expect("in-memory chunk length must fit in u64"); let projected = head - .batch_user_op_count + .batch_user_op_count() .checked_add(included_count) .expect("batch user-op count overflow: contract-impossible"); - if user_op_count_to_bytes::(projected) >= head.max_batch_user_op_bytes { + if user_op_count_to_bytes::(projected) >= head.max_batch_user_op_bytes() { return Ok(TurnOutcome::HitBatchTarget); } } @@ -446,7 +483,7 @@ fn user_op_count_to_bytes(user_op_count: u64) -> u64 { /// Lane-local state threaded through every loop iteration. /// /// `head` and `next_safe_input_index` stay in lockstep — every safe-frontier -/// advance updates both `head.safe_block` (persisted in the open frame) and +/// advance updates both the head's safe block (persisted in the open frame) and /// `next_safe_input_index` (in-memory drain cursor). /// /// `last_frontier_check` is the time gate's bookkeeping; `None` initially so diff --git a/sequencer/src/ingress/inclusion_lane/snapshot.rs b/sequencer/src/ingress/inclusion_lane/snapshot.rs index f817f4f9..dbe4c02c 100644 --- a/sequencer/src/ingress/inclusion_lane/snapshot.rs +++ b/sequencer/src/ingress/inclusion_lane/snapshot.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use super::dump_info::{self, DumpInfo}; -use crate::storage::{Storage, WriteHead}; +use crate::storage::{SnapshotLeases, Storage, WriteHead}; use sequencer_core::application::Application; #[derive(Debug, thiserror::Error)] @@ -25,11 +25,11 @@ pub enum GcError { Storage(#[from] rusqlite::Error), } -pub(super) fn run_gc(storage: &mut Storage) -> Result { - let removed = storage.gc_unreferenced_dumps()?; - for row in &removed { - if let Err(err) = dump_info::delete_dump_dir(&row.prefix) { - tracing::warn!(error = %err, prefix = ?row.prefix, +pub(super) fn run_gc(storage: &mut Storage, leases: &SnapshotLeases) -> Result { + let removed = storage.collect_snapshots(leases)?; + for prefix in &removed { + if let Err(err) = dump_info::delete_dump_dir(prefix) { + tracing::warn!(error = %err, ?prefix, "GC: filesystem delete failed; orphan left for next startup sweep"); } } @@ -46,33 +46,24 @@ pub(super) fn close_batch_with_snapshot( app: &mut A, storage: &mut Storage, head: &mut WriteHead, - next_safe_block: u64, dumps_dir: &Path, ) -> Result<(), TakeDumpError> { - let batch_index = head.batch_index; - let nonce = storage.batch_nonce(batch_index)?; + let nonce = head.batch_nonce(); let dump_dir = make_dump_dir(dumps_dir, nonce); let started = Instant::now(); dump_info::create_dump_dir_with_info(app, &dump_dir, &DumpInfo::at_batch_close(nonce))?; let snapshot_elapsed = started.elapsed(); - storage.close_frame_and_batch_with_snapshot( - head, - next_safe_block, - &dump_dir, - app.executed_input_count(), - )?; + storage.close_frame_and_batch_with_snapshot(head, &dump_dir, app.l2_tx_count())?; let snapshot_ms = snapshot_elapsed.as_millis(); if snapshot_elapsed >= SLOW_SNAPSHOT { tracing::warn!( - batch_index, - nonce, + batch_nonce = nonce, snapshot_ms, "slow batch-close snapshot blocked user-op admission" ); } else { tracing::debug!( - batch_index, - nonce, + batch_nonce = nonce, snapshot_ms, "batch closed with snapshot" ); @@ -82,7 +73,7 @@ pub(super) fn close_batch_with_snapshot( fn make_dump_dir(dumps_dir: &Path, nonce: u64) -> PathBuf { // Unique per call within a process: nonce + nanos + atomic counter. - // Nonces can be reused across recovery cascades, so they alone + // Several dumps can carry one nonce, so it alone // don't guarantee uniqueness; the nanos+counter pair does. The name // is opaque — checkpoint metadata lives in the dir's `info.toml`, // never in the path. diff --git a/sequencer/src/ingress/inclusion_lane/tests.rs b/sequencer/src/ingress/inclusion_lane/tests.rs index 33634720..2c73156f 100644 --- a/sequencer/src/ingress/inclusion_lane/tests.rs +++ b/sequencer/src/ingress/inclusion_lane/tests.rs @@ -10,18 +10,24 @@ use tokio::sync::{mpsc, oneshot}; use crate::runtime::shutdown::RuntimeScope; use crate::storage::test_helpers::{ - SENDER_A, default_protocol_timing, record_canonical_divergence, set_one_op_batch_target, + SENDER_A, WriteHeadParts, record_canonical_divergence, set_one_op_batch_target, +}; +use crate::storage::{ + DirectInputExecution, IngestedSafeInput, L2TxContext, SafeInputRange, Storage, StoredSafeInput, + WriteHead, }; -use crate::storage::{DirectInputExecution, SafeInputRange, Storage, StoredSafeInput, WriteHead}; use crate::test_support::{pin_test_deployment_identity, temp_db}; -use sequencer_core::application::{AppError, Application}; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx}; +use alloy_primitives::B256; +use sequencer_core::application::{ + Admission, AppError, AppOutputs, Application, SchedulerLimits, SkipReason, +}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use sequencer_core::user_op::{SignedUserOp, UserOp}; use crate::test_support::{ReplayEvent, StubApp, shutdown_lane, wait_until}; -use super::catch_up::{catch_up_application_paged, catch_up_snapshot}; +use super::catch_up::catch_up_application_paged; use super::dequeue_and_execute_user_op_chunk; use super::error::CatchUpError; use super::{ @@ -37,6 +43,7 @@ fn default_test_config() -> InclusionLaneConfig { dumps_dir: tempfile::tempdir() .expect("create dumps_dir tempdir") .keep(), + snapshot_leases: crate::storage::SnapshotLeases::default(), max_user_ops_per_chunk: 16, safe_input_buffer_capacity: 16, max_batch_open: Duration::MAX, @@ -68,7 +75,7 @@ fn register_genesis_snapshot(app: &mut A, storage: &mut Storage, ) .expect("create genesis dump"); storage - .insert_baseline_snapshot(&dump_dir, ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&dump_dir, L2TxCount::ZERO) .expect("insert finalized snapshot"); } @@ -83,7 +90,7 @@ async fn start_lane( let mut storage = Storage::open(db_path).expect("open storage"); pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) .expect("seed observed safe head"); let mut app = StubApp::default(); register_genesis_snapshot(&mut app, &mut storage, &config.dumps_dir); @@ -109,7 +116,7 @@ fn make_pending_user_op( let user_op = UserOp { nonce: 0, // Must be >= the DB default recommended_fee (1356) to pass the - // protocol-level max_fee >= fee_price check in the trait default. + // lane's max_fee >= frame fee check. max_fee: u16::MAX, data: vec![seed; 4].into(), }; @@ -131,7 +138,7 @@ fn make_included_user_op(seed: u8, offset: u64) -> IncludedUserOp { let (pending, _response) = make_pending_user_op(seed); IncludedUserOp { pending, - executed_input_offset: ExecutedInputCount::new(offset), + l2_tx_index: L2TxCount::new(offset), } } @@ -140,7 +147,7 @@ fn fast_turn_processes_at_most_one_rejected_chunk() { let db = temp_db("one-rejected-chunk-per-fast-turn"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("seed observed safe head"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -186,7 +193,7 @@ async fn sustained_rejected_queue_cannot_starve_poisoned_frontier() { let db = temp_db("reject-flood-does-not-starve-frontier"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("seed observed safe head"); storage.ensure_open_tip().expect("establish open tip"); @@ -211,8 +218,7 @@ async fn sustained_rejected_queue_cannot_starve_poisoned_frontier() { storage, config, }; - let mut lane_handle = - tokio::task::spawn_blocking(move || lane.run_forever(ExecutedInputCount::ZERO)); + let mut lane_handle = tokio::task::spawn_blocking(move || lane.run_forever(L2TxCount::ZERO)); let producer_tx = tx.clone(); let producer = tokio::spawn(async move { @@ -275,7 +281,7 @@ fn reconciliation_digests_an_epoch_sized_outage_backlog_in_one_turn() { let config = default_test_config(); pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) .expect("seed observed safe head"); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -292,8 +298,7 @@ fn reconciliation_digests_an_epoch_sized_outage_backlog_in_one_turn() { .append_safe_inputs( JUMP_TARGET_BLOCK, backlog.as_slice(), - SENDER_A, - &default_protocol_timing(), + Address::repeat_byte(0xff), ) .expect("seed the outage backlog at the jump target"); @@ -336,7 +341,7 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { let config = default_test_config(); pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) .expect("seed observed safe head"); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -360,8 +365,7 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { payload: vec![0xaa], block_number: 4, }], - SENDER_A, - &default_protocol_timing(), + Address::repeat_byte(0xff), ) .expect("append below-threshold direct"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) @@ -370,7 +374,7 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { assert_eq!(lane.storage.next_undrained_safe_input_index().unwrap(), 0); lane.storage - .append_safe_inputs(5, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(5, &[], Address::repeat_byte(0xff)) .expect("advance to first clock tick"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("rotate at block 5"); @@ -390,35 +394,35 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { assert_eq!(sequenced.len(), 2); assert!(matches!( &sequenced[0].context, - crate::storage::L2TxContext::DirectInput { - tx: DirectInput { - block_number: 4, - .. - }, + L2TxContext::Direct { + tx: Direct { block: 4, .. }, .. } )); assert!(matches!( &sequenced[1].context, - crate::storage::L2TxContext::UserOp { safe_block: 5, .. } + L2TxContext::UserOp { + tx: UserOpTx { block: 5, .. }, + .. + } )); lane.storage - .append_safe_inputs(32, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(32, &[], Address::repeat_byte(0xff)) .expect("jump safe head"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("collapse jump to one frame"); assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0, 5, 32]); lane.storage - .append_safe_inputs(36, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(36, &[], Address::repeat_byte(0xff)) .expect("advance below reset threshold"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("observe block 36"); assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0, 5, 32]); lane.storage - .append_safe_inputs(37, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(37, &[], Address::repeat_byte(0xff)) .expect("advance to reset threshold"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("rotate at block 37"); @@ -431,7 +435,7 @@ fn structural_batch_frame_does_not_reset_frame_clock_anchor() { let db = temp_db("batch-frame-does-not-reset-clock"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("seed observed safe head"); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -448,18 +452,17 @@ fn structural_batch_frame_does_not_reset_frame_clock_anchor() { let mut safe_inputs = Vec::new(); lane.storage - .append_safe_inputs(4, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(4, &[], SENDER_A) .expect("advance below threshold"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("observe block 4"); - let unchanged_clock = state.head.safe_block; lane.storage - .close_frame_and_batch(&mut state.head, unchanged_clock) + .close_frame_and_batch(&mut state.head) .expect("create successor batch frame"); assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0, 0]); lane.storage - .append_safe_inputs(5, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(5, &[], SENDER_A) .expect("reach original five-block threshold"); lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) .expect("clock tick after structural frame"); @@ -471,7 +474,7 @@ fn poisoned_frontier_outranks_frame_clock_and_closes_intake() { let db = temp_db("poisoned-frontier-outranks-clock"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("seed observed safe head"); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -527,7 +530,7 @@ fn seed_replay_fixture(db_path: &str) -> Vec { let user_op_a = make_included_user_op(0x51, 0); let user_op_b = make_included_user_op(0x52, 1); storage - .append_executed_user_ops_chunk(&mut head, &[user_op_a, user_op_b]) + .append_included_user_ops(&mut head, &[user_op_a, user_op_b]) .expect("append attributed first-frame user ops"); storage .append_safe_inputs( @@ -538,7 +541,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { block_number: 10, }], batch_submitter_address, - &default_protocol_timing(), ) .expect("append first direct input"); storage @@ -548,14 +550,14 @@ fn seed_replay_fixture(db_path: &str) -> Vec { SafeInputRange::new(0, 1), &[DirectInputExecution { safe_input_index: 0, - executed_input_offset: ExecutedInputCount::new(2), + l2_tx_index: L2TxCount::new(2), }], ) .expect("close first frame with direct attribution"); let user_op_c = make_included_user_op(0x53, 3); storage - .append_executed_user_ops_chunk(&mut head, &[user_op_c]) + .append_included_user_ops(&mut head, &[user_op_c]) .expect("append attributed second-frame user op"); storage .append_safe_inputs( @@ -566,7 +568,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { block_number: 20, }], batch_submitter_address, - &default_protocol_timing(), ) .expect("append second direct input"); storage @@ -576,7 +577,7 @@ fn seed_replay_fixture(db_path: &str) -> Vec { SafeInputRange::new(1, 2), &[DirectInputExecution { safe_input_index: 1, - executed_input_offset: ExecutedInputCount::new(4), + l2_tx_index: L2TxCount::new(4), }], ) .expect("close second frame with direct attribution"); @@ -590,7 +591,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { block_number: 30, }], batch_submitter_address, - &default_protocol_timing(), ) .expect("append third direct input"); storage @@ -600,7 +600,7 @@ fn seed_replay_fixture(db_path: &str) -> Vec { SafeInputRange::new(2, 3), &[DirectInputExecution { safe_input_index: 2, - executed_input_offset: ExecutedInputCount::new(5), + l2_tx_index: L2TxCount::new(5), }], ) .expect("close third frame with direct attribution"); @@ -615,8 +615,10 @@ fn seed_replay_fixture(db_path: &str) -> Vec { data: vec![0x52; 4], }, ReplayEvent::DirectInput { + input_index: 0, sender: Address::ZERO, block_number: 10, + timestamp: 0, payload: vec![0xaa], }, ReplayEvent::UserOp { @@ -624,13 +626,17 @@ fn seed_replay_fixture(db_path: &str) -> Vec { data: vec![0x53; 4], }, ReplayEvent::DirectInput { + input_index: 1, sender: Address::ZERO, block_number: 20, + timestamp: 0, payload: vec![0xbb], }, ReplayEvent::DirectInput { + input_index: 2, sender: Address::ZERO, block_number: 30, + timestamp: 0, payload: vec![0xcc], }, ] @@ -643,14 +649,14 @@ fn read_count(db_path: &str, table: &str) -> i64 { .expect("count rows") } -fn read_frame_direct_count(db_path: &str, batch_index: i64, frame_in_batch: i64) -> i64 { +fn read_frame_direct_count(db_path: &str, batch_nonce: i64, frame_in_batch: i64) -> i64 { let conn = Storage::open_connection(db_path).expect("open sqlite reader"); conn.query_row( "SELECT COUNT(*) FROM application_inputs - WHERE batch_index = ?1 + WHERE batch_nonce = ?1 AND frame_in_batch = ?2 AND safe_input_index IS NOT NULL", - params![batch_index, frame_in_batch], + params![batch_nonce, frame_in_batch], |row| row.get(0), ) .expect("query frame direct count") @@ -661,7 +667,7 @@ fn read_frame_safe_blocks(db_path: &str) -> Vec { let mut statement = conn .prepare( "SELECT safe_block FROM frames \ - ORDER BY batch_index ASC, frame_in_batch ASC", + ORDER BY batch_nonce ASC, frame_in_batch ASC", ) .expect("prepare frame-clock query"); statement @@ -708,8 +714,7 @@ async fn direct_inputs_close_frame_and_persist_drain() { payload: vec![0xaa], block_number: 10, }], - SENDER_A, - &default_protocol_timing(), + Address::repeat_byte(0xff), ) .expect("append safe direct input"); @@ -729,11 +734,11 @@ async fn sequenced_safe_inputs_are_drained_but_not_executed() { let db = temp_db("sequenced-safe-inputs-skip"); let batch_submitter_address = Address::from([0xfe; 20]); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + pin_test_deployment_identity(&mut storage, batch_submitter_address); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], batch_submitter_address) .expect("seed observed safe head"); let config = default_test_config(); - pin_test_deployment_identity(&mut storage, batch_submitter_address); { let mut app = StubApp::default(); register_genesis_snapshot(&mut app, &mut storage, &config.dumps_dir); @@ -766,7 +771,6 @@ async fn sequenced_safe_inputs_are_drained_but_not_executed() { block_number: 10, }], batch_submitter_address, - &default_protocol_timing(), ) .expect("append safe batch-submitter input"); @@ -783,10 +787,10 @@ async fn sequenced_safe_inputs_are_drained_but_not_executed() { assert!(history_rows(&mut storage, 16).is_empty()); assert_eq!(read_frame_direct_count(db.path.as_str(), 0, 1), 0); let mut fresh_app = StubApp::default(); - catch_up_application_paged(&mut fresh_app, &mut storage, ExecutedInputCount::ZERO, 16) + catch_up_application_paged(&mut fresh_app, &mut storage, L2TxCount::ZERO, 16) .expect("catch up"); assert_eq!( - fresh_app.executed_input_count().get(), + fresh_app.l2_tx_count().get(), 0, "batch-submitter safe input should be skipped by the local app" ); @@ -809,7 +813,7 @@ async fn direct_inputs_are_paginated_by_buffer_capacity() { }); } feeder_storage - .append_safe_inputs(10, directs.as_slice(), SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, directs.as_slice(), Address::repeat_byte(0xff)) .expect("append safe direct inputs"); let drained = wait_until(Duration::from_secs(2), || { @@ -837,8 +841,7 @@ async fn safe_inputs_already_available_are_sequenced_before_later_user_ops() { payload: vec![0xaa], block_number: 10, }], - SENDER_A, - &default_protocol_timing(), + Address::repeat_byte(0xff), ) .expect("append safe direct input"); @@ -858,7 +861,7 @@ async fn safe_inputs_already_available_are_sequenced_before_later_user_ops() { .expect("wait for ack") .expect("ack channel open"); - let replay: Vec = { + let replay: Vec = { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); history_rows(&mut storage, 1_000_000) .into_iter() @@ -871,11 +874,11 @@ async fn safe_inputs_already_available_are_sequenced_before_later_user_ops() { assert_eq!(replay.len(), 2); assert!(matches!( replay.first(), - Some(SequencedL2Tx::Direct(direct)) if direct.payload.as_slice() == [0xaa] + Some(L2Tx::Direct(direct)) if direct.payload.as_slice() == [0xaa] )); assert!(matches!( replay.get(1), - Some(SequencedL2Tx::UserOp(user_op)) if user_op.data.as_slice() == [0x31, 0x31, 0x31, 0x31] + Some(L2Tx::UserOp(user_op)) if user_op.data.as_slice() == [0x31, 0x31, 0x31, 0x31] )); } @@ -959,16 +962,12 @@ async fn batch_closes_when_max_user_op_bytes_is_reached() { /// in `dequeue_and_execute_user_op_chunk` never triggers from the size check /// alone. Tests that want to exercise the size check construct their own. fn unbounded_head() -> WriteHead { - WriteHead { - batch_index: 0, + WriteHeadParts { batch_created_at: SystemTime::now(), - frame_fee: 0, - safe_block: 0, - batch_user_op_count: 0, - open_frame_user_op_count: 0, - frame_in_batch: 0, max_batch_user_op_bytes: u64::MAX, + ..WriteHeadParts::ZERO } + .build() } #[test] @@ -1050,16 +1049,13 @@ fn catch_up_replays_multiple_pages() { assert_eq!(mappings, vec![0, 1, 2, 3, 4, 5]); let mut app = StubApp::default(); - catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) + catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2) .expect("catch up in pages"); + // The expected events carry each direct's index, block and timestamp, so + // they also pin what replay hands the application. assert_eq!(app.replayed, expected); - assert_eq!(app.executed_input_count().get(), expected.len() as u64); - assert_eq!( - app.last_executed_safe_block(), - 30, - "catch-up must advance application-owned progress through execution" - ); + assert_eq!(app.l2_tx_count().get(), expected.len() as u64); } #[test] @@ -1068,10 +1064,10 @@ fn catch_up_rejects_history_gap_before_execution() { seed_replay_fixture(&db.path); let mut storage = Storage::open(&db.path).unwrap(); let conn = Storage::open_connection(&db.path).unwrap(); - conn.execute_batch("DROP TRIGGER trg_protect_valid_application_input_delete; DELETE FROM application_inputs WHERE offset=1;").unwrap(); + conn.execute_batch("DROP TRIGGER trg_application_inputs_not_deletable; DELETE FROM application_inputs WHERE l2_tx_count=1;").unwrap(); let mut app = StubApp::default(); let failure = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2); + let _ = catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2); })); assert!(failure.is_err()); assert!(app.replayed.is_empty()); @@ -1082,9 +1078,9 @@ fn catch_up_rejects_wrong_snapshot_count_before_execution() { let db = temp_db("catch-up-wrong-mapping"); seed_replay_fixture(db.path.as_str()); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut app = StubApp::with_progress(3, 0); + let mut app = StubApp::with_progress(3); - let err = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) + let err = catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2) .expect_err("mapping from a different application boundary must stop catch-up"); assert!(matches!( @@ -1098,7 +1094,7 @@ fn catch_up_rejects_wrong_snapshot_count_before_execution() { app.replayed.is_empty(), "mapping is checked before execution" ); - assert_eq!(app.executed_input_count(), ExecutedInputCount::new(3)); + assert_eq!(app.l2_tx_count(), L2TxCount::new(3)); assert!(InclusionLaneError::CatchUp { source: err }.is_terminal_invariant()); } @@ -1109,252 +1105,19 @@ fn catch_up_handles_mixed_user_ops_and_direct_inputs_across_page_boundary() { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let mut app = StubApp::default(); - catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 4) + catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 4) .expect("catch up across page boundary"); assert_eq!(app.replayed, expected); } -#[test] -fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { - let db = temp_db("standard-recovery-history-boundary"); - let dumps_dir = tempfile::tempdir().expect("create dump directory"); - let batch_submitter = Address::from([0xff; 20]); - let direct_sender = Address::repeat_byte(0x44); - let protocol = default_protocol_timing(); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, batch_submitter); - storage - .append_safe_inputs(0, &[], batch_submitter, &protocol) - .expect("seed genesis safe head"); - storage.ensure_open_tip().expect("open genesis Tip"); - let mut head = storage.open_state().unwrap().unwrap(); - let initial_history = storage.history_state().expect("initial history"); - let mut live_app = StubApp::default(); - - // Batch 0 is the surviving prefix. Its pending snapshot reflects exactly - // input 0 and becomes the recovery checkpoint once the batch lands. - let (prefix_op, _response) = make_pending_user_op(0x51); - let mut included = Vec::new(); - super::execute_user_op( - &mut live_app, - prefix_op, - head.frame_fee, - head.safe_block, - &mut included, - ) - .expect("execute prefix user op"); - assert_eq!(included[0].executed_input_offset, ExecutedInputCount::ZERO); - storage - .append_executed_user_ops_chunk(&mut head, &included) - .expect("persist prefix user op"); - super::snapshot::close_batch_with_snapshot( - &mut live_app, - &mut storage, - &mut head, - 0, - dumps_dir.path(), - ) - .expect("close prefix batch with snapshot"); - - let prefix_landing = crate::storage::test_helpers::local_batch_payload(&mut storage, 0); - let direct = DirectInput { - sender: direct_sender, - block_number: 10, - payload: vec![0xd1], - }; - storage - .append_safe_inputs( - 10, - &[ - StoredSafeInput { - sender: batch_submitter, - payload: prefix_landing, - block_number: 10, - }, - StoredSafeInput { - sender: direct.sender, - payload: direct.payload.clone(), - block_number: direct.block_number, - }, - ], - batch_submitter, - &protocol, - ) - .expect("observe prefix landing and direct input"); - let direct_receipt = sequencer_core::application::execute_direct_input(&mut live_app, &direct) - .expect("execute direct input in doomed suffix"); - assert_eq!(direct_receipt.offset, ExecutedInputCount::new(1)); - storage - .close_frame_only_with_executions( - &mut head, - 10, - SafeInputRange::new(0, 2), - &[DirectInputExecution { - safe_input_index: 1, - executed_input_offset: direct_receipt.offset, - }], - ) - .expect("drain direct and promote prefix snapshot"); - let finalized_before = storage - .finalized_dump() - .expect("read finalized prefix") - .expect("prefix snapshot promoted"); - assert_eq!( - finalized_before.executed_input_count, - ExecutedInputCount::new(1) - ); - - // The direct and this user op are both beyond the finalized checkpoint. - // Closing batch 1 makes it the first non-gold recovery pivot. - let (doomed_op, _response) = make_pending_user_op(0x52); - included.clear(); - super::execute_user_op( - &mut live_app, - doomed_op, - head.frame_fee, - head.safe_block, - &mut included, - ) - .expect("execute doomed user op"); - assert_eq!( - included[0].executed_input_offset, - ExecutedInputCount::new(2) - ); - storage - .append_executed_user_ops_chunk(&mut head, &included) - .expect("persist doomed user op"); - super::snapshot::close_batch_with_snapshot( - &mut live_app, - &mut storage, - &mut head, - 10, - dumps_dir.path(), - ) - .expect("close doomed batch with snapshot"); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::new(3) - ); - - let before_recovery = history_rows(&mut storage, 32); - assert_eq!( - before_recovery - .iter() - .map(|row| row.offset.get()) - .collect::>(), - vec![0, 1, 2] - ); - let invalidated = storage - .recover_post_flush_for_recovery(10, &protocol, crate::clock::unix_now_ms()) - .expect("standard recovery cascade"); - assert_eq!(invalidated, vec![1, 2]); - let recovered_history = storage.history_state().expect("recovered history"); - assert_eq!( - recovered_history.version.era_id, initial_history.version.era_id, - "standard recovery must stay in the same era" - ); - assert_eq!(recovered_history.version.recovery_generation.get(), 1); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::new(2), - "H rolls back the doomed user op while the direct is re-drained" - ); - assert_eq!( - storage.finalized_dump().unwrap().unwrap(), - finalized_before, - "the accepted prefix checkpoint must survive the cascade" - ); - assert_eq!( - storage.latest_snapshot().unwrap().unwrap().dump, - finalized_before.dump - ); - let after_recovery = history_rows(&mut storage, 32); - assert_eq!(after_recovery.len(), 2); - assert!(matches!( - &after_recovery[1].context, - crate::storage::L2TxContext::DirectInput { input_index: 1, .. } - )); - assert_eq!(after_recovery[1].offset, ExecutedInputCount::new(1)); - - let checkpoint = catch_up_snapshot(&mut storage).expect("select surviving checkpoint"); - assert_eq!( - checkpoint.executed_input_count, - finalized_before.executed_input_count - ); - let mut restarted = StubApp::from_dump(&super::dump_info::app_prefix(&checkpoint.dump.prefix)) - .expect("load surviving application checkpoint"); - assert_eq!(restarted.executed_input_count(), ExecutedInputCount::new(1)); - catch_up_application_paged( - &mut restarted, - &mut storage, - checkpoint.executed_input_count, - 2, - ) - .expect("catch up through recovery re-drain"); - assert_eq!(restarted.executed_input_count(), ExecutedInputCount::new(2)); - - // The replacement suffix reuses offset 2, rolling H forward without - // retaining the invalidated op that previously occupied that coordinate. - let mut recovery_head = storage.open_state().unwrap().unwrap(); - let (replacement_op, _response) = make_pending_user_op(0x53); - included.clear(); - super::execute_user_op( - &mut restarted, - replacement_op, - recovery_head.frame_fee, - recovery_head.safe_block, - &mut included, - ) - .expect("execute replacement user op"); - assert_eq!( - included[0].executed_input_offset, - ExecutedInputCount::new(2) - ); - storage - .append_executed_user_ops_chunk(&mut recovery_head, &included) - .expect("persist replacement suffix"); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::new(3) - ); - - let valid = history_rows(&mut storage, 32); - let mappings: Vec<_> = valid.iter().map(|row| row.offset.get()).collect(); - assert_eq!(mappings, vec![0, 1, 2]); - let user_seeds: Vec<_> = valid - .iter() - .filter_map(|row| match &row.context { - crate::storage::L2TxContext::UserOp { tx, .. } => Some(tx.data[0]), - crate::storage::L2TxContext::DirectInput { .. } => None, - }) - .collect(); - assert_eq!(user_seeds, vec![0x51, 0x53]); - - let mut restarted_again = - StubApp::from_dump(&super::dump_info::app_prefix(&checkpoint.dump.prefix)) - .expect("reload surviving checkpoint"); - catch_up_application_paged( - &mut restarted_again, - &mut storage, - checkpoint.executed_input_count, - 2, - ) - .expect("catch up through replacement suffix"); - assert_eq!( - restarted_again.executed_input_count(), - ExecutedInputCount::new(3) - ); - assert_eq!(restarted_again.last_executed_safe_block(), 10); -} - #[test] fn catch_up_load_error_reports_offset() { let db = temp_db("catch-up-load-error"); let mut storage = Storage::open_writer(db.path.as_str()).expect("open raw storage"); let mut app = StubApp::default(); - let err = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) + let err = catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2) .expect_err("catch up should fail without schema"); assert!(matches!(err, CatchUpError::LoadReplay { offset: 0, .. })); @@ -1367,15 +1130,10 @@ async fn lane_refuses_snapshot_whose_application_count_disagrees_with_storage() let mut storage = Storage::open(db.path.as_str()).expect("open storage"); pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage - .append_safe_inputs( - 0, - &[], - Address::repeat_byte(0xff), - &default_protocol_timing(), - ) + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) .expect("seed observed safe head"); - let mut app = StubApp::with_progress(1, 0); + let mut app = StubApp::with_progress(1); register_genesis_snapshot(&mut app, &mut storage, &config.dumps_dir); storage.ensure_open_tip().expect("establish genesis tip"); @@ -1401,7 +1159,7 @@ async fn lane_refuses_snapshot_whose_application_count_disagrees_with_storage() fn read_dump_counter(dump_dir: &Path) -> u64 { StubApp::from_dump(&super::dump_info::app_prefix(dump_dir)) .expect("decode dump progress") - .executed_input_count() + .l2_tx_count() .get() } @@ -1424,7 +1182,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { // Batch size target < one user op, so every op closes its batch. set_one_op_batch_target(&mut storage); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("seed observed safe head"); } @@ -1468,7 +1226,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { let mut s = Storage::open(db.path.as_str()).expect("open"); s.latest_snapshot() .expect("read pending") - .map(|p| read_dump_counter(&p.dump.prefix) == 3) + .map(|p| read_dump_counter(&p.prefix) == 3) .unwrap_or(false) }) .await; @@ -1482,7 +1240,8 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { s.finalized_dump() .expect("finalized") .expect("genesis finalized exists") - .executed_input_count + .snapshot + .l2_tx_count .get(), 0, "finalized must still be genesis (nothing was promoted)" @@ -1514,7 +1273,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { let mut s = Storage::open(db.path.as_str()).expect("open"); s.latest_snapshot() .expect("read pending") - .map(|p| read_dump_counter(&p.dump.prefix) == 4) + .map(|p| read_dump_counter(&p.prefix) == 4) .unwrap_or(false) }) .await; @@ -1522,7 +1281,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { let mut s = Storage::open(db.path.as_str()).expect("open"); s.latest_snapshot() .expect("read pending") - .map(|p| read_dump_counter(&p.dump.prefix)) + .map(|p| read_dump_counter(&p.prefix)) }; shutdown_lane(&shutdown2, handle2).await; @@ -1543,32 +1302,27 @@ fn application_dump_failure_leaves_tip_and_latest_snapshot_unchanged() { let dumps = tempfile::tempdir().unwrap(); let mut app = StubApp::default(); register_genesis_snapshot(&mut app, &mut storage, dumps.path()); - super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 0, dumps.path()) + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, dumps.path()) .unwrap(); let snapshot = storage.latest_snapshot().unwrap().unwrap(); - let tip = head.batch_index; - let dump_count = storage.list_dump_rows().unwrap().len(); + let tip = head.batch_nonce(); + let dump_count = storage.list_snapshots().unwrap().len(); app.fail_dump = true; - let error = super::snapshot::close_batch_with_snapshot( - &mut app, - &mut storage, - &mut head, - 0, - dumps.path(), - ) - .expect_err("an application dump failure must precede the batch seal"); + let error = + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, dumps.path()) + .expect_err("an application dump failure must precede the batch seal"); assert!(matches!( error, super::snapshot::TakeDumpError::CreateDump(super::dump_info::CreateDumpDirError::App( AppError::Io(ref source) )) if source.to_string() == "injected application dump failure" )); - assert_eq!(head.batch_index, tip); - assert_eq!(storage.open_state().unwrap().unwrap().batch_index, tip); - assert_eq!(storage.latest_batch_index().unwrap(), Some(tip)); + assert_eq!(head.batch_nonce(), tip); + assert_eq!(storage.open_state().unwrap().unwrap().batch_nonce(), tip); + assert_eq!(storage.latest_batch_nonce().unwrap(), Some(tip)); assert_eq!(storage.latest_snapshot().unwrap().unwrap(), snapshot); - assert_eq!(storage.list_dump_rows().unwrap().len(), dump_count); + assert_eq!(storage.list_snapshots().unwrap().len(), dump_count); } #[test] @@ -1577,19 +1331,18 @@ fn empty_batch_snapshot_preserves_application_count() { let expected = seed_replay_fixture(&db.path); let mut storage = Storage::open(&db.path).unwrap(); let mut app = StubApp::default(); - catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2).unwrap(); + catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2).unwrap(); assert_eq!(app.replayed, expected); let mut head = storage.open_state().unwrap().unwrap(); let dumps = tempfile::tempdir().unwrap(); - super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 30, dumps.path()) + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, dumps.path()) .unwrap(); - super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 30, dumps.path()) + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, dumps.path()) .unwrap(); let snapshot = storage.latest_snapshot().unwrap().unwrap(); - assert_eq!(snapshot.executed_input_count.get(), 6); - let restored = - StubApp::from_dump(&super::dump_info::app_prefix(&snapshot.dump.prefix)).unwrap(); - assert_eq!(restored.executed_input_count().get(), 6); + assert_eq!(snapshot.l2_tx_count.get(), 6); + let restored = StubApp::from_dump(&super::dump_info::app_prefix(&snapshot.prefix)).unwrap(); + assert_eq!(restored.l2_tx_count().get(), 6); } fn history_rows(storage: &mut Storage, limit: usize) -> Vec { @@ -1597,7 +1350,7 @@ fn history_rows(storage: &mut Storage, limit: usize) -> Vec Vec SequencedL2Tx { +fn context_tx(context: L2TxContext) -> L2Tx { match context { - crate::storage::L2TxContext::UserOp { tx, .. } => SequencedL2Tx::UserOp(tx), - crate::storage::L2TxContext::DirectInput { tx, .. } => SequencedL2Tx::Direct(tx), + L2TxContext::UserOp { tx, .. } => L2Tx::UserOp(tx), + L2TxContext::Direct { tx, .. } => L2Tx::Direct(tx), } } @@ -1623,7 +1376,7 @@ fn lane_with_closed_queue_and_due_direct( let mut storage = Storage::open(db.path.as_str()).expect("open storage"); pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) .expect("seed observed safe head"); storage.ensure_open_tip().expect("establish open tip"); storage @@ -1634,8 +1387,7 @@ fn lane_with_closed_queue_and_due_direct( payload: vec![0xaa], block_number: 5, }], - SENDER_A, - &default_protocol_timing(), + Address::repeat_byte(0xff), ) .expect("append a direct at the first clock tick"); @@ -1660,7 +1412,7 @@ fn lane_stops_cleanly_when_its_queue_closes_during_shutdown() { let (_dir, mut lane, shutdown) = lane_with_closed_queue_and_due_direct("closed-queue-during-shutdown", true); - let result = lane.run_forever(ExecutedInputCount::ZERO); + let result = lane.run_forever(L2TxCount::ZERO); assert!(shutdown.is_shutdown_requested()); assert!(result.is_ok(), "{result:?}"); @@ -1671,7 +1423,393 @@ fn lane_reports_a_closed_queue_without_shutdown() { let (_dir, mut lane, _shutdown) = lane_with_closed_queue_and_due_direct("closed-queue-without-shutdown", false); - let result = lane.run_forever(ExecutedInputCount::ZERO); + let result = lane.run_forever(L2TxCount::ZERO); assert!(matches!(result, Err(InclusionLaneError::ChannelClosed))); } + +/// Run one queued op through a dequeue chunk whose frame price is 10 and +/// return the 422 text it was refused with. Nothing may execute. +fn refusal_text(app: &mut StubApp, user_op: UserOp) -> String { + let (tx, mut rx) = mpsc::channel::(1); + let (mut pending, mut response) = make_pending_user_op(0x61); + pending.signed.user_op = user_op; + tx.try_send(pending).expect("enqueue refused op"); + let head = WriteHeadParts { + frame_fee: 10, + ..WriteHeadParts::of(&unbounded_head()) + } + .build(); + let count_before = app.l2_tx_count(); + let mut included = Vec::new(); + + dequeue_and_execute_user_op_chunk(&mut rx, app, 1, &head, &mut included) + .expect("a refusal never stops the lane"); + + assert!(included.is_empty(), "a refused op is never persisted"); + assert_eq!(app.l2_tx_count(), count_before); + assert!( + app.replayed.is_empty(), + "a refused op never reaches the app" + ); + match response.try_recv() { + Ok(Err(SequencerError::Invalid(message))) => message, + other => panic!("expected a 422 refusal, got {other:?}"), + } +} + +#[test] +fn lane_refuses_with_each_refusal_text_in_check_order() { + let sender = Address::from_slice(&[0x61; 20]); + let op = |nonce: u32, max_fee: u16| UserOp { + nonce, + max_fee, + data: vec![0x61; 4].into(), + }; + let refusing = || StubApp { + refuse_user_ops: Some("sender is frozen by policy".to_owned()), + ..StubApp::default() + }; + + // Fee cap first: it outranks a wrong nonce and a policy refusal. + assert_eq!( + refusal_text(&mut refusing(), op(3, 9)), + "max fee 9 below base fee 10" + ); + // Then the executor's nonce rule, ahead of admission. + assert_eq!( + refusal_text(&mut refusing(), op(3, 10)), + "bad nonce: expected 0, got 3" + ); + let mut exhausted = refusing(); + exhausted.nonces.insert(sender, u32::MAX); + assert_eq!( + refusal_text(&mut exhausted, op(u32::MAX, 10)), + "nonce 4294967295 has no successor" + ); + // Admission last, its text verbatim. + assert_eq!( + refusal_text(&mut refusing(), op(0, 10)), + "sender is frozen by policy" + ); +} + +/// Reports every sender one nonce higher after its first read, so the +/// executor disagrees with the lane's pre-check about an op it admitted. +#[derive(Default)] +struct NonceDriftApp { + inner: StubApp, + reads: std::cell::Cell, +} + +impl Application for NonceDriftApp { + fn user_nonce(&self, sender: Address) -> Result { + let drift = self.reads.get().min(1); + self.reads.set(self.reads.get().saturating_add(1)); + Ok(self.inner.user_nonce(sender)? + drift) + } + + fn apply_user_op(&mut self, op: &UserOpTx) -> Result { + self.inner.apply_user_op(op) + } + + fn apply_direct(&mut self, direct: &Direct) -> Result { + self.inner.apply_direct(direct) + } + + fn l2_tx_count(&self) -> L2TxCount { + self.inner.l2_tx_count() + } + + fn max_method_payload_bytes() -> usize { + StubApp::max_method_payload_bytes() + } + + fn scheduler_limits() -> SchedulerLimits { + StubApp::scheduler_limits() + } + + fn admit_user_op(&self, sender: Address, op: &UserOp, fee: u16) -> Result { + self.inner.admit_user_op(sender, op, fee) + } + + fn from_dump(_prefix: &Path) -> Result { + unreachable!("the drift double is never restored") + } + + fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { + unreachable!("the drift double is never dumped") + } + + fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { + StubApp::state_file_in_dump(prefix) + } + + fn from_canonical_state(_state_file: &Path, _l2_tx_count: L2TxCount) -> Result { + unreachable!("the drift double is never restored") + } +} + +#[test] +fn admitted_op_that_the_executor_skips_is_a_terminal_lane_error() { + let (tx, mut rx) = mpsc::channel::(2); + let (pending, response) = make_pending_user_op(0x62); + tx.blocking_send(pending).expect("enqueue admitted op"); + let (later, mut later_response) = make_pending_user_op(0x63); + tx.blocking_send(later).expect("enqueue later op"); + let mut app = NonceDriftApp::default(); + let mut included = Vec::new(); + + let err = + dequeue_and_execute_user_op_chunk(&mut rx, &mut app, 16, &unbounded_head(), &mut included) + .expect_err("a skip after admission must stop the lane"); + + assert!(matches!( + err, + InclusionLaneError::AdmittedOpSkipped { + reason: SkipReason::WrongNonce { + expected: 1, + got: 0 + } + } + )); + assert!(err.is_terminal_invariant()); + assert!(included.is_empty()); + assert_eq!(app.l2_tx_count(), L2TxCount::ZERO); + // The client gets the fixed 500 text, never the skip reason. + let response = response + .blocking_recv() + .expect("lane answers the skipped op") + .expect_err("skipped op gets an internal error"); + assert!(matches!( + response, + SequencerError::Internal(message) if message == "application internal error" + )); + assert!( + matches!( + later_response.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + ), + "the chunk stops at the faulting op" + ); +} + +#[test] +fn catch_up_stops_on_a_stored_op_the_application_would_skip() { + let db = temp_db("catch-up-replay-skipped"); + seed_replay_fixture(db.path.as_str()); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut app = StubApp::default(); + // The restored application already moved the first sender past its + // stored op, so history and application disagree. + app.nonces.insert(Address::from_slice(&[0x51; 20]), 1); + + let err = catch_up_application_paged(&mut app, &mut storage, L2TxCount::ZERO, 2) + .expect_err("a replayed skip must stop catch-up"); + + assert!(matches!( + err, + CatchUpError::ReplaySkipped { + offset: 0, + reason: SkipReason::WrongNonce { + expected: 1, + got: 0 + } + } + )); + assert!(app.replayed.is_empty()); + assert_eq!(app.l2_tx_count(), L2TxCount::ZERO); + assert!(InclusionLaneError::CatchUp { source: err }.is_terminal_invariant()); +} + +fn ingested_direct(block: u64, timestamp: u64, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender: Address::repeat_byte(0x77), + payload, + block_number: block, + block_timestamp: timestamp, + transaction_hash: B256::ZERO, + } +} + +#[test] +fn direct_over_the_payload_cap_never_executes_and_later_directs_do() { + let cap = usize::try_from(StubApp::scheduler_limits().max_direct_payload_bytes) + .expect("payload cap fits usize"); + let db = temp_db("direct-payload-cap"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); + storage + .append_safe_inputs(0, &[], Address::repeat_byte(0xff)) + .expect("seed observed safe head"); + let head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize open state"); + let (_tx, rx) = mpsc::channel(1); + let mut lane = InclusionLane { + rx, + shutdown: RuntimeScope::default(), + app: StubApp::default(), + storage, + config: default_test_config(), + }; + let mut state = LaneState::new(0, head); + let mut safe_inputs = Vec::new(); + + lane.storage + .append_ingested_safe_inputs_with_timestamp( + 5, + 1_005, + &[ + ingested_direct(3, 1_003, vec![0xa1; cap]), + ingested_direct(4, 1_004, vec![0xee; cap + 1]), + ingested_direct(5, 1_005, vec![0xa3]), + ], + ) + .expect("append directs around the payload cap"); + lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) + .expect("drain the range and close the frame"); + + let at_cap = ReplayEvent::DirectInput { + input_index: 0, + sender: Address::repeat_byte(0x77), + block_number: 3, + timestamp: 1_003, + payload: vec![0xa1; cap], + }; + let after_dropped = ReplayEvent::DirectInput { + input_index: 2, + sender: Address::repeat_byte(0x77), + block_number: 5, + timestamp: 1_005, + payload: vec![0xa3], + }; + // Each direct reaches the application with its own L1 index and block + // timestamp; the over-cap one never reaches it at all. + assert_eq!( + lane.app.replayed, + vec![at_cap.clone(), after_dropped.clone()] + ); + assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0, 5]); + assert_eq!(read_frame_direct_count(db.path.as_str(), 0, 1), 2); + assert_eq!(lane.storage.next_undrained_safe_input_index().unwrap(), 3); + + // A later frame keeps executing directs at the next contiguous offset. + lane.storage + .append_ingested_safe_inputs_with_timestamp( + 10, + 1_010, + &[ingested_direct(10, 1_010, vec![0xb1])], + ) + .expect("append a later direct"); + lane.maybe_advance_safe_frontier(&mut state, &mut safe_inputs) + .expect("drain the later direct"); + let later = ReplayEvent::DirectInput { + input_index: 3, + sender: Address::repeat_byte(0x77), + block_number: 10, + timestamp: 1_010, + payload: vec![0xb1], + }; + assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0, 5, 10]); + let rows = history_rows(&mut lane.storage, 16); + let stored: Vec<_> = rows + .iter() + .map(|row| match &row.context { + L2TxContext::Direct { tx, .. } => (row.offset.get(), tx.input_index), + L2TxContext::UserOp { .. } => panic!("no user op was submitted"), + }) + .collect(); + assert_eq!(stored, vec![(0, 0), (1, 2), (2, 3)]); + + // Replay hands a restarted application the same directs, index and + // timestamp included. + let mut restarted = StubApp::default(); + catch_up_application_paged(&mut restarted, &mut lane.storage, L2TxCount::ZERO, 2) + .expect("catch up over the dropped direct"); + assert_eq!(restarted.replayed, vec![at_cap, after_dropped, later]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lane_in_a_rebuilt_era_drains_from_the_eras_first_queued_input() { + use crate::egress::test_era; + + let db = temp_db("lane-rebuilt-era"); + let config = default_test_config(); + let baseline = config.dumps_dir.join("era-baseline"); + super::dump_info::create_dump_dir_with_info( + &mut StubApp::with_progress(test_era::REBUILT_END), + &baseline, + &super::dump_info::DumpInfo::at_baseline(test_era::NEXT_BATCH_NONCE), + ) + .expect("create the era's baseline dump"); + let mut storage = test_era::publish_rebuilt_era(&db.path, &baseline); + // Input 3 sits at the stop block but is still queued, so the drain starts + // there, not after the stop block's last input. + assert_eq!( + storage.next_undrained_safe_input_index().unwrap(), + test_era::FIRST_QUEUED_INPUT + ); + // `run`'s startup: the root frame drains the queued direct. + storage.ensure_open_tip().expect("open the root frame"); + assert_eq!( + storage.next_undrained_safe_input_index().unwrap(), + test_era::FIRST_QUEUED_INPUT + 1 + ); + + let shutdown = RuntimeScope::default(); + let (_tx, handle) = InclusionLane::::start(16, shutdown.clone(), storage, config); + let later = IngestedSafeInput { + sender: test_era::DIRECT_SENDER, + payload: vec![0xb4], + block_number: test_era::STOP_BLOCK + 6, + block_timestamp: 1_016, + transaction_hash: B256::repeat_byte(0xb4), + }; + Storage::open(&db.path) + .expect("open feeder storage") + .append_ingested_safe_inputs_with_timestamp(test_era::STOP_BLOCK + 6, 1_016, &[later]) + .expect("land a direct after the root frame"); + let head = |db_path: &str| { + Storage::open_read_only(db_path) + .unwrap() + .history_bounds() + .unwrap() + .head + .get() + }; + let drained = wait_until(Duration::from_secs(2), || { + head(&db.path) == test_era::REBUILT_END + 2 + }) + .await; + // A clean stop means catch-up replayed the queued direct onto the + // baseline, so the lane's own direct executed at the next count. + shutdown_lane(&shutdown, handle).await; + assert!(drained, "the lane must drain the later direct"); + + let mut storage = Storage::open_read_only(&db.path).unwrap(); + let bounds = storage.history_bounds().unwrap(); + let live: Vec<_> = storage + .canonical_history_page( + sequencer_core::history::HistoryClaim { + era_id: bounds.era_id, + next_input: L2TxCount::new(test_era::REBUILT_END), + }, + 16, + ) + .unwrap() + .rows + .into_iter() + .map(|row| match row.context { + L2TxContext::Direct { tx, .. } => (row.offset.get(), tx.input_index), + L2TxContext::UserOp { .. } => panic!("no user op was submitted"), + }) + .collect(); + assert_eq!( + live, + vec![ + (test_era::REBUILT_END, test_era::FIRST_QUEUED_INPUT), + (test_era::REBUILT_END + 1, test_era::FIRST_QUEUED_INPUT + 1), + ] + ); +} diff --git a/sequencer/src/ingress/inclusion_lane/types.rs b/sequencer/src/ingress/inclusion_lane/types.rs index 51715ded..dae32c12 100644 --- a/sequencer/src/ingress/inclusion_lane/types.rs +++ b/sequencer/src/ingress/inclusion_lane/types.rs @@ -6,7 +6,8 @@ use std::time::SystemTime; -use sequencer_core::history::ExecutedInputCount; +use crate::storage::ExecutedUserOp; +use sequencer_core::history::L2TxCount; use sequencer_core::user_op::SignedUserOp; use thiserror::Error; use tokio::sync::oneshot; @@ -27,12 +28,37 @@ pub struct PendingUserOp { #[derive(Debug)] pub(crate) struct IncludedUserOp { pub pending: PendingUserOp, - pub executed_input_offset: ExecutedInputCount, + pub l2_tx_index: L2TxCount, +} + +impl IncludedUserOp { + /// The borrowed view storage persists. + pub(crate) fn executed(&self) -> ExecutedUserOp<'_> { + ExecutedUserOp { + signed: &self.pending.signed, + received_at: self.pending.received_at, + l2_tx_index: self.l2_tx_index, + } + } +} + +#[cfg(test)] +impl crate::storage::Storage { + /// Test shorthand: append a chunk of lane-included ops. + pub(crate) fn append_included_user_ops( + &mut self, + head: &mut crate::storage::WriteHead, + user_ops: &[IncludedUserOp], + ) -> rusqlite::Result<()> { + self.append_executed_user_ops_chunk(head, user_ops.iter().map(IncludedUserOp::executed)) + } } /// Per-op outcome reported back to the API caller via the response channel. /// -/// - `Invalid` — application rejected the op (nonce mismatch, fee too low, etc.); maps to HTTP 4xx. +/// - `Invalid` — the lane refused the op: its fee cap is below the frame's price, +/// the executor would skip it (wrong or exhausted nonce), or the application's +/// admission refused it; the text is the refusal's own; maps to HTTP 422. /// - `Unavailable` — sequencer can't currently accept (shutting down, queue full); maps to HTTP 503/429. /// - `Internal` — bug or unrecoverable failure; maps to HTTP 500. #[derive(Debug, Error, Clone)] diff --git a/sequencer/src/integration_tests/batch_submitter.rs b/sequencer/src/integration_tests/batch_submitter.rs index 5df9c05d..678068c0 100644 --- a/sequencer/src/integration_tests/batch_submitter.rs +++ b/sequencer/src/integration_tests/batch_submitter.rs @@ -14,7 +14,6 @@ use sequencer::l1::watermark::WalletNonceWatermarkSink; use sequencer::runtime::shutdown::RuntimeScope; use sequencer::storage::{SafeInputRange, Storage}; use sequencer_core::batch::Batch; -use sequencer_core::protocol::ProtocolTiming; use crate::test_support::{TestDb, temp_db}; @@ -82,13 +81,13 @@ impl BatchPoster for TestMock { let mut tx_hashes = Vec::with_capacity(payloads.len()); for payload in payloads { - let batch_index = ssz::Decode::from_ssz_bytes(payload.as_slice()) + let batch_nonce = ssz::Decode::from_ssz_bytes(payload.as_slice()) .map(|b: Batch| b.nonce) .unwrap_or(0); self.submissions .lock() .expect("lock") - .push((batch_index, payload.len())); + .push((batch_nonce, payload.len())); tx_hashes.push(TxHash::ZERO); } Ok(SubmitBatchesOutcome::Submitted(tx_hashes)) @@ -108,15 +107,13 @@ impl BatchPoster for TestMock { } } -/// Mirrors what the startup recovery procedure establishes in production: persist a real +/// Mirrors what setup's sync establishes in production: persist a real /// safe-head observation so `submitter_frontier` has a row to read. Without /// this, the submitter's first tick errors out on `current_safe_block_required` /// and the loop exits before submitting anything. fn seed_observed_safe_head(storage: &mut Storage) { - let timing = - ProtocolTiming::try_new(sequencer_core::MAX_WAIT_BLOCKS, 75, 900, 12).expect("test timing"); storage - .append_safe_inputs(0, &[], alloy_primitives::Address::ZERO, &timing) + .append_safe_inputs(0, &[], alloy_primitives::Address::ZERO) .expect("seed observed safe head"); } @@ -127,15 +124,14 @@ fn seed_two_closed_batches(db_path: &str) { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - let next_safe = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 0"); storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 1"); storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 2"); } @@ -146,9 +142,8 @@ fn seed_one_closed_batch(db_path: &str) { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - let next_safe = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 0"); } @@ -159,9 +154,8 @@ fn close_current_tip(db_path: &str) { .open_state() .expect("load open state") .expect("open Tip exists"); - let next_safe = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close current Tip"); } diff --git a/sequencer/src/integration_tests/chain_id_validation.rs b/sequencer/src/integration_tests/chain_id_validation.rs index da5b5d29..0b59a4ab 100644 --- a/sequencer/src/integration_tests/chain_id_validation.rs +++ b/sequencer/src/integration_tests/chain_id_validation.rs @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! `run`'s bootstrap guards on the identity `setup` pinned: no setup, a key for -//! another submitter, and a wrong-chain RPC (against Anvil) are refused, and a -//! matching chain is not. Setup's on-chain discovery is covered by the e2e suite. +//! another submitter, a binary with another direct-payload cap, and a +//! wrong-chain RPC (against Anvil) are refused, and a matching chain is not. +//! Setup's on-chain discovery is covered by the e2e suite. use std::time::Duration; @@ -11,9 +12,9 @@ use alloy_primitives::{Address, address}; use clap::Parser; use sequencer::commands::{BootstrapError, CommandError, IdentityError}; use sequencer::l1::reader::InputReaderError; -use sequencer::recovery::{RecoveryError, RecoveryFailure}; use sequencer::storage::{DeploymentIdentity, Storage}; use sequencer::{Cli, Command}; +use sequencer_core::application::Application; use tempfile::TempDir; // Anvil's default devnet private key #0 and its address. @@ -59,37 +60,44 @@ fn run_config(data_dir: &str, eth_rpc_url: &str, key: &str) -> sequencer::RunCon } } +/// The scheduler limits of the binary these tests run. +fn wallet_limits() -> sequencer_core::application::SchedulerLimits { + app_core::application::WalletApp::scheduler_limits() +} + /// Seed a pinned deployment identity (chain id `chain_id`, submitter -/// `submitter`), finalized-snapshot fact, and completed setup lifecycle — the -/// minimal local state needed to reach `run`'s initial L1 sync without doing -/// `setup`'s on-chain discovery. Uses the same typed controller path `setup` +/// `submitter`, scheduler limits `limits`), finalized-snapshot fact, and +/// completed setup lifecycle — the minimal local state needed to reach `run`'s +/// initial L1 sync without doing `setup`'s on-chain discovery. Uses the same +/// typed controller path `setup` /// itself records through (admission → facts → completion), so this seed /// cannot drift from the lifecycle schema or encode a state the controller /// would never write. -fn seed_setup_complete(db_path: &str, chain_id: u64, submitter: Address) { +fn seed_setup_complete( + db_path: &str, + chain_id: u64, + submitter: Address, + limits: sequencer_core::application::SchedulerLimits, +) { use sequencer::storage::LifecycleCommand; let mut storage = Storage::initialize_for_command(db_path, LifecycleCommand::Setup) .expect("initialize setup lifecycle"); storage - .load_or_insert_deployment_identity(DeploymentIdentity { + .pin_deployment_identity(DeploymentIdentity { chain_id, app_address: TEST_APP_ADDR, input_box_address: TEST_INPUT_BOX, app_deployment_block: 100, batch_submitter_address: submitter, + max_direct_payload_bytes: limits.max_direct_payload_bytes, + max_wait_blocks: limits.max_wait_blocks, fee_oracle: sequencer::storage::FeeOracleIdentity::Fixed { log_gas_price: 0 }, }) .expect("seed deployment identity"); let snapshot_prefix = std::path::Path::new(db_path).with_file_name("seed-finalized"); storage - .complete_baseline_setup( - &snapshot_prefix, - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, - ) + .publish_era(&snapshot_prefix, &sequencer::storage::NewEra::EMPTY_GENESIS) .expect("seed finalized snapshot fact"); } @@ -125,7 +133,7 @@ async fn run_refuses_on_submitter_key_mismatch() { let dir = TempDir::new().expect("tempdir"); let data_dir = dir.path().to_str().unwrap(); let db_path = format!("{data_dir}/sequencer.db"); - seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR, wallet_limits()); let config = run_config(data_dir, "http://127.0.0.1:1", OTHER_KEY); let result = tokio::time::timeout( @@ -144,6 +152,54 @@ async fn run_refuses_on_submitter_key_mismatch() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn run_refuses_scheduler_limits_other_than_the_pinned_ones() { + // Setup pinned a cap one byte larger than the binary's: storage would + // classify directs the lane's application does not execute. Or it pinned + // a window one block longer: acceptance would be judged by another rule + // than the template's. Either way `run` must fail loud before any L1 + // contact. + let binary = wallet_limits(); + let cases = [ + ( + sequencer_core::application::SchedulerLimits { + max_direct_payload_bytes: binary.max_direct_payload_bytes + 1, + ..binary + }, + "max_direct_payload_bytes", + ), + ( + sequencer_core::application::SchedulerLimits { + max_wait_blocks: binary.max_wait_blocks + 1, + ..binary + }, + "max_wait_blocks", + ), + ]; + for (pinned, field) in cases { + let dir = TempDir::new().expect("tempdir"); + let data_dir = dir.path().to_str().unwrap(); + let db_path = format!("{data_dir}/sequencer.db"); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR, pinned); + + let config = run_config(data_dir, "http://127.0.0.1:1", ANVIL_KEY); + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() must return quickly on a limits mismatch"); + + match result { + Err(CommandError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { + fields, + .. + }))) => assert_eq!(fields, field), + other => panic!("expected {field} mismatch, got: {other:?}"), + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn run_refuses_on_wrong_chain_rpc() { // Pinned chain id 31337, but the (reachable) RPC reports a different @@ -155,7 +211,7 @@ async fn run_refuses_on_wrong_chain_rpc() { let db_path = format!("{data_dir}/sequencer.db"); // Submitter = anvil key address so the key check passes and we reach the // chain-id check. - seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR, wallet_limits()); let config = run_config(data_dir, &anvil.endpoint(), ANVIL_KEY); let result = tokio::time::timeout( @@ -166,17 +222,14 @@ async fn run_refuses_on_wrong_chain_rpc() { .expect("run() must return quickly on chain-id mismatch"); match result { - Err(CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::Refuse(failure)))) => { - match *failure { - RecoveryFailure::InputReader(InputReaderError::ChainIdMismatch { - rpc, - expected, - }) => { - assert_eq!(rpc, 99); - assert_eq!(expected, 31_337); - } - other => panic!("expected input-reader chain-id mismatch, got: {other:?}"), - } + Err( + error @ CommandError::Bootstrap(BootstrapError::StartupSync( + InputReaderError::ChainIdMismatch { rpc, expected }, + )), + ) => { + assert_eq!(rpc, 99); + assert_eq!(expected, 31_337); + assert_eq!(error.exit_code(), sequencer::commands::error::EXIT_TERMINAL); } other => panic!("expected ChainIdMismatch, got: {other:?}"), } @@ -193,7 +246,7 @@ async fn run_accepts_matching_chain_rpc() { let dir = TempDir::new().expect("tempdir"); let data_dir = dir.path().to_str().unwrap(); let db_path = format!("{data_dir}/sequencer.db"); - seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR, wallet_limits()); let config = run_config(data_dir, &anvil.endpoint(), ANVIL_KEY); let result = tokio::time::timeout( @@ -210,3 +263,78 @@ async fn run_accepts_matching_chain_rpc() { "matching chain id must reach snapshot preparation, got: {result:?}" ); } + +/// Seed a void history on a completed setup: one open batch whose first frame +/// is at safe block 10, under a safe head from which no later block can +/// accept it (`H + 1 - s == max_wait_blocks`). +fn seed_void_history(db_path: &str, limits: sequencer_core::application::SchedulerLimits) { + let mut storage = Storage::open(db_path).expect("open storage"); + storage + .append_safe_inputs(10, &[], ANVIL_ADDR) + .expect("first safe head"); + storage + .initialize_open_state(10, sequencer::storage::SafeInputRange::empty_at(0)) + .expect("open the era's first batch"); + storage + .append_safe_inputs(10 + limits.max_wait_blocks - 1, &[], ANVIL_ADDR) + .expect("the safe head that voids the batch"); +} + +/// The batches as `(nonce, sealed)`, in nonce order. +fn batch_rows(db_path: &str) -> Vec<(i64, bool)> { + Storage::open(db_path) + .expect("open storage") + .raw_read(|tx| { + tx.prepare("SELECT nonce, sealed_at_ms IS NOT NULL FROM batches ORDER BY nonce")? + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect() + }) + .expect("read batches") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn run_refuses_a_void_history_with_l1_unreachable_on_every_start() { + // The provider is unreachable (port 1): the void history is a fact of + // the database alone, so `run` refuses before any L1 call, before any + // batch is opened, and again on the next start. + let dir = TempDir::new().expect("tempdir"); + let data_dir = dir.path().to_str().unwrap(); + let db_path = format!("{data_dir}/sequencer.db"); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR, wallet_limits()); + seed_void_history(&db_path, wallet_limits()); + let before = batch_rows(&db_path); + + for start in ["first", "second"] { + let config = run_config(data_dir, "http://127.0.0.1:1", ANVIL_KEY); + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() must refuse promptly without L1"); + match result { + Err( + ref error @ CommandError::Bootstrap(BootstrapError::TerminalFact( + sequencer::storage::TerminalFact::VoidHistory { + lag, + max_wait_blocks, + }, + )), + ) => { + assert_eq!(lag.first_frame_safe_block, 10, "{start} start"); + assert_eq!(max_wait_blocks, wallet_limits().max_wait_blocks); + assert_eq!(error.exit_code(), sequencer::commands::error::EXIT_TERMINAL); + assert!( + error.to_string().contains("the history is void"), + "{start} start: {error}" + ); + } + other => panic!("{start} start: expected a void history, got: {other:?}"), + } + assert_eq!( + batch_rows(&db_path), + before, + "{start} start changed the batches" + ); + } +} diff --git a/sequencer/src/integration_tests/e2e_sequencer.rs b/sequencer/src/integration_tests/e2e_sequencer.rs index c66ca45d..f9ce6b5b 100644 --- a/sequencer/src/integration_tests/e2e_sequencer.rs +++ b/sequencer/src/integration_tests/e2e_sequencer.rs @@ -6,12 +6,15 @@ use std::time::Duration; use crate::storage::{ApplicationInputRow, L2TxContext}; use alloy_primitives::{Address, U256}; use alloy_sol_types::Eip712Domain; -use app_core::application::{MAX_METHOD_PAYLOAD_BYTES, Method, WalletConfig, Withdrawal}; +use app_core::application::{ + MAX_METHOD_PAYLOAD_BYTES, Method, WalletApp, WalletConfig, Withdrawal, +}; use futures_util::StreamExt; use k256::ecdsa::SigningKey; use sequencer::storage::{Storage, StoredSafeInput}; use sequencer_core::api::{TxRequest, TxResponse, WsTxMessage}; -use sequencer_core::history::{ExecutedInputCount, HistoryClaim}; +use sequencer_core::application::Application; +use sequencer_core::history::{HistoryClaim, L2TxCount}; use sequencer_core::test_support::sign_user_op; use sequencer_core::user_op::UserOp; use sequencer_rust_client::SequencerClient; @@ -884,11 +887,10 @@ async fn api_serves_the_next_nonce_after_each_ack_and_the_signing_domain() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn api_rejects_user_op_when_balance_below_fee_cost() { // if sender's balance < `fee_to_linear(current_frame_fee)` the - // user op must be rejected with 422 `InsufficientFeeBalance` and leave - // state unchanged. Exercises the balance check in - // `WalletApp::validate_user_op` (app-core). A fresh sender with no - // deposits has balance 0, well below `fee_to_linear(1356)` (the - // bootstrapped frame fee). + // user op must be refused with 422 and leave state unchanged. The + // refusal is `WalletApp::admit_user_op`'s `Underfunded` policy, whose + // text clients parse. A fresh sender with no deposits has balance 0, + // well below `fee_to_linear(1356)` (the bootstrapped frame fee). let db = temp_db("insufficient-fee-balance"); let domain = test_domain(); let signing_key = SigningKey::from_bytes((&[11_u8; 32]).into()).expect("create signing key"); @@ -924,14 +926,120 @@ async fn api_rejects_user_op_when_balance_below_fee_cost() { status, 422, "insufficient-balance must produce 422, got {status}: {body}", ); - assert!( - body.contains("insufficient balance for fee"), - "expected InsufficientFeeBalance message, got: {body}", + let envelope: serde_json::Value = serde_json::from_str(&body).expect("response is JSON"); + assert_eq!(envelope["code"], "EXECUTION_REJECTED", "{body}"); + assert_eq!( + envelope["message"], + format!( + "insufficient balance for fee: required {}, available 0", + sequencer_core::fee::fee_to_linear(1356) + ), + "the refusal's text reaches the client verbatim: {body}", + ); + assert_eq!( + client + .get_nonce(sender) + .await + .expect("GET /nonce") + .next_nonce, + 0, + "a refused op leaves the sender's nonce unchanged" ); runtime.shutdown().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn over_cap_direct_never_executes_and_survives_restart() { + // A direct longer than the template's cap never joins the fridge: it gets + // no execution row, the next direct takes the next offset with its own + // InputBox index, and a restarted lane replays the same history. + let db = temp_db("over-cap-direct"); + let domain = test_domain(); + let signing_key = SigningKey::from_bytes((&[7_u8; 32]).into()).expect("create signing key"); + let sender = address_from_signing_key(&signing_key); + bootstrap_open_frame_with_deposits(db.path.as_str(), &[(sender, U256::from(1_000_000_u64))]); + let cap = usize::try_from(WalletApp::scheduler_limits().max_direct_payload_bytes) + .expect("cap fits usize"); + seed_safe_direct_input(db.path.as_str(), 10, vec![0xbb; cap + 1]); + seed_safe_direct_input(db.path.as_str(), 11, vec![0xcc; cap]); + + let runtime = start_full_server(db.path.as_str(), domain.clone()).await; + let client = SequencerClient::new_with_timeout( + format!("http://{}", runtime.public_addr), + Duration::from_secs(2), + ) + .expect("build sequencer client"); + let ws_url = SequencerClient::new(format!("http://{}", runtime.internal_addr)) + .expect("build feed client") + .ws_subscribe_url(history_claim(&db.path, 0)); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(ws_url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); + let mut live = vec![ + recv_ws_message(&mut ws).await, + recv_ws_message(&mut ws).await, + ]; + let (status, body) = client + .submit_tx_with_status(&make_valid_request(&domain)) + .await + .expect("submit tx"); + assert_eq!(status, 200, "a later op still executes: {body}"); + live.push(recv_ws_message(&mut ws).await); + drop(ws); + + let expected = all_ordered_l2_txs(db.path.as_str()); + let direct_indexes: Vec = expected + .iter() + .filter_map(|row| match &row.context { + L2TxContext::Direct { tx, .. } => Some(tx.input_index), + L2TxContext::UserOp { .. } => None, + }) + .collect(); + assert_eq!( + direct_indexes, + vec![0, 2], + "the over-cap direct (index 1) has no execution row" + ); + assert_eq!(expected.len(), 3, "deposit, at-cap direct, user op"); + for (offset, (message, row)) in live.into_iter().zip(&expected).enumerate() { + assert_ws_message_matches_tx(message, row, offset as u64); + } + runtime.shutdown().await; + + let restarted = start_full_server(db.path.as_str(), domain.clone()).await; + let user_op = UserOp { + nonce: 1, + max_fee: TEST_MAX_FEE, + data: ssz::Encode::as_ssz_bytes(&Method::Withdrawal(Withdrawal { + amount: U256::from(0_u64), + })) + .into(), + }; + let request = TxRequest { + signature: sign_user_op_hex(&domain, &user_op, &signing_key), + sender: sender.to_string(), + message: user_op, + }; + let (status, body) = SequencerClient::new_with_timeout( + format!("http://{}", restarted.public_addr), + Duration::from_secs(2), + ) + .expect("build sequencer client after restart") + .submit_tx_with_status(&request) + .await + .expect("submit tx after restart"); + assert_eq!( + status, 200, + "the restarted lane replays without the over-cap direct: {body}" + ); + let after = all_ordered_l2_txs(db.path.as_str()); + assert_eq!(after.len(), 4); + assert_eq!(after[3].offset, L2TxCount::new(3)); + restarted.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn api_concurrent_same_nonce_leaves_exactly_one_committed() { // two concurrent POSTs for the same (sender, nonce) — one @@ -1091,8 +1199,8 @@ fn bootstrap_open_frame_with_deposits(db_path: &str, deposits: &[(Address, U256) let config = WalletConfig::default(); pin_test_deployment_identity(&mut storage, TEST_BATCH_SUBMITTER); - // Always record a safe-head observation: production callers are gated by - // the startup recovery procedure, so storage paths like `safe_input_frontier` + // Always record a safe-head observation: in production setup's sync + // leaves one before `run` starts, so storage paths like `safe_input_frontier` // assume a row exists. With no deposits we still write an empty advance // so the lane can start without `current_safe_block_required` failing. let safe_inputs: Vec = deposits @@ -1110,12 +1218,7 @@ fn bootstrap_open_frame_with_deposits(db_path: &str, deposits: &[(Address, U256) }) .collect(); storage - .append_safe_inputs( - 1, - &safe_inputs, - TEST_BATCH_SUBMITTER, - &crate::storage::test_helpers::default_protocol_timing(), - ) + .append_safe_inputs(1, &safe_inputs, TEST_BATCH_SUBMITTER) .expect("seed safe head (and any deposits)"); } @@ -1152,7 +1255,6 @@ fn seed_safe_direct_input(db_path: &str, safe_block: u64, payload: Vec) { block_number: safe_block, }], TEST_BATCH_SUBMITTER, - &crate::storage::test_helpers::default_protocol_timing(), ) .expect("append safe direct input"); } @@ -1195,16 +1297,20 @@ fn assert_ws_message_matches_tx( sender, block_number, payload, + input_index, + block_timestamp, .. }, - L2TxContext::DirectInput { tx: expected, .. }, + L2TxContext::Direct { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( decode_hex_prefixed(sender.as_str()), expected.sender.as_slice() ); - assert_eq!(block_number, expected.block_number); + assert_eq!(block_number, expected.block); + assert_eq!(input_index, expected.input_index); + assert_eq!(block_timestamp, expected.timestamp); assert_eq!( decode_hex_prefixed(payload.as_str()), expected.payload.as_slice() @@ -1320,11 +1426,7 @@ fn test_domain() -> Eip712Domain { fn history_claim(db_path: &str, next: u64) -> HistoryClaim { HistoryClaim { - version: Storage::open_read_only(db_path) - .unwrap() - .history_state() - .unwrap() - .version, - next_input: ExecutedInputCount::new(next), + era_id: Storage::open_read_only(db_path).unwrap().era().unwrap().id, + next_input: L2TxCount::new(next), } } diff --git a/sequencer/src/integration_tests/historical_bootstrap.rs b/sequencer/src/integration_tests/historical_bootstrap.rs deleted file mode 100644 index 9d5cbc47..00000000 --- a/sequencer/src/integration_tests/historical_bootstrap.rs +++ /dev/null @@ -1,560 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! A reader restores its own transfer history, replays raw L1 through the -//! canonical scheduler, and crosses the rebuilt baseline into the live feed. - -mod recovery_compatibility; - -use std::net::SocketAddr; -use std::time::Duration; - -use alloy::signers::{SignerSync, local::PrivateKeySigner}; -use alloy_primitives::{Address, B256, U256}; -use alloy_sol_types::{Eip712Domain, SolCall, SolStruct}; -use app_core::application::{ - DepositNotice, Method, Transfer, TransferNotice, WalletApp, WalletConfig, -}; -use futures_util::StreamExt; -use sequencer_core::api::WsTxMessage; -use sequencer_core::application::{ - AppOutput, AppOutputs, Application, CanonicalState, execute_direct_input, -}; -use sequencer_core::batch::{Batch, Frame, WireUserOp}; -use sequencer_core::history::{ExecutedInputCount, HistoryClaim, HistoryPolicyError}; -use sequencer_core::history_api::{AcceptedCheckpoint, HistoricalL1InputStart}; -use sequencer_core::l2_tx::DirectInput; -use sequencer_core::scheduler::{ - BatchRejectReason, ProcessOutcome, Scheduler, SchedulerConfig, SchedulerInput, -}; -use sequencer_core::user_op::UserOp; -use sequencer_rust_client::{HistoryReadError, SequencerClient}; -use ssz::Encode; -use tokio::sync::mpsc; -use tokio_tungstenite::tungstenite::Message; - -use crate::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; -use crate::http::{self, ApiConfig}; -use crate::ingress::inclusion_lane::{PendingUserOp, dump_info}; -use crate::runtime::shutdown::RuntimeScope; -use crate::storage::test_helpers::default_protocol_timing; -use crate::storage::{ - DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, - Storage, -}; -use crate::test_support::{pin_test_deployment_identity, temp_db}; - -const SUBMITTER: Address = Address::repeat_byte(0x33); -const APP: Address = Address::repeat_byte(0x11); -const RECIPIENT: Address = Address::repeat_byte(0x44); -const STOP: u64 = 1240; - -fn domain() -> Eip712Domain { - sequencer_core::build_input_domain(1, APP) -} - -fn config() -> WalletConfig { - WalletConfig { - sequencer_address: SUBMITTER, - ..WalletConfig::default() - } -} - -fn deposit(block: u64, recipient: Address, amount: u64) -> IngestedSafeInput { - let mut payload = Vec::new(); - payload.extend_from_slice(config().supported_erc20_token.as_slice()); - payload.extend_from_slice(recipient.as_slice()); - payload.extend_from_slice(&U256::from(amount).to_be_bytes::<32>()); - raw(config().erc20_portal_address, block, payload) -} - -fn raw(sender: Address, block: u64, payload: Vec) -> IngestedSafeInput { - IngestedSafeInput { - sender, - payload, - block_number: block, - block_timestamp: 1_700_000_000 + block * 12, - transaction_hash: B256::repeat_byte(u8::try_from(block % 251).unwrap()), - } -} - -fn transfer_batch( - signer: &PrivateKeySigner, - nonce: u32, - block: u64, - safe_block: u64, - amount: u64, -) -> IngestedSafeInput { - let op = UserOp { - nonce, - max_fee: 0, - data: Method::Transfer(Transfer { - amount: U256::from(amount), - to: RECIPIENT, - }) - .as_ssz_bytes() - .into(), - }; - let signature = signer - .sign_hash_sync(&op.eip712_signing_hash(&domain())) - .unwrap(); - raw( - SUBMITTER, - block, - Batch { - nonce: u64::from(nonce), - frames: vec![Frame { - safe_block, - fee_price: 0, - user_ops: vec![WireUserOp { - nonce, - max_fee: op.max_fee, - data: op.data.to_vec(), - signature: signature.as_bytes().to_vec(), - }], - }], - } - .as_ssz_bytes(), - ) -} - -fn notices(outputs: AppOutputs) -> Vec> { - outputs - .into_iter() - .map(|output| match output { - AppOutput::Notice(bytes) => bytes, - other => panic!("unexpected output in transfer-history fixture: {other:?}"), - }) - .collect() -} - -fn execute( - scheduler: &mut Scheduler, - input: &IngestedSafeInput, -) -> sequencer_core::scheduler::ProcessResult { - scheduler - .process_input(SchedulerInput { - sender: input.sender, - inclusion_block: input.block_number, - domain: domain(), - payload: input.payload.clone(), - }) - .unwrap() -} - -struct Server { - addr: SocketAddr, - shutdown: RuntimeScope, - task: Option, - _rx: mpsc::Receiver, -} - -impl Drop for Server { - fn drop(&mut self) { - self.shutdown.request_shutdown(); - if let Some(task) = self.task.take() { - task.abort(); - } - } -} - -impl Server { - async fn start(db_path: &str) -> Self { - let (listeners, _public_addr, addr) = - crate::test_support::server::bind_api_listeners().await; - let shutdown = RuntimeScope::default(); - let (tx, rx) = mpsc::channel(1); - let feed = L2TxFeed::new( - db_path.to_owned(), - shutdown.clone(), - L2TxFeedConfig::default(), - ); - let task = http::start_on_listeners( - listeners, - tx, - shutdown.clone(), - feed, - ApiConfig::new(domain(), WalletApp::max_method_payload_bytes()), - http::SnapshotState { - db_path: db_path.to_owned(), - state_file_in_dump: |prefix| { - WalletApp::state_file_in_dump(&dump_info::app_prefix(prefix)) - }, - }, - ); - Self { - addr, - shutdown, - task: Some(task), - _rx: rx, - } - } - - async fn stop(mut self) { - self.shutdown.request_shutdown(); - tokio::time::timeout(Duration::from_secs(3), self.task.take().unwrap()) - .await - .unwrap() - .unwrap() - .unwrap(); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn historical_bootstrap_restores_transfer_history_and_hands_off_at_baseline() { - let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); - let user = signer.address(); - let inputs = vec![ - deposit(5, user, 100), - deposit(12, user, 20), - transfer_batch(&signer, 0, 20, 10, 7), - deposit(20, user, 30), - deposit(24, user, 40), - raw(SUBMITTER, 1230, vec![0]), - transfer_batch(&signer, 1, 1232, 1228, 11), - deposit(1235, user, 50), - ]; - - let mut reference = Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); - let mut expected_history = Vec::new(); - for input in &inputs { - expected_history.extend(notices(execute(&mut reference, input).outputs)); - } - expected_history.extend(notices(reference.drain_covered_at(STOP).unwrap())); - let (mut reference_app, reference_nonce) = reference.finish(); - assert_eq!(reference_app.executed_input_count().get(), 7); - assert_eq!(reference_app.last_executed_safe_block(), 1235); - assert_eq!(reference_nonce, 2); - let deposit_notice = |amount| { - DepositNotice { - token: config().supported_erc20_token, - sender: user, - amount: U256::from(amount), - } - .abi_encode() - }; - let transfer_notice = |amount| { - TransferNotice { - sender: user, - recipient: RECIPIENT, - amount: U256::from(amount), - } - .abi_encode() - }; - assert_eq!( - expected_history, - vec![ - deposit_notice(100_u64), - transfer_notice(7_u64), - deposit_notice(20), - deposit_notice(30), - deposit_notice(40), - transfer_notice(11), - deposit_notice(50) - ] - ); - - let db = temp_db("historical-projection-bootstrap"); - let dumps = tempfile::tempdir().unwrap(); - let baseline_dump = dumps.path().join("baseline"); - dump_info::create_dump_dir_with_info( - &mut reference_app, - &baseline_dump, - &dump_info::DumpInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: reference_nonce, - }, - ) - .unwrap(); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - pin_test_deployment_identity(&mut storage, SUBMITTER); - storage - .append_ingested_safe_inputs_with_timestamp( - STOP, - 1_700_000_000 + STOP * 12, - &inputs, - SUBMITTER, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, - ) - .unwrap(); - storage - .complete_baseline_setup( - &baseline_dump, - reference_app.executed_input_count(), - STOP, - reference_nonce, - true, - ) - .unwrap(); - let server = Server::start(&db.path).await; - let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); - let metadata = client.history(None, None).await.unwrap(); - let era = metadata.history.version.era_id; - assert_eq!(metadata.history.available_from.get(), 7); - assert_eq!(metadata.history.head.get(), 7); - assert!(matches!( - client.subscribe(HistoryClaim { - version: metadata.history.version, - next_input: ExecutedInputCount::new(6), - }).await, - Err(sequencer_rust_client::SubscribeError::History(HistoryPolicyError::HistoryUnavailable { available_from })) - if available_from == metadata.history.available_from - )); - assert_eq!(metadata.baseline.l1_stop_block, STOP); - assert_eq!(metadata.baseline.l1_end_input_index, 8); - assert_eq!(metadata.baseline.next_batch_nonce, reference_nonce); - assert_eq!(metadata.accepted_checkpoint, None); - assert_eq!(metadata.deployment.batch_submitter_address, SUBMITTER); - assert_eq!( - sequencer_core::build_input_domain( - metadata.deployment.chain_id, - metadata.deployment.app_address - ), - domain() - ); - let mut other_era_bytes = *era.as_bytes(); - other_era_bytes[0] ^= 1; - let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); - assert!(matches!( - client.history(Some(other_era), None).await, - Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current })) - if current == metadata.history.version - )); - assert!(matches!( - client - .historical_l1_inputs( - other_era, - HistoricalL1InputStart::NextInputIndex(u64::MAX), - None - ) - .await, - Err(HistoryReadError::History( - HistoryPolicyError::EraChanged { .. } - )) - )); - assert!(matches!( - client - .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(9), None) - .await, - Err(HistoryReadError::Http { status: 400, .. }) - )); - - // The reader's saved projection includes notices that the core wallet dump - // cannot reconstruct. Prepare a trusted end-of-block B=20 checkpoint. - let mut checkpoint_scheduler = - Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); - let mut saved_history = Vec::new(); - for input in &inputs[..4] { - saved_history.extend(notices(execute(&mut checkpoint_scheduler, input).outputs)); - } - assert_eq!(checkpoint_scheduler.queued_direct_len(), 2); - let (mut checkpoint_app, next_batch_nonce) = checkpoint_scheduler.finish(); - let receipt = AcceptedCheckpoint { - inclusion_block: 20, - executed_input_count: checkpoint_app.executed_input_count(), - next_batch_nonce, - }; - assert_eq!(receipt.executed_input_count.get(), 2); - assert_eq!(checkpoint_app.last_executed_safe_block(), 10); - let backup = dumps.path().join("reader-checkpoint"); - std::fs::create_dir(&backup).unwrap(); - checkpoint_app.create_dump(&backup.join("wallet")).unwrap(); - std::fs::write( - backup.join("projection.json"), - serde_json::to_vec(&(receipt, &saved_history)).unwrap(), - ) - .unwrap(); - drop(checkpoint_app); - drop(saved_history); - - let restored_app = WalletApp::from_dump(&backup.join("wallet")).unwrap(); - let (receipt, mut history): (AcceptedCheckpoint, Vec>) = - serde_json::from_slice(&std::fs::read(backup.join("projection.json")).unwrap()).unwrap(); - assert_eq!( - restored_app.executed_input_count(), - receipt.executed_input_count - ); - let mut start = HistoricalL1InputStart::AfterBlock(restored_app.last_executed_safe_block()); - let mut replay = Scheduler::resume_at( - restored_app, - SchedulerConfig::new(SUBMITTER), - receipt.next_batch_nonce, - ); - std::fs::remove_dir_all(backup).unwrap(); - - let mut seen = Vec::new(); - loop { - // One row per page forces a page split between the batch at B and the - // direct arriving later in that same block. - let page = client - .historical_l1_inputs(era, start, Some(1)) - .await - .unwrap(); - assert_eq!(page.era_id, era); - assert_eq!(page.l1_stop_block, STOP); - assert_eq!(page.end_input_index, 8); - for item in page.items { - let source = &inputs[usize::try_from(item.input_index).unwrap()]; - assert_eq!(item.payload.as_ref(), source.payload); - assert_eq!(item.sender, source.sender); - assert_eq!(item.block_timestamp, source.block_timestamp); - assert_eq!(item.transaction_hash, source.transaction_hash); - seen.push(item.input_index); - if item.block_number <= receipt.inclusion_block { - if item.sender != metadata.deployment.batch_submitter_address { - replay.enqueue_direct(item.sender, item.block_number, item.payload.to_vec()); - } - } else { - let result = replay - .process_input(SchedulerInput { - sender: item.sender, - inclusion_block: item.block_number, - domain: domain(), - payload: item.payload.to_vec(), - }) - .unwrap(); - if item.input_index == 5 { - assert_eq!( - result.outcome, - ProcessOutcome::BatchRejected(BatchRejectReason::DecodeFailed) - ); - assert_eq!( - result.outputs.len(), - 3, - "malformed batch still drains overdue directs" - ); - assert_eq!(replay.next_expected_batch_nonce(), 1); - } - history.extend(notices(result.outputs)); - } - } - if page.next_input_index == page.end_input_index { - break; - } - start = HistoricalL1InputStart::NextInputIndex(page.next_input_index); - } - assert_eq!(seen, vec![1, 2, 3, 4, 5, 6, 7]); - assert_eq!( - history.len(), - 6, - "the young final direct waits until raw EOF" - ); - assert_eq!(replay.queued_direct_len(), 1); - history.extend(notices( - replay - .drain_covered_at(metadata.baseline.l1_stop_block) - .unwrap(), - )); - let (mut reader_app, nonce) = replay.finish(); - assert_eq!(nonce, metadata.baseline.next_batch_nonce); - assert_eq!( - reader_app.executed_input_count(), - metadata.history.available_from - ); - assert_eq!( - reader_app.canonical_snapshot_bytes().unwrap(), - reference_app.canonical_snapshot_bytes().unwrap() - ); - assert_eq!(history, expected_history); - let eof = client - .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(8), None) - .await - .unwrap(); - assert!(eof.items.is_empty()); - assert_eq!(eof.next_input_index, eof.end_input_index); - - let current = client.history(Some(era), None).await.unwrap(); - assert_eq!(current.baseline, metadata.baseline); - let mut stream = client - .subscribe(HistoryClaim { - version: current.history.version, - next_input: reader_app.executed_input_count(), - }) - .await - .unwrap(); - let live = deposit(1245, user, 60); - storage - .append_ingested_safe_inputs_with_timestamp( - 1245, - live.block_timestamp, - std::slice::from_ref(&live), - SUBMITTER, - &default_protocol_timing(), - FrontierMode::Populate, - ) - .unwrap(); - let mut head = storage.open_state().unwrap().unwrap(); - let execution = execute_direct_input( - &mut reference_app, - &DirectInput { - sender: live.sender, - block_number: live.block_number, - payload: live.payload.clone(), - }, - ) - .unwrap(); - expected_history.extend(notices(execution.outputs)); - storage - .close_frame_only_with_executions( - &mut head, - 1245, - SafeInputRange::new(8, 9), - &[DirectInputExecution { - safe_input_index: 8, - executed_input_offset: execution.offset, - }], - ) - .unwrap(); - let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) - .await - .unwrap() - .unwrap() - .unwrap(); - let Message::Text(text) = message else { - panic!("expected application input") - }; - let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); - let WsTxMessage::DirectInput { - offset, - sender, - block_number, - payload, - input_index, - batch_nonce, - .. - } = message - else { - panic!("expected post-baseline deposit") - }; - assert_eq!(offset, 7, "terminal-drained D4 is not delivered again"); - assert_eq!(input_index, 8); - assert_eq!(batch_nonce, 2); - let execution = execute_direct_input( - &mut reader_app, - &DirectInput { - sender: sender.parse().unwrap(), - block_number, - payload: alloy_primitives::hex::decode(payload).unwrap(), - }, - ) - .unwrap(); - assert_eq!(execution.offset, ExecutedInputCount::new(7)); - history.extend(notices(execution.outputs)); - assert_eq!(history, expected_history); - assert_eq!( - reader_app.canonical_snapshot_bytes().unwrap(), - reference_app.canonical_snapshot_bytes().unwrap() - ); - let fixed = client - .historical_l1_inputs(era, HistoricalL1InputStart::AfterBlock(STOP), None) - .await - .unwrap(); - assert!( - fixed.items.is_empty(), - "new L1 inputs never extend this era's historical prefix" - ); - assert_eq!(fixed.end_input_index, 8); - drop(stream); - server.stop().await; -} diff --git a/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs b/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs deleted file mode 100644 index f701e9cc..00000000 --- a/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs +++ /dev/null @@ -1,402 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -use std::path::{Path, PathBuf}; -use std::time::SystemTime; - -use sequencer_core::application::{ExecutionOutcome, validate_and_execute_user_op}; -use sequencer_core::history::{HistoryVersion, RecoveryGeneration}; -use sequencer_core::user_op::SignedUserOp; -use sequencer_rust_client::{ - HistoryPolicyError, HistoryReadError, SubscribeError, SubscribeStream, -}; -use tokio::sync::oneshot; - -use super::*; -use crate::ingress::inclusion_lane::IncludedUserOp; -use crate::storage::WriteHead; -use crate::storage::test_helpers::local_batch_payload; - -struct Backup { - claim: HistoryClaim, - path: PathBuf, -} - -impl Backup { - fn save( - directory: &Path, - name: &str, - app: &mut WalletApp, - history: &[Vec], - version: HistoryVersion, - ) -> Self { - let claim = HistoryClaim { - version, - next_input: app.executed_input_count(), - }; - let path = directory.join(name); - std::fs::create_dir(&path).unwrap(); - app.create_dump(&path.join("wallet")).unwrap(); - std::fs::write( - path.join("projection.json"), - serde_json::to_vec(&(claim, history)).unwrap(), - ) - .unwrap(); - Self { claim, path } - } - - fn restore(&self) -> (WalletApp, Vec>) { - let app = WalletApp::from_dump(&self.path.join("wallet")).unwrap(); - let (claim, history): (HistoryClaim, Vec>) = - serde_json::from_slice(&std::fs::read(self.path.join("projection.json")).unwrap()) - .unwrap(); - assert_eq!(claim, self.claim); - assert_eq!(app.executed_input_count(), claim.next_input); - (app, history) - } -} - -fn append_transfer( - storage: &mut Storage, - head: &mut WriteHead, - app: &mut WalletApp, - history: &mut Vec>, - signer: &PrivateKeySigner, - amount: u64, -) { - let op = UserOp { - nonce: app.current_user_nonce(signer.address()), - max_fee: head.frame_fee, - data: Method::Transfer(Transfer { - to: RECIPIENT, - amount: U256::from(amount), - }) - .as_ssz_bytes() - .into(), - }; - let outcome = - validate_and_execute_user_op(app, signer.address(), &op, head.frame_fee, head.safe_block) - .unwrap(); - let ExecutionOutcome::Included(execution) = outcome else { - panic!("fixture transfer rejected: {outcome:?}") - }; - history.extend(notices(execution.outputs)); - let (respond_to, _response) = oneshot::channel(); - let included = IncludedUserOp { - pending: PendingUserOp { - signed: SignedUserOp { - sender: signer.address(), - signature: signer - .sign_hash_sync(&op.eip712_signing_hash(&domain())) - .unwrap(), - user_op: op, - }, - respond_to, - received_at: SystemTime::now(), - }, - executed_input_offset: execution.offset, - }; - storage - .append_executed_user_ops_chunk(head, &[included]) - .unwrap(); -} - -fn close_and_accept( - storage: &mut Storage, - head: &mut WriteHead, - app: &mut WalletApp, - dumps: &Path, - inclusion_block: u64, -) { - let index = head.batch_index; - let nonce = storage.batch_nonce(index).unwrap(); - let prefix = dumps.join(format!("accepted-{index}")); - dump_info::create_dump_dir_with_info( - app, - &prefix, - &dump_info::DumpInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: nonce + 1, - }, - ) - .unwrap(); - storage - .close_frame_and_batch_with_snapshot( - head, - head.safe_block, - &prefix, - app.executed_input_count(), - ) - .unwrap(); - let payload = local_batch_payload(storage, nonce); - storage - .append_safe_inputs( - inclusion_block, - &[crate::storage::StoredSafeInput { - sender: SUBMITTER, - block_number: inclusion_block, - payload, - }], - SUBMITTER, - &default_protocol_timing(), - ) - .unwrap(); -} - -fn recover_tip(storage: &mut Storage, head: &mut WriteHead, safe_block: u64) { - let protocol = default_protocol_timing(); - storage - .append_safe_inputs(safe_block, &[], SUBMITTER, &protocol) - .unwrap(); - let invalidated = storage - .recover_aging_tip_for_recovery(head.batch_index, &protocol, crate::clock::unix_now_ms()) - .unwrap(); - assert_eq!(invalidated, vec![head.batch_index]); - *head = storage.open_state().unwrap().unwrap(); -} - -async fn replay_one(stream: &mut SubscribeStream, app: &mut WalletApp, history: &mut Vec>) { - let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) - .await - .unwrap() - .unwrap() - .unwrap(); - let Message::Text(text) = message else { - panic!("expected application input") - }; - let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); - let WsTxMessage::UserOp { - offset, - sender, - nonce, - fee, - data, - safe_block, - .. - } = message - else { - panic!("expected transfer") - }; - assert_eq!(offset, app.executed_input_count().get()); - let op = UserOp { - nonce, - max_fee: fee, - data: alloy_primitives::hex::decode(data).unwrap().into(), - }; - let outcome = - validate_and_execute_user_op(app, sender.parse().unwrap(), &op, fee, safe_block).unwrap(); - let ExecutionOutcome::Included(execution) = outcome else { - panic!("feed transfer rejected: {outcome:?}") - }; - history.extend(notices(execution.outputs)); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn checkpoint_compatibility_survives_missed_recoveries_and_retries_admission_races() { - let db = temp_db("reader-checkpoint-compatibility"); - let dumps = tempfile::tempdir().unwrap(); - let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); - let mut app = WalletApp::new(config()); - let mut history = Vec::new(); - let baseline = dumps.path().join("genesis"); - dump_info::create_dump_dir_with_info( - &mut app, - &baseline, - &dump_info::DumpInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: 0, - }, - ) - .unwrap(); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); - pin_test_deployment_identity(&mut storage, SUBMITTER); - storage - .complete_baseline_setup(&baseline, ExecutedInputCount::ZERO, 0, 0, false) - .unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - let directs = [ - deposit(5, signer.address(), 1_000_000_000_000_000), - deposit(6, RECIPIENT, 100), - ]; - storage - .append_ingested_safe_inputs_with_timestamp( - 10, - crate::clock::unix_now_ms() / 1000, - &directs, - SUBMITTER, - &default_protocol_timing(), - FrontierMode::Populate, - ) - .unwrap(); - let mut executions = Vec::new(); - for (index, direct) in directs.into_iter().enumerate() { - let execution = execute_direct_input( - &mut app, - &DirectInput { - sender: direct.sender, - payload: direct.payload, - block_number: direct.block_number, - }, - ) - .unwrap(); - history.extend(notices(execution.outputs)); - executions.push(DirectInputExecution { - safe_input_index: index as u64, - executed_input_offset: execution.offset, - }); - } - storage - .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 2), &executions) - .unwrap(); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 10); - close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 20); - let generation_zero = storage.history_state().unwrap().version; - let good_zero = Backup::save(dumps.path(), "g0-at3", &mut app, &history, generation_zero); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 11); - let bad_zero = Backup::save(dumps.path(), "g0-at4", &mut app, &history, generation_zero); - - let server = Server::start(&db.path).await; - let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); - let era = generation_zero.era_id; - recover_tip(&mut storage, &mut head, 1500); - (app, history) = good_zero.restore(); - let generation_one = storage.history_state().unwrap().version; - assert_eq!(generation_one.recovery_generation.get(), 1); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 20); - let middle_one = Backup::save(dumps.path(), "g1-at4", &mut app, &history, generation_one); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 21); - close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 1501); - let good_one = Backup::save(dumps.path(), "g1-at5", &mut app, &history, generation_one); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 22); - let bad_one = Backup::save(dumps.path(), "g1-at6", &mut app, &history, generation_one); - - recover_tip(&mut storage, &mut head, 3000); - (app, history) = good_one.restore(); - for amount in [30, 31, 32] { - append_transfer( - &mut storage, - &mut head, - &mut app, - &mut history, - &signer, - amount, - ); - } - let current = client.history(Some(era), None).await.unwrap(); - assert_eq!(current.history.head.get(), 8); - assert_eq!(current.history.version.recovery_generation.get(), 2); - assert_eq!(current.compatibility, None); - let same = client - .history(Some(era), Some(RecoveryGeneration::new(2))) - .await - .unwrap(); - assert_eq!(same.compatibility.unwrap().preserved_input_count.get(), 8); - let future = Some(RecoveryGeneration::new(u64::MAX)); - assert!(matches!( - client.history(Some(era), future).await, - Err(HistoryReadError::Http { status: 400, .. }) - )); - let mut other_era_bytes = *era.as_bytes(); - other_era_bytes[0] ^= 1; - let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); - assert!(matches!( - client.history(Some(other_era), future).await, - Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current: version })) - if version == current.history.version - )); - - // Every backup is evaluated under its own saved generation. Latest-cut-only - // matching would incorrectly resurrect the discarded g0 transfer at offset 3. - let mut selected = None; - for (candidate, eligible, cut) in [ - (&good_zero, true, 3), - (&bad_zero, false, 3), - (&middle_one, true, 5), - (&good_one, true, 5), - (&bad_one, false, 5), - ] { - let info = client - .history(Some(era), Some(candidate.claim.version.recovery_generation)) - .await - .unwrap(); - let compatibility = info.compatibility.unwrap(); - assert_eq!( - compatibility.from_generation, - candidate.claim.version.recovery_generation - ); - assert_eq!(compatibility.preserved_input_count.get(), cut); - let survives = candidate.claim.next_input >= info.history.available_from - && candidate.claim.next_input <= compatibility.preserved_input_count; - assert_eq!(survives, eligible); - if survives - && selected.is_none_or(|previous: &Backup| { - previous.claim.next_input < candidate.claim.next_input - }) - { - selected = Some(candidate); - } - } - let selected = selected.unwrap(); - assert_eq!(selected.claim, good_one.claim); - let (mut reader_app, mut reader_history) = selected.restore(); - let mut stream = client - .subscribe(HistoryClaim { - version: current.history.version, - next_input: selected.claim.next_input, - }) - .await - .unwrap(); - for _ in 5..8 { - replay_one(&mut stream, &mut reader_app, &mut reader_history).await; - } - assert_eq!( - reader_app.canonical_snapshot_bytes().unwrap(), - app.canonical_snapshot_bytes().unwrap() - ); - assert_eq!( - reader_history, history, - "restored projection loses invalidated transfers and follows their replacements" - ); - drop(stream); - - let lookup = client - .history(Some(era), Some(good_one.claim.version.recovery_generation)) - .await - .unwrap(); - assert_eq!(lookup.compatibility.unwrap().preserved_input_count.get(), 5); - recover_tip(&mut storage, &mut head, 4500); - let stale = client - .subscribe(HistoryClaim { - version: lookup.history.version, - next_input: good_one.claim.next_input, - }) - .await; - assert!( - matches!(stale, Err(SubscribeError::History(HistoryPolicyError::StaleGeneration { current })) if current.recovery_generation.get() == 3) - ); - let fresh = client - .history(Some(era), Some(good_one.claim.version.recovery_generation)) - .await - .unwrap(); - assert_eq!(fresh.compatibility.unwrap().preserved_input_count.get(), 5); - (reader_app, reader_history) = good_one.restore(); - (app, history) = good_one.restore(); - let mut stream = client - .subscribe(HistoryClaim { - version: fresh.history.version, - next_input: good_one.claim.next_input, - }) - .await - .unwrap(); - append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 40); - replay_one(&mut stream, &mut reader_app, &mut reader_history).await; - assert_eq!( - reader_app.canonical_snapshot_bytes().unwrap(), - app.canonical_snapshot_bytes().unwrap() - ); - assert_eq!(reader_history, history); - drop(stream); - server.stop().await; -} diff --git a/sequencer/src/integration_tests/mod.rs b/sequencer/src/integration_tests/mod.rs index 26b078d4..b94ed05d 100644 --- a/sequencer/src/integration_tests/mod.rs +++ b/sequencer/src/integration_tests/mod.rs @@ -4,6 +4,5 @@ mod batch_submitter; mod chain_id_validation; mod e2e_sequencer; -mod historical_bootstrap; mod snapshot_endpoints; mod ws_broadcaster; diff --git a/sequencer/src/integration_tests/snapshot_endpoints.rs b/sequencer/src/integration_tests/snapshot_endpoints.rs index 7e075978..73e95bcc 100644 --- a/sequencer/src/integration_tests/snapshot_endpoints.rs +++ b/sequencer/src/integration_tests/snapshot_endpoints.rs @@ -12,7 +12,7 @@ use app_core::application::WalletApp; use futures_util::StreamExt; use sequencer::ingress::inclusion_lane::dump_info; use sequencer::storage::Storage; -use sequencer_core::application::Application; +use sequencer_core::application::{Application, CanonicalState}; use crate::test_support::server::{PUBLIC_ROUTES, TestServer}; use crate::test_support::temp_db; @@ -44,22 +44,19 @@ fn register_accepted( ) -> i64 { let mut storage = Storage::open(db_path).unwrap(); let sender = alloy_primitives::Address::repeat_byte(0x7f); - crate::test_support::pin_test_deployment_identity(&mut storage, sender); + // A test registers several accepted batches on one database. + if storage.deployment_identity().unwrap().is_none() { + crate::test_support::pin_test_deployment_identity(&mut storage, sender); + } let mut head = storage.open_state().unwrap().unwrap_or_else(|| { storage .initialize_open_state(inclusion_block, crate::storage::SafeInputRange::empty_at(0)) .unwrap() }); - let nonce = storage.batch_nonce(head.batch_index).unwrap(); + let nonce = head.batch_nonce(); let prefix = write_state(dir, name, bytes, nonce + 1); - let safe_block = head.safe_block; storage - .close_frame_and_batch_with_snapshot( - &mut head, - safe_block, - &prefix, - crate::storage::ExecutedInputCount::ZERO, - ) + .close_frame_and_batch_with_snapshot(&mut head, &prefix, crate::storage::L2TxCount::ZERO) .unwrap(); crate::storage::test_helpers::seed_safe_inputs_with_batch_nonces( &mut storage, @@ -67,31 +64,42 @@ fn register_accepted( inclusion_block, &[nonce], ); - storage.finalized_dump().unwrap().unwrap().dump.id + storage.finalized_dump().unwrap().unwrap().snapshot.id } fn register_optimistic(db_path: &str, dir: &Path, name: &str, bytes: &[u8], nonce: u64) -> i64 { let mut storage = Storage::open(db_path).unwrap(); let mut head = storage.open_state().unwrap().unwrap(); - while storage.batch_nonce(head.batch_index).unwrap() <= nonce { - let current = storage.batch_nonce(head.batch_index).unwrap(); + while head.batch_nonce() <= nonce { + let current = head.batch_nonce(); let dump_name = if current == nonce { name.to_owned() } else { format!("intermediate-{current}") }; let prefix = write_state(dir, &dump_name, bytes, current + 1); - let safe_block = head.safe_block; storage .close_frame_and_batch_with_snapshot( &mut head, - safe_block, &prefix, - crate::storage::ExecutedInputCount::ZERO, + crate::storage::L2TxCount::ZERO, ) .unwrap(); } - storage.latest_snapshot().unwrap().unwrap().dump.id + storage.latest_snapshot().unwrap().unwrap().id +} + +/// The response's `X-` headers, sorted: the claim travels in the era and the +/// count, and nothing else names the history. +fn history_headers(response: &reqwest::Response) -> Vec<&str> { + let mut names: Vec<_> = response + .headers() + .keys() + .map(|name| name.as_str()) + .filter(|name| name.starts_with("x-")) + .collect(); + names.sort_unstable(); + names } fn archive_state(bytes: &[u8]) -> Vec { @@ -105,11 +113,11 @@ fn archive_state(bytes: &[u8]) -> Vec { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn same_block_divergence_refuses_every_finalized_endpoint() { - use crate::storage::test_helpers::{default_protocol_timing, local_batch_payload}; - use crate::storage::{ExecutedInputCount, SafeInputRange, StoredSafeInput}; + use crate::storage::test_helpers::local_batch_payload; + use crate::storage::{L2TxCount, SafeInputRange, StoredSafeInput}; use crate::test_support::pin_test_deployment_identity; use sequencer_core::batch::{Batch, Frame}; - use sequencer_core::scheduler::{Scheduler, SchedulerConfig, SchedulerInput}; + use sequencer_core::scheduler::{CanonicalRunner, L1Input, SchedulerConfig}; use ssz::Encode; let db = temp_db("same-block-divergence-http"); @@ -122,7 +130,7 @@ async fn same_block_divergence_refuses_every_finalized_endpoint() { .unwrap(); let prefix = write_state(root.path(), "local", b"local checkpoint", 1); storage - .close_frame_and_batch_with_snapshot(&mut head, 5, &prefix, ExecutedInputCount::ZERO) + .close_frame_and_batch_with_snapshot(&mut head, &prefix, L2TxCount::ZERO) .unwrap(); let snapshot = storage.latest_snapshot().unwrap().unwrap(); let inputs = vec![ @@ -150,28 +158,33 @@ async fn same_block_divergence_refuses_every_finalized_endpoint() { block_number: 10, }, ]; - storage - .append_safe_inputs(10, &inputs, submitter, &default_protocol_timing()) - .unwrap(); + storage.append_safe_inputs(10, &inputs, submitter).unwrap(); assert!(storage.canonical_divergence().unwrap().is_some()); // The foreign batch consumes the next nonce and drains a direct that the // matching local batch did not: its snapshot is not the end of block 10. - let mut canonical = Scheduler::new(WalletApp::default(), SchedulerConfig::new(submitter)); - for input in inputs { + let mut canonical = CanonicalRunner::genesis( + WalletApp::default(), + SchedulerConfig::for_application::(submitter), + ); + for (index, input) in (0_u64..).zip(inputs) { canonical - .process_input(SchedulerInput { + .process_input(L1Input { + index, sender: input.sender, + block: input.block_number, + timestamp: 0, + chain_id: 1, + app_contract: alloy_primitives::Address::ZERO, payload: input.payload, - inclusion_block: input.block_number, - domain: sequencer_core::build_input_domain(1, alloy_primitives::Address::ZERO), }) .unwrap(); } - let (app, nonce) = canonical.finish(); - assert_eq!(nonce, 2); - assert_eq!(app.executed_input_count().get(), 1); - assert_eq!(snapshot.executed_input_count, ExecutedInputCount::ZERO); + let export = canonical.export(); + assert_eq!(export.next_batch_nonce, 2); + assert_eq!(export.l2_tx_count, 1); + assert_eq!(export.fridge_len, 0); + assert_eq!(snapshot.l2_tx_count, L2TxCount::ZERO); let server = TestServer::start(&db.path).await; let client = reqwest::Client::new(); @@ -195,90 +208,25 @@ async fn same_block_divergence_refuses_every_finalized_endpoint() { response.json::().await.unwrap()["code"], "UNAVAILABLE" ); - assert_eq!(storage.dump_lease_count(snapshot.dump.id).unwrap(), Some(0)); - } -} - -/// Transient WAL lock contention vs. a real read failure. -/// -/// SQLite readers can see `SQLITE_BUSY` in WAL mode during last-connection -/// cleanup / checkpoint (see -/// ). -/// Production RO opens use a 50ms `busy_timeout` by design; tests must retry -/// that case rather than `.expect` through it. Non-lock failures are preserved -/// so diagnostics are not swallowed. -enum LeaseReadError { - Busy, - Other(String), -} - -impl std::fmt::Display for LeaseReadError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Busy => write!(f, "database busy"), - Self::Other(msg) => write!(f, "{msg}"), - } - } -} - -fn sqlite_is_lock_contention(err: &rusqlite::Error) -> bool { - matches!( - err.sqlite_error_code(), - Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) - ) -} - -fn open_is_lock_contention(err: &sequencer::storage::StorageOpenError) -> bool { - match err { - sequencer::storage::StorageOpenError::Sqlite(e) => sqlite_is_lock_contention(e), - _ => false, - } -} - -fn try_lease_count(db_path: &str, dump_id: i64) -> Result { - let mut storage = match Storage::open_read_only(db_path) { - Ok(storage) => storage, - Err(err) if open_is_lock_contention(&err) => return Err(LeaseReadError::Busy), - Err(err) => return Err(LeaseReadError::Other(err.to_string())), - }; - match storage.dump_lease_count(dump_id) { - Ok(n) => Ok(n.unwrap_or(0)), - Err(err) if sqlite_is_lock_contention(&err) => Err(LeaseReadError::Busy), - Err(err) => Err(LeaseReadError::Other(err.to_string())), + assert_eq!(lease_count(&server, snapshot.id), 0); } } -fn lease_count(db_path: &str, dump_id: i64) -> i64 { - for _ in 0..50 { - match try_lease_count(db_path, dump_id) { - Ok(n) => return n, - Err(LeaseReadError::Busy) => std::thread::sleep(Duration::from_millis(10)), - Err(err) => panic!("read lease: {err}"), - } - } - match try_lease_count(db_path, dump_id) { - Ok(n) => n, - Err(err) => panic!("read lease: {err}"), - } +/// The server's lease count on one snapshot; a missing entry counts as zero. +fn lease_count(server: &TestServer, snapshot_id: i64) -> u32 { + server.snapshot_leases.count(snapshot_id).unwrap_or(0) } -async fn wait_for_lease(db_path: &str, dump_id: i64, want: i64) -> bool { +/// The server drops a body, and its lease, after the client's last read. +async fn wait_for_lease(server: &TestServer, snapshot_id: i64, want: u32) -> bool { let started = tokio::time::Instant::now(); while started.elapsed() < Duration::from_secs(3) { - match try_lease_count(db_path, dump_id) { - Ok(got) if got == want => return true, - // Still waiting for the drop-guard release, or transient WAL - // cleanup busy — keep polling. - Ok(_) | Err(LeaseReadError::Busy) => {} - Err(err) => panic!("read lease: {err}"), + if lease_count(server, snapshot_id) == want { + return true; } tokio::time::sleep(Duration::from_millis(10)).await; } - match try_lease_count(db_path, dump_id) { - Ok(got) => got == want, - Err(LeaseReadError::Busy) => false, - Err(err) => panic!("read lease: {err}"), - } + lease_count(server, snapshot_id) == want } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -286,7 +234,7 @@ async fn finalized_state_round_trips_bytes_and_headers() { let db = temp_db("snap-finalized-roundtrip"); let dir = tempfile::tempdir().expect("dumps dir"); let state = b"the-canonical-finalized-state".to_vec(); - let dump_id = register_accepted(db.path.as_str(), dir.path(), "fin", &state, 4242); + let snapshot_id = register_accepted(db.path.as_str(), dir.path(), "fin", &state, 4242); let server = TestServer::start(db.path.as_str()).await; let resp = reqwest::Client::new() @@ -299,7 +247,14 @@ async fn finalized_state_round_trips_bytes_and_headers() { assert_eq!(resp.headers()["x-inclusion-block"], "4242"); assert_eq!(resp.headers()["x-executed-input-count"], "0"); assert!(resp.headers().contains_key("x-history-era")); - assert_eq!(resp.headers()["x-recovery-generation"], "0"); + assert_eq!( + history_headers(&resp), + [ + "x-executed-input-count", + "x-history-era", + "x-inclusion-block" + ] + ); assert_eq!(resp.headers()["etag"], "\"block-4242\""); let body = resp.bytes().await.expect("body"); assert_eq!( @@ -309,7 +264,7 @@ async fn finalized_state_round_trips_bytes_and_headers() { ); assert!( - wait_for_lease(db.path.as_str(), dump_id, 0).await, + wait_for_lease(&server, snapshot_id, 0).await, "lease released after the stream completed" ); } @@ -359,7 +314,7 @@ async fn endpoints_404_when_no_finalized_snapshot() { async fn finalized_state_if_none_match_returns_304() { let db = temp_db("snap-etag"); let dir = tempfile::tempdir().expect("dumps dir"); - let dump_id = register_accepted(db.path.as_str(), dir.path(), "fin", b"abc", 500); + let snapshot_id = register_accepted(db.path.as_str(), dir.path(), "fin", b"abc", 500); let server = TestServer::start(db.path.as_str()).await; let resp = reqwest::Client::new() @@ -373,7 +328,7 @@ async fn finalized_state_if_none_match_returns_304() { assert!(body.is_empty(), "304 has no body"); assert!( - wait_for_lease(db.path.as_str(), dump_id, 0).await, + wait_for_lease(&server, snapshot_id, 0).await, "the briefly-held lease is released even on a 304" ); } @@ -392,7 +347,10 @@ async fn latest_snapshot_prefers_newest_valid_batch() { assert_eq!(resp.status().as_u16(), 200); assert_eq!(resp.headers()["x-executed-input-count"], "0"); assert!(resp.headers().contains_key("x-history-era")); - assert_eq!(resp.headers()["x-recovery-generation"], "0"); + assert_eq!( + history_headers(&resp), + ["x-executed-input-count", "x-history-era"] + ); let body = resp.bytes().await.expect("body"); assert_eq!( archive_state(body.as_ref()).as_slice(), @@ -400,7 +358,7 @@ async fn latest_snapshot_prefers_newest_valid_batch() { "serves the latest pending, not the finalized fallback" ); - assert!(wait_for_lease(db.path.as_str(), pending_id, 0).await); + assert!(wait_for_lease(&server, pending_id, 0).await); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -408,7 +366,7 @@ async fn finalized_state_streams_large_file() { let db = temp_db("snap-large"); let dir = tempfile::tempdir().expect("dumps dir"); let big = vec![0x5Au8; 8 * 1024 * 1024]; - let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); + let snapshot_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); let server = TestServer::start(db.path.as_str()).await; let resp = reqwest::get(server.url("/finalized_state")) @@ -419,7 +377,7 @@ async fn finalized_state_streams_large_file() { assert_eq!(body.len(), big.len(), "full large file streamed"); assert_eq!(body.as_ref(), big.as_slice()); - assert!(wait_for_lease(db.path.as_str(), dump_id, 0).await); + assert!(wait_for_lease(&server, snapshot_id, 0).await); } /// The test that justifies the drop-guard: a client that disconnects @@ -433,7 +391,7 @@ async fn finalized_state_disconnect_mid_stream_releases_lease() { // 8 MiB so the body is still streaming (not fully in the socket buffer) // when we disconnect. let big = vec![0xABu8; 8 * 1024 * 1024]; - let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 7); + let snapshot_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 7); let server = TestServer::start(db.path.as_str()).await; let resp = reqwest::Client::new() @@ -444,7 +402,7 @@ async fn finalized_state_disconnect_mid_stream_releases_lease() { assert_eq!(resp.status().as_u16(), 200); // Lease is held for the duration of the stream (X semantics). assert_eq!( - lease_count(db.path.as_str(), dump_id), + lease_count(&server, snapshot_id), 1, "lease held while streaming" ); @@ -459,9 +417,9 @@ async fn finalized_state_disconnect_mid_stream_releases_lease() { drop(stream); assert!( - wait_for_lease(db.path.as_str(), dump_id, 0).await, + wait_for_lease(&server, snapshot_id, 0).await, "drop-guard must release the lease on disconnect (still {})", - lease_count(db.path.as_str(), dump_id) + lease_count(&server, snapshot_id) ); } @@ -470,7 +428,7 @@ async fn finalized_state_concurrent_streams_count_and_release_independently() { let db = temp_db("snap-concurrent"); let dir = tempfile::tempdir().expect("dumps dir"); let big = vec![0xCDu8; 8 * 1024 * 1024]; - let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); + let snapshot_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); let server = TestServer::start(db.path.as_str()).await; // Two concurrent readers of the same finalized dump. @@ -493,19 +451,19 @@ async fn finalized_state_concurrent_streams_count_and_release_independently() { s2.next().await.expect("chunk 2").expect("ok"); assert!( - wait_for_lease(db.path.as_str(), dump_id, 2).await, - "two live streams hold lease_count == 2 (got {})", - lease_count(db.path.as_str(), dump_id) + wait_for_lease(&server, snapshot_id, 2).await, + "two live streams hold two leases (got {})", + lease_count(&server, snapshot_id) ); drop(s1); assert!( - wait_for_lease(db.path.as_str(), dump_id, 1).await, + wait_for_lease(&server, snapshot_id, 1).await, "dropping one stream steps the lease down to 1" ); drop(s2); assert!( - wait_for_lease(db.path.as_str(), dump_id, 0).await, + wait_for_lease(&server, snapshot_id, 0).await, "dropping the second steps it to 0" ); } @@ -527,32 +485,39 @@ async fn live_stream_blocks_gc_until_it_drops() { assert_eq!(resp.status().as_u16(), 200); let mut stream = resp.bytes_stream(); stream.next().await.expect("chunk").expect("ok"); - assert!(wait_for_lease(db.path.as_str(), served_id, 1).await); + assert!(wait_for_lease(&server, served_id, 1).await); // Acceptance advances independently of the lane's L1 reconciliation. register_accepted(db.path.as_str(), dir.path(), "new", b"new-finalized", 2); - // GC must skip the superseded dump while the stream holds its lease. + // GC, sharing the server's registry, must skip the superseded snapshot + // while the stream holds its lease. + let served = dir.path().join("served"); let removed = { let mut storage = Storage::open(db.path.as_str()).expect("open"); - storage.gc_unreferenced_dumps().expect("gc") + storage + .collect_snapshots(&server.snapshot_leases) + .expect("gc") }; assert!( - !removed.iter().any(|row| row.id == served_id), - "live stream's lease must block GC of the superseded dump" + !removed.contains(&served), + "live stream's lease must block GC of the superseded snapshot" ); - assert_eq!(lease_count(db.path.as_str(), served_id), 1, "still leased"); + assert_eq!(lease_count(&server, served_id), 1, "still leased"); // Drop the stream → lease released → the next GC reaps it. drop(stream); - assert!(wait_for_lease(db.path.as_str(), served_id, 0).await); + assert!(wait_for_lease(&server, served_id, 0).await); let removed_after = { let mut storage = Storage::open(db.path.as_str()).expect("open"); - storage.gc_unreferenced_dumps().expect("gc") + storage + .collect_snapshots(&server.snapshot_leases) + .expect("gc") }; - assert!( - removed_after.iter().any(|row| row.id == served_id), - "GC reaps the dump once the stream releases its lease" + assert_eq!( + removed_after, + vec![served], + "GC reaps the snapshot once the stream releases its lease" ); } @@ -593,7 +558,10 @@ async fn latest_snapshot_restores_accepted_batch_when_it_is_newest() { assert_eq!(resp.status().as_u16(), 200); assert_eq!(resp.headers()["x-executed-input-count"], "0"); assert!(resp.headers().contains_key("x-history-era")); - assert_eq!(resp.headers()["x-recovery-generation"], "0"); + assert_eq!( + history_headers(&resp), + ["x-executed-input-count", "x-history-era"] + ); let body = resp.bytes().await.expect("body"); assert_eq!( archive_state(body.as_ref()).as_slice(), @@ -601,7 +569,7 @@ async fn latest_snapshot_restores_accepted_batch_when_it_is_newest() { "restores finalized artifact" ); - assert!(wait_for_lease(db.path.as_str(), fin_id, 0).await); + assert!(wait_for_lease(&server, fin_id, 0).await); } /// Each route answers only on its own listener: a wrong-method probe gets 405 @@ -609,10 +577,9 @@ async fn latest_snapshot_restores_accepted_batch_when_it_is_newest() { /// handler's state. #[tokio::test] async fn every_route_is_served_on_exactly_its_own_listener() { - const INTERNAL_ROUTES: [&str; 11] = [ + const INTERNAL_ROUTES: [&str; 10] = [ "/ws/subscribe", "/history", - "/historical-l1-inputs", "/finalized_state", "/finalized_state/inclusion_block", "/finalized_state/digest", @@ -736,7 +703,7 @@ async fn cors_permits_browser_preflight_on_fee() { async fn cors_is_limited_to_ingress_and_covers_rejections() { let db = temp_db("cors-route-scope"); let dumps = tempfile::tempdir().expect("snapshot directory"); - let dump_id = register_accepted(&db.path, dumps.path(), "finalized", b"canonical", 1); + let snapshot_id = register_accepted(&db.path, dumps.path(), "finalized", b"canonical", 1); let server = TestServer::start(db.path.as_str()).await; let client = reqwest::Client::new(); let rejected = client @@ -797,7 +764,7 @@ async fn cors_is_limited_to_ingress_and_covers_rejections() { assert!(response.status().is_success()); let _ = response.bytes().await.expect("response body"); } - assert!(wait_for_lease(&db.path, dump_id, 0).await); + assert!(wait_for_lease(&server, snapshot_id, 0).await); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -823,7 +790,7 @@ async fn sdk_snapshot_restores_application_and_recovery_export_is_self_contained .unpack(restored_dir.path()) .unwrap(); let restored = WalletApp::from_dump(&dump_info::app_prefix(restored_dir.path())).unwrap(); - assert_eq!(restored.executed_input_count(), claim.next_input); + assert_eq!(restored.l2_tx_count(), claim.next_input); assert!(!restored_dir.path().join("checkpoint.toml").exists()); let mut stream = client.subscribe(claim).await.unwrap(); stream.close(None).await.unwrap(); @@ -832,6 +799,18 @@ async fn sdk_snapshot_restores_application_and_recovery_export_is_self_contained .await .unwrap(); assert_eq!(export.headers()["x-inclusion-block"], "1"); + assert_eq!( + history_headers(&export), + [ + "x-executed-input-count", + "x-history-era", + "x-inclusion-block" + ] + ); + assert_eq!( + export.headers()["x-history-era"].to_str().unwrap(), + claim.era_id.to_string() + ); let export_dir = tempfile::tempdir().unwrap(); tar::Archive::new(export.bytes().await.unwrap().as_ref()) .unpack(export_dir.path()) @@ -845,11 +824,11 @@ async fn sdk_snapshot_restores_application_and_recovery_export_is_self_contained .next_batch_nonce, receipt.next_batch_nonce ); + let exported = WalletApp::from_dump(&dump_info::app_prefix(export_dir.path())).unwrap(); + assert_eq!(exported.l2_tx_count(), app.l2_tx_count()); assert_eq!( - WalletApp::from_dump(&dump_info::app_prefix(export_dir.path())) - .unwrap() - .progress(), - app.progress() + exported.canonical_snapshot_bytes().unwrap(), + app.canonical_snapshot_bytes().unwrap() ); assert!( !dir.path().join("accepted/checkpoint.toml").exists(), @@ -866,10 +845,10 @@ async fn archive_disconnect_releases_both_stream_and_producer_lease_owners() { let response = reqwest::get(server.url("/latest_snapshot")).await.unwrap(); let mut stream = response.bytes_stream(); stream.next().await.unwrap().unwrap(); - assert!(wait_for_lease(&db.path, id, 1).await); + assert!(wait_for_lease(&server, id, 1).await); drop(stream); assert!( - wait_for_lease(&db.path, id, 0).await, + wait_for_lease(&server, id, 0).await, "disconnected archive producer must terminate and release its lease" ); } diff --git a/sequencer/src/integration_tests/ws_broadcaster.rs b/sequencer/src/integration_tests/ws_broadcaster.rs index 59781399..1b84bb64 100644 --- a/sequencer/src/integration_tests/ws_broadcaster.rs +++ b/sequencer/src/integration_tests/ws_broadcaster.rs @@ -13,7 +13,7 @@ use sequencer::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp, Sequence use sequencer::runtime::shutdown::RuntimeScope; use sequencer::storage::{SafeInputRange, Storage, StoredSafeInput}; use sequencer_core::api::WsTxMessage; -use sequencer_core::history::{EraId, ExecutedInputCount, HistoryClaim, HistoryPolicyError}; +use sequencer_core::history::{EraId, HistoryClaim, HistoryPolicyError, L2TxCount}; use sequencer_core::user_op::{SignedUserOp, UserOp}; use sequencer_rust_client::SequencerClient; use tokio::sync::{mpsc, oneshot}; @@ -206,17 +206,45 @@ async fn ws_subscribe_requires_a_valid_history_claim_before_upgrade() { let runtime = start_test_server(&db.path).await; let client = SequencerClient::new(format!("http://{}", runtime.addr)).unwrap(); let mut start = history_claim(&db.path, 3); + // A valid claim with any extra field, such as a revision the server does + // not check, is refused before the claim is read. + for extra in ["recovery_generation=0", "unrecognized=1"] { + let url = format!( + "ws://{}/ws/subscribe?era_id={}&next_input=0&{extra}", + runtime.addr, start.era_id + ); + let refused = connect_async(url).await; + assert!( + matches!(&refused, Err(tokio_tungstenite::tungstenite::Error::Http(response)) if response.status().as_u16() == 400), + "{extra}: {refused:?}" + ); + } assert!(matches!(client.subscribe(start).await, Err(sequencer_rust_client::SubscribeError::History(HistoryPolicyError::AheadOfHead { head })) if head.get() == 2)); - start.next_input = ExecutedInputCount::ZERO; - start.version.era_id = + start.next_input = L2TxCount::ZERO; + let current_era = start.era_id; + start.era_id = EraId::from_bytes([0x11, 0, 0, 0, 0, 0, 0x40, 0, 0x80, 0, 0, 0, 0, 0, 0, 1]).unwrap(); assert!(matches!( client.subscribe(start).await, Err(sequencer_rust_client::SubscribeError::History( - HistoryPolicyError::EraChanged { .. } - )) + HistoryPolicyError::EraChanged { era_id } + )) if era_id == current_era )); + // The refusal's body and header carry the same exact JSON. + let refused = connect_async(format!( + "ws://{}/ws/subscribe?era_id={}&next_input=0", + runtime.addr, start.era_id + )) + .await; + let Err(tokio_tungstenite::tungstenite::Error::Http(response)) = refused else { + panic!("expected an HTTP refusal, got {refused:?}"); + }; + assert_eq!(response.status().as_u16(), 409); + let expected = serde_json::json!({ "code": "ERA_CHANGED", "era_id": current_era.to_string() }); + let header: serde_json::Value = + serde_json::from_slice(response.headers()["X-History-Error"].as_bytes()).unwrap(); + assert_eq!(header, expected); let legacy = connect_async(format!("ws://{}/ws/subscribe?from_offset=0", runtime.addr)).await; assert!( matches!(legacy, Err(tokio_tungstenite::tungstenite::Error::Http(response)) if response.status().as_u16() == 400) @@ -224,6 +252,80 @@ async fn ws_subscribe_requires_a_valid_history_claim_before_upgrade() { shutdown_runtime(runtime).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_client_of_an_older_era_adopts_the_rebuilt_era_at_its_count_and_follows_live_rows() { + use crate::egress::test_era; + use sequencer_rust_client::{AdoptEraError, AdoptError}; + + let db = temp_db("ws-adopt-rebuilt-era"); + let mut storage = + test_era::seed_rebuilt_era(&db.path, std::path::Path::new("/unused/baseline")); + let first_live_op = test_era::append_live_user_op(&mut storage, 2); + let runtime = start_test_server(&db.path).await; + let client = SequencerClient::new(format!("http://{}", runtime.addr)).unwrap(); + + let info = client.history(None).await.unwrap(); + assert_eq!(info.history.available_from.get(), test_era::BASE); + assert_eq!( + info.baseline.rebuilt_history_end.get(), + test_era::REBUILT_END + ); + assert_eq!(info.history.head.get(), test_era::REBUILT_END + 2); + + // The client's state sits at count 43 of the era the recovery replaced. + let count = L2TxCount::new(43); + let old_era = HistoryClaim { + era_id: EraId::from_bytes([0x22, 0, 0, 0, 0, 0, 0x40, 0, 0x80, 0, 0, 0, 0, 0, 0, 2]) + .unwrap(), + next_input: count, + }; + assert!(matches!( + client.subscribe(old_era).await, + Err(sequencer_rust_client::SubscribeError::History( + HistoryPolicyError::EraChanged { era_id } + )) if era_id == info.history.era_id + )); + let claim = client.adopt_current_era(count).await.unwrap(); + assert_eq!(claim.era_id, info.history.era_id); + assert_eq!(claim.next_input, count); + + // The feed crosses the seam from the rebuilt rows into the live ones, + // then keeps following new rows. + let mut ws = client.subscribe(claim).await.unwrap(); + let mut expected: Vec<_> = (43..=test_era::REBUILT_END) + .map(test_era::expected_feed_message) + .collect(); + expected.push(first_live_op); + let mut received = Vec::new(); + for _ in 0..expected.len() { + received.push(serde_json::to_value(recv_tx_message(&mut ws).await).unwrap()); + } + assert_eq!(received, expected); + let next_live_op = test_era::append_live_user_op(&mut storage, 3); + assert_eq!( + serde_json::to_value(recv_tx_message(&mut ws).await).unwrap(), + next_live_op + ); + drop(ws); + + // Counts outside the base..=rebuilt-end range cannot adopt, even below + // the head. + assert!(matches!( + client + .adopt_current_era(L2TxCount::new(test_era::BASE - 1)) + .await, + Err(AdoptEraError::Adopt(AdoptError::BelowBase { .. })) + )); + assert!(matches!( + client + .adopt_current_era(L2TxCount::new(test_era::REBUILT_END + 1)) + .await, + Err(AdoptEraError::Adopt(AdoptError::PastRebuiltEnd { .. })) + )); + drop(storage); + shutdown_runtime(runtime).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_closes_on_oversized_inbound_message() { let db = temp_db("ws-oversized-inbound"); @@ -278,13 +380,13 @@ fn seed_ordered_txs(db_path: &str) { received_at: SystemTime::now(), }; - let executed_input_offset = storage.next_executed_input_count().expect("next offset"); + let l2_tx_index = storage.next_l2_tx_count().expect("next offset"); storage - .append_executed_user_ops_chunk( + .append_included_user_ops( &mut head, &[IncludedUserOp { pending, - executed_input_offset, + l2_tx_index, }], ) .expect("append user-op chunk"); @@ -297,7 +399,6 @@ fn seed_ordered_txs(db_path: &str) { block_number: 10, }], Address::repeat_byte(0x7f), - &crate::storage::test_helpers::default_protocol_timing(), ) .expect("append direct input"); storage @@ -328,7 +429,6 @@ fn append_drained_direct_input(db_path: &str, payload: Vec) { block_number: safe_block, }], Address::repeat_byte(0x7f), - &crate::storage::test_helpers::default_protocol_timing(), ) .expect("append direct input"); storage @@ -396,7 +496,9 @@ async fn start_test_server_with_limits( http::SnapshotState { db_path: db_path.to_string(), state_file_in_dump: snapshot_state_file, + leases: crate::storage::SnapshotLeases::default(), }, + crate::landing::test_feed(1200), ); WsServerRuntime { @@ -465,7 +567,7 @@ fn ws_subscribe_url(db_path: &str, addr: std::net::SocketAddr, from_offset: u64) fn ordered_l2_tx_count(db_path: &str) -> u64 { let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); storage - .next_executed_input_count() + .next_l2_tx_count() .expect("query application head") .get() } @@ -512,16 +614,20 @@ fn assert_ws_message_matches_tx( sender, block_number, payload, + input_index, + block_timestamp, .. }, - L2TxContext::DirectInput { tx: expected, .. }, + L2TxContext::Direct { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( decode_hex_prefixed(sender.as_str()), expected.sender.as_slice() ); - assert_eq!(block_number, expected.block_number); + assert_eq!(block_number, expected.block); + assert_eq!(input_index, expected.input_index); + assert_eq!(block_timestamp, expected.timestamp); assert_eq!( decode_hex_prefixed(payload.as_str()), expected.payload.as_slice() @@ -535,11 +641,7 @@ fn assert_ws_message_matches_tx( fn history_claim(db_path: &str, next: u64) -> HistoryClaim { HistoryClaim { - version: Storage::open_read_only(db_path) - .unwrap() - .history_state() - .unwrap() - .version, - next_input: ExecutedInputCount::new(next), + era_id: Storage::open_read_only(db_path).unwrap().era().unwrap().id, + next_input: L2TxCount::new(next), } } diff --git a/sequencer/src/l1/fee_oracle/worker.rs b/sequencer/src/l1/fee_oracle/worker.rs index 222ac4ac..6c2774de 100644 --- a/sequencer/src/l1/fee_oracle/worker.rs +++ b/sequencer/src/l1/fee_oracle/worker.rs @@ -363,6 +363,28 @@ mod tests { let _ = Storage::open(path).expect("initialize storage schema"); } + /// The stored price, or `None` while the database is busy. A running + /// worker opens and closes its own writer on every refresh, and a reader + /// that meets it gives up after its 50 ms busy timeout, so a poll takes + /// busy as "not yet". + fn poll_log_gas_price(path: &str) -> Option { + match Storage::open_read_only(path).unwrap().log_gas_price() { + Ok(price) => Some(price), + Err(error) + if matches!( + error.sqlite_error_code(), + Some( + rusqlite::ffi::ErrorCode::DatabaseBusy + | rusqlite::ffi::ErrorCode::DatabaseLocked + ) + ) => + { + None + } + Err(error) => panic!("reading the gas price: {error}"), + } + } + fn oracle_with( path: &str, gas: Box, @@ -590,21 +612,20 @@ mod tests { result = &mut handle => panic!("fee oracle exited early: {result:?}"), _ = tokio::time::sleep(Duration::from_millis(10)) => {} } - let price = Storage::open_read_only(&db.path) - .unwrap() - .log_gas_price() - .unwrap(); + let price = poll_log_gas_price(&db.path); let calls = gas_calls.load(Ordering::SeqCst); - if price == expected_log && calls >= 2 { + if price == Some(expected_log) && calls >= 2 { break; } if tokio::time::Instant::now() >= deadline { panic!( "timed out waiting for retained fee-oracle price \ - (expected {expected_log}, last read {price}, gas_calls={calls})" + (expected {expected_log}, last read {price:?}, gas_calls={calls})" ); } } + // The worker's only write finished before its second gas call, so + // nothing else holds the database now. let storage = Storage::open_read_only(&db.path).unwrap(); assert_eq!(storage.log_gas_price().unwrap(), expected_log); drop(storage); @@ -636,12 +657,7 @@ mod tests { tokio::time::timeout(Duration::from_secs(2), async { loop { - if Storage::open_read_only(&db.path) - .unwrap() - .log_gas_price() - .unwrap() - == expected_log - { + if poll_log_gas_price(&db.path) == Some(expected_log) { break; } tokio::time::sleep(Duration::from_millis(10)).await; diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/l1/flusher.rs similarity index 90% rename from sequencer/src/recovery/flusher.rs rename to sequencer/src/l1/flusher.rs index e001aba3..f6e830d9 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/l1/flusher.rs @@ -16,7 +16,7 @@ use tracing::{debug, info, warn}; use crate::l1::eip1559::{estimate_fees, pad_gas_estimate}; use crate::l1::provider::{KeyedProviders, SendError, error_chain}; use crate::l1::watermark::{ - StorageWatermarkSink, WalletNonceWatermarkError, WalletNonceWatermarkSink, + Covered, StorageWatermarkSink, WalletNonceWatermarkError, WalletNonceWatermarkSink, cover, }; #[derive(Debug, Error)] @@ -164,10 +164,11 @@ impl MempoolFlusher { /// 1. Submit 0-ETH self-transfers for wallet nonces in /// `[Latest, max(Pending, watermark + 1))` — every slot not yet /// mined that we might ever have used. The sink raises the persisted - /// watermark before the broadcast (uniform write-before-broadcast, - /// though no-ops never exceed the existing watermark under the - /// invariant). These compete with any of our txs still in the - /// network; whichever wins, the slot advances. + /// watermark to the top of that range before the broadcast, and each + /// send requires the resulting witness (uniform write-before-broadcast, + /// though no-ops above the given watermark only appear if another + /// process used our key). These compete with any of our txs still in + /// the network; whichever wins, the slot advances. /// 2. Wait until every slot below the flush end is mined, by an original /// or a no-op: the slot, not a particular tx, is what must be consumed. /// 3. Sleep to let the safe head advance, then re-check the loop condition. @@ -175,8 +176,8 @@ impl MempoolFlusher { /// have been dropped or underpriced). /// /// Returns the L1 **safe block number** at which resolution was observed: - /// the caller must not cascade until its own re-synced view reaches at - /// least this block. + /// the caller must not act on its own re-synced view until it reaches at + /// least this block (setup pins it as the stop block). pub async fn flush_and_wait( &self, watermark: Option, @@ -246,13 +247,14 @@ impl MempoolFlusher { // Uniform write-before-broadcast; covers the no-ops we are // about to send (a no-op above the current watermark can only // happen if someone else used our key — over-covering then is - // exactly right). + // exactly right). An empty range sends nothing and raises nothing. let highest_wallet_nonce = flush_end .checked_sub(1) .expect("a non-empty flush range must have a highest nonce"); - sink.raise_to(highest_wallet_nonce)?; + let covered = cover(sink, highest_wallet_nonce)?; + self.submit_noops(latest_wallet_nonce, flush_end, &covered) + .await?; } - self.submit_noops(latest_wallet_nonce, flush_end).await?; if !self.wait_for_slots_mined(flush_end).await? { continue; @@ -276,11 +278,13 @@ impl MempoolFlusher { } /// Submit 0-ETH self-transfers for wallet nonces - /// `from_wallet_nonce..to_wallet_nonce`. + /// `from_wallet_nonce..to_wallet_nonce`; each send asserts `covered` + /// reaches its nonce. async fn submit_noops( &self, from_wallet_nonce: u64, to_wallet_nonce: u64, + covered: &Covered, ) -> Result<(), FlushError> { if from_wallet_nonce >= to_wallet_nonce { return Ok(()); @@ -337,6 +341,7 @@ impl MempoolFlusher { .with_max_fee_per_gas(max_fee) .with_max_priority_fee_per_gas(priority_fee); + covered.assert_covers(wallet_nonce); match self.providers.submit.send(tx).await { Ok(tx_hash) => { debug!(wallet_nonce, %tx_hash, gas, "flush no-op submitted"); @@ -774,6 +779,89 @@ mod tests { ); } + /// Records every `raise_to` and fails it, so a test sees the raise happen + /// and can check that nothing was broadcast before it. + struct FailingRecordingSink { + calls: std::sync::Mutex>, + } + impl WalletNonceWatermarkSink for FailingRecordingSink { + fn raise_to(&self, highest: u64) -> Result<(), WalletNonceWatermarkError> { + self.calls.lock().expect("lock").push(highest); + Err(WalletNonceWatermarkError::Storage( + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::DatabaseBusy, + extended_code: 5, + }, + None, + ), + )) + } + } + + /// Write-before-broadcast for the flusher: the raise comes before the + /// first no-op. A failing sink aborts the pass; had a no-op gone out + /// first, the pending wallet nonce would have moved past 0. + #[tokio::test] + async fn flush_raises_watermark_before_the_first_noop() { + require_anvil(); + + let anvil = spawn_anvil(); + let providers = keyed_providers(&anvil); + let provider = providers.read.clone(); + let addr = anvil.addresses()[0]; + let sink = FailingRecordingSink { + calls: std::sync::Mutex::new(Vec::new()), + }; + + // Watermark 0 with an empty pool: slot 0 needs a no-op. + let flusher = MempoolFlusher::new(providers.clone(), addr, 12) + .with_timeouts(Duration::from_secs(2), Duration::from_millis(100)); + let result = tokio::time::timeout( + Duration::from_secs(5), + flusher.flush_and_wait(Some(0), &sink), + ) + .await + .expect("a failing raise must end the flush promptly"); + + assert!( + matches!( + result, + Err(FlushError::Watermark(WalletNonceWatermarkError::Storage(_))) + ), + "a failing watermark sink must abort the flush, got {result:?}" + ); + assert_eq!( + *sink.calls.lock().expect("lock"), + vec![0], + "raise_to must run once, to the top of the no-op range" + ); + let pending = provider + .get_transaction_count(addr) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .expect("pending nonce"); + assert_eq!( + pending, 0, + "raise_to must run before any no-op; a broadcast would have bumped the pending nonce" + ); + } + + /// A no-op above its witness trips the assert before it is sent. + #[tokio::test] + #[should_panic(expected = "above the durably covered watermark")] + async fn noop_above_the_covered_watermark_panics() { + require_anvil(); + + let anvil = spawn_anvil(); + let providers = keyed_providers(&anvil); + let addr = anvil.addresses()[0]; + let flusher = MempoolFlusher::new(providers, addr, 12); + let covered = cover(&NoopWatermarkSink, 0).expect("noop sink covers"); + + let _ = flusher.submit_noops(0, 2, &covered).await; + } + #[tokio::test] async fn slot_wait_counts_a_slot_mined_by_any_transaction() { require_anvil(); diff --git a/sequencer/src/l1/mod.rs b/sequencer/src/l1/mod.rs index 295f3ce6..a60b8cf9 100644 --- a/sequencer/src/l1/mod.rs +++ b/sequencer/src/l1/mod.rs @@ -2,11 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! L1 client surface: reads InputBox events into storage (`reader`), submits -//! batches back out (`submitter`), and shares L1 utilities (`provider`, -//! `partition`). +//! batches back out (`submitter`), settles the submitter's wallet slots +//! (`flusher`), and shares L1 utilities (`provider`, `partition`). pub mod eip1559; pub mod fee_oracle; +pub mod flusher; pub mod partition; pub mod provider; pub mod reader; diff --git a/sequencer/src/l1/partition.rs b/sequencer/src/l1/partition.rs index da91e00b..d950efb7 100644 --- a/sequencer/src/l1/partition.rs +++ b/sequencer/src/l1/partition.rs @@ -437,7 +437,7 @@ mod tests { const CONSENSUS_VECTORS: &str = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../tests/fixtures/consensus_vectors_v1.json" + "/../tests/fixtures/consensus_vectors_v2.json" )); #[derive(Debug, Deserialize)] @@ -482,14 +482,14 @@ mod tests { assert_eq!(decoded.payload, fields.payload, "{case}"); } - /// Pinned in `consensus_vectors_v1.json`, never regenerated. The variants + /// Pinned in `consensus_vectors_v2.json`, never regenerated. The variants /// pin today's lenient decode: trailing bytes and dirty padding decode to /// the canonical fields. A flip to strict decoding changes `accepted`, /// which is a finding, not a fixture update. #[test] fn evm_advance_decode_matches_the_pinned_vector() { let vector = serde_json::from_str::(CONSENSUS_VECTORS) - .expect("parse consensus_vectors_v1.json") + .expect("parse consensus_vectors_v2.json") .evm_advance; let decoded = decode_evm_advance_input(&vector.calldata).expect("decode pinned calldata"); assert_fields(&decoded, &vector.fields, "canonical"); diff --git a/sequencer/src/l1/reader.rs b/sequencer/src/l1/reader.rs index 91b4f2b6..5a12a20a 100644 --- a/sequencer/src/l1/reader.rs +++ b/sequencer/src/l1/reader.rs @@ -21,8 +21,7 @@ use crate::l1::partition::{ use crate::l1::provider::error_chain; use crate::runtime::process_lock::{ProcessLock, spawn_blocking_with_lock}; use crate::runtime::shutdown::ShutdownSignal; -use crate::storage::{FrontierMode, IngestedSafeInput, Storage, StorageOpenError}; -use sequencer_core::protocol::ProtocolTiming; +use crate::storage::{IngestedSafeInput, Storage, StorageOpenError}; #[derive(Debug, Clone)] pub struct InputReaderConfig { @@ -39,7 +38,7 @@ pub struct InputReaderConfig { /// contact — the RPC URL is an operator CLI/env arg that may be repointed /// across restarts (token rotation, provider swap), so `run` cannot assume /// it still matches the pinned identity. The check lives on the reader so - /// both startup recovery and the long-lived worker verify identity before + /// both the startup sync and the long-lived worker verify identity before /// ingesting from the first successful provider contact. pub expected_chain_id: u64, } @@ -104,8 +103,8 @@ impl InputReaderError { } /// Whether a failure in a startup phase is worth retrying: setup's - /// discovery and initial sync, and run's `InitialSync` and - /// `PostFlushSync` (both through `sync_to_current_safe_head`). + /// discovery and initial sync, and run's startup sync (through + /// `sync_to_current_safe_head`). /// `ChainIdMismatch` and `StorageTaskPanicked` are refused at startup and /// terminal in the live worker alike. Two variants are phase-dependent by /// design and differ from [`Self::is_terminal_invariant`]: @@ -163,17 +162,6 @@ pub struct InputReader { /// 0 and the InputBox count at the scanned safe head. app_deployment_block: u64, db_path: String, - /// Scheduler-acceptance identity — passed into - /// `Storage::append_ingested_safe_inputs_with_timestamp` so the persisted - /// scheduler-accepted frontier filters by the right sender. - batch_submitter: Address, - /// Protocol timing used to keep `safe_accepted_batches` consistent with - /// every safe-input write. - timing: ProtocolTiming, - /// Whether each sync also populates the scheduler-accepted frontier - /// ([`FrontierMode::Populate`] normally; `setup --recovery` defers it — see - /// [`Self::set_frontier_mode`]). - frontier_mode: FrontierMode, /// Set once the provider's chain id has been verified against /// [`InputReaderConfig::expected_chain_id`] on a successful RPC contact. /// Until then every `advance_once` re-attempts the check, so an unreachable @@ -189,8 +177,6 @@ impl InputReader { pub(crate) async fn new( db_path: impl Into, config: InputReaderConfig, - batch_submitter: Address, - timing: ProtocolTiming, process_lock: ProcessLock, ) -> Result { let provider = @@ -231,8 +217,6 @@ impl InputReader { input_box_address, app_deployment_block, db_path.into(), - batch_submitter, - timing, process_lock, )) } @@ -242,8 +226,6 @@ impl InputReader { input_box_address: Address, app_deployment_block: u64, db_path: String, - batch_submitter: Address, - timing: ProtocolTiming, process_lock: ProcessLock, ) -> Self { Self { @@ -251,22 +233,11 @@ impl InputReader { input_box_address, app_deployment_block, db_path, - batch_submitter, - timing, - frontier_mode: FrontierMode::Populate, chain_id_verified: false, process_lock, } } - /// Set whether subsequent syncs populate the scheduler-accepted frontier. - /// `setup --recovery` sets [`FrontierMode::DeferUntilAnchorSet`] for the - /// syncs that feed the fold (the frontier is populated by `run`'s first - /// sync, once the anchor = `N'`). - pub fn set_frontier_mode(&mut self, mode: FrontierMode) { - self.frontier_mode = mode; - } - pub fn input_box_address(&self) -> Address { self.input_box_address } @@ -292,9 +263,10 @@ impl InputReader { } /// Validate the storage dependency without starting a task. Runtime - /// startup calls this for every worker before launching any of them. + /// startup calls this for every worker before launching any of them, so + /// no writer runs beside this read-only open. pub(crate) fn preflight_storage(&self) -> Result<(), StorageOpenError> { - let _ = Storage::open(self.db_path.as_str())?; + let _ = Storage::open_read_only(self.db_path.as_str())?; Ok(()) } @@ -378,9 +350,9 @@ impl InputReader { /// Reads `[cursor, safe]` as one span for one commit, with the InputBox /// count at `safe` as its completeness witness: a commit at a lower head /// would rest on index contiguity alone, which a dropped tail input passes - /// (I6). In `run` the span stays under about `l1_read_stale_after_blocks`, - /// since the danger detector stops `run` first; setup, rebuild and startup - /// syncs read the whole gap, from the app's deployment on a fresh database. + /// (I6). In `run` the span is one poll's worth of blocks, or the whole + /// outage once the provider returns; setup and startup syncs read the + /// whole gap, from the app's deployment on a fresh database. /// Bounding those is a known optimization in `docs/review/register.md`. async fn read_safe_inputs( &mut self, @@ -411,7 +383,7 @@ impl InputReader { if current_safe_block < start_block { // A head behind the persisted one is environmental (a lagging, // resyncing or repointed endpoint) and is held as no progress; log - // it, or only the stale-view danger check surfaces it, hours later. + // it, or only the safe head's age surfaces it. if let Some(persisted_safe_block) = previous_safe_block && current_safe_block < persisted_safe_block { @@ -547,7 +519,7 @@ impl InputReader { /// on the first successful contact. A transport failure is retryable /// (`Provider`) and leaves the flag unset, so the check re-fires on the next /// tick until L1 is reachable; a value mismatch is fatal (`ChainIdMismatch`) - /// and propagates out of the run loop / aborts a recovery sync. Keeping the + /// and propagates out of the run loop / aborts the startup sync. Keeping the /// check on the reader itself also covers a warm boot whose initial refresh /// tolerated an unreachable provider: a later reconnect to the wrong chain /// cannot ingest address-filtered foreign logs unnoticed. @@ -587,12 +559,13 @@ impl InputReader { /// safe inputs (= the next local `safe_input_index`, the expected on-chain /// index of the next ingested input). Reading them together is safe: /// this reader is the sole writer of both, and `advance_once` holds - /// `&mut self` on a single-task loop. + /// `&mut self` on a single-task loop. A writer handle, for its 5 s busy + /// timeout: the lane commits beside this read. async fn scan_cursor(&self) -> Result<(Option, u64), InputReaderError> { let db_path = self.db_path.clone(); let process_lock = self.process_lock.clone(); spawn_blocking_with_lock(process_lock, move || { - let mut storage = Storage::open(&db_path)?; + let mut storage = Storage::open_writer(&db_path)?; Ok(( storage.current_safe_block()?, storage.safe_input_end_exclusive()?, @@ -608,20 +581,14 @@ impl InputReader { batch: Vec, ) -> Result<(), InputReaderError> { let db_path = self.db_path.clone(); - let batch_submitter = self.batch_submitter; - let timing = self.timing; - let frontier_mode = self.frontier_mode; let process_lock = self.process_lock.clone(); spawn_blocking_with_lock(process_lock, move || { - let mut storage = Storage::open(&db_path)?; + let mut storage = Storage::open_writer(&db_path)?; storage .append_ingested_safe_inputs_with_timestamp( current_safe_head.block_number, current_safe_head.block_timestamp, &batch, - batch_submitter, - &timing, - frontier_mode, ) .map_err(InputReaderError::from) }) @@ -935,16 +902,24 @@ mod tests { use alloy::sol_types::SolCall; use tempfile::NamedTempFile; - fn test_timing() -> ProtocolTiming { - crate::storage::test_helpers::default_protocol_timing() - } - + /// A reader over `db_path` with a deployment identity pinned (submitter + /// `Address::ZERO`) unless a seeding append already pinned it, as setup + /// pins it before any sync. fn test_reader( db_path: String, rpc_url: String, app_deployment_block: u64, poll_interval: Duration, ) -> InputReader { + let mut storage = Storage::open(&db_path).expect("open storage"); + if storage + .deployment_identity() + .expect("read identity") + .is_none() + { + crate::test_support::pin_test_deployment_identity(&mut storage, Address::ZERO); + } + drop(storage); InputReader::from_parts( InputReaderConfig { rpc_url, @@ -958,8 +933,6 @@ mod tests { Address::ZERO, app_deployment_block, db_path, - Address::ZERO, - test_timing(), ProcessLock::test(), ) } @@ -1081,8 +1054,6 @@ mod tests { Address::ZERO, 0, db_file.path().to_string_lossy().into_owned(), - Address::ZERO, - test_timing(), ProcessLock::test(), ); let provider = alloy::providers::ProviderBuilder::new() @@ -1169,8 +1140,6 @@ mod tests { poll_interval: Duration::from_secs(1), expected_chain_id: 31337, }, - Address::ZERO, - test_timing(), ProcessLock::test(), ) .await; @@ -1243,14 +1212,9 @@ mod tests { let db_file = NamedTempFile::new().expect("temp file"); let db_path = db_file.path().to_string_lossy().into_owned(); let mut storage = Storage::open(&db_path).expect("open storage"); - let timing = test_timing(); storage - .append_safe_inputs(1000, &[], Address::ZERO, &timing) + .append_safe_inputs_with_timestamp(1000, 42, &[], Address::ZERO) .expect("set safe head ahead of chain"); - let recorded_sync = storage - .last_safe_progress_ms() - .expect("read safe-progress timestamp") - .expect("append_safe_inputs should stamp safe progress"); drop(storage); let mut reader = test_reader( @@ -1271,14 +1235,14 @@ mod tests { "safe head should remain unchanged when already ahead of chain" ); - let storage = + let mut storage = Storage::open(db_file.path().to_string_lossy().as_ref()).expect("re-open storage"); assert_eq!( storage - .last_safe_progress_ms() - .expect("read unchanged safe-progress timestamp"), - Some(recorded_sync), - "same-head polls must not refresh the safe-progress marker" + .current_safe_block_timestamp() + .expect("read unchanged safe-block timestamp"), + Some(42), + "same-head polls must not rewrite the safe head" ); } @@ -1836,7 +1800,6 @@ mod tests { block_number: 5, }], Address::ZERO, - &test_timing(), ) .expect("seed one stored direct input"); drop(storage); @@ -1934,7 +1897,7 @@ mod tests { let db_path = db_file.path().to_string_lossy().into_owned(); let mut storage = Storage::open(&db_path).expect("open storage"); storage - .append_safe_inputs(9, &[], Address::ZERO, &test_timing()) + .append_safe_inputs(9, &[], Address::ZERO) .expect("seed a persisted safe head"); drop(storage); diff --git a/sequencer/src/l1/submitter/config.rs b/sequencer/src/l1/submitter/config.rs index 8812f758..f01c80f6 100644 --- a/sequencer/src/l1/submitter/config.rs +++ b/sequencer/src/l1/submitter/config.rs @@ -8,11 +8,10 @@ use std::time::Duration; /// [`crate::l1::L1Config`] and its `identity`). These fields are parsed as /// part of `RunConfig` and passed through at runtime. /// -/// Danger-zone tuning (`max_wait_blocks`, `preemptive_margin_blocks`) lives -/// in `ProtocolTiming`, not here — the submitter doesn't read it; the -/// [`crate::recovery::DangerDetector`] worker owns that. The one timing value -/// the poster does read is `seconds_per_block`, carried on -/// `BatchPosterConfig` for its confirmation timeout. +/// The staleness window is the pinned identity's, not here — the submitter +/// doesn't read it; the [`crate::landing::LandingWatch`] worker owns the void +/// history. The one timing value the poster does read is `seconds_per_block`, +/// carried on `BatchPosterConfig` for its confirmation timeout. #[derive(Debug, Clone)] pub struct BatchSubmitterConfig { /// How often the submitter polls for new work when idle, and how long it diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index a84f5a58..ec175edd 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -19,7 +19,7 @@ use tracing::{debug, info, warn}; use crate::l1::eip1559::{Eip1559Fees, estimate_fees, pad_gas_estimate}; use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::l1::provider::{KeyedProviders, SendError, error_chain}; -use crate::l1::watermark::{WalletNonceWatermarkError, WalletNonceWatermarkSink}; +use crate::l1::watermark::{Covered, WalletNonceWatermarkError, WalletNonceWatermarkSink, cover}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -257,7 +257,9 @@ impl EthereumBatchPoster { payload: Vec, wallet_nonce: u64, fees: &Eip1559Fees, + covered: &Covered, ) -> Result { + covered.assert_covers(wallet_nonce); #[cfg(test)] { if let Some(message) = self @@ -444,10 +446,10 @@ impl BatchPoster for EthereumBatchPoster { // Write-before-broadcast: durably cover every wallet nonce this // tick will use before the first send. One raise to the highest - // covers the whole consecutive range. + // covers the whole consecutive range; each send requires its witness. let highest_wallet_nonce = checked_highest_wallet_nonce(first_wallet_nonce, payloads.len())?; - watermark.raise_to(highest_wallet_nonce)?; + let covered = cover(watermark, highest_wallet_nonce)?; let mut broadcast = Vec::with_capacity(payloads.len()); let mut any_refused = false; @@ -464,7 +466,7 @@ impl BatchPoster for EthereumBatchPoster { .expect("validated wallet-nonce range must not overflow"); self.track_pending(wallet_nonce); match self - .send_batch_at_wallet_nonce(payload, wallet_nonce, &fees) + .send_batch_at_wallet_nonce(payload, wallet_nonce, &fees, &covered) .await { Ok(tx_hash) => { @@ -651,13 +653,13 @@ pub(crate) mod mock { let waiting_after = *self.waiting_after.lock().expect("lock"); let mut tx_hashes = Vec::with_capacity(payloads.len()); for (index, payload) in payloads.into_iter().enumerate() { - let batch_index = ssz::Decode::from_ssz_bytes(payload.as_ref()) + let batch_nonce = ssz::Decode::from_ssz_bytes(payload.as_ref()) .map(|b: Batch| b.nonce) .unwrap_or(0); self.submissions .lock() .expect("lock") - .push((batch_index, payload.len())); + .push((batch_nonce, payload.len())); if waiting_after.is_none_or(|k| index < k) { tx_hashes.push(TxHash::ZERO); } @@ -934,6 +936,41 @@ mod tests { ); } + /// The send wrapper refuses a wallet nonce above its witness before any + /// RPC: the endpoint is never dialled, so no node is needed. + #[tokio::test] + #[should_panic(expected = "above the durably covered watermark")] + async fn send_above_the_covered_watermark_panics() { + let providers = crate::l1::provider::create_keyed_providers( + "http://127.0.0.1:1", + SUBMITTER_KEY, + 31337, + false, + ) + .expect("keyed providers"); + let config = BatchPosterConfig { + l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), + app_address: alloy_primitives::Address::repeat_byte(0x22), + batch_submitter_address: SUBMITTER, + start_block: 0, + confirmation_depth: 0, + seconds_per_block: 1, + expected_chain_id: 31337, + }; + let poster = EthereumBatchPoster::new(providers, config); + let covered = + super::cover(&RecordingWatermarkSink::passing(), 4).expect("passing sink covers"); + let fees = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 1, + max_fee_per_gas: 2, + }; + + let _ = poster + .send_batch_at_wallet_nonce(vec![0u8; 4], 5, &fees, &covered) + .await; + } + /// Keyed-write chain-id gate: a long-lived submitter pointed at an RPC that /// serves a different chain than the pinned one must refuse to submit, before /// any productive work — no watermark raise, no broadcast. Anvil's chain id diff --git a/sequencer/src/l1/submitter/worker.rs b/sequencer/src/l1/submitter/worker.rs index 9771b3ad..ba01d13d 100644 --- a/sequencer/src/l1/submitter/worker.rs +++ b/sequencer/src/l1/submitter/worker.rs @@ -3,8 +3,9 @@ //! Batch submitter worker: stateless, at-least-once submission. Each tick reads //! the accepted frontier from SQLite, advances past batch nonces already seen on -//! L1 above the safe block, and submits every remaining valid closed batch. It -//! never observes danger; a cancelled tick's sends are found by the next scan. +//! L1 above the safe block, and submits every remaining closed batch, a +//! void one included, until the process stops. A cancelled tick's sends are +//! found by the next scan. use std::sync::Arc; use std::time::Duration; @@ -67,8 +68,9 @@ pub(crate) enum TickOutcome { /// observed nonce equal to the next expected one. The observed nonces are our own /// `InputAdded` events past the frontier's safe block up to latest, in event (so /// wallet-nonce, so submission) order; matching by equality advances once for a -/// nonce repeated across recovery generations. A resubmission heuristic, not a -/// scheduler mirror: it reads unsafe events and applies no staleness rule. +/// nonce L1 carries twice, such as an earlier era's batch. A resubmission +/// heuristic, not a scheduler mirror: it reads unsafe events and applies no +/// staleness rule. fn decide_submit_start(frontier: SubmitterFrontier, recently_observed_nonces: &[u64]) -> u64 { let mut expected = frontier.accepted_next_nonce; for &nonce in recently_observed_nonces { @@ -184,11 +186,10 @@ impl BatchSubmitter

{ pub(crate) async fn tick_once(&self) -> Result { let frontier = self.load_frontier().await?; - // Must start scanning at `safe_block + 1`: after a danger-zone shutdown - // the flusher only returns once `Pending <= Safe`, so any wallet-nonce - // slots backed by blocks at or below the safe head are already - // resolved and folded into `accepted_next_nonce`. Re-scanning those - // blocks here would double-count the finalized prefix. + // Must start scanning at `safe_block + 1`: landings at or below the + // safe head are already judged and folded into `accepted_next_nonce`. + // Re-scanning those blocks here would double-count the accepted + // prefix. let scan_start = frontier .safe_block .checked_add(1) @@ -206,8 +207,7 @@ impl BatchSubmitter

{ for batch in &pending { debug!( - batch_index = batch.batch_index, - nonce = batch.nonce, + batch_nonce = batch.nonce, "queueing batch for L1 submission" ); } @@ -299,16 +299,9 @@ mod tests { use crate::runtime::shutdown::RuntimeScope; use crate::storage::{SafeInputRange, Storage, StoredSafeInput, SubmitterFrontier}; use crate::test_support::{TestDb, temp_db}; - use sequencer_core::protocol::ProtocolTiming; const BATCH_SUBMITTER_ADDRESS: Address = Address::repeat_byte(0x11); - /// Protocol pinned to `BATCH_SUBMITTER_ADDRESS` — worker tests use that as - /// their test submitter, so populate sees the seeded safe_inputs. - fn submitter_test_protocol() -> ProtocolTiming { - crate::storage::test_helpers::default_protocol_timing() - } - fn default_test_config() -> BatchSubmitterConfig { BatchSubmitterConfig { idle_poll_interval_ms: 1000, @@ -318,20 +311,19 @@ mod tests { fn seed_two_closed_batches(db_path: &str) { let mut storage = Storage::open(db_path).expect("open storage"); storage - .append_safe_inputs(0, &[], BATCH_SUBMITTER_ADDRESS, &submitter_test_protocol()) + .append_safe_inputs(0, &[], BATCH_SUBMITTER_ADDRESS) .expect("record observed safe head"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - let next_safe = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 0"); storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 1"); storage - .close_frame_and_batch(&mut head, next_safe) + .close_frame_and_batch(&mut head) .expect("close batch 2"); } @@ -348,12 +340,7 @@ mod tests { }) .collect(); storage - .append_safe_inputs( - safe_block, - inputs.as_slice(), - BATCH_SUBMITTER_ADDRESS, - &submitter_test_protocol(), - ) + .append_safe_inputs(safe_block, inputs.as_slice(), BATCH_SUBMITTER_ADDRESS) .expect("append safe submitted batches"); } @@ -651,16 +638,16 @@ mod tests { } #[test] - fn decide_submit_start_advances_once_per_matching_nonce_across_recovery_generations() { - // Batch nonces repeat across recovery generations (a cascade reuses the - // last valid ancestor's `nonce + 1`), so the stream can carry one twice; - // only the occurrence that matches `expected` advances. + fn decide_submit_start_advances_once_per_matching_nonce_across_duplicates() { + // A batch nonce can land twice (a resubmission, or an older era's + // batch), so the stream can carry one twice; only the occurrence that + // matches `expected` advances. let from_nonce = decide_submit_start( SubmitterFrontier { safe_block: 10, accepted_next_nonce: 2, }, - // Two events reporting nonce=2 (one per generation), then nonce=3. + // Two events reporting nonce=2, then nonce=3. &[2, 2, 3], ); // 2 matches expected=2 → advance to 3. Second 2 doesn't match diff --git a/sequencer/src/l1/watermark.rs b/sequencer/src/l1/watermark.rs index 9d4c4ba8..56eb7eb0 100644 --- a/sequencer/src/l1/watermark.rs +++ b/sequencer/src/l1/watermark.rs @@ -1,10 +1,10 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Write-before-broadcast hook (I14): the poster and the flusher durably raise the -//! wallet-nonce watermark to the highest nonce they are about to send, then send, -//! so a flush consumes every slot ever used without trusting the node's mempool. -//! A crash between the two only over-covers. +//! Write-before-broadcast (I14): the poster and the flusher durably raise the +//! wallet-nonce watermark, and their sends require the [`Covered`] witness only that +//! raise mints, so a flush consumes every slot ever used without trusting the node's +//! mempool. A crash between the raise and the send only over-covers. use thiserror::Error; @@ -37,6 +37,39 @@ pub trait WalletNonceWatermarkSink: Send + Sync { fn raise_to(&self, highest_wallet_nonce: u64) -> Result<(), WalletNonceWatermarkError>; } +/// Witness that the watermark durably covers wallet nonces `<= up_to`. Only +/// [`cover`] mints one, and only after `raise_to` succeeds, so every send +/// wrapper that takes `&Covered` and checks [`Covered::assert_covers`] cannot +/// broadcast at a nonce the watermark has not reached (I14). +#[derive(Debug)] +pub(crate) struct Covered { + up_to: u64, +} + +impl Covered { + /// Panics if `wallet_nonce` is above the covered range: a send there + /// would escape the flush's coverage, an impossible state by construction. + pub(crate) fn assert_covers(&self, wallet_nonce: u64) { + assert!( + wallet_nonce <= self.up_to, + "broadcast at wallet nonce {wallet_nonce} above the durably covered watermark {}", + self.up_to + ); + } +} + +/// Durably raise the watermark to `highest_wallet_nonce` and return the +/// witness that it covers every wallet nonce up to it. +pub(crate) fn cover( + sink: &dyn WalletNonceWatermarkSink, + highest_wallet_nonce: u64, +) -> Result { + sink.raise_to(highest_wallet_nonce)?; + Ok(Covered { + up_to: highest_wallet_nonce, + }) +} + /// Sink backed by the sequencer DB's `wallet_nonce_watermark` singleton. /// Opens a short-lived writer connection per raise — raises happen at most /// once per submitter tick / flush pass, well off the hot path. diff --git a/sequencer/src/landing.rs b/sequencer/src/landing.rs new file mode 100644 index 00000000..96a46c5f --- /dev/null +++ b/sequencer/src/landing.rs @@ -0,0 +1,263 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The landing watch: every `poll_interval` it reads +//! [`Storage::terminal_fact`] and [`Storage::landing_lag`]. A terminal +//! fact (a divergence or a void history) ends the worker with +//! [`LandingWatchExit::Terminal`], which stops the process terminally; +//! otherwise it publishes the reading for `/healthz` and logs it. It is its own +//! worker so slow L1 submission never delays the stop (I15). + +use std::time::Duration; + +use thiserror::Error; +use tokio::sync::watch; +use tokio::time::Instant; + +use crate::runtime::process_lock::{ProcessLock, spawn_blocking_with_lock}; +use crate::runtime::shutdown::ShutdownSignal; +use crate::storage::{LandingLag, Storage, StorageOpenError, TerminalFact}; + +/// How often the reading is logged at `info`, after the first tick; every +/// tick logs at `debug`. +const INFO_LOG_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// What the landing watch publishes and logs each tick: the landing lag beside +/// the window, and the safe block's L1 timestamp for the safe head's age. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LandingReading { + pub lag: Option, + /// The window `setup` pinned with the deployment identity. + pub max_wait_blocks: u64, + /// L1 timestamp (Unix seconds) of the persisted safe block; `None` before + /// the first sync. + pub safe_block_timestamp: Option, +} + +impl LandingReading { + /// `landing_lag_blocks`: `H + 1 - s` ([`LandingLag::blocks`]), void at + /// `max_wait_blocks`; `None` when no batch is unaccepted or there is no + /// safe head yet. + pub fn landing_lag_blocks(&self) -> Option { + self.lag.map(|lag| lag.blocks()) + } + + /// `safe_head_age_seconds`: the wall clock minus the safe block's L1 + /// timestamp, saturating at 0. It keeps growing while the provider is + /// down, when the landing lag freezes. + pub fn safe_head_age_seconds(&self, now_ms: u64) -> Option { + self.safe_block_timestamp + .map(|timestamp| (now_ms / 1000).saturating_sub(timestamp)) + } +} + +/// The latest landing reading, for `/healthz`. +pub(crate) type LandingFeed = watch::Receiver; + +/// How the watch's loop exited. +#[derive(Debug)] +pub enum LandingWatchExit { + /// Shutdown signal fired before any terminal fact held. + Shutdown, + /// A fact that ends the era holds. The runtime stops terminally; every + /// later `run` refuses on the same fact. + Terminal(TerminalFact), +} + +#[derive(Debug, Error)] +pub enum LandingWatchError { + #[error(transparent)] + OpenStorage(#[from] StorageOpenError), + #[error(transparent)] + Storage(#[from] rusqlite::Error), + #[error("storage task panicked while reading the landing lag: persistent invariant failure")] + StorageTaskPanicked, + #[error("landing watch join error: {0}")] + Join(String), +} + +impl LandingWatchError { + /// Whether this error poisons the run rather than restarting. Named + /// arms, no wildcard: a new variant must classify itself here. + pub(crate) fn is_terminal_invariant(&self) -> bool { + match self { + Self::Storage(source) => crate::storage::is_persistent_storage_error(source), + Self::OpenStorage(source) => crate::storage::is_persistent_storage_open_error(source), + Self::StorageTaskPanicked => true, + // A non-panic inner-task join is shutdown-path cancellation. + Self::Join(_) => false, + } + } +} + +/// One tick's result: a terminal fact, or the reading to publish. +enum Tick { + Terminal(TerminalFact), + Reading(LandingReading), +} + +pub struct LandingWatch { + db_path: String, + max_wait_blocks: u64, + poll_interval: Duration, + publish: watch::Sender, + /// Retains data-directory exclusivity in detached blocking reads. + /// Required at construction. + process_lock: ProcessLock, +} + +impl LandingWatch { + /// The watch and the feed `/healthz` reads. Until the first tick the feed + /// carries the pinned window and no lag. + pub(crate) fn new( + db_path: impl Into, + max_wait_blocks: u64, + poll_interval: Duration, + process_lock: ProcessLock, + ) -> (Self, LandingFeed) { + let (publish, feed) = watch::channel(LandingReading { + lag: None, + max_wait_blocks, + safe_block_timestamp: None, + }); + ( + Self { + db_path: db_path.into(), + max_wait_blocks, + poll_interval, + publish, + process_lock, + }, + feed, + ) + } + + #[cfg(test)] + pub(crate) fn start( + self, + shutdown: ShutdownSignal, + ) -> Result< + tokio::task::JoinHandle>, + StorageOpenError, + > { + self.preflight_storage()?; + Ok(self.start_preflighted(shutdown)) + } + + /// Validate the storage dependency without starting a task. + pub(crate) fn preflight_storage(&self) -> Result<(), StorageOpenError> { + let _ = Storage::open_read_only(self.db_path.as_str())?; + Ok(()) + } + + /// Spawn after [`Self::preflight_storage`] succeeded. Infallible so the + /// runtime can launch all workers in one non-yielding ownership step. + pub(crate) fn start_preflighted( + self, + shutdown: ShutdownSignal, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { self.run_forever(shutdown).await }) + } + + /// The biased shutdown arm returns promptly; an in-flight blocking read + /// retains its own process-lock clone until it stops. + async fn run_forever( + self, + shutdown: ShutdownSignal, + ) -> Result { + tokio::select! { + biased; + _ = shutdown.wait_for_shutdown() => Ok(LandingWatchExit::Shutdown), + result = self.run_loop() => result, + } + } + + async fn run_loop(self) -> Result { + let mut last_info: Option = None; + loop { + match self.check_once().await? { + Tick::Terminal(fact) => { + tracing::error!(%fact, "terminal fact: stopping the sequencer for good"); + return Ok(LandingWatchExit::Terminal(fact)); + } + Tick::Reading(reading) => { + let now = Instant::now(); + let info = + last_info.is_none_or(|at| now.duration_since(at) >= INFO_LOG_INTERVAL); + if info { + last_info = Some(now); + } + log_reading(&reading, info); + self.publish.send_replace(reading); + } + } + tokio::time::sleep(self.poll_interval).await; + } + } + + async fn check_once(&self) -> Result { + let db_path = self.db_path.clone(); + let max_wait_blocks = self.max_wait_blocks; + let process_lock = self.process_lock.clone(); + spawn_blocking_with_lock(process_lock, move || { + let mut storage = Storage::open_read_only(&db_path)?; + if let Some(fact) = storage.terminal_fact()? { + return Ok(Tick::Terminal(fact)); + } + Ok(Tick::Reading(LandingReading { + lag: storage.landing_lag()?, + max_wait_blocks, + safe_block_timestamp: storage.current_safe_block_timestamp()?, + })) + }) + .await + .map_err(map_storage_task_join)? + } +} + +fn log_reading(reading: &LandingReading, info: bool) { + let batch_nonce = reading.lag.map(|lag| lag.batch_nonce); + let landing_lag_blocks = reading.landing_lag_blocks(); + let max_wait_blocks = reading.max_wait_blocks; + let safe_head_age_seconds = reading.safe_head_age_seconds(crate::clock::unix_now_ms()); + if info { + tracing::info!( + batch_nonce, + landing_lag_blocks, + max_wait_blocks, + safe_head_age_seconds, + "landing lag" + ); + } else { + tracing::debug!( + batch_nonce, + landing_lag_blocks, + max_wait_blocks, + safe_head_age_seconds, + "landing lag" + ); + } +} + +/// A feed with no watch behind it: the window and no reading, for servers +/// started without a runtime. +#[cfg(test)] +pub(crate) fn test_feed(max_wait_blocks: u64) -> LandingFeed { + watch::channel(LandingReading { + lag: None, + max_wait_blocks, + safe_block_timestamp: None, + }) + .1 +} + +fn map_storage_task_join(err: tokio::task::JoinError) -> LandingWatchError { + if err.is_panic() { + LandingWatchError::StorageTaskPanicked + } else { + LandingWatchError::Join(err.to_string()) + } +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/landing/tests.rs b/sequencer/src/landing/tests.rs new file mode 100644 index 00000000..96c91b55 --- /dev/null +++ b/sequencer/src/landing/tests.rs @@ -0,0 +1,165 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::*; +use crate::storage::test_helpers::{ + LandingFixture, SENDER_A, open_temp_storage, record_canonical_divergence, +}; +use crate::storage::{Divergence, SafeInputRange}; +use crate::test_support::TestDb; + +/// The window the test identity pins (the wallet's). +const W: u64 = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; +const POLL: Duration = Duration::from_millis(20); + +/// One open batch whose first frame is at safe block 10, under a safe head +/// at `safe_block` stamped with L1 timestamp `timestamp`. +fn open_batch_at_10(name: &str, safe_block: u64, timestamp: u64) -> TestDb { + let (db, mut storage) = open_temp_storage(name); + storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + storage + .append_safe_inputs_with_timestamp(safe_block, timestamp, &[], SENDER_A) + .expect("safe head"); + db +} + +fn start( + db: &TestDb, +) -> ( + tokio::task::JoinHandle>, + LandingFeed, + ShutdownSignal, +) { + let (watch, feed) = LandingWatch::new(db.path.clone(), W, POLL, ProcessLock::test()); + let shutdown = ShutdownSignal::default(); + let handle = watch.start(shutdown.clone()).expect("start the watch"); + (handle, feed, shutdown) +} + +async fn exit_of( + handle: tokio::task::JoinHandle>, +) -> LandingWatchExit { + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("the watch exits within the timeout") + .expect("join") + .expect("the watch's result") +} + +/// Inside the window the watch publishes the reading and keeps running until +/// shutdown. +#[tokio::test] +async fn publishes_the_reading_inside_the_window_and_stops_on_shutdown() { + let db = open_batch_at_10("watch-inside-window", 100, 1_000); + let (handle, mut feed, shutdown) = start(&db); + assert_eq!( + feed.borrow().lag, + None, + "before the first tick only the window is known" + ); + assert_eq!(feed.borrow().max_wait_blocks, W); + + tokio::time::timeout(Duration::from_secs(5), feed.changed()) + .await + .expect("the first tick publishes") + .expect("the watch is alive"); + let reading = *feed.borrow(); + assert_eq!( + reading, + LandingReading { + lag: Some(LandingLag { + batch_nonce: 0, + first_frame_safe_block: 10, + safe_block: 100, + }), + max_wait_blocks: W, + safe_block_timestamp: Some(1_000), + } + ); + assert_eq!(reading.landing_lag_blocks(), Some(91)); + assert_eq!(reading.safe_head_age_seconds(1_500_999), Some(500)); + assert_eq!( + reading.safe_head_age_seconds(999_000), + Some(0), + "a safe block ahead of the clock saturates at 0" + ); + + // Several more ticks inside the window: still running. + tokio::time::sleep(POLL * 5).await; + assert!(!handle.is_finished()); + shutdown.request_shutdown(); + assert!(matches!(exit_of(handle).await, LandingWatchExit::Shutdown)); +} + +/// The watch stops the process on a void history that arises while it runs: +/// the safe head reaches the block from which no later block can accept the +/// oldest unaccepted batch. +#[tokio::test] +async fn stops_on_a_void_history_reached_while_running() { + let db = open_batch_at_10("watch-void-while-running", 10 + W - 2, 1_000); + let (handle, mut feed, _shutdown) = start(&db); + tokio::time::timeout(Duration::from_secs(5), feed.changed()) + .await + .expect("the first tick publishes") + .expect("the watch is alive"); + assert_eq!(feed.borrow().landing_lag_blocks(), Some(W - 1)); + assert!( + !handle.is_finished(), + "one block short of void, it keeps running" + ); + + Storage::open(&db.path) + .expect("open writer") + .advance_safe_head(10 + W - 1); + match exit_of(handle).await { + LandingWatchExit::Terminal(TerminalFact::VoidHistory { + lag, + max_wait_blocks, + }) => { + assert_eq!(lag.batch_nonce, 0); + assert_eq!(lag.first_frame_safe_block, 10); + assert_eq!(lag.safe_block, 10 + W - 1); + assert_eq!(max_wait_blocks, W); + } + other => panic!("expected a void history, got {other:?}"), + } +} + +#[tokio::test] +async fn stops_on_a_divergence() { + let db = open_batch_at_10("watch-divergence", 100, 1_000); + let (handle, mut feed, _shutdown) = start(&db); + tokio::time::timeout(Duration::from_secs(5), feed.changed()) + .await + .expect("the first tick publishes") + .expect("the watch is alive"); + + let mut storage = Storage::open(&db.path).expect("open writer"); + record_canonical_divergence(&mut storage, 7, 3); + drop(storage); + assert!(matches!( + exit_of(handle).await, + LandingWatchExit::Terminal(TerminalFact::Divergence(Divergence { + nonce: 7, + safe_input_index: 3, + })) + )); +} + +/// A void history already recorded stops the watch on its first tick. +#[tokio::test] +async fn stops_on_its_first_tick_over_a_void_history() { + let db = open_batch_at_10("watch-void-at-start", 10 + W, 1_000); + let (handle, feed, _shutdown) = start(&db); + assert!(matches!( + exit_of(handle).await, + LandingWatchExit::Terminal(TerminalFact::VoidHistory { .. }) + )); + assert_eq!( + feed.borrow().lag, + None, + "a terminal tick publishes no reading" + ); +} diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 52a5f2e0..379374e2 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -12,9 +12,10 @@ pub mod harness; pub mod http; pub mod ingress; pub mod l1; -pub mod recovery; +pub(crate) mod landing; +pub mod machine; pub mod runtime; -pub mod storage; +pub(crate) mod storage; #[cfg(test)] extern crate self as sequencer; diff --git a/sequencer/src/machine.rs b/sequencer/src/machine.rs new file mode 100644 index 00000000..9d4f71ec --- /dev/null +++ b/sequencer/src/machine.rs @@ -0,0 +1,975 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Reads a stored canonical machine's export: the `scheduler` and `state` +//! inspect reports. Setup's base comes through here. The machine is loaded +//! privately (every file mapped copy-on-write), so the stored directory is +//! never written; the queries run on the in-memory copy, which is dropped. +//! `docs/cartesi-machine.md` ("Reading a stored machine's export") has the +//! emulator facts this relies on. + +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use cartesi_machine::cartesi_machine_sys::{ + CM_HTIF_DEV_YIELD, CM_HTIF_YIELD_CMD_MANUAL, CM_REG_HTIF_TOHOST_CMD, CM_REG_HTIF_TOHOST_DEV, + CM_REG_HTIF_TOHOST_REASON, CM_REG_IFLAGS_H, CM_REG_IFLAGS_Y, CM_REG_MCYCLE, +}; +use cartesi_machine::config::runtime::RuntimeConfig; +use cartesi_machine::constants::{break_reason, cmio::tohost}; +use cartesi_machine::error::MachineError; +use cartesi_machine::types::cmio::{ + AutomaticReason, CmioRequest, CmioResponseReason, ManualReason, +}; +use cartesi_machine::types::{BreakReason, Register}; +use sequencer_core::scheduler::{ + SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY, SchedulerExport, SchedulerExportLengthError, +}; +use thiserror::Error; + +/// Cycles one query may run before the machine counts as stuck: minutes of +/// emulation, far past what encoding a report that fits the 2 MiB TX buffer +/// costs. It bounds a guest that never answers. +const QUERY_MCYCLE_BUDGET: u64 = 1 << 36; + +/// The longest exception message an error carries; the guest's can be 2 MiB. +const MAX_EXCEPTION_MESSAGE_CHARS: usize = 512; + +#[derive(Debug, Error)] +pub enum MachineExportError { + #[error("cannot load the machine at {path}: {source}")] + Load { + path: PathBuf, + #[source] + source: MachineError, + }, + #[error("the machine at {path} is not stopped between inputs: {state}")] + NotBetweenInputs { path: PathBuf, state: String }, + #[error("the emulator failed while the machine answered the `{query}` query: {source}")] + Emulator { + query: &'static str, + #[source] + source: MachineError, + }, + #[error("the machine did not answer the `{query}` query: {reason}")] + Unanswered { query: &'static str, reason: String }, + #[error("the machine answered the `{query}` query with {count} reports; expected one")] + ReportCount { query: &'static str, count: usize }, + #[error("the machine emitted an output while answering the `{query}` query")] + OutputEmitted { query: &'static str }, + #[error("the machine's `scheduler` report is malformed: {0}")] + MalformedExport(#[from] SchedulerExportLengthError), + #[error("cannot write the machine's state to {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, +} + +/// Loads the stored machine at `machine_dir`, answers its `scheduler` query +/// and writes its `state` report to `state_file`, replacing that file. The +/// directory itself is only read. Blocking and `!Send` inside: an async +/// caller runs it on a blocking thread. +pub fn read_machine_export( + machine_dir: &Path, + state_file: &Path, +) -> Result { + let mut machine = cartesi_machine::Machine::load(machine_dir, &RuntimeConfig::quiet_console()) + .map_err(|source| MachineExportError::Load { + path: machine_dir.to_path_buf(), + source, + })?; + export(&mut machine, machine_dir, state_file) +} + +/// The emulator calls the loader makes. A seam so that every failure arm can +/// be driven by a scripted machine; the flag and counter reads go through +/// `read_reg`, exactly as the bindings' own getters do. +trait Emulator { + fn read_reg(&mut self, reg: Register) -> Result; + fn send_cmio_response( + &mut self, + reason: CmioResponseReason, + data: &[u8], + ) -> Result<(), MachineError>; + fn run(&mut self, mcycle_end: u64) -> Result; + fn receive_cmio_request(&mut self) -> Result; + + fn iflags_h(&mut self) -> Result { + Ok(self.read_reg(CM_REG_IFLAGS_H)? != 0) + } + fn iflags_y(&mut self) -> Result { + Ok(self.read_reg(CM_REG_IFLAGS_Y)? != 0) + } + fn mcycle(&mut self) -> Result { + self.read_reg(CM_REG_MCYCLE) + } +} + +impl Emulator for cartesi_machine::Machine { + fn read_reg(&mut self, reg: Register) -> Result { + cartesi_machine::Machine::read_reg(self, reg) + } + fn send_cmio_response( + &mut self, + reason: CmioResponseReason, + data: &[u8], + ) -> Result<(), MachineError> { + cartesi_machine::Machine::send_cmio_response(self, reason, data, None) + } + fn run(&mut self, mcycle_end: u64) -> Result { + cartesi_machine::Machine::run(self, mcycle_end) + } + fn receive_cmio_request(&mut self) -> Result { + cartesi_machine::Machine::receive_cmio_request(self) + } +} + +fn export( + machine: &mut impl Emulator, + machine_dir: &Path, + state_file: &Path, +) -> Result { + let not_between = between_inputs(machine).map_err(|source| MachineExportError::Load { + path: machine_dir.to_path_buf(), + source, + })?; + if let Some(state) = not_between { + return Err(MachineExportError::NotBetweenInputs { + path: machine_dir.to_path_buf(), + state, + }); + } + + let export = SchedulerExport::decode(&inspect(machine, "scheduler", SCHEDULER_INSPECT_QUERY)?)?; + let state = inspect(machine, "state", STATE_INSPECT_QUERY)?; + write_state(state_file, &state)?; + Ok(export) +} + +/// Why the machine is not at an `RX_ACCEPTED` manual yield, if it is not, +/// read as the step reads it (the flags, then tohost's device, command and +/// reason), so a malformed yield is an answer, never a panic in the bindings. +fn between_inputs(machine: &mut impl Emulator) -> Result, MachineError> { + if machine.iflags_h()? { + return Ok(Some("it is halted".to_owned())); + } + if !machine.iflags_y()? { + return Ok(Some("it is not at a manual yield".to_owned())); + } + let (dev, cmd, reason) = tohost_header(machine)?; + if dev != u64::from(CM_HTIF_DEV_YIELD) || cmd != u64::from(CM_HTIF_YIELD_CMD_MANUAL) { + return Ok(Some(format!( + "its pending yield has device {dev} and command {cmd}" + ))); + } + if reason != u64::from(tohost::manual::RX_ACCEPTED) { + return Ok(Some(format!( + "its pending manual yield has reason {reason}, not rx-accepted" + ))); + } + Ok(None) +} + +fn tohost_header(machine: &mut impl Emulator) -> Result<(u64, u64, u64), MachineError> { + Ok(( + machine.read_reg(CM_REG_HTIF_TOHOST_DEV)?, + machine.read_reg(CM_REG_HTIF_TOHOST_CMD)?, + machine.read_reg(CM_REG_HTIF_TOHOST_REASON)?, + )) +} + +/// Sends one inspect query and runs the machine back to `RX_ACCEPTED`. The +/// guest answers with exactly one report; a second one fails at once, so a +/// runaway guest cannot grow the host's memory. +fn inspect( + machine: &mut impl Emulator, + name: &'static str, + query: &[u8], +) -> Result, MachineExportError> { + let emulator = |source| MachineExportError::Emulator { + query: name, + source, + }; + let unanswered = |reason: String| MachineExportError::Unanswered { + query: name, + reason, + }; + + machine + .send_cmio_response(CmioResponseReason::Inspect, query) + .map_err(emulator)?; + // An undeliverable response is a silent no-op; only delivery clears Y. + if machine.iflags_y().map_err(emulator)? { + return Err(unanswered("the machine did not take the query".to_owned())); + } + + let mcycle_end = machine + .mcycle() + .map_err(emulator)? + .checked_add(QUERY_MCYCLE_BUDGET) + .ok_or_else(|| unanswered("its mcycle is at the end of the counter".to_owned()))?; + let mut report = None; + loop { + match machine.run(mcycle_end).map_err(emulator)? { + break_reason::YIELDED_AUTOMATICALLY => { + let reason = machine + .read_reg(CM_REG_HTIF_TOHOST_REASON) + .map_err(emulator)?; + if reason == u64::from(tohost::automatic::PROGRESS) { + continue; + } + if reason == u64::from(tohost::automatic::TX_OUTPUT) { + return Err(MachineExportError::OutputEmitted { query: name }); + } + if reason != u64::from(tohost::automatic::TX_REPORT) { + return Err(unanswered(format!( + "it yielded automatically with unknown reason {reason}" + ))); + } + if report.is_some() { + return Err(MachineExportError::ReportCount { + query: name, + count: 2, + }); + } + match machine.receive_cmio_request().map_err(emulator)? { + CmioRequest::Automatic(AutomaticReason::TxReport { data }) => { + report = Some(data) + } + _ => return Err(unanswered("its report could not be read".to_owned())), + } + } + break_reason::YIELDED_MANUALLY => { + let reason = machine + .read_reg(CM_REG_HTIF_TOHOST_REASON) + .map_err(emulator)?; + if reason == u64::from(tohost::manual::RX_ACCEPTED) { + return report.ok_or(MachineExportError::ReportCount { + query: name, + count: 0, + }); + } + return Err(unanswered(manual_yield(machine, reason).map_err(emulator)?)); + } + // A soft yield only hands control back; the next run resumes. + break_reason::YIELDED_SOFTLY => {} + break_reason::HALTED => return Err(unanswered("the machine halted".to_owned())), + break_reason::MCYCLE_OVERFLOW => { + return Err(unanswered( + "the machine exhausted its cycle budget".to_owned(), + )); + } + break_reason::REACHED_TARGET_MCYCLE => { + return Err(unanswered(format!( + "no answer within {QUERY_MCYCLE_BUDGET} cycles" + ))); + } + other => return Err(unanswered(format!("the run broke with reason {other}"))), + } + } +} + +/// Describes a manual yield other than `RX_ACCEPTED`. +fn manual_yield(machine: &mut impl Emulator, reason: u64) -> Result { + if reason == u64::from(tohost::manual::RX_REJECTED) { + return Ok("the guest rejected the query".to_owned()); + } + if reason == u64::from(tohost::manual::TX_EXCEPTION) { + let message = match machine.receive_cmio_request()? { + CmioRequest::Manual(ManualReason::TxException { message }) => message, + _ => "unreadable".to_owned(), + }; + let message: String = message.chars().take(MAX_EXCEPTION_MESSAGE_CHARS).collect(); + return Ok(format!("the guest raised an exception: {message}")); + } + Ok(format!("the guest yielded manually with reason {reason}")) +} + +fn write_state(path: &Path, state: &[u8]) -> Result<(), MachineExportError> { + let io = |source| MachineExportError::Io { + path: path.to_path_buf(), + source, + }; + let mut file = File::create(path).map_err(io)?; + file.write_all(state).map_err(io)?; + file.flush().map_err(io) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use cartesi_machine::cartesi_machine_sys::{CM_HTIF_DEV_CONSOLE, CM_HTIF_YIELD_CMD_AUTOMATIC}; + + use super::*; + + const MACHINE_DIR: &str = "/stored/machine"; + const MCYCLE: u64 = 1_000; + const STATE: &[u8] = b"the application's state bytes"; + + /// One `run` of the scripted machine: how it breaks, the tohost reason it + /// leaves, and the request `receive_cmio_request` then returns. + struct Step { + break_reason: BreakReason, + tohost_reason: u16, + request: Option, + } + + fn manual(reason: u16, request: Option) -> Step { + Step { + break_reason: break_reason::YIELDED_MANUALLY, + tohost_reason: reason, + request, + } + } + + fn automatic(reason: u16, request: Option) -> Step { + Step { + break_reason: break_reason::YIELDED_AUTOMATICALLY, + tohost_reason: reason, + request, + } + } + + fn broke(reason: BreakReason) -> Step { + Step { + break_reason: reason, + tohost_reason: 0, + request: None, + } + } + + fn accepted() -> Step { + manual(tohost::manual::RX_ACCEPTED, None) + } + + fn report(data: &[u8]) -> Step { + automatic( + tohost::automatic::TX_REPORT, + Some(CmioRequest::Automatic(AutomaticReason::TxReport { + data: data.to_vec(), + })), + ) + } + + fn exception(message: &str) -> Step { + manual( + tohost::manual::TX_EXCEPTION, + Some(CmioRequest::Manual(ManualReason::TxException { + message: message.to_owned(), + })), + ) + } + + fn sample_export() -> SchedulerExport { + SchedulerExport { + l1_input_count: 7, + l2_tx_count: 40, + next_batch_nonce: 3, + clock: 28, + first_queued_input: 6, + fridge_len: 1, + } + } + + /// The `scheduler` query answered with the sample export; the `state` + /// query's steps follow. + fn scheduler_answered() -> Vec { + vec![report(&sample_export().encode()), accepted()] + } + + fn both_answered() -> Vec { + let mut steps = scheduler_answered(); + steps.extend([report(STATE), accepted()]); + steps + } + + /// A machine that plays a script: its registers at load, then one step + /// per `run`. It records what the loader sent and the run targets. + struct Scripted { + halted: bool, + yielded: bool, + tohost: [u64; 3], + mcycle: u64, + takes_queries: bool, + failing_reg: Option, + /// The call that fails, and on which of its calls, counting from 0. + failing_call: Option<(Call, usize)>, + calls: Vec, + steps: VecDeque, + pending: Option, + queries: Vec>, + run_targets: Vec, + } + + impl Scripted { + /// Stopped at an `RX_ACCEPTED` manual yield, as a stored canonical + /// machine is. + fn between_inputs(steps: Vec) -> Self { + Self { + halted: false, + yielded: true, + tohost: [ + u64::from(CM_HTIF_DEV_YIELD), + u64::from(CM_HTIF_YIELD_CMD_MANUAL), + u64::from(tohost::manual::RX_ACCEPTED), + ], + mcycle: MCYCLE, + takes_queries: true, + failing_reg: None, + failing_call: None, + calls: Vec::new(), + steps: steps.into(), + pending: None, + queries: Vec::new(), + run_targets: Vec::new(), + } + } + + fn queries(&self) -> Vec<&[u8]> { + self.queries.iter().map(Vec::as_slice).collect() + } + + /// Records `call`, and fails it when the script says so. + fn call(&mut self, call: Call) -> Result<(), MachineError> { + let nth = self.calls.iter().filter(|made| **made == call).count(); + self.calls.push(call); + if self.failing_call == Some((call, nth)) { + return Err(MachineError { + code: -1, + message: format!("{call:?} failed"), + }); + } + Ok(()) + } + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Call { + SendCmioResponse, + Run, + ReceiveCmioRequest, + } + + impl Emulator for Scripted { + fn read_reg(&mut self, reg: Register) -> Result { + if self.failing_reg == Some(reg) { + return Err(MachineError { + code: -1, + message: format!("register {reg} is unreadable"), + }); + } + Ok(match reg { + CM_REG_IFLAGS_H => u64::from(self.halted), + CM_REG_IFLAGS_Y => u64::from(self.yielded), + CM_REG_HTIF_TOHOST_DEV => self.tohost[0], + CM_REG_HTIF_TOHOST_CMD => self.tohost[1], + CM_REG_HTIF_TOHOST_REASON => self.tohost[2], + CM_REG_MCYCLE => self.mcycle, + other => panic!("the loader reads no register {other}"), + }) + } + + fn send_cmio_response( + &mut self, + reason: CmioResponseReason, + data: &[u8], + ) -> Result<(), MachineError> { + assert_eq!(reason, CmioResponseReason::Inspect); + self.call(Call::SendCmioResponse)?; + self.queries.push(data.to_vec()); + if self.yielded && self.takes_queries { + self.yielded = false; + } + Ok(()) + } + + fn run(&mut self, mcycle_end: u64) -> Result { + assert!(!self.yielded, "a run starts only after the query is taken"); + self.call(Call::Run)?; + self.run_targets.push(mcycle_end); + let step = self.steps.pop_front().expect("the script has another step"); + self.tohost[2] = u64::from(step.tohost_reason); + self.pending = step.request; + match step.break_reason { + break_reason::YIELDED_MANUALLY => { + self.yielded = true; + self.tohost[1] = u64::from(CM_HTIF_YIELD_CMD_MANUAL); + } + break_reason::YIELDED_AUTOMATICALLY => { + self.tohost[1] = u64::from(CM_HTIF_YIELD_CMD_AUTOMATIC); + } + _ => {} + } + Ok(step.break_reason) + } + + fn receive_cmio_request(&mut self) -> Result { + self.call(Call::ReceiveCmioRequest)?; + Ok(self.pending.take().expect("the step scripted a request")) + } + } + + /// Runs the loader on `machine`; returns its result and the state file's + /// bytes afterwards, which start as a stale file the loader must replace. + fn run_export( + machine: &mut Scripted, + ) -> (Result, Vec) { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("state"); + std::fs::write(&state_file, b"stale").expect("stale state"); + let result = export(machine, Path::new(MACHINE_DIR), &state_file); + let state = std::fs::read(&state_file).expect("state file"); + (result, state) + } + + fn not_between(machine: &mut Scripted) -> String { + let (result, state) = run_export(machine); + assert_eq!(state, b"stale", "a refused machine writes no state"); + assert!( + machine.queries.is_empty(), + "a refused machine gets no query" + ); + match result { + Err(MachineExportError::NotBetweenInputs { path, state }) => { + assert_eq!(path, Path::new(MACHINE_DIR)); + state + } + other => panic!("expected NotBetweenInputs, got {other:?}"), + } + } + + /// The loader's error for a script that fails during a query; no state + /// is written. + fn failed(machine: &mut Scripted) -> MachineExportError { + let (result, state) = run_export(machine); + assert_eq!(state, b"stale", "a failed query writes no state"); + result.expect_err("the script fails a query") + } + + fn unanswered(steps: Vec) -> (&'static str, String) { + match failed(&mut Scripted::between_inputs(steps)) { + MachineExportError::Unanswered { query, reason } => (query, reason), + other => panic!("expected Unanswered, got {other:?}"), + } + } + + #[test] + fn a_missing_machine_is_a_load_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = dir.path().join("no-machine"); + let state_file = dir.path().join("state"); + + let err = read_machine_export(&missing, &state_file).expect_err("nothing to load"); + + assert!( + matches!(&err, MachineExportError::Load { path, .. } if *path == missing), + "{err}" + ); + assert!(!state_file.exists(), "a failed load writes no state"); + } + + #[test] + fn a_machine_between_inputs_answers_both_queries_and_its_state_replaces_the_file() { + let mut machine = Scripted::between_inputs(both_answered()); + + let (result, state) = run_export(&mut machine); + + assert_eq!(result.expect("export"), sample_export()); + assert_eq!(state, STATE); + assert_eq!( + machine.queries(), + [SCHEDULER_INSPECT_QUERY, STATE_INSPECT_QUERY] + ); + let target = MCYCLE + QUERY_MCYCLE_BUDGET; + assert_eq!(machine.run_targets, [target; 4]); + assert!(machine.steps.is_empty()); + } + + #[test] + fn progress_and_soft_yields_are_passed_over() { + let progress = || { + automatic( + tohost::automatic::PROGRESS, + Some(CmioRequest::Automatic(AutomaticReason::Progress { + mille_progress: 500, + })), + ) + }; + let mut machine = Scripted::between_inputs(vec![ + progress(), + broke(break_reason::YIELDED_SOFTLY), + report(&sample_export().encode()), + broke(break_reason::YIELDED_SOFTLY), + progress(), + accepted(), + report(STATE), + accepted(), + ]); + + let (result, state) = run_export(&mut machine); + + assert_eq!(result.expect("export"), sample_export()); + assert_eq!(state, STATE); + assert!(machine.steps.is_empty()); + } + + #[test] + fn a_halted_machine_is_not_between_inputs() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.halted = true; + assert_eq!(not_between(&mut machine), "it is halted"); + } + + #[test] + fn a_machine_not_at_a_manual_yield_is_not_between_inputs() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.yielded = false; + assert_eq!(not_between(&mut machine), "it is not at a manual yield"); + } + + #[test] + fn a_pending_yield_with_another_device_or_command_is_not_between_inputs() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.tohost[0] = u64::from(CM_HTIF_DEV_CONSOLE); + assert_eq!( + not_between(&mut machine), + format!( + "its pending yield has device {CM_HTIF_DEV_CONSOLE} and command {CM_HTIF_YIELD_CMD_MANUAL}" + ) + ); + + let mut machine = Scripted::between_inputs(Vec::new()); + machine.tohost[1] = u64::from(CM_HTIF_YIELD_CMD_AUTOMATIC); + assert_eq!( + not_between(&mut machine), + format!( + "its pending yield has device {CM_HTIF_DEV_YIELD} and command {CM_HTIF_YIELD_CMD_AUTOMATIC}" + ) + ); + } + + #[test] + fn a_manual_yield_with_another_reason_is_not_between_inputs() { + for reason in [tohost::manual::RX_REJECTED, tohost::manual::TX_EXCEPTION] { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.tohost[2] = u64::from(reason); + assert_eq!( + not_between(&mut machine), + format!("its pending manual yield has reason {reason}, not rx-accepted") + ); + } + } + + #[test] + fn an_unreadable_register_at_load_is_a_load_error() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.failing_reg = Some(CM_REG_HTIF_TOHOST_REASON); + + let err = failed(&mut machine); + + assert!( + matches!(&err, MachineExportError::Load { path, .. } if path == Path::new(MACHINE_DIR)), + "{err}" + ); + assert!(machine.queries.is_empty()); + } + + #[test] + fn an_unreadable_register_during_a_query_is_an_emulator_error() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.failing_reg = Some(CM_REG_MCYCLE); + + let err = failed(&mut machine); + + assert!( + matches!( + err, + MachineExportError::Emulator { + query: "scheduler", + .. + } + ), + "{err}" + ); + } + + /// A failing emulator call is an emulator error naming the query it + /// served, and writes no state. + #[test] + fn a_failing_emulator_call_is_an_emulator_error() { + let cases = [ + (Call::SendCmioResponse, 0, "scheduler", both_answered()), + (Call::Run, 0, "scheduler", both_answered()), + (Call::ReceiveCmioRequest, 0, "scheduler", both_answered()), + (Call::SendCmioResponse, 1, "state", both_answered()), + (Call::Run, 2, "state", both_answered()), + (Call::ReceiveCmioRequest, 1, "state", both_answered()), + // The exception's message is read inside the manual yield's arm. + ( + Call::ReceiveCmioRequest, + 0, + "scheduler", + vec![exception("out of gas")], + ), + ]; + for (call, nth, query, steps) in cases { + let mut machine = Scripted::between_inputs(steps); + machine.failing_call = Some((call, nth)); + + let err = failed(&mut machine); + + assert!( + matches!(&err, MachineExportError::Emulator { query: failed_query, .. } + if *failed_query == query), + "{call:?} #{nth}: {err}" + ); + } + } + + #[test] + fn a_query_the_machine_does_not_take_is_unanswered() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.takes_queries = false; + + let err = failed(&mut machine); + + assert!( + matches!(&err, MachineExportError::Unanswered { query: "scheduler", reason } + if reason == "the machine did not take the query"), + "{err}" + ); + assert!(machine.run_targets.is_empty(), "nothing runs"); + } + + #[test] + fn an_output_during_a_query_fails_it() { + let output = automatic( + tohost::automatic::TX_OUTPUT, + Some(CmioRequest::Automatic(AutomaticReason::TxOutput { + data: vec![1], + })), + ); + let err = failed(&mut Scripted::between_inputs(vec![output])); + assert!( + matches!( + err, + MachineExportError::OutputEmitted { query: "scheduler" } + ), + "{err}" + ); + } + + #[test] + fn an_automatic_yield_with_an_unknown_reason_is_unanswered() { + // The request is never received: the bindings panic on an unknown reason. + let (query, reason) = unanswered(vec![automatic(7, None)]); + assert_eq!(query, "scheduler"); + assert_eq!(reason, "it yielded automatically with unknown reason 7"); + } + + /// The pinned bindings decode the request's variant from tohost, so a + /// report yield always reads as a report; this arm guards a change in + /// the bindings, and the script fakes a request they cannot return. + #[test] + fn a_report_that_cannot_be_read_is_unanswered() { + let not_a_report = automatic( + tohost::automatic::TX_REPORT, + Some(CmioRequest::Manual(ManualReason::RxRejected)), + ); + let (_, reason) = unanswered(vec![not_a_report]); + assert_eq!(reason, "its report could not be read"); + } + + #[test] + fn a_second_report_fails_the_query_at_once() { + let mut machine = Scripted::between_inputs(vec![report(b"one"), report(b"two")]); + + let err = failed(&mut machine); + + assert!( + matches!( + err, + MachineExportError::ReportCount { + query: "scheduler", + count: 2 + } + ), + "{err}" + ); + assert_eq!( + machine.pending, + Some(CmioRequest::Automatic(AutomaticReason::TxReport { + data: b"two".to_vec() + })), + "the second report is never read" + ); + } + + #[test] + fn no_report_fails_the_query() { + let err = failed(&mut Scripted::between_inputs(vec![accepted()])); + assert!( + matches!( + err, + MachineExportError::ReportCount { + query: "scheduler", + count: 0 + } + ), + "{err}" + ); + } + + #[test] + fn a_rejected_query_is_unanswered() { + let (_, reason) = unanswered(vec![manual(tohost::manual::RX_REJECTED, None)]); + assert_eq!(reason, "the guest rejected the query"); + } + + #[test] + fn an_exception_message_is_cut_at_its_character_cap() { + let long = "é".repeat(MAX_EXCEPTION_MESSAGE_CHARS + 100); + let (_, reason) = unanswered(vec![exception(&long)]); + let kept = "é".repeat(MAX_EXCEPTION_MESSAGE_CHARS); + assert_eq!(reason, format!("the guest raised an exception: {kept}")); + + let (_, reason) = unanswered(vec![exception("out of gas")]); + assert_eq!(reason, "the guest raised an exception: out of gas"); + } + + /// As for a report: the pinned bindings always read an exception yield as + /// an exception, so this arm guards a change in the bindings. + #[test] + fn an_exception_that_cannot_be_read_is_described_as_unreadable() { + let garbled = manual( + tohost::manual::TX_EXCEPTION, + Some(CmioRequest::Manual(ManualReason::RxRejected)), + ); + let (_, reason) = unanswered(vec![garbled]); + assert_eq!(reason, "the guest raised an exception: unreadable"); + } + + #[test] + fn a_manual_yield_with_another_reason_during_a_query_is_unanswered() { + let gio = manual( + 0x10, + Some(CmioRequest::Manual(ManualReason::GIO { + domain: 0x10, + data: Vec::new(), + })), + ); + let (_, reason) = unanswered(vec![gio]); + assert_eq!(reason, "the guest yielded manually with reason 16"); + } + + #[test] + fn a_halt_during_a_query_is_unanswered() { + let (_, reason) = unanswered(vec![report(b"partial"), broke(break_reason::HALTED)]); + assert_eq!(reason, "the machine halted"); + } + + #[test] + fn an_exhausted_cycle_budget_is_unanswered() { + let (_, reason) = unanswered(vec![broke(break_reason::REACHED_TARGET_MCYCLE)]); + assert_eq!( + reason, + format!("no answer within {QUERY_MCYCLE_BUDGET} cycles") + ); + + let (_, reason) = unanswered(vec![broke(break_reason::MCYCLE_OVERFLOW)]); + assert_eq!(reason, "the machine exhausted its cycle budget"); + + let (_, reason) = unanswered(vec![broke(break_reason::FAILED)]); + assert_eq!( + reason, + format!("the run broke with reason {}", break_reason::FAILED) + ); + } + + #[test] + fn a_mcycle_at_the_end_of_the_counter_is_unanswered() { + let mut machine = Scripted::between_inputs(Vec::new()); + machine.mcycle = u64::MAX - QUERY_MCYCLE_BUDGET + 1; + + let err = failed(&mut machine); + + assert!( + matches!(&err, MachineExportError::Unanswered { query: "scheduler", reason } + if reason == "its mcycle is at the end of the counter"), + "{err}" + ); + assert!(machine.run_targets.is_empty(), "nothing runs"); + } + + #[test] + fn the_budget_may_end_on_the_counters_last_value() { + let mut machine = Scripted::between_inputs(both_answered()); + machine.mcycle = u64::MAX - QUERY_MCYCLE_BUDGET; + + let (result, _) = run_export(&mut machine); + + assert_eq!(result.expect("export"), sample_export()); + assert!(machine.run_targets.iter().all(|&target| target == u64::MAX)); + } + + #[test] + fn a_scheduler_report_of_the_wrong_length_is_malformed() { + let short = &sample_export().encode()[..SchedulerExport::ENCODED_LEN - 1]; + let mut machine = Scripted::between_inputs(vec![report(short), accepted()]); + + let err = failed(&mut machine); + + assert!( + matches!( + err, + MachineExportError::MalformedExport(SchedulerExportLengthError { got }) + if got == SchedulerExport::ENCODED_LEN - 1 + ), + "{err}" + ); + assert_eq!( + machine.queries(), + [SCHEDULER_INSPECT_QUERY], + "no state query" + ); + } + + #[test] + fn a_failed_state_query_names_it_and_writes_no_state() { + let mut steps = scheduler_answered(); + steps.push(accepted()); + let mut machine = Scripted::between_inputs(steps); + + let err = failed(&mut machine); + + assert!( + matches!( + err, + MachineExportError::ReportCount { + query: "state", + count: 0 + } + ), + "{err}" + ); + } + + #[test] + fn a_state_file_that_cannot_be_written_is_an_io_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("missing-dir").join("state"); + let mut machine = Scripted::between_inputs(both_answered()); + + let err = export(&mut machine, Path::new(MACHINE_DIR), &state_file) + .expect_err("no directory to write into"); + + assert!( + matches!(&err, MachineExportError::Io { path, .. } if *path == state_file), + "{err}" + ); + } +} diff --git a/sequencer/src/recovery/detector.rs b/sequencer/src/recovery/detector.rs deleted file mode 100644 index eb4ced36..00000000 --- a/sequencer/src/recovery/detector.rs +++ /dev/null @@ -1,350 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! Runtime danger detector: every `poll_interval` it reads [`Storage::check_danger`] -//! and exits with [`DetectorExit::RecoveryRequired`] on any non-`Safe` status. It -//! is its own worker so slow L1 submission never delays detection; the preemptive -//! margin absorbs its lag of up to one interval behind the input reader. - -use std::time::Duration; - -use thiserror::Error; -use tracing::debug; - -use crate::clock::unix_now_ms; -use crate::runtime::process_lock::{ProcessLock, spawn_blocking_with_lock}; -use crate::runtime::shutdown::ShutdownSignal; -use crate::storage::{DangerStatus, Storage, StorageOpenError}; -use sequencer_core::protocol::ProtocolTiming; - -/// How the detector's loop exited. -/// -/// `RecoveryRequired` is a *deliberate* exit — not an error. The runtime maps -/// it to a distinct `CommandError` variant so operators can tell "time to recover -/// or refuse startup" apart from "something crashed". -#[derive(Debug)] -pub enum DetectorExit { - /// Shutdown signal fired before any danger was detected. - Shutdown, - /// A non-safe danger status was observed. Stop and let startup re-derive - /// the recovery/refusal path from fresh facts. - RecoveryRequired { status: DangerStatus }, -} - -#[derive(Debug, Error)] -pub enum DangerDetectorError { - #[error(transparent)] - OpenStorage(#[from] StorageOpenError), - #[error(transparent)] - Storage(#[from] rusqlite::Error), - #[error("storage task panicked while checking danger: persistent invariant failure")] - StorageTaskPanicked, - #[error("danger detector join error: {0}")] - Join(String), -} - -impl DangerDetectorError { - /// Whether this error poisons the run rather than restarting. Named - /// arms, no wildcard: a new variant must classify itself here. - pub(crate) fn is_terminal_invariant(&self) -> bool { - match self { - Self::Storage(source) => crate::storage::is_persistent_storage_error(source), - Self::OpenStorage(source) => crate::storage::is_persistent_storage_open_error(source), - Self::StorageTaskPanicked => true, - // A non-panic inner-task join is shutdown-path cancellation. - Self::Join(_) => false, - } - } -} - -pub struct DangerDetector { - db_path: String, - protocol: ProtocolTiming, - poll_interval: Duration, - /// Retains data-directory exclusivity in detached blocking checks. - /// Required at construction. - process_lock: ProcessLock, -} - -impl DangerDetector { - pub(crate) fn new( - db_path: impl Into, - protocol: ProtocolTiming, - poll_interval: Duration, - process_lock: ProcessLock, - ) -> Self { - Self { - db_path: db_path.into(), - protocol, - poll_interval, - process_lock, - } - } - - /// Spawn the detector loop. The `shutdown` signal is what the loop - /// respects; passing it at start time (instead of construction time) keeps - /// the construction phase pure. The notification half suffices: the - /// detector externalizes nothing and contains nothing, and it holds its - /// data-directory ownership as the construction-required `ProcessLock`. - #[cfg(test)] - pub(crate) fn start( - self, - shutdown: ShutdownSignal, - ) -> Result>, StorageOpenError> - { - self.preflight_storage()?; - Ok(self.start_preflighted(shutdown)) - } - - /// Validate the storage dependency without starting a task. - pub(crate) fn preflight_storage(&self) -> Result<(), StorageOpenError> { - let _ = Storage::open_read_only(self.db_path.as_str())?; - Ok(()) - } - - /// Spawn after [`Self::preflight_storage`] succeeded. Infallible so the - /// runtime can launch all workers in one non-yielding ownership step. - pub(crate) fn start_preflighted( - self, - shutdown: ShutdownSignal, - ) -> tokio::task::JoinHandle> { - tokio::spawn(async move { self.run_forever(shutdown).await }) - } - - /// Top-level driver. The biased shutdown arm returns promptly; an in-flight - /// blocking check retains its own process-lock clone until it stops. - async fn run_forever( - self, - shutdown: ShutdownSignal, - ) -> Result { - tokio::select! { - biased; - _ = shutdown.wait_for_shutdown() => Ok(DetectorExit::Shutdown), - result = self.run_loop() => result, - } - } - - /// Tick → sleep → tick. Returns `RecoveryRequired` when a non-Safe danger - /// status fires. - async fn run_loop(self) -> Result { - loop { - match self.check_once().await? { - DangerStatus::Safe => { - debug!("danger check: safe"); - } - status => { - // All non-Safe variants exit. The reducer re-inspects on - // the next startup and selects one phase, Retry, or Refuse - // from the then-current facts. - tracing::warn!( - ?status, - danger_threshold = self.protocol.danger_threshold(), - l1_read_stale_after_blocks = self.protocol.l1_read_stale_after_blocks(), - "danger detected — triggering shutdown for startup recovery" - ); - return Ok(DetectorExit::RecoveryRequired { status }); - } - } - tokio::time::sleep(self.poll_interval).await; - } - } - - async fn check_once(&self) -> Result { - let db_path = self.db_path.clone(); - let protocol = self.protocol; - let now_ms = unix_now_ms(); - let process_lock = self.process_lock.clone(); - spawn_blocking_with_lock(process_lock, move || { - let mut storage = Storage::open_read_only(&db_path)?; - storage - .check_danger(&protocol, now_ms) - .map_err(DangerDetectorError::from) - }) - .await - .map_err(map_storage_task_join)? - } -} - -fn map_storage_task_join(err: tokio::task::JoinError) -> DangerDetectorError { - if err.is_panic() { - DangerDetectorError::StorageTaskPanicked - } else { - DangerDetectorError::Join(err.to_string()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::storage::test_helpers::{SENDER_A, default_protocol_timing}; - use crate::storage::{SafeInputRange, Storage, StoredSafeInput}; - use crate::test_support::temp_db; - use std::time::Duration; - - #[tokio::test] - async fn exits_on_shutdown_when_safe() { - let db = temp_db("detector-shutdown"); - let mut storage = Storage::open(&db.path).expect("open storage"); - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .append_safe_inputs(10, &[], SENDER_A, &default_protocol_timing()) - .expect("record fresh safe-head observation"); - drop(storage); - - let shutdown = ShutdownSignal::default(); - let detector = DangerDetector::new( - db.path.clone(), - default_protocol_timing(), - Duration::from_millis(50), - ProcessLock::test(), - ); - let handle = detector.start(shutdown.clone()).expect("start detector"); - - tokio::time::sleep(Duration::from_millis(20)).await; - shutdown.request_shutdown(); - let exit = tokio::time::timeout(Duration::from_secs(2), handle) - .await - .expect("detector exits within timeout") - .expect("join") - .expect("detector result"); - assert!(matches!(exit, DetectorExit::Shutdown)); - } - - #[tokio::test] - async fn exits_with_recovery_required_when_observed_closed_check_fires() { - // Closed frontier batch is aged past `danger_threshold` against the - // observed safe block — the closed-batch arm of `check_danger` trips. - let db = temp_db("detector-observed-closed-danger"); - let mut storage = Storage::open(&db.path).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 1"); - - let protocol = default_protocol_timing(); - let landed = crate::storage::test_helpers::local_batch_payload(&mut storage, 0); - storage - .append_safe_inputs( - 1135, - &[StoredSafeInput { - sender: SENDER_A, - payload: landed, - block_number: 20, - }], - SENDER_A, - &protocol, - ) - .expect("append"); - drop(storage); - - let shutdown = ShutdownSignal::default(); - let detector = DangerDetector::new( - db.path.clone(), - protocol, - Duration::from_millis(50), - ProcessLock::test(), - ); - let handle = detector.start(shutdown).expect("start detector"); - - let exit = tokio::time::timeout(Duration::from_secs(2), handle) - .await - .expect("detector exits within timeout") - .expect("join") - .expect("detector result"); - match exit { - DetectorExit::RecoveryRequired { status } => { - assert_eq!(status, DangerStatus::ClosedBatchInDanger(1)); - } - other => panic!("expected recovery-required exit, got {other:?}"), - } - } - - #[tokio::test] - async fn exits_with_recovery_required_when_wall_clock_fallback_fires() { - // Safe head appears frozen — observed block-based checks wouldn't trip - // (ages look fine against the last observed safe block), but the - // wall-clock-adjusted check infers extended L1 silence and lowers the - // effective threshold. - // - // The detector treats observed and estimated danger identically (both - // exit for startup recovery), but the estimated path goes through - // `wall_clock_adjusted_danger_threshold` - // — a completely separate code path that deserves its own test. - let db = temp_db("detector-estimated-danger"); - let mut storage = Storage::open(&db.path).expect("open storage"); - let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch 0"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch 1"); - - let protocol = default_protocol_timing(); - let landed = crate::storage::test_helpers::local_batch_payload(&mut storage, 0); - storage - .append_safe_inputs( - 1200, - &[StoredSafeInput { - sender: SENDER_A, - payload: landed, - block_number: 200, - }], - SENDER_A, - &protocol, - ) - .expect("append accepted batch 0"); - - // Observed check: batch 1's first_frame_safe_block = 100, current - // safe = 1200. age = 1100 < danger_threshold (MAX_WAIT_BLOCKS 1200 - - // margin 75 = 1125), so observed - // closed-batch danger would NOT fire. - // - // Rewind synced_at_ms by 25 blocks' worth of wall-clock time so the - // wall-clock arm shaves 25 off the threshold (1125 → 1100). At 1100, - // batch 1's age = 1100 trips `>=`. Estimated batch danger fires. - let now_ms = crate::clock::unix_now_ms(); - drop(storage); - let rewind_conn = - Storage::open_connection(&db.path).expect("open raw connection to rewind synced_at_ms"); - rewind_conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms.saturating_sub(25 * 12 * 1000)).unwrap_or(i64::MAX)], - ) - .expect("rewind safe-progress timestamp"); - drop(rewind_conn); - - let shutdown = ShutdownSignal::default(); - let detector = DangerDetector::new( - db.path.clone(), - protocol, - Duration::from_millis(50), - ProcessLock::test(), - ); - let handle = detector.start(shutdown).expect("start detector"); - - let exit = tokio::time::timeout(Duration::from_secs(2), handle) - .await - .expect("detector exits within timeout") - .expect("join") - .expect("detector result"); - match exit { - DetectorExit::RecoveryRequired { status } => { - assert_eq!(status, DangerStatus::EstimatedBatchInDanger(1)); - } - other => { - panic!("expected recovery-required exit from wall-clock fallback, got {other:?}") - } - } - } -} diff --git a/sequencer/src/recovery/mod.rs b/sequencer/src/recovery/mod.rs deleted file mode 100644 index 476c3ae5..00000000 --- a/sequencer/src/recovery/mod.rs +++ /dev/null @@ -1,1083 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! Startup recovery while the process lock excludes other writers and no -//! workers are running. Local inspection selects one repair: Tip replacement -//! or Flush → Sync → Cascade. A fresh check follows repair and runtime -//! preparation; only the final check authorizes worker launch. - -mod detector; -mod flusher; - -use thiserror::Error; - -use crate::l1::L1Config; -use crate::l1::reader::{InputReader, InputReaderError}; -use crate::storage::{ - self, DangerStatus, RecoveryInspection, RecoveryMutationError, StorageOpenError, -}; -pub use detector::{DangerDetector, DangerDetectorError, DetectorExit}; -pub use flusher::{FlushError, MempoolFlusher}; -use sequencer_core::protocol::ProtocolTiming; - -/// A startup recovery failure is already classified when it leaves the -/// controller. Runtime lifecycle settlement projects only this outer class; -/// it never reinterprets raw provider/storage/phase errors. -#[derive(Debug, Error)] -pub enum RecoveryError { - #[error("startup recovery should retry: {0}")] - Retry(Box), - #[error("startup recovery refused: {0}")] - Refuse(Box), -} - -/// Diagnostic provenance retained underneath the controller's retry/refuse -/// verdict. -#[derive(Debug, Error)] -pub enum RecoveryFailure { - #[error(transparent)] - PolicyRetry(#[from] RecoveryRetryReason), - #[error(transparent)] - PolicyRefusal(#[from] RecoveryRefusalReason), - #[error("open storage: {0}")] - OpenStorage(#[source] StorageOpenError), - #[error("storage: {0}")] - Storage(#[source] rusqlite::Error), - #[error("flush: {0}")] - Flush(#[source] FlushError), - #[error("input reader: {0}")] - InputReader(#[source] InputReaderError), - /// The chain-id read on the flush's signer provider failed: transport, - /// so retry. - #[error("provider unreachable: {0}")] - ProviderUnreachable(String), - /// The flush's signer provider could not be constructed (bad RPC URL or - /// key): deterministic misconfiguration, so refuse. Kept distinct from - /// [`Self::ProviderUnreachable`] so the verdict is a function of the - /// value, never of the construction site. - #[error("signer provider misconfiguration: {0}")] - SignerMisconfig(String), - #[error("recovery flush chain-id mismatch: rpc {rpc} != pinned {expected}")] - ChainIdMismatch { rpc: u64, expected: u64 }, -} - -#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] -pub enum RecoveryRetryReason { - #[error("the persisted L1 view is stale")] - L1ViewStale, - #[error("batch {batch_index} is in danger only under wall-clock estimation")] - EstimatedBatchInDanger { batch_index: u64 }, - #[error("danger persists after repair: {status:?}")] - DangerPersists { status: DangerStatus }, - #[error( - "post-flush re-sync reached safe block {resynced_safe_block}, behind the flush observation at {flush_observed_safe_block}" - )] - ResyncBehindFlushView { - resynced_safe_block: u64, - flush_observed_safe_block: u64, - }, - #[error( - "post-flush stopping block {stop_block} predates checkpoint block {checkpoint_block}; \ - synchronize the RPC node and retry recovery" - )] - CheckpointAheadOfStop { - checkpoint_block: u64, - stop_block: u64, - }, - #[error("local recovery facts changed before phase execution: {status:?}")] - StaleDecision { status: DangerStatus }, - #[error("runtime preparation outlived its clean admission decision ({decision})")] - AdmissionChanged { decision: &'static str }, -} - -#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] -pub enum RecoveryRefusalReason { - /// A fully accepted L1 landing failed content identity. Standard recovery - /// assumes the opposite and is forbidden. - #[error("canonical divergence at batch nonce {nonce}")] - CanonicalDivergence { nonce: u64 }, - #[error("the completed setup has no recovery checkpoint")] - MissingRecoveryCheckpoint, - #[error("post-sync recovery has no persisted safe head")] - MissingSafeHead, - /// A Tip appeared between the no-Tip inspection and the `EnsureOpenTip` - /// transaction. Under the process lock nothing else writes, so this is a - /// lock violation or an inspection bug; retrying would admit over it. - #[error("the Tip was already open when the EnsureOpenTip phase ran")] - TipAlreadyOpen, - /// The `EnsureOpenTip` transaction violated its open-Tip postcondition. - #[error("the EnsureOpenTip phase left no valid open Tip")] - TipMissingAfterOpen, -} - -/// Single-use proof that the final check found clean, consistent facts after -/// all fallible preparation. Only [`admit_runtime`] can construct it. -#[must_use = "runtime admission must be consumed by PreparedRuntime::launch"] -#[derive(Debug)] -pub(crate) struct RuntimeAdmission { - _private: (), -} - -impl RecoveryError { - pub(crate) fn retry(failure: impl Into) -> Self { - Self::Retry(Box::new(failure.into())) - } - - pub(crate) fn refuse(failure: impl Into) -> Self { - Self::Refuse(Box::new(failure.into())) - } - - pub(crate) fn is_retryable(&self) -> bool { - matches!(self, Self::Retry(_)) - } -} - -/// The post-flush resync coherence check — the resynced safe block must -/// reach the flush observation before cascade — shared with -/// `setup --recovery`. Runtime recovery also enforces this inside the -/// guarded cascade transaction. -pub(crate) fn assert_resync_caught_up( - resynced_safe_block: u64, - flush_observed_safe_block: u64, -) -> Result<(), RecoveryError> { - if resynced_safe_block < flush_observed_safe_block { - return Err(RecoveryError::retry( - RecoveryRetryReason::ResyncBehindFlushView { - resynced_safe_block, - flush_observed_safe_block, - }, - )); - } - Ok(()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RecoveryAction { - Ready, - EnsureOpenTip, - RecoverTip { batch_index: u64 }, - Flush, -} - -impl RecoveryAction { - fn label(self) -> &'static str { - match self { - Self::Ready => "admit", - Self::EnsureOpenTip => "ensure_open_tip", - Self::RecoverTip { .. } => "recover_tip", - Self::Flush => "flush", - } - } -} - -fn refuse_local_terminal(facts: RecoveryInspection) -> Result<(), RecoveryError> { - if let DangerStatus::CanonicalDivergence(nonce) = facts.danger { - return Err(RecoveryError::refuse( - RecoveryRefusalReason::CanonicalDivergence { nonce }, - )); - } - if !facts.has_recovery_checkpoint { - return Err(RecoveryError::refuse( - RecoveryRefusalReason::MissingRecoveryCheckpoint, - )); - } - Ok(()) -} - -/// The startup dispatch and final admission use the same exhaustive policy. -fn select_recovery(facts: RecoveryInspection) -> Result { - refuse_local_terminal(facts)?; - match facts.danger { - DangerStatus::Safe if facts.has_open_tip => Ok(RecoveryAction::Ready), - DangerStatus::Safe => Ok(RecoveryAction::EnsureOpenTip), - DangerStatus::ClosedBatchInDanger(_) => Ok(RecoveryAction::Flush), - DangerStatus::TipInDanger(batch_index) => Ok(RecoveryAction::RecoverTip { batch_index }), - DangerStatus::L1ViewStale => Err(RecoveryError::retry(RecoveryRetryReason::L1ViewStale)), - DangerStatus::EstimatedBatchInDanger(batch_index) => Err(RecoveryError::retry( - RecoveryRetryReason::EstimatedBatchInDanger { batch_index }, - )), - DangerStatus::CanonicalDivergence(_) => { - unreachable!("terminal facts were checked before dispatch") - } - } -} - -fn inspect_recovery( - db_path: &str, - protocol: &ProtocolTiming, -) -> Result { - let mut storage = storage::Storage::open_writer(db_path).map_err(classify_open)?; - storage - .inspect_recovery(protocol, crate::clock::unix_now_ms()) - .map_err(classify_storage) -} - -/// Only L1 operations are abstracted: tests keep the real local inspections -/// and repair transactions and substitute the external system. -trait RecoveryL1 { - async fn sync(&mut self) -> Result<(), InputReaderError>; - async fn flush(&mut self) -> Result; -} - -struct StartupL1<'a> { - db_path: &'a str, - input_reader: &'a mut InputReader, - l1_config: &'a L1Config, - protocol: &'a ProtocolTiming, -} - -impl RecoveryL1 for StartupL1<'_> { - async fn sync(&mut self) -> Result<(), InputReaderError> { - self.input_reader.sync_to_current_safe_head().await - } - - async fn flush(&mut self) -> Result { - let providers = crate::l1::provider::create_verified_keyed_providers( - &self.l1_config.eth_rpc_url, - self.l1_config.batch_submitter_private_key.expose_secret(), - self.l1_config.identity.chain_id, - self.l1_config.allow_insecure_rpc, - ) - .await - .map_err(classify_signer_provider)?; - - let watermark = { - let mut storage = storage::Storage::open_writer(self.db_path).map_err(classify_open)?; - storage.wallet_nonce_watermark().map_err(classify_storage)? - }; - MempoolFlusher::flush_to_safe( - providers, - self.l1_config.identity.batch_submitter_address, - self.protocol.seconds_per_block(), - self.db_path, - watermark, - ) - .await - .map_err(classify_flush) - } -} - -/// Repair once, then require a clean view. Flush cannot change local recovery -/// facts except the wallet watermark; Sync is the only external operation that -/// can discover divergence. The guarded repair transaction checks those new -/// facts before mutation, including the post-flush safe-head floor. -async fn recover_startup( - db_path: &str, - protocol: &ProtocolTiming, - l1: &mut impl RecoveryL1, -) -> Result<(), RecoveryError> { - refuse_local_terminal(inspect_recovery(db_path, protocol)?)?; - match l1.sync().await { - Ok(()) => tracing::info!("L1 safe head synced"), - // An unreachable provider does not invalidate a still-fresh persisted - // view. Post-flush Sync has no such fallback: cascade needs its result. - Err(InputReaderError::Provider(error)) => tracing::warn!( - error = %error, - "L1 unreachable during initial startup sync; inspecting persisted view" - ), - Err(error) => return Err(classify_input_reader(error)), - } - - let facts = inspect_recovery(db_path, protocol)?; - let action = select_recovery(facts)?; - tracing::info!( - danger_status = facts.danger.label(), - danger_batch_index = ?facts.danger.batch_index(), - recovery_decision = action.label(), - "startup recovery decision" - ); - let invalidated = match action { - RecoveryAction::Ready => return Ok(()), - RecoveryAction::EnsureOpenTip => { - let mut storage = storage::Storage::open_writer(db_path).map_err(classify_open)?; - storage - .ensure_open_tip_for_recovery(protocol, crate::clock::unix_now_ms()) - .map_err(classify_mutation)?; - Vec::new() - } - RecoveryAction::RecoverTip { batch_index } => { - let mut storage = storage::Storage::open_writer(db_path).map_err(classify_open)?; - storage - .recover_aging_tip_for_recovery(batch_index, protocol, crate::clock::unix_now_ms()) - .map_err(classify_mutation)? - } - RecoveryAction::Flush => { - // The observation exists only on this invocation's stack. A crash - // or retry loses it; a new attempt must flush again before cascade. - let observed_safe_block = l1.flush().await?; - l1.sync().await.map_err(classify_input_reader)?; - let mut storage = storage::Storage::open_writer(db_path).map_err(classify_open)?; - storage - .recover_post_flush_for_recovery( - observed_safe_block, - protocol, - crate::clock::unix_now_ms(), - ) - .map_err(classify_mutation)? - } - }; - if invalidated.is_empty() { - tracing::info!("startup recovery completed without invalidation"); - } else { - tracing::warn!( - invalidated_count = invalidated.len(), - batches = ?invalidated, - "startup recovery invalidated the doomed suffix" - ); - } - - let facts = inspect_recovery(db_path, protocol)?; - refuse_local_terminal(facts)?; - // Observed repair can expose a surviving clock/view refusal. Never treat - // successful mutation as admission or begin another repair in this boot. - match facts.danger { - DangerStatus::Safe => { - assert!(facts.has_open_tip, "recovery committed without an open Tip"); - Ok(()) - } - status @ (DangerStatus::ClosedBatchInDanger(_) - | DangerStatus::TipInDanger(_) - | DangerStatus::L1ViewStale - | DangerStatus::EstimatedBatchInDanger(_)) => { - Err(RecoveryError::retry(RecoveryRetryReason::DangerPersists { - status, - })) - } - DangerStatus::CanonicalDivergence(_) => { - unreachable!("terminal facts were checked after repair") - } - } -} - -/// Finish startup recovery before fallible, task-free runtime preparation. -/// This grants no runtime authority; [`admit_runtime`] checks again afterwards. -pub(crate) async fn run_startup_recovery( - db_path: &str, - input_reader: &mut InputReader, - l1_config: &L1Config, - protocol: &ProtocolTiming, -) -> Result<(), RecoveryError> { - recover_startup( - db_path, - protocol, - &mut StartupL1 { - db_path, - input_reader, - l1_config, - protocol, - }, - ) - .await -} - -/// Check current facts after preparation. The process lock and task-free -/// preparation exclude another writer, but elapsed time can stale the view. -/// Launch consumes the resulting witness without yielding or further fallible -/// preparation. -pub(crate) fn admit_runtime( - db_path: &str, - protocol: &ProtocolTiming, -) -> Result { - match select_recovery(inspect_recovery(db_path, protocol)?)? { - RecoveryAction::Ready => Ok(RuntimeAdmission { _private: () }), - action => Err(RecoveryError::retry( - RecoveryRetryReason::AdmissionChanged { - decision: action.label(), - }, - )), - } -} - -fn classify_open(error: StorageOpenError) -> RecoveryError { - let persistent = storage::is_persistent_storage_open_error(&error); - let failure = RecoveryFailure::OpenStorage(error); - if persistent { - RecoveryError::refuse(failure) - } else { - RecoveryError::retry(failure) - } -} - -fn classify_storage(error: rusqlite::Error) -> RecoveryError { - let persistent = storage::is_persistent_storage_error(&error); - let failure = RecoveryFailure::Storage(error); - if persistent { - RecoveryError::refuse(failure) - } else { - RecoveryError::retry(failure) - } -} - -/// The flush's signer-provider errors, classified at birth. The -/// terminal/transient split must agree with `From -/// for BootstrapError` in `commands/error.rs` (mismatch and construction are -/// terminal; the chain-id read is transient); this is recovery's own -/// retry/refuse polarity over the same facts, pinned beside the others. -fn classify_signer_provider( - error: crate::l1::provider::VerifiedSignerProviderError, -) -> RecoveryError { - use crate::l1::provider::VerifiedSignerProviderError; - match error { - VerifiedSignerProviderError::ChainIdMismatch { rpc, expected } => { - RecoveryError::refuse(RecoveryFailure::ChainIdMismatch { rpc, expected }) - } - VerifiedSignerProviderError::ChainIdRpc(message) => { - RecoveryError::retry(RecoveryFailure::ProviderUnreachable(message)) - } - VerifiedSignerProviderError::Create(message) => { - RecoveryError::refuse(RecoveryFailure::SignerMisconfig(message)) - } - } -} - -/// Input-reader errors met during the startup phases, classified at birth by -/// [`InputReaderError::is_retryable_at_startup`]; storage keeps its provenance. -fn classify_input_reader(error: InputReaderError) -> RecoveryError { - match error { - InputReaderError::OpenStorage(source) => classify_open(source), - InputReaderError::Storage(source) => classify_storage(source), - error if error.is_retryable_at_startup() => { - RecoveryError::retry(RecoveryFailure::InputReader(error)) - } - error => RecoveryError::refuse(RecoveryFailure::InputReader(error)), - } -} - -fn classify_flush(error: FlushError) -> RecoveryError { - let terminal = error.is_terminal_invariant(); - let failure = RecoveryFailure::Flush(error); - if terminal { - RecoveryError::refuse(failure) - } else { - RecoveryError::retry(failure) - } -} - -fn classify_mutation(error: RecoveryMutationError) -> RecoveryError { - match error { - RecoveryMutationError::Storage(source) => classify_storage(source), - RecoveryMutationError::CanonicalDivergence { nonce } => { - RecoveryError::refuse(RecoveryRefusalReason::CanonicalDivergence { nonce }) - } - RecoveryMutationError::MissingRecoveryCheckpoint => { - RecoveryError::refuse(RecoveryRefusalReason::MissingRecoveryCheckpoint) - } - RecoveryMutationError::MissingSafeHead => { - RecoveryError::refuse(RecoveryRefusalReason::MissingSafeHead) - } - RecoveryMutationError::ResyncBehindFlushView { - resynced_safe_block, - flush_observed_safe_block, - } => RecoveryError::retry(RecoveryRetryReason::ResyncBehindFlushView { - resynced_safe_block, - flush_observed_safe_block, - }), - RecoveryMutationError::StaleDecision { actual, .. } => { - RecoveryError::retry(RecoveryRetryReason::StaleDecision { status: actual }) - } - RecoveryMutationError::TipAlreadyOpen => { - RecoveryError::refuse(RecoveryRefusalReason::TipAlreadyOpen) - } - RecoveryMutationError::TipMissingAfterOpen => { - RecoveryError::refuse(RecoveryRefusalReason::TipMissingAfterOpen) - } - } -} - -#[cfg(test)] -mod tests { - use std::collections::VecDeque; - - use super::*; - - fn sqlite_failure(code: rusqlite::ffi::ErrorCode, extended_code: i32) -> rusqlite::Error { - rusqlite::Error::SqliteFailure( - rusqlite::ffi::Error { - code, - extended_code, - }, - None, - ) - } - - fn assert_retry(error: RecoveryError) { - assert!( - matches!(error, RecoveryError::Retry(_)), - "expected retry, got {error:?}" - ); - } - - fn assert_refuse(error: RecoveryError) { - assert!( - matches!(error, RecoveryError::Refuse(_)), - "expected refusal, got {error:?}" - ); - } - - /// The reader's predicate and the startup classifier agree on every - /// variant, including both storage polarities. - #[test] - fn startup_reader_polarity_has_one_owner() { - let busy = || sqlite_failure(rusqlite::ffi::ErrorCode::DatabaseBusy, 5); - let corrupt = || sqlite_failure(rusqlite::ffi::ErrorCode::NotADatabase, 26); - let errors: Vec InputReaderError> = vec![ - || InputReaderError::Provider("offline".into()), - || InputReaderError::InconsistentL1Response("gap".into()), - || InputReaderError::ChainIdMismatch { - rpc: 1, - expected: 2, - }, - || InputReaderError::Bootstrap("bad url".into()), - || InputReaderError::StorageTaskPanicked { operation: "sync" }, - || InputReaderError::Join("lost".into()), - ]; - let mut cases: Vec = errors.into_iter().map(|make| make()).collect(); - cases.push(InputReaderError::Storage(busy())); - cases.push(InputReaderError::Storage(corrupt())); - cases.push(InputReaderError::OpenStorage(StorageOpenError::Sqlite( - busy(), - ))); - cases.push(InputReaderError::OpenStorage(StorageOpenError::Sqlite( - corrupt(), - ))); - for error in cases { - let expected = error.is_retryable_at_startup(); - let description = error.to_string(); - assert_eq!( - classify_input_reader(error).is_retryable(), - expected, - "{description}" - ); - } - } - - #[test] - fn error_classifiers_pin_retry_and_refuse_polarity() { - use crate::l1::watermark::WalletNonceWatermarkError; - - let busy = || sqlite_failure(rusqlite::ffi::ErrorCode::DatabaseBusy, 5); - let corrupt = || sqlite_failure(rusqlite::ffi::ErrorCode::NotADatabase, 26); - - assert_retry(classify_open(StorageOpenError::Sqlite(busy()))); - assert_refuse(classify_open(StorageOpenError::Sqlite(corrupt()))); - assert_retry(classify_storage(busy())); - assert_refuse(classify_storage(rusqlite::Error::QueryReturnedNoRows)); - - assert_retry(classify_input_reader(InputReaderError::Provider( - "offline".into(), - ))); - assert_retry(classify_input_reader( - InputReaderError::InconsistentL1Response("gap".into()), - )); - assert_refuse(classify_input_reader(InputReaderError::ChainIdMismatch { - rpc: 1, - expected: 2, - })); - // Phase-dependent: refused at startup, non-terminal in the live - // worker (`InputReaderError::is_terminal_invariant`). - assert_refuse(classify_input_reader(InputReaderError::Bootstrap( - "plaintext rpc url".into(), - ))); - assert_refuse(classify_input_reader(InputReaderError::Join( - "sync task lost".into(), - ))); - assert!(!InputReaderError::Bootstrap("x".into()).is_terminal_invariant()); - assert!(!InputReaderError::Join("x".into()).is_terminal_invariant()); - - { - use crate::commands::error::BootstrapError; - use crate::l1::provider::VerifiedSignerProviderError as Signer; - // The split's payloads are load-bearing, not only its polarity: - // swapping the two provider variants must go red. - assert!(matches!( - classify_signer_provider(Signer::ChainIdRpc("timeout".into())), - RecoveryError::Retry(f) if matches!(*f, RecoveryFailure::ProviderUnreachable(_)) - )); - assert!(matches!( - classify_signer_provider(Signer::Create("bad url".into())), - RecoveryError::Refuse(f) if matches!(*f, RecoveryFailure::SignerMisconfig(_)) - )); - assert!(matches!( - classify_signer_provider(Signer::ChainIdMismatch { - rpc: 1, - expected: 2, - }), - RecoveryError::Refuse(f) if matches!(*f, RecoveryFailure::ChainIdMismatch { .. }) - )); - // The doc's "must agree with the `BootstrapError` projection": - // recovery retries exactly the arm that projection calls - // transient. - let mint = || { - [ - Signer::ChainIdRpc("timeout".into()), - Signer::Create("bad url".into()), - Signer::ChainIdMismatch { - rpc: 1, - expected: 2, - }, - ] - }; - for (recovery_side, projection_side) in mint().into_iter().zip(mint()) { - let retried = matches!( - classify_signer_provider(recovery_side), - RecoveryError::Retry(_) - ); - let transient = matches!( - BootstrapError::from(projection_side), - BootstrapError::ChainIdRpc { .. } - ); - assert_eq!(retried, transient); - } - } - - assert_retry(classify_flush(FlushError::Provider("offline".into()))); - assert_refuse(classify_flush(FlushError::UnsignableTransaction( - "no signer".into(), - ))); - assert_refuse(classify_flush(FlushError::Watermark( - WalletNonceWatermarkError::Storage(rusqlite::Error::QueryReturnedNoRows), - ))); - - assert_refuse(classify_mutation(RecoveryMutationError::TipAlreadyOpen)); - assert_refuse(classify_mutation( - RecoveryMutationError::TipMissingAfterOpen, - )); - assert_retry(classify_mutation(RecoveryMutationError::StaleDecision { - expected: DangerStatus::Safe, - actual: DangerStatus::L1ViewStale, - })); - assert_retry(classify_mutation( - RecoveryMutationError::ResyncBehindFlushView { - resynced_safe_block: 10, - flush_observed_safe_block: 11, - }, - )); - assert_refuse(classify_mutation( - RecoveryMutationError::CanonicalDivergence { nonce: 7 }, - )); - assert_refuse(classify_mutation( - RecoveryMutationError::MissingRecoveryCheckpoint, - )); - assert_refuse(classify_mutation(RecoveryMutationError::MissingSafeHead)); - } - - fn admission_fixture( - name: &str, - has_recovery_checkpoint: bool, - has_open_tip: bool, - ) -> (crate::test_support::TestDb, ProtocolTiming) { - use crate::storage::test_helpers::{SENDER_A, default_protocol_timing}; - use crate::test_support::temp_db; - - let db = temp_db(name); - let protocol = default_protocol_timing(); - let now_ms = crate::clock::unix_now_ms(); - let mut storage = - storage::Storage::initialize_for_command(&db.path, storage::LifecycleCommand::Setup) - .expect("initialize setup"); - crate::test_support::pin_test_deployment_identity(&mut storage, SENDER_A); - storage - .append_safe_inputs_with_timestamp( - 0, - now_ms / 1_000, - &[], - SENDER_A, - &protocol, - storage::FrontierMode::Populate, - ) - .expect("seed fresh safe head"); - let prefix = db._dir.path().join("finalized"); - storage - .complete_baseline_setup( - &prefix, - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, - ) - .expect("seed recovery checkpoint"); - if has_open_tip { - storage - .initialize_open_state(0, storage::SafeInputRange::empty_at(0)) - .expect("seed open Tip"); - } - if !has_recovery_checkpoint { - storage - .write(|tx| { - tx.execute("DELETE FROM snapshots WHERE batch_index IS NULL", [])?; - Ok(()) - }) - .expect("simulate post-setup snapshot loss"); - } - drop(storage); - (db, protocol) - } - - enum SyncStep { - Keep, - Fail(InputReaderError), - Observe { - block: u64, - inputs: Vec, - }, - } - - struct TestL1<'a> { - db_path: &'a str, - protocol: &'a ProtocolTiming, - syncs: VecDeque, - flush_block: u64, - calls: Vec<&'static str>, - } - - impl<'a> TestL1<'a> { - fn new(db_path: &'a str, protocol: &'a ProtocolTiming, syncs: Vec) -> Self { - Self { - db_path, - protocol, - syncs: syncs.into(), - flush_block: protocol.danger_threshold(), - calls: Vec::new(), - } - } - } - - impl RecoveryL1 for TestL1<'_> { - async fn sync(&mut self) -> Result<(), InputReaderError> { - self.calls.push("sync"); - match self.syncs.pop_front().expect("unexpected L1 sync") { - SyncStep::Keep => Ok(()), - SyncStep::Fail(error) => Err(error), - SyncStep::Observe { block, inputs } => storage::Storage::open_writer(self.db_path) - .expect("open fixture storage") - .append_safe_inputs_with_timestamp( - block, - crate::clock::unix_now_ms() / 1_000, - &inputs, - crate::storage::test_helpers::SENDER_A, - self.protocol, - storage::FrontierMode::Populate, - ) - .map_err(InputReaderError::Storage), - } - } - - async fn flush(&mut self) -> Result { - self.calls.push("flush"); - Ok(self.flush_block) - } - } - - fn danger_fixture(name: &str, closed: bool) -> (crate::test_support::TestDb, ProtocolTiming) { - let (db, protocol) = admission_fixture(name, true, true); - let mut storage = storage::Storage::open_writer(&db.path).expect("open writer"); - if closed { - let mut head = storage.open_state().unwrap().unwrap(); - storage.close_frame_and_batch(&mut head, 0).unwrap(); - } - storage - .append_safe_inputs_with_timestamp( - protocol.danger_threshold(), - crate::clock::unix_now_ms() / 1_000, - &[], - crate::storage::test_helpers::SENDER_A, - &protocol, - storage::FrontierMode::Populate, - ) - .expect("advance into observed danger"); - (db, protocol) - } - - fn invalidated_batches(db_path: &str) -> Vec { - let mut storage = storage::Storage::open_writer(db_path).unwrap(); - storage - .read(|tx| { - tx.prepare("SELECT batch_index FROM batches WHERE invalidated_at_ms IS NOT NULL ORDER BY batch_index")? - .query_map([], |row| { - Ok(u64::try_from(row.get::<_, i64>(0)?).expect("nonnegative batch index")) - })? - .collect() - }) - .unwrap() - } - - #[tokio::test] - async fn initial_provider_failure_uses_only_a_fresh_persisted_view() { - let (db, protocol) = admission_fixture("recovery-warm-provider-failure", true, true); - let offline = || SyncStep::Fail(InputReaderError::Provider("offline".into())); - let mut l1 = TestL1::new(&db.path, &protocol, vec![offline()]); - recover_startup(&db.path, &protocol, &mut l1).await.unwrap(); - assert_eq!(l1.calls, ["sync"]); - - storage::Storage::open_writer(&db.path) - .unwrap() - .write(|tx| tx.execute("UPDATE l1_safe_head SET block_timestamp = 0", [])) - .unwrap(); - let mut l1 = TestL1::new(&db.path, &protocol, vec![offline()]); - assert_retry( - recover_startup(&db.path, &protocol, &mut l1) - .await - .unwrap_err(), - ); - assert_eq!(l1.calls, ["sync"]); - } - - #[tokio::test] - async fn local_terminal_facts_refuse_before_l1() { - for diverged in [false, true] { - let (db, protocol) = admission_fixture("recovery-local-terminal", diverged, true); - if diverged { - let mut storage = storage::Storage::open_writer(&db.path).unwrap(); - crate::storage::test_helpers::record_canonical_divergence(&mut storage, 7, 0); - } - let mut l1 = TestL1::new(&db.path, &protocol, vec![]); - assert_refuse( - recover_startup(&db.path, &protocol, &mut l1) - .await - .unwrap_err(), - ); - assert!(l1.calls.is_empty()); - } - } - - #[tokio::test] - async fn fresh_start_opens_tip_and_tip_repair_never_flushes() { - let (db, protocol) = admission_fixture("recovery-ensure-tip", true, false); - let mut l1 = TestL1::new(&db.path, &protocol, vec![SyncStep::Keep]); - recover_startup(&db.path, &protocol, &mut l1).await.unwrap(); - assert!(inspect_recovery(&db.path, &protocol).unwrap().has_open_tip); - assert_eq!(l1.calls, ["sync"]); - - let (db, protocol) = danger_fixture("recovery-tip", false); - let mut l1 = TestL1::new(&db.path, &protocol, vec![SyncStep::Keep]); - recover_startup(&db.path, &protocol, &mut l1).await.unwrap(); - assert_eq!(invalidated_batches(&db.path), [0]); - assert_eq!(l1.calls, ["sync"]); - assert_eq!( - inspect_recovery(&db.path, &protocol).unwrap().danger, - DangerStatus::Safe - ); - } - - #[tokio::test] - async fn post_flush_cascade_runs_even_when_the_refreshed_view_is_safe() { - use crate::storage::test_helpers::{SENDER_A, local_batch_payload}; - - let (db, protocol) = admission_fixture("recovery-unconditional-cascade", true, true); - let block = protocol.danger_threshold(); - let mut storage = storage::Storage::open_writer(&db.path).unwrap(); - let mut head = storage.open_state().unwrap().unwrap(); - storage - .close_frame_only(&mut head, block, storage::SafeInputRange::empty_at(0)) - .unwrap(); - storage.close_frame_and_batch(&mut head, block).unwrap(); - storage - .insert_batch_snapshot(&db._dir.path().join("batch0"), 0) - .unwrap(); - storage.close_frame_and_batch(&mut head, block).unwrap(); - storage - .insert_batch_snapshot(&db._dir.path().join("batch1"), 1) - .unwrap(); - storage - .append_safe_inputs_with_timestamp( - block, - crate::clock::unix_now_ms() / 1_000, - &[], - SENDER_A, - &protocol, - storage::FrontierMode::Populate, - ) - .unwrap(); - let landed = storage::StoredSafeInput { - sender: SENDER_A, - payload: local_batch_payload(&mut storage, 0), - block_number: block, - }; - drop(storage); - let mut l1 = TestL1::new( - &db.path, - &protocol, - vec![ - SyncStep::Keep, - SyncStep::Observe { - block: block + 1, - inputs: vec![landed], - }, - ], - ); - recover_startup(&db.path, &protocol, &mut l1).await.unwrap(); - assert_eq!(l1.calls, ["sync", "flush", "sync"]); - // Batch 0 became gold; the young unresolved batch 1 still belongs to - // the flushed suffix even though it no longer trips observed danger. - assert_eq!(invalidated_batches(&db.path), [1, 2]); - } - - #[tokio::test] - async fn sync_discovered_divergence_refuses_before_cascade() { - use crate::storage::test_helpers::SENDER_A; - let (db, protocol) = danger_fixture("recovery-sync-divergence", true); - let block = protocol.danger_threshold(); - let foreign = storage::StoredSafeInput { - sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 0, - frames: vec![], - }), - block_number: block + 1, - }; - let mut l1 = TestL1::new( - &db.path, - &protocol, - vec![ - SyncStep::Keep, - SyncStep::Observe { - block: block + 1, - inputs: vec![foreign], - }, - ], - ); - assert_refuse( - recover_startup(&db.path, &protocol, &mut l1) - .await - .unwrap_err(), - ); - assert_eq!(l1.calls, ["sync", "flush", "sync"]); - assert!(invalidated_batches(&db.path).is_empty()); - } - - #[tokio::test] - async fn failed_post_flush_sync_cannot_use_the_initial_sync_fallback() { - let (db, protocol) = danger_fixture("recovery-post-flush-offline", true); - let mut l1 = TestL1::new( - &db.path, - &protocol, - vec![ - SyncStep::Keep, - SyncStep::Fail(InputReaderError::Provider("offline".into())), - ], - ); - assert_retry( - recover_startup(&db.path, &protocol, &mut l1) - .await - .unwrap_err(), - ); - assert_eq!(l1.calls, ["sync", "flush", "sync"]); - assert!(invalidated_batches(&db.path).is_empty()); - } - - #[tokio::test] - async fn retry_discards_the_flush_observation_and_flushes_again() { - let (db, protocol) = danger_fixture("recovery-flush-floor", true); - let mut interrupted = - TestL1::new(&db.path, &protocol, vec![SyncStep::Keep, SyncStep::Keep]); - interrupted.flush_block += 1; - let error = recover_startup(&db.path, &protocol, &mut interrupted) - .await - .unwrap_err(); - assert!(matches!(error, RecoveryError::Retry(failure) - if matches!(*failure, RecoveryFailure::PolicyRetry(RecoveryRetryReason::ResyncBehindFlushView { .. })))); - assert_eq!(interrupted.calls, ["sync", "flush", "sync"]); - assert!(invalidated_batches(&db.path).is_empty()); - drop(interrupted); - - let mut restarted = TestL1::new(&db.path, &protocol, vec![SyncStep::Keep, SyncStep::Keep]); - recover_startup(&db.path, &protocol, &mut restarted) - .await - .unwrap(); - assert_eq!(restarted.calls, ["sync", "flush", "sync"]); - assert_eq!(invalidated_batches(&db.path), [0, 1]); - } - - #[tokio::test] - async fn observed_tip_repair_still_refuses_a_surviving_clock_fault() { - let (db, protocol) = danger_fixture("recovery-tip-clock", false); - let ahead = crate::clock::unix_now_ms() + protocol.seconds_per_block() * 2_000; - storage::Storage::open_writer(&db.path) - .unwrap() - .write(|tx| { - tx.execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1", - [i64::try_from(ahead).unwrap()], - ) - }) - .unwrap(); - let mut l1 = TestL1::new(&db.path, &protocol, vec![SyncStep::Keep]); - let error = recover_startup(&db.path, &protocol, &mut l1) - .await - .unwrap_err(); - assert!(matches!(error, RecoveryError::Retry(failure) - if matches!(*failure, RecoveryFailure::PolicyRetry(RecoveryRetryReason::DangerPersists { status: DangerStatus::L1ViewStale })))); - assert_eq!(invalidated_batches(&db.path), [0]); - assert_eq!(l1.calls, ["sync"]); - } - - #[test] - fn final_admission_rechecks_view_freshness_after_preparation() { - let (db, protocol) = admission_fixture("admit-stale-view", true, true); - let _admission = admit_runtime(&db.path, &protocol).unwrap(); - storage::Storage::open_writer(&db.path) - .unwrap() - .write(|tx| tx.execute("UPDATE l1_safe_head SET block_timestamp = 0", [])) - .unwrap(); - assert_retry(admit_runtime(&db.path, &protocol).unwrap_err()); - } - - #[test] - fn final_admission_refuses_new_divergence() { - let (db, protocol) = admission_fixture("admit-divergence", true, true); - let mut storage = storage::Storage::open_writer(&db.path).expect("open writer"); - crate::storage::test_helpers::record_canonical_divergence(&mut storage, 7, 0); - drop(storage); - - let error = - admit_runtime(&db.path, &protocol).expect_err("divergence must refuse final admission"); - assert!(matches!( - error, - RecoveryError::Refuse(failure) - if matches!( - *failure, - RecoveryFailure::PolicyRefusal( - RecoveryRefusalReason::CanonicalDivergence { nonce: 7 } - ) - ) - )); - } - - #[test] - fn final_admission_retries_when_tip_disappeared() { - let (db, protocol) = admission_fixture("admit-no-tip", true, false); - - let error = admit_runtime(&db.path, &protocol) - .expect_err("a missing Tip requires a fresh recovery attempt"); - assert!(matches!( - error, - RecoveryError::Retry(failure) - if matches!( - *failure, - RecoveryFailure::PolicyRetry( - RecoveryRetryReason::AdmissionChanged { - decision: "ensure_open_tip" - } - ) - ) - )); - } - - #[test] - fn final_admission_refuses_missing_snapshot() { - let (db, protocol) = admission_fixture("admit-no-snapshot", false, true); - - let error = admit_runtime(&db.path, &protocol) - .expect_err("a missing recovery checkpoint must refuse final admission"); - assert!(matches!( - error, - RecoveryError::Refuse(failure) - if matches!( - *failure, - RecoveryFailure::PolicyRefusal( - RecoveryRefusalReason::MissingRecoveryCheckpoint - ) - ) - )); - } -} diff --git a/sequencer/src/storage/safe_accepted_batches.rs b/sequencer/src/storage/acceptance.rs similarity index 57% rename from sequencer/src/storage/safe_accepted_batches.rs rename to sequencer/src/storage/acceptance.rs index 4b935da5..a2f3773c 100644 --- a/sequencer/src/storage/safe_accepted_batches.rs +++ b/sequencer/src/storage/acceptance.rs @@ -1,67 +1,49 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! `safe_accepted_batches`: the prefix of submitted batches the on-chain scheduler -//! would accept, simulated with `ProtocolTiming::scheduler_accepts`. Its only -//! writer is [`populate_safe_accepted_batches`], inside the input reader's append -//! transaction; a divergence marker freezes it (I15). +//! Batch acceptance: `batches.accepted_input`, the prefix of our batches +//! the on-chain scheduler accepted, simulated with `protocol::scheduler_accepts` +//! under the pinned window. +//! Its only writer is [`record_accepted_batches`], inside the input reader's +//! append transaction; a divergence marker freezes it (I9, I15). -use alloy_primitives::Address; use rusqlite::{Connection, OptionalExtension, Result, params}; use super::convert::{i64_to_u64, now_unix_ms, u64_to_i64}; -use super::mutations::batch_tree_anchor_in; -use sequencer_core::protocol::{AcceptedBatch, ProtocolTiming, SafeInputView}; - -/// One row of `safe_accepted_batches`, exposing just the columns the -/// frontier-read code paths need. -#[derive(Debug, Clone, Copy)] -pub(super) struct SafeAcceptedBatchRow { - pub safe_input_index: i64, - pub nonce: i64, -} - -/// The most recently accepted row, or `None` if the view is empty. -pub(super) fn query_latest_safe_accepted_batch( - conn: &Connection, -) -> Result> { - conn.query_row( - "SELECT safe_input_index, nonce FROM safe_accepted_batches \ - ORDER BY safe_input_index DESC LIMIT 1", - [], - |row| { - Ok(SafeAcceptedBatchRow { - safe_input_index: row.get(0)?, - nonce: row.get(1)?, - }) - }, - ) - .optional() +use super::history::{era_in, find_era_in}; +use super::l1_inputs::query_deployment_identity; +use sequencer_core::protocol::{AcceptedBatch, SafeInputView, scheduler_accepts}; + +/// The safe input of the greatest acceptance, or `None` before any. +fn latest_accepted_input(conn: &Connection) -> Result> { + conn.query_row("SELECT MAX(accepted_input) FROM batches", [], |row| { + row.get(0) + }) } /// Next nonce the scheduler is expected to accept — the gold frontier's /// "expected next" cursor. /// -/// Returns `latest_accepted.nonce + 1` if any batch has been accepted, else the -/// deployment anchor (`0` for genesis, `N'` after rebuild). -/// Equivalently, the nonce that the very next valid closed batch (the cascade -/// pivot, when one exists) will carry, by the contiguity invariant on the -/// valid path (`trg_enforce_nonce_contiguity`). +/// The greatest accepted batch's nonce + 1, else the era's anchor +/// (`anchor_batch_nonce`, its next batch nonce at the stop block). By the +/// contiguity of batch nonces this is the nonce of the first unaccepted +/// closed batch, the one the void history judges, when one exists (I16). pub(super) fn frontier_nonce(conn: &Connection) -> Result { - match query_latest_safe_accepted_batch(conn)? { - Some(row) => Ok(next_expected_nonce(i64_to_u64(row.nonce))), - // Empty accepted table ⇒ the frontier sits at the batch-tree anchor: 0 - // for a genesis deployment, N' for a cockroach-recovered one. Reading - // the anchor (not a hard-coded 0) keeps this in step with - // `populate_safe_accepted_batches`, which seeds `expected` from the same - // anchor. Without it, a freshly recovered deployment — whose frontier - // stays empty until `run`'s first sync lands an accepted row — would - // have its submitter fold from 0 and re-submit its first post-recovery - // batch (nonce N') every tick until then. The cascade reader - // (`first_non_gold_closed_batch`) is unaffected: no valid batch sits - // below the anchor (I16), so `nonce >= 0` and `nonce >= N'` select the - // same row. (See I16 / docs/recovery/cockroach.md.) - None => batch_tree_anchor_in(conn), + let latest: Option = conn + .query_row( + "SELECT nonce FROM batches \ + WHERE accepted_input = (SELECT MAX(accepted_input) FROM batches)", + [], + |row| row.get(0), + ) + .optional()?; + match latest { + Some(nonce) => Ok(next_expected_nonce(i64_to_u64(nonce))), + // Before any acceptance the frontier sits at the anchor, the same + // value `record_accepted_batches` seeds `expected` from: otherwise a + // recovered deployment's submitter would fold from 0 and re-submit its + // first batch every tick until the first acceptance (I16). + None => Ok(era_in(conn)?.anchor_batch_nonce), } } @@ -71,79 +53,56 @@ fn next_expected_nonce(nonce: u64) -> u64 { .expect("accepted batch nonce overflow: contract-impossible") } -/// Mirror scheduler acceptance over new batch-submitter inputs into -/// `safe_accepted_batches`; an accepted landing that is foreign or +/// Mirror scheduler acceptance over new batch-submitter inputs onto +/// `batches.accepted_input`; an accepted landing that is foreign or /// byte-different records `canonical_divergence` instead (I9, I15). The only -/// persistent cursor is the latest accepted row: recovery can reuse a nonce -/// after rejected rows, so those are rescanned until a later acceptance. The -/// expected-nonce advance stays inline because it interleaves with the -/// content-identity check and the divergence freeze. -pub(super) fn populate_safe_accepted_batches( - conn: &Connection, - batch_submitter: Address, - timing: &ProtocolTiming, -) -> Result<()> { +/// persistent cursor is the greatest accepted input: rows rejected at the +/// expected nonce are rescanned until a later acceptance. The expected-nonce +/// advance stays inline because it interleaves with the content-identity +/// check and the divergence freeze. Runs exactly when the era exists; the +/// submitter and the staleness window are the pinned deployment identity's. +pub(super) fn record_accepted_batches(conn: &Connection) -> Result<()> { const PAGE_SIZE: i64 = 256; const SELECT_SQL: &str = "SELECT safe_input_index, payload, block_number, transaction_hash \ FROM safe_inputs \ WHERE sender = ?1 AND safe_input_index > ?2 \ AND block_number > ?3 \ ORDER BY safe_input_index ASC LIMIT ?4"; - const INSERT_SQL: &str = "INSERT INTO safe_accepted_batches \ - (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ - VALUES (?1, ?2, ?3, ?4)"; + // The content-identity check found this closed batch; the schema + // refuses an acceptance that does not extend the accepted prefix. + const ACCEPT_SQL: &str = "UPDATE batches SET accepted_input = ?1 \ + WHERE nonce = ?2 AND sealed_at_ms IS NOT NULL"; // A persisted divergence marker freezes the acceptance frontier (I15). if canonical_divergence_in(conn)?.is_some() { return Ok(()); } - // The recovered prefix is opaque. Replaying it with the final nonce N' - // could accept an old future-nonce submission that the scheduler rejected - // at its original position. Only inputs after the baseline belong here. - let base_safe_block: Option = conn - .query_row( - "SELECT base_safe_block FROM history_state WHERE singleton_id = 0", - [], - |row| row.get(0), - ) - .optional()?; - let base_safe_block = match base_safe_block { - Some(block) => block, - None => { - let setup_complete: bool = - conn.query_row("SELECT EXISTS(SELECT 1 FROM setup_complete)", [], |row| { - row.get(0) - })?; - assert!(!setup_complete, "completed setup has no history baseline"); - // Plain setup detects prior activity before installing genesis. - // Rebuild ingestion defers this projection until its baseline exists. - 0 - } + // The prefix through the stop block is the baseline's. Replaying it from + // the anchor could accept an old future-nonce submission the scheduler + // rejected at its original position. Only inputs after it belong here. + let Some(era) = find_era_in(conn)? else { + // Setup's sync, before the era exists: its stop block and anchor are + // not known yet, and judging the old instance's landings against an + // empty tree would record a false divergence (I15). + return Ok(()); }; - let anchor = batch_tree_anchor_in(conn)?; - let latest_accepted = query_latest_safe_accepted_batch(conn)?; - let mut cursor = latest_accepted - .map(|row| row.safe_input_index) - .unwrap_or(-1); - let mut expected = latest_accepted - .map(|row| next_expected_nonce(i64_to_u64(row.nonce))) - .unwrap_or(anchor); + let stop_block = u64_to_i64(era.stop_block); + let identity = query_deployment_identity(conn)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; + let batch_submitter = identity.batch_submitter_address; + let max_wait_blocks = identity.max_wait_blocks; + let mut cursor = latest_accepted_input(conn)?.unwrap_or(-1); + let mut expected = frontier_nonce(conn)?; loop { - // Materialize one page before executing any INSERTs. rusqlite's row - // iterator borrows the prepared statement, so we can't INSERT on the + // Materialize one page before executing any UPDATEs. rusqlite's row + // iterator borrows the prepared statement, so we can't write on the // same connection while iterating. Once the page is collected and the - // statement is dropped, the connection is free for inserts. + // statement is dropped, the connection is free for writes. let page: Vec<(i64, Vec, i64, Vec)> = { let mut stmt = conn.prepare_cached(SELECT_SQL)?; stmt.query_map( - params![ - batch_submitter.as_slice(), - cursor, - base_safe_block, - PAGE_SIZE - ], + params![batch_submitter.as_slice(), cursor, stop_block, PAGE_SIZE], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), )? .collect::>()? @@ -154,7 +113,7 @@ pub(super) fn populate_safe_accepted_batches( } let page_len = page.len() as i64; - let mut insert_stmt = conn.prepare_cached(INSERT_SQL)?; + let mut accept_stmt = conn.prepare_cached(ACCEPT_SQL)?; for (safe_input_index, payload, block_number, transaction_hash) in &page { cursor = *safe_input_index; let input = SafeInputView { @@ -163,7 +122,9 @@ pub(super) fn populate_safe_accepted_batches( payload: payload.as_slice(), inclusion_block: i64_to_u64(*block_number), }; - let Some(accepted) = timing.scheduler_accepts(batch_submitter, input, expected) else { + let Some(accepted) = + scheduler_accepts(batch_submitter, max_wait_blocks, input, expected) + else { continue; }; @@ -189,17 +150,15 @@ pub(super) fn populate_safe_accepted_batches( ); // Stop scanning; the marker (committed with this sync) // freezes the frontier and routes every subsequent boot and - // detector tick to refusal, and the lane's next time-gated + // landing-watch tick to the terminal stop, and the lane's next time-gated // frontier read (`Storage::safe_frontier_state`) returns // `CanonicalDivergence` instead of an open frontier (I15). return Ok(()); } - let changed = insert_stmt.execute(params![ + let changed = accept_stmt.execute(params![ u64_to_i64(accepted.safe_input_index), u64_to_i64(accepted.nonce), - u64_to_i64(accepted.first_frame_safe_block), - u64_to_i64(accepted.inclusion_block), ])?; if changed != 1 { return Err(rusqlite::Error::StatementChangedRows(changed)); @@ -218,11 +177,11 @@ pub(super) fn populate_safe_accepted_batches( /// How an accepted landing failed the content-identity check. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum DivergenceKind { - /// No valid closed local batch exists at the accepted nonce — we never - /// (validly) submitted one, so the landing is a zombie or foreign batch - /// from our key. + /// No closed local batch exists at the accepted nonce — we never + /// submitted one, so the landing is a zombie or foreign batch from our + /// key. Foreign, - /// A valid closed local batch exists, but the landed wire bytes hash + /// A closed local batch exists, but the landed wire bytes hash /// differently from the bytes we sealed. Mismatch, } @@ -237,7 +196,7 @@ impl DivergenceKind { } /// Content-identity check proper: compare a fully-accepted landing against -/// our valid closed batch at the same nonce. `None` = identical (the normal +/// our closed batch at the same nonce. `None` = identical (the normal /// case); a mismatch also returns our payload hash. /// /// Why content (not identity) suffices: accepted content-equal copies are @@ -251,7 +210,7 @@ fn content_identity_violation( ) -> Result>)>> { let ours: Option>> = conn .query_row( - "SELECT payload_hash FROM valid_closed_batches WHERE nonce = ?1", + "SELECT payload_hash FROM closed_batches WHERE nonce = ?1", params![u64_to_i64(accepted.nonce)], |row| row.get(0), ) @@ -267,7 +226,7 @@ fn content_identity_violation( Err(rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CORRUPT), Some(format!( - "valid closed batch at nonce {} has no payload_hash", + "closed batch at nonce {} has no payload_hash", accepted.nonce )), )) @@ -283,16 +242,24 @@ fn content_identity_violation( } } -/// The persisted divergence marker, if any: `(nonce, safe_input_index)`. -pub(super) fn canonical_divergence_in(conn: &Connection) -> Result> { +/// The persisted canonical-divergence marker (I15): the diverged landing's +/// batch nonce and the safe input that carried it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Divergence { + pub nonce: u64, + pub safe_input_index: u64, +} + +/// The persisted divergence marker, if any. +pub(super) fn canonical_divergence_in(conn: &Connection) -> Result> { conn.query_row( "SELECT nonce, safe_input_index FROM canonical_divergence WHERE singleton_id = 0", [], |row| { - Ok(( - i64_to_u64(row.get::<_, i64>(0)?), - i64_to_u64(row.get::<_, i64>(1)?), - )) + Ok(Divergence { + nonce: i64_to_u64(row.get::<_, i64>(0)?), + safe_input_index: i64_to_u64(row.get::<_, i64>(1)?), + }) }, ) .optional() @@ -301,7 +268,7 @@ pub(super) fn canonical_divergence_in(conn: &Connection) -> Result( - 0 - )) - .expect("accepted count"), - 1 - ); - } - } - - fn insert_safe_input_zero(storage: &Storage) { - storage - .conn - .execute( - "INSERT INTO safe_inputs \ - (safe_input_index, sender, payload, block_number, block_timestamp, transaction_hash) \ - VALUES (0, ?1, X'', 0, 0, ?2)", - params![Address::ZERO.as_slice(), [0_u8; 32].as_slice()], - ) - .expect("insert parent safe input"); - } - - #[test] - fn safe_accepted_batch_nonce_constraint_rejects_negative_values() { - let db = temp_db("safe-accepted-negative-nonce"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); - insert_safe_input_zero(&storage); - - let err = storage - .conn - .execute( - "INSERT INTO safe_accepted_batches \ - (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ - VALUES (0, -1, 0, 0)", - [], - ) - .expect_err("negative accepted-batch nonce must violate the schema"); - - assert!( - matches!( - err, - rusqlite::Error::SqliteFailure( - rusqlite::ffi::Error { - code: rusqlite::ErrorCode::ConstraintViolation, - .. - }, - _ - ) - ), - "unexpected error: {err}" - ); - } - - #[test] - fn accepted_batch_insert_conflicts_fail_loud() { - let db = temp_db("safe-accepted-duplicate"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); - insert_safe_input_zero(&storage); - storage - .conn - .execute( - "INSERT INTO safe_accepted_batches \ - (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ - VALUES (0, 0, 0, 0)", - [], - ) - .expect("insert accepted row"); - - let err = storage - .conn - .execute( - "INSERT INTO safe_accepted_batches \ - (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ - VALUES (0, 0, 0, 0)", - [], - ) - .expect_err("duplicate accepted row must not be ignored"); - - assert!( - matches!( - err, - rusqlite::Error::SqliteFailure( - rusqlite::ffi::Error { - code: rusqlite::ErrorCode::ConstraintViolation, - .. - }, - _ + let accepted: (i64, i64, i64) = storage + .conn + .query_row( + "SELECT COUNT(*), MAX(accepted_input), MAX(nonce) FROM batches \ + WHERE accepted_input IS NOT NULL", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) - ), - "unexpected error: {err}" - ); + .expect("read acceptances"); + assert_eq!(accepted, (1, 2, 1), "exactly one acceptance"); + } } #[test] @@ -550,17 +432,16 @@ mod tests { let _ = next_expected_nonce(u64::MAX); } #[test] - fn populate_safe_accepted_batches_resumes_from_latest_row() { + fn record_accepted_batches_resumes_from_latest_row() { let db = temp_db("safe-accepted-frontier-resume"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); for nonce in 0..4 { storage - .close_frame_and_batch(&mut head, 10) + .close_frame_and_batch(&mut head) .unwrap_or_else(|e| panic!("close {nonce}: {e}")); } seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0, 1]); @@ -588,7 +469,7 @@ mod tests { }, ]; storage - .append_safe_inputs(11, second_wave.as_slice(), SENDER_A, &protocol) + .append_safe_inputs(11, second_wave.as_slice(), SENDER_A) .expect("append second wave"); let frontier = storage.submitter_frontier().expect("submitter frontier"); @@ -597,60 +478,107 @@ mod tests { let accepted_count: i64 = storage .conn - .query_row("SELECT COUNT(*) FROM safe_accepted_batches", [], |row| { - row.get(0) - }) + .query_row( + "SELECT COUNT(*) FROM batches WHERE accepted_input IS NOT NULL", + [], + |row| row.get(0), + ) .expect("count accepted rows"); assert_eq!(accepted_count, 4); } + /// Acceptance judges staleness by the pinned window, not the wallet's: + /// under a pinned window of 50, a batch landing `s + 49` is accepted and + /// one landing `s + 50` is not. #[test] - fn safe_accepted_frontier_skips_stale_payloads() { - let db = temp_db("safe-accepted-frontier-skip-stale"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); - - // Local batch 0 (first frame safe_block=1900) and batch 1 (first frame - // safe_block=1900). Their landings carry the real wire bytes so the - // content-identity check accepts them. + fn acceptance_uses_the_pinned_window() { + let db = temp_db("accepted-pinned-window"); + let mut storage = + crate::storage::test_helpers::set_up_storage_with_window(&db.path, SENDER_A, 50); let mut head = storage - .initialize_open_state(1900, SafeInputRange::empty_at(0)) + .initialize_open_state(100, SafeInputRange::empty_at(0)) .expect("initialize"); + storage.close_frame_and_batch(&mut head).expect("close 0"); + storage.close_frame_and_batch(&mut head).expect("close 1"); + let first = local_batch_payload(&mut storage, 0); + let second = local_batch_payload(&mut storage, 1); storage - .close_frame_and_batch(&mut head, 1900) - .expect("close 0"); - storage - .close_frame_and_batch(&mut head, 1900) - .expect("close 1"); + .append_safe_inputs( + 150, + &[ + StoredSafeInput { + sender: SENDER_A, + payload: first, + block_number: 149, + }, + StoredSafeInput { + sender: SENDER_A, + payload: second, + block_number: 150, + }, + ], + SENDER_A, + ) + .expect("append"); + assert_eq!( + storage + .submitter_frontier() + .expect("frontier") + .accepted_next_nonce, + 1, + "batch 0 at s + 49 is accepted; batch 1 at s + 50 is stale" + ); + assert_eq!(storage.canonical_divergence().expect("marker"), None); + } + + #[test] + fn accepted_frontier_skips_stale_payloads() { + let db = temp_db("safe-accepted-frontier-skip-stale"); + let mut storage = set_up_storage(&db.path, SENDER_A); + + // Local batch 0 and batch 1, both with first frame at `first_frame`. + // Their landings carry the real wire bytes so the content-identity + // check accepts them. Every block derives from the window, so the + // stale copy is stale and the real landings fresh whatever it is. + let first_frame = WINDOW + 700; + let landing = first_frame + 100; + let stale_safe_block = 100; + assert!(landing - stale_safe_block >= WINDOW); + let mut head = storage + .initialize_open_state(first_frame, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage.close_frame_and_batch(&mut head).expect("close 0"); + storage.close_frame_and_batch(&mut head).expect("close 1"); - // Non-stale landing of batch 0 (block 1950 - safe_block 1900 < 1200). + // Fresh landing of batch 0, 50 blocks after its first frame. let non_stale_payload = local_batch_payload(&mut storage, 0); - // Stale copy at nonce 1 (safe_block=100, block 2000 → age >= 1200): - // a scheduler no-op, so its content is never compared — synthetic - // bytes are deliberate here. - let stale_payload = super::super::test_helpers::make_stale_batch_payload(1, 100); - // Non-stale landing of batch 1 (block 2000 - safe_block 1900 < 1200). + // Stale copy at nonce 1 (its first frame a window or more before its + // landing): a scheduler no-op, so its content is never compared — + // synthetic bytes are deliberate here. + let stale_payload = + super::super::test_helpers::make_stale_batch_payload(1, stale_safe_block); + // Fresh landing of batch 1, 100 blocks after its first frame. let non_stale_payload_2 = local_batch_payload(&mut storage, 1); let inputs = vec![ StoredSafeInput { sender: SENDER_A, payload: non_stale_payload, - block_number: 1950, + block_number: first_frame + 50, }, StoredSafeInput { sender: SENDER_A, payload: stale_payload, - block_number: 2000, + block_number: landing, }, StoredSafeInput { sender: SENDER_A, payload: non_stale_payload_2, - block_number: 2000, + block_number: landing, }, ]; storage - .append_safe_inputs(2000, inputs.as_slice(), SENDER_A, &protocol) + .append_safe_inputs(landing, inputs.as_slice(), SENDER_A) .expect("append"); let frontier = storage.submitter_frontier().expect("submitter frontier"); @@ -663,18 +591,18 @@ mod tests { // be the keccak256 of exactly the bytes the submitter will broadcast // (`pending_batches`'s encoding) — same code path, by construction. let db = temp_db("seal-stamps-payload-hash"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_frame_and_batch(&mut head) .expect("close batch 0"); let stamped: Vec = storage .conn .query_row( - "SELECT payload_hash FROM batches WHERE batch_index = 0", + "SELECT payload_hash FROM batches WHERE nonce = 0", [], |row| row.get(0), ) @@ -694,13 +622,12 @@ mod tests { // 'mismatch' marker atomically with the sync and freezes the // frontier; later syncs stay frozen. let db = temp_db("mismatch-divergence"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_frame_and_batch(&mut head) .expect("close batch 0"); // Same nonce + same first-frame safe block (fresh by inclusion), @@ -715,7 +642,6 @@ mod tests { block_number: 20, }], SENDER_A, - &protocol, ) .expect("sync commits; the marker rides the same transaction"); @@ -743,7 +669,6 @@ mod tests { block_number: 21, }], SENDER_A, - &protocol, ) .expect("append after marker"); let frontier = storage.submitter_frontier().expect("frontier"); @@ -752,17 +677,18 @@ mod tests { #[test] fn sim_accepted_foreign_batch_freezes_frontier_with_divergence_marker() { - // `scheduler_accepts` deliberately omits the two structural - // rejections (future safe_block, non-monotonic frames) under - // self-trust — the sequencer never produces them. Before the + // `scheduler_accepts` deliberately omits the canonical scheduler's + // structural rejections (a frame safe_block not below the inclusion + // block, non-monotonic frames) under self-trust — the sequencer never + // produces them. Before the // content-identity check, such a foreign batch at the // expected nonce would be sim-accepted and silently desync the // frontier forever. With the check, it fails the local-batch lookup // (kind = foreign), the poison marker persists atomically with the - // sync, the frontier freezes, and `check_danger` reports - // `CanonicalDivergence` ahead of every other arm. + // sync, the frontier freezes, and `terminal_fact` reports the + // divergence. let db = temp_db("frontier-future-safe-block"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, Address::repeat_byte(0xCC)); let future_safe_block_payload = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { nonce: 0, @@ -774,14 +700,13 @@ mod tests { }); let batch_submitter = Address::repeat_byte(0xCC); - let protocol = ProtocolTiming::try_new(u64::MAX, 75, 900, 12).expect("valid timing"); let inputs = vec![StoredSafeInput { sender: batch_submitter, payload: future_safe_block_payload, block_number: 100, }]; storage - .append_safe_inputs(200, inputs.as_slice(), batch_submitter, &protocol) + .append_safe_inputs(200, inputs.as_slice(), batch_submitter) .expect("append commits; the marker rides the same transaction"); let frontier = storage.submitter_frontier().expect("submitter frontier"); @@ -789,41 +714,50 @@ mod tests { frontier.accepted_next_nonce, 0, "the frontier must freeze before the diverged landing" ); - let status = storage - .check_danger(&protocol, unix_now_ms()) - .expect("check_danger"); assert_eq!( - status, - crate::storage::DangerStatus::CanonicalDivergence(0), - "the marker outranks every other danger arm" + storage.terminal_fact().expect("terminal fact"), + Some(crate::storage::TerminalFact::Divergence(Divergence { + nonce: 0, + safe_input_index: 0, + })), + "the marker is the era's terminal fact" ); } #[test] - fn populate_accepts_post_recovery_batch_at_anchor() { + fn acceptance_records_post_recovery_batch_at_anchor() { // A cockroach-recovered deployment is anchored at N' > 0; its first - // post-recovery batch lands at nonce N'. populate must seed `expected` + // post-recovery batch lands at nonce N'. The writer must seed `expected` // from the anchor and accept it (frontier → N'+1), content-identity - // passing against the genuine local batch. Every other populate test + // passing against the genuine local batch. Every other acceptance test // uses anchor=0, so the N' > 0 acceptance path had no coverage. let db = temp_db("populate-at-anchor"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = Storage::open_with_era( + db.path.as_str(), + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); let submitter = SENDER_A; - // Root the tree at N' = 7, then build + seal one local batch — it carries - // nonce 7 (compute_next_nonce(None) reads the anchor). - storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); + // The era is anchored at N' = 7: build + seal one local batch — with + // no batch before it, it carries nonce 7 (the anchor). let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); + assert_eq!( + head.batch_nonce(), + 7, + "the era's first batch carries the anchor nonce N'" + ); storage - .close_frame_and_batch(&mut head, 10) + .close_frame_and_batch(&mut head) .expect("close the nonce-7 batch"); assert_eq!( - storage.batch_nonce(0).expect("nonce"), - 7, - "the root tip carries the anchor nonce N'" + head.batch_nonce(), + 8, + "the successor carries the next nonce" ); // Its genuine wire bytes (looked up by nonce N'=7) land from the @@ -838,7 +772,6 @@ mod tests { block_number: 20, }], submitter, - &protocol, ) .expect("sync the N' landing"); @@ -863,20 +796,24 @@ mod tests { } #[test] - fn populate_skips_below_anchor_collapsed_history_without_divergence() { - // Below the anchor the local tree has no batches (that history was folded - // into S'). Old-generation L1 landings at nonces < N' are trusted + fn acceptance_skips_below_anchor_collapsed_history_without_divergence() { + // Below the anchor the local list has no batches (that history was folded + // into S'). Earlier-era L1 landings at nonces < N' are trusted // collapsed history: skipped by nonce-mismatch BEFORE the content-identity // check, so they must NOT be flagged Foreign (which would freeze the // frontier and force a false recovery). Pins the anchor-aware-frontier // (I15) skip arm — the inverse of the foreign-batch freeze test above. let db = temp_db("populate-below-anchor"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = Storage::open_with_era( + db.path.as_str(), + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); let submitter = SENDER_A; - storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); - // Old-generation batches at nonces 0, 1, 2 (< anchor) land from the submitter. + // Earlier-era batches at nonces 0, 1, 2 (< anchor) land from the submitter. let mk = |nonce: u64| { ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { nonce, @@ -895,7 +832,7 @@ mod tests { }) .collect(); storage - .append_safe_inputs(30, &landings, submitter, &protocol) + .append_safe_inputs(30, &landings, submitter) .expect("sync below-anchor landings"); let divergence: i64 = storage @@ -919,15 +856,14 @@ mod tests { } #[test] - fn populate_safe_accepted_batches_skips_duplicate_nonces() { + fn record_accepted_batches_skips_duplicate_nonces() { let db = temp_db("populate-dup-nonces"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("init"); - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_frame_and_batch(&mut head).expect("close"); let landed = local_batch_payload(&mut storage, 0); storage @@ -946,7 +882,6 @@ mod tests { }, ], SENDER_A, - &protocol, ) .expect("append"); @@ -958,15 +893,14 @@ mod tests { } #[test] - fn populate_safe_accepted_batches_handles_large_nonce_gap() { + fn record_accepted_batches_handles_large_nonce_gap() { let db = temp_db("populate-nonce-gap"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("init"); - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_frame_and_batch(&mut head).expect("close"); storage .append_safe_inputs( @@ -977,7 +911,6 @@ mod tests { block_number: 20, }], SENDER_A, - &protocol, ) .expect("append"); @@ -986,18 +919,15 @@ mod tests { } #[test] - fn populate_safe_accepted_batches_out_of_order_arrivals_stalls_frontier() { + fn record_accepted_batches_out_of_order_arrivals_stalls_frontier() { let db = temp_db("populate-out-of-order"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("init"); - storage.close_frame_and_batch(&mut head, 10).expect("close"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close 2"); + storage.close_frame_and_batch(&mut head).expect("close"); + storage.close_frame_and_batch(&mut head).expect("close 2"); storage .append_safe_inputs( @@ -1008,7 +938,6 @@ mod tests { block_number: 20, }], SENDER_A, - &protocol, ) .expect("append"); @@ -1028,7 +957,6 @@ mod tests { block_number: 21, }], SENDER_A, - &protocol, ) .expect("append nonce 0"); @@ -1038,4 +966,155 @@ mod tests { "frontier must remain stalled" ); } + + fn accepted_count(storage: &Storage) -> i64 { + storage + .conn + .query_row( + "SELECT COUNT(*) FROM batches WHERE accepted_input IS NOT NULL", + [], + |row| row.get(0), + ) + .expect("count accepted rows") + } + + /// A submitter batch at nonce `nonce` with no frames: the simulated + /// scheduler accepts it at the expected nonce, and no local batch matches. + fn empty_batch(nonce: u64) -> Vec { + ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { + nonce, + frames: Vec::new(), + }) + } + + #[test] + fn an_append_before_the_era_leaves_the_frontier_empty() { + use crate::storage::LifecycleCommand; + + let db = temp_db("accepted-before-era"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup) + .expect("initialize setup"); + // Judged against the empty tree, this landing would read as Foreign. + assert!( + scheduler_accepts( + SENDER_A, + WINDOW, + SafeInputView { + safe_input_index: 0, + sender: SENDER_A, + payload: &empty_batch(0), + inclusion_block: 10, + }, + 0 + ) + .is_some() + ); + storage + .append_safe_inputs( + 20, + &[ + StoredSafeInput { + sender: SENDER_A, + payload: empty_batch(0), + block_number: 10, + }, + StoredSafeInput { + sender: SENDER_B, + payload: vec![0xaa], + block_number: 15, + }, + ], + SENDER_A, + ) + .expect("setup's sync appends"); + + assert_eq!(storage.safe_input_end_exclusive().expect("inputs"), 2); + assert_eq!(storage.current_safe_block().expect("head"), Some(20)); + assert_eq!(accepted_count(&storage), 0); + assert_eq!(storage.canonical_divergence().expect("marker"), None); + } + + #[test] + fn an_append_with_an_era_and_no_pinned_identity_fails_and_rolls_back() { + use crate::storage::IngestedSafeInput; + + let db = temp_db("accepted-era-without-identity"); + // An era with no identity pinned. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + let err = storage + .append_ingested_safe_inputs_with_timestamp( + 20, + 240, + &[IngestedSafeInput { + sender: SENDER_B, + payload: vec![0xaa], + block_number: 15, + block_timestamp: 180, + transaction_hash: alloy_primitives::B256::repeat_byte(0x01), + }], + ) + .expect_err("the era's frontier needs the pinned submitter"); + assert!( + crate::storage::is_persistent_storage_error(&err), + "a missing identity is persistent, got {err:?}" + ); + assert_eq!(storage.safe_input_end_exclusive().expect("inputs"), 0); + assert_eq!(storage.current_safe_block().expect("head"), None); + } + + #[test] + fn the_first_append_after_the_era_judges_every_input_past_the_stop_block() { + use crate::storage::{LifecycleCommand, NewEra}; + + let db = temp_db("accepted-first-after-era"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup) + .expect("initialize setup"); + // Setup's sync ingests past the stop block 30: a submitter landing at + // the anchor nonce 1 inside the baseline (opaque) and one after it. + storage + .append_safe_inputs( + 40, + &[ + StoredSafeInput { + sender: SENDER_A, + payload: empty_batch(1), + block_number: 25, + }, + StoredSafeInput { + sender: SENDER_A, + payload: empty_batch(1), + block_number: 35, + }, + ], + SENDER_A, + ) + .expect("setup's sync appends"); + assert_eq!(accepted_count(&storage), 0); + assert_eq!(storage.canonical_divergence().expect("marker"), None); + + storage + .publish_era( + &db._dir.path().join("baseline"), + &NewEra { + anchor_batch_nonce: 1, + ..crate::storage::test_helpers::era(0, 0, 30) + }, + ) + .expect("publish the era"); + // `run`'s first sync lands nothing new, yet judges the landing past + // the stop block that setup's sync ingested: no local batch carries + // nonce 1, so it is Foreign. + storage + .append_safe_inputs(41, &[], SENDER_A) + .expect("run's first sync"); + assert_eq!( + storage.canonical_divergence().expect("marker"), + Some(super::Divergence { + nonce: 1, + safe_input_index: 1 + }), + "the post-stop landing is judged; the baseline one is not" + ); + assert_eq!(accepted_count(&storage), 0); + } } diff --git a/sequencer/src/storage/egress/historical.rs b/sequencer/src/storage/egress/historical.rs index a988f8e0..a599c70f 100644 --- a/sequencer/src/storage/egress/historical.rs +++ b/sequencer/src/storage/egress/historical.rs @@ -1,36 +1,25 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Era-pinned raw L1 pages and their immutable handoff to application history. +//! `/history`'s metadata: the era, its baseline and the accepted checkpoint, in one read. -use alloy_primitives::{Address, B256}; -use rusqlite::{Connection, OptionalExtension, params}; -use sequencer_core::history::{ - EraId, ExecutedInputCount, HistoryBounds, HistoryPolicyError, RecoveryGeneration, -}; +use sequencer_core::history::{EraId, HistoryPolicyError}; use sequencer_core::history_api::{ - AcceptedCheckpoint, HISTORICAL_INPUT_MAX_ITEMS, HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES, - HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryBaseline, - HistoryCompatibility, HistoryDeployment, HistoryInfo, + AcceptedCheckpoint, HistoryBaseline, HistoryDeployment, HistoryInfo, }; use crate::storage::Storage; -use crate::storage::convert::{i64_to_u64, u64_to_i64}; -use crate::storage::history::{ - next_executed_input_count_in, preserved_input_count_in, query_history_state, -}; +use crate::storage::history::{era_in, history_bounds_from}; use crate::storage::l1_inputs::query_deployment_identity; -use crate::storage::mutations::batch_tree_anchor_in; -use crate::storage::snapshot_dumps::{ - FinalizedSelectionError, finalized_dump_in, has_rollback_safe_snapshot_in, +use crate::storage::queries::safe_input_count_through_block_in; +use crate::storage::snapshots::{ + FinalizedSelectionError, finalized_dump_in, has_accepted_snapshot_in, }; #[derive(Debug, thiserror::Error)] pub(crate) enum HistoricalReadError { #[error(transparent)] Policy(#[from] HistoryPolicyError), - #[error("{0}")] - BadRequest(String), #[error(transparent)] Checkpoint(#[from] FinalizedSelectionError), #[error("reading history: {0}")] @@ -41,25 +30,11 @@ impl Storage { pub(crate) fn history_info( &mut self, expected_era: Option, - from_generation: Option, ) -> Result { self.read(|tx| { - let state = query_history_state(tx)?; - if expected_era.is_some_and(|era| era != state.version.era_id) { - return Ok(Err(HistoryPolicyError::EraChanged { - current: state.version, - } - .into())); - } - if from_generation.is_some() && expected_era.is_none() { - return Ok(Err(HistoricalReadError::BadRequest( - "era_id is required with from_generation".to_owned(), - ))); - } - if from_generation.is_some_and(|from| from > state.version.recovery_generation) { - return Ok(Err(HistoricalReadError::BadRequest( - "from_generation exceeds the current recovery generation".to_owned(), - ))); + let era = era_in(tx)?; + if expected_era.is_some_and(|expected| expected != era.id) { + return Ok(Err(HistoryPolicyError::EraChanged { era_id: era.id }.into())); } let accepted = match finalized_dump_in(tx) { Ok(accepted) => accepted, @@ -69,21 +44,10 @@ impl Storage { query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; if accepted.is_none() { assert!( - has_rollback_safe_snapshot_in(tx)?, - "history has no rollback-safe snapshot" + has_accepted_snapshot_in(tx)?, + "history has no accepted snapshot" ); } - let head = next_executed_input_count_in(tx)?; - let compatibility = from_generation - .map(|from| { - preserved_input_count_in(tx, from, state.version.recovery_generation, head).map( - |preserved_input_count| HistoryCompatibility { - from_generation: from, - preserved_input_count, - }, - ) - }) - .transpose()?; Ok(Ok(HistoryInfo { deployment: HistoryDeployment { chain_id: deployment.chain_id, @@ -92,171 +56,21 @@ impl Storage { app_deployment_block: deployment.app_deployment_block, batch_submitter_address: deployment.batch_submitter_address, }, - history: HistoryBounds { - version: state.version, - available_from: ExecutedInputCount::new(state.base_executed_input_count), - head, - }, + history: history_bounds_from(tx, &era)?, baseline: HistoryBaseline { - l1_stop_block: state.base_safe_block, - l1_end_input_index: historical_end_in(tx, state.base_safe_block)?, - next_batch_nonce: batch_tree_anchor_in(tx)?, + l1_stop_block: era.stop_block, + l1_end_input_index: safe_input_count_through_block_in(tx, era.stop_block)?, + next_batch_nonce: era.anchor_batch_nonce, + rebuilt_history_end: era.rebuilt_end, }, - accepted_checkpoint: accepted.map(|snapshot| AcceptedCheckpoint { - inclusion_block: snapshot.inclusion_block, - executed_input_count: snapshot.executed_input_count, - next_batch_nonce: snapshot.next_batch_nonce, + accepted_checkpoint: accepted.map(|finalized| AcceptedCheckpoint { + inclusion_block: finalized.inclusion_block, + executed_input_count: finalized.snapshot.l2_tx_count, + next_batch_nonce: finalized.next_batch_nonce, }), - compatibility, })) })? } - - pub(crate) fn historical_l1_inputs( - &mut self, - era: EraId, - start: HistoricalL1InputStart, - limit: usize, - ) -> Result { - self.read(|tx| { - let state = query_history_state(tx)?; - if era != state.version.era_id { - return Ok(Err(HistoryPolicyError::EraChanged { - current: state.version, - } - .into())); - } - let end = historical_end_in(tx, state.base_safe_block)?; - if !(1..=HISTORICAL_INPUT_MAX_ITEMS).contains(&limit) { - return Ok(Err(HistoricalReadError::BadRequest(format!( - "limit must be between 1 and {HISTORICAL_INPUT_MAX_ITEMS}" - )))); - } - let next = match start { - HistoricalL1InputStart::NextInputIndex(next) if next <= end => next, - HistoricalL1InputStart::AfterBlock(block) if block <= state.base_safe_block => { - first_input_after_block_in(tx, block, state.base_safe_block)?.unwrap_or(end) - } - HistoricalL1InputStart::NextInputIndex(_) => { - return Ok(Err(HistoricalReadError::BadRequest(format!( - "next_input_index exceeds historical end {end}" - )))); - } - HistoricalL1InputStart::AfterBlock(_) => { - return Ok(Err(HistoricalReadError::BadRequest(format!( - "after_block exceeds historical stop block {}", - state.base_safe_block - )))); - } - }; - raw_page_in(tx, era, state.base_safe_block, end, next, limit).map(Ok) - })? - } -} - -fn historical_end_in(conn: &Connection, stop: u64) -> rusqlite::Result { - let last: Option = conn - .query_row( - "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 \ - ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", - [u64_to_i64(stop)], - |row| row.get(0), - ) - .optional()?; - Ok(last.map_or(0, |index| { - i64_to_u64(index) - .checked_add(1) - .expect("historical input index overflow") - })) -} - -fn first_input_after_block_in( - conn: &Connection, - block: u64, - stop: u64, -) -> rusqlite::Result> { - conn.query_row( - "SELECT safe_input_index FROM safe_inputs WHERE block_number > ?1 AND block_number <= ?2 \ - ORDER BY block_number, safe_input_index LIMIT 1", - params![u64_to_i64(block), u64_to_i64(stop)], - |row| Ok(i64_to_u64(row.get(0)?)), - ) - .optional() -} - -fn raw_page_in( - conn: &Connection, - era_id: EraId, - stop: u64, - end: u64, - mut next: u64, - limit: usize, -) -> rusqlite::Result { - let expected_len = (end - next).min(limit as u64); - let mut items = Vec::new(); - if expected_len > 0 { - let mut stmt = conn.prepare_cached( - "SELECT safe_input_index, sender, payload, block_number, block_timestamp, \ - transaction_hash, length(payload) \ - FROM safe_inputs WHERE safe_input_index >= ?1 AND safe_input_index <= ?2 \ - ORDER BY safe_input_index LIMIT ?3", - )?; - let mut rows = stmt.query(params![ - u64_to_i64(next), - u64_to_i64(end - 1), - u64_to_i64(expected_len), - ])?; - let mut payload_bytes = 0_u64; - let mut byte_limited = false; - while let Some(row) = rows.next()? { - let input_index = i64_to_u64(row.get(0)?); - assert_eq!(input_index, next, "historical L1 page has an input gap"); - let payload_len = i64_to_u64(row.get(6)?); - // Budget before copying the BLOB out of SQLite. The first row may - // exceed the target so every valid L1 input remains consumable. - if !items.is_empty() - && payload_len > (HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 - payload_bytes) - { - byte_limited = true; - break; - } - let block_number = i64_to_u64(row.get(3)?); - assert!( - block_number <= stop, - "historical L1 page exceeds its stop block" - ); - items.push(HistoricalL1Input { - input_index, - sender: Address::from_slice(&row.get::<_, Vec>(1)?), - payload: row.get::<_, Vec>(2)?.into(), - block_number, - block_timestamp: i64_to_u64(row.get(4)?), - transaction_hash: B256::from_slice(&row.get::<_, Vec>(5)?), - }); - next = next - .checked_add(1) - .expect("historical input index overflow"); - payload_bytes += payload_len; - if payload_bytes > HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 { - byte_limited = true; - break; - } - } - if !byte_limited { - assert_eq!( - items.len() as u64, - expected_len, - "historical L1 page ended before its recorded end" - ); - } - } - Ok(HistoricalL1InputsPage { - era_id, - l1_stop_block: stop, - end_input_index: end, - next_input_index: next, - items, - }) } #[cfg(test)] diff --git a/sequencer/src/storage/egress/historical/tests.rs b/sequencer/src/storage/egress/historical/tests.rs index eaa331e2..7c630c18 100644 --- a/sequencer/src/storage/egress/historical/tests.rs +++ b/sequencer/src/storage/egress/historical/tests.rs @@ -3,29 +3,26 @@ use std::path::Path; +use alloy_primitives::{Address, B256}; +use sequencer_core::history::L2TxCount; + use super::*; -use crate::storage::history::{advance_recovery_generation_in, initialize_history_in}; -use crate::storage::test_helpers::{ - SENDER_A, SENDER_B, default_protocol_timing, local_batch_payload, -}; -use crate::storage::{ - DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, -}; +use crate::storage::history::insert_era_in; +use crate::storage::test_helpers::{SENDER_A, SENDER_B, era as era_record, local_batch_payload}; +use crate::storage::{IngestedSafeInput, LifecycleCommand, NewEra, SafeInputRange}; use crate::test_support::{TestDb, pin_test_deployment_identity, temp_db}; fn fixture(name: &str, stop: u64, count: u64, nonce: u64) -> (TestDb, Storage) { let db = temp_db(name); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); pin_test_deployment_identity(&mut storage, SENDER_A); + let record = NewEra { + anchor_batch_nonce: nonce, + ..era_record(count, count, stop) + }; + storage.write(|tx| insert_era_in(tx, &record)).unwrap(); storage - .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(count), stop)) - .unwrap(); - storage.set_batch_tree_anchor(nonce).unwrap(); - storage - .insert_baseline_snapshot( - Path::new("/snapshot/baseline"), - ExecutedInputCount::new(count), - ) + .insert_baseline_snapshot(Path::new("/snapshot/baseline"), L2TxCount::new(count)) .unwrap(); (db, storage) } @@ -42,52 +39,35 @@ fn input(block: u64, sender: Address, payload: Vec) -> IngestedSafeInput { fn append(storage: &mut Storage, safe_block: u64, inputs: &[IngestedSafeInput]) { storage - .append_ingested_safe_inputs_with_timestamp( - safe_block, - safe_block * 12, - inputs, - SENDER_A, - &default_protocol_timing(), - FrontierMode::DeferUntilAnchorSet, - ) + .append_ingested_safe_inputs_with_timestamp(safe_block, safe_block * 12, inputs) .unwrap(); } -fn era(storage: &Storage) -> EraId { - storage.history_state().unwrap().version.era_id -} - -fn page(storage: &mut Storage, next: u64, limit: usize) -> HistoricalL1InputsPage { - storage - .historical_l1_inputs( - era(storage), - HistoricalL1InputStart::NextInputIndex(next), - limit, - ) - .unwrap() +fn era(storage: &mut Storage) -> EraId { + storage.era().unwrap().id } #[test] -fn genesis_history_has_an_empty_raw_prefix_even_after_live_inputs_arrive() { +fn genesis_metadata_is_unmoved_by_live_inputs() { let (_db, mut storage) = fixture("historical-genesis", 0, 0, 0); append(&mut storage, 20, &[input(10, SENDER_B, vec![1])]); - let info = storage.history_info(None, None).unwrap(); - assert_eq!(info.history.available_from, ExecutedInputCount::ZERO); - assert_eq!(info.history.head, ExecutedInputCount::ZERO); - assert_eq!(info.compatibility, None); + let info = storage.history_info(None).unwrap(); + assert_eq!(info.history.available_from, L2TxCount::ZERO); + assert_eq!(info.history.head, L2TxCount::ZERO); assert_eq!( info.baseline, HistoryBaseline { l1_stop_block: 0, l1_end_input_index: 0, - next_batch_nonce: 0 + next_batch_nonce: 0, + rebuilt_history_end: L2TxCount::ZERO, } ); assert_eq!( info.accepted_checkpoint, Some(AcceptedCheckpoint { inclusion_block: 0, - executed_input_count: ExecutedInputCount::ZERO, + executed_input_count: L2TxCount::ZERO, next_batch_nonce: 0, }) ); @@ -99,13 +79,10 @@ fn genesis_history_has_an_empty_raw_prefix_even_after_live_inputs_arrive() { ); assert_eq!(info.deployment.app_deployment_block, 0); assert_eq!(info.deployment.batch_submitter_address, SENDER_A); - let raw = page(&mut storage, 0, 10); - assert!(raw.items.is_empty()); - assert_eq!((raw.end_input_index, raw.next_input_index), (0, 0)); } #[test] -fn rebuilt_history_preserves_raw_payloads_metadata_and_its_fixed_prefix() { +fn a_rebuilt_eras_baseline_survives_new_inputs() { let (_db, mut storage) = fixture("historical-rebuilt", 20, 41, 7); let inputs = [ input(5, SENDER_B, vec![0xaa, 0xbb]), @@ -114,206 +91,43 @@ fn rebuilt_history_preserves_raw_payloads_metadata_and_its_fixed_prefix() { input(21, SENDER_B, vec![0xcc]), ]; append(&mut storage, 25, &inputs); - let info = storage.history_info(None, None).unwrap(); - assert_eq!(info.history.available_from, ExecutedInputCount::new(41)); - assert_eq!(info.history.head, ExecutedInputCount::new(41)); + let info = storage.history_info(None).unwrap(); + assert_eq!(info.history.available_from, L2TxCount::new(41)); + assert_eq!(info.history.head, L2TxCount::new(41)); assert_eq!(info.baseline.l1_stop_block, 20); assert_eq!(info.baseline.l1_end_input_index, 3); assert_eq!(info.baseline.next_batch_nonce, 7); - assert_eq!(info.accepted_checkpoint, None); - - let first = page(&mut storage, 0, 2); - assert_eq!((first.end_input_index, first.next_input_index), (3, 2)); - for (index, actual) in first.items.iter().enumerate() { - let expected = &inputs[index]; - assert_eq!(actual.input_index, index as u64); - assert_eq!(actual.sender, expected.sender); - assert_eq!(actual.payload.as_ref(), expected.payload); - assert_eq!(actual.block_number, expected.block_number); - assert_eq!(actual.block_timestamp, expected.block_timestamp); - assert_eq!(actual.transaction_hash, expected.transaction_hash); - } - let last = page(&mut storage, first.next_input_index, 2); - assert_eq!(last.items.len(), 1); - assert_eq!(last.items[0].input_index, 2); - assert!(last.items[0].payload.is_empty()); - assert_eq!((last.end_input_index, last.next_input_index), (3, 3)); - assert!(page(&mut storage, 3, 2).items.is_empty()); + assert_eq!(info.baseline.rebuilt_history_end, L2TxCount::new(41)); + // The baseline is canonical at the stop block: the first comparison checkpoint. + assert_eq!( + info.accepted_checkpoint, + Some(AcceptedCheckpoint { + inclusion_block: 20, + executed_input_count: L2TxCount::new(41), + next_batch_nonce: 7, + }) + ); append(&mut storage, 40, &[input(35, SENDER_B, vec![9])]); - storage.write(advance_recovery_generation_in).unwrap(); - assert_eq!(page(&mut storage, 0, 2), first); - let updated = storage - .history_info(Some(info.history.version.era_id), None) - .unwrap(); + let updated = storage.history_info(Some(info.history.era_id)).unwrap(); assert_eq!(updated.baseline, info.baseline); - assert_eq!(updated.history.version.recovery_generation.get(), 1); } #[test] -fn after_block_skips_the_whole_block_and_paging_keeps_its_remaining_rows() { - let (_db, mut storage) = fixture("historical-block-seek", 30, 9, 2); - append( - &mut storage, - 40, - &[ - input(5, SENDER_B, vec![0]), - input(12, SENDER_A, vec![1]), - input(12, SENDER_B, vec![2]), - input(20, SENDER_B, vec![3]), - input(20, SENDER_A, vec![4]), - input(31, SENDER_B, vec![5]), - ], - ); - let current = era(&storage); - let first = storage - .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(12), 1) - .unwrap(); - assert_eq!(first.items[0].input_index, 3); - assert_eq!(first.next_input_index, 4); - assert_eq!(page(&mut storage, 4, 1).items[0].input_index, 4); - for block in [20, 25, 30] { - let eof = storage - .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(block), 1) - .unwrap(); - assert!(eof.items.is_empty()); - assert_eq!((eof.end_input_index, eof.next_input_index), (5, 5)); - } -} - -#[test] -fn era_validation_precedes_numeric_bounds_even_for_empty_history() { +fn metadata_refuses_another_era() { let (_db, mut storage) = fixture("historical-claim", 0, 0, 0); - let current = storage.history_state().unwrap().version; + let current = era(&mut storage); let other: EraId = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); - assert_ne!(other, current.era_id); - for start in [ - HistoricalL1InputStart::NextInputIndex(0), - HistoricalL1InputStart::NextInputIndex(u64::MAX), - HistoricalL1InputStart::AfterBlock(u64::MAX), - ] { - for limit in [0, 1, usize::MAX] { - assert!(matches!( - storage.historical_l1_inputs(other, start, limit), - Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) - if actual == current - )); - } - } + assert_ne!(other, current); assert!(matches!( - storage.history_info(Some(other), None), - Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) - if actual == current + storage.history_info(Some(other)), + Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { era_id })) + if era_id == current )); - for (start, limit) in [ - (HistoricalL1InputStart::NextInputIndex(0), 0), - ( - HistoricalL1InputStart::NextInputIndex(0), - HISTORICAL_INPUT_MAX_ITEMS + 1, - ), - (HistoricalL1InputStart::NextInputIndex(1), 1), - (HistoricalL1InputStart::NextInputIndex(u64::MAX), 1), - (HistoricalL1InputStart::AfterBlock(1), 1), - (HistoricalL1InputStart::AfterBlock(u64::MAX), 1), - ] { - assert!(matches!( - storage.historical_l1_inputs(current.era_id, start, limit), - Err(HistoricalReadError::BadRequest(_)) - )); - } -} - -#[test] -fn byte_budget_makes_progress_through_oversized_inputs_without_skipping_them() { - let (_db, mut storage) = fixture("historical-byte-budget", 20, 1, 1); - let target = HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES; - append( - &mut storage, - 20, - &[ - input(1, SENDER_B, vec![1; target / 2]), - input(2, SENDER_B, vec![2; target / 2 + 1]), - input(3, SENDER_A, vec![3; target + 1]), - input(4, SENDER_B, vec![4]), - ], - ); - for (start, expected_len) in [ - (0, target / 2), - (1, target / 2 + 1), - (2, target + 1), - (3, 1), - ] { - let raw = page(&mut storage, start, HISTORICAL_INPUT_MAX_ITEMS); - assert_eq!(raw.items.len(), 1); - assert_eq!(raw.items[0].input_index, start); - assert_eq!(raw.items[0].payload.len(), expected_len); - assert_eq!(raw.next_input_index, start + 1); - assert_eq!(raw.end_input_index, 4); - } -} - -#[test] -fn exact_byte_target_and_item_cap_have_independent_boundaries() { - let (_db, mut storage) = fixture("historical-item-budget", 20, 1, 1); - let mut inputs = vec![input( - 1, - SENDER_B, - vec![7; HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES], - )]; - inputs.extend((0..HISTORICAL_INPUT_MAX_ITEMS).map(|_| input(2, SENDER_B, vec![]))); - append(&mut storage, 20, &inputs); - let first = page(&mut storage, 0, HISTORICAL_INPUT_MAX_ITEMS); - assert_eq!(first.items.len(), HISTORICAL_INPUT_MAX_ITEMS); - assert_eq!(first.next_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64); - assert_eq!(first.end_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64 + 1); - let last = page( - &mut storage, - first.next_input_index, - HISTORICAL_INPUT_MAX_ITEMS, - ); - assert_eq!(last.items.len(), 1); - assert_eq!(last.next_input_index, last.end_input_index); -} - -#[test] -#[should_panic(expected = "historical L1 page has an input gap")] -fn missing_raw_input_is_an_invariant_fault() { - let (_db, mut storage) = fixture("historical-gap", 20, 3, 1); - append( - &mut storage, - 20, - &[ - input(1, SENDER_B, vec![1]), - input(2, SENDER_B, vec![2]), - input(3, SENDER_B, vec![3]), - ], - ); - storage - .conn - .execute_batch( - "DROP TRIGGER trg_safe_inputs_not_deletable; \ - DELETE FROM safe_inputs WHERE safe_input_index=1;", - ) - .unwrap(); - let _ = page(&mut storage, 0, 3); -} - -#[test] -fn maximum_sqlite_index_has_an_unclamped_exclusive_end() { - let (_db, mut storage) = fixture("historical-max-index", 20, 1, 1); - storage.conn.execute( - "INSERT INTO safe_inputs (safe_input_index,sender,payload,block_number,block_timestamp,transaction_hash) \ - VALUES (?1,?2,?3,20,240,?4)", - params![i64::MAX, SENDER_B.as_slice(), &[1_u8][..], B256::ZERO.as_slice()], - ).unwrap(); - let max = i64::MAX as u64; - let last = page(&mut storage, max, 10); - assert_eq!(last.items.len(), 1); assert_eq!( - (last.end_input_index, last.next_input_index), - (max + 1, max + 1) + storage.history_info(Some(current)).unwrap().history.era_id, + current ); - assert!(page(&mut storage, max + 1, 10).items.is_empty()); } #[test] @@ -326,9 +140,8 @@ fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { storage .close_frame_and_batch_with_snapshot( &mut head, - 20, Path::new(&format!("/snapshot/{index}")), - ExecutedInputCount::new(41), + L2TxCount::new(41), ) .unwrap(); } @@ -344,13 +157,12 @@ fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { input(29, SENDER_A, payloads[0].clone()), input(30, SENDER_A, payloads[1].clone()), ], - SENDER_A, - &default_protocol_timing(), - FrontierMode::Populate, ) .unwrap(); - storage.gc_unreferenced_dumps().unwrap(); - let info = storage.history_info(None, None).unwrap(); + storage + .collect_snapshots(&crate::storage::SnapshotLeases::default()) + .unwrap(); + let info = storage.history_info(None).unwrap(); assert_eq!(info.baseline.next_batch_nonce, 7); assert_eq!(info.baseline.l1_stop_block, 20); assert_eq!(info.baseline.l1_end_input_index, 0); @@ -358,16 +170,16 @@ fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { info.accepted_checkpoint, Some(AcceptedCheckpoint { inclusion_block: 30, - executed_input_count: ExecutedInputCount::new(41), + executed_input_count: L2TxCount::new(41), next_batch_nonce: 9, }) ); storage .conn - .execute("DELETE FROM snapshots WHERE batch_index=1", []) + .execute("DELETE FROM snapshots WHERE batch_nonce=8", []) .unwrap(); assert!(matches!( - storage.history_info(None, None), + storage.history_info(None), Err(HistoricalReadError::Checkpoint( FinalizedSelectionError::Storage(rusqlite::Error::QueryReturnedNoRows) )) @@ -386,7 +198,7 @@ fn canonical_divergence_cannot_be_advertised_as_an_accepted_receipt() { ) .unwrap(); assert!(matches!( - storage.history_info(None, None), + storage.history_info(None), Err(HistoricalReadError::Checkpoint( FinalizedSelectionError::CanonicalDivergence )) @@ -394,106 +206,9 @@ fn canonical_divergence_cannot_be_advertised_as_an_accepted_receipt() { } #[test] -#[should_panic(expected = "history has no rollback-safe snapshot")] +#[should_panic(expected = "history has no accepted snapshot")] fn missing_baseline_is_not_reported_as_an_absent_accepted_checkpoint() { let (_db, mut storage) = fixture("historical-missing-baseline", 20, 41, 7); storage.conn.execute("DELETE FROM snapshots", []).unwrap(); - let _ = storage.history_info(None, None); -} - -#[test] -fn compatibility_requires_an_era_and_rejects_future_generations_after_era_validation() { - let (_db, mut storage) = fixture("historical-generation-query", 0, 0, 0); - let current = era(&storage); - assert!(matches!( - storage.history_info(None, Some(RecoveryGeneration::new(0))), - Err(HistoricalReadError::BadRequest(_)) - )); - for from in [1, u64::MAX] { - assert!(matches!( - storage.history_info(Some(current), Some(RecoveryGeneration::new(from))), - Err(HistoricalReadError::BadRequest(_)) - )); - } - let other = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); - assert!(matches!( - storage.history_info(Some(other), Some(RecoveryGeneration::new(u64::MAX))), - Err(HistoricalReadError::Policy( - HistoryPolicyError::EraChanged { .. } - )) - )); - let info = storage - .history_info(Some(current), Some(RecoveryGeneration::new(0))) - .unwrap(); - assert_eq!( - info.compatibility, - Some(HistoryCompatibility { - from_generation: RecoveryGeneration::new(0), - preserved_input_count: ExecutedInputCount::ZERO, - }) - ); -} - -#[test] -fn full_recovery_preserves_nonzero_baseline_before_replacement_directs() { - let (_db, mut storage) = fixture("historical-generation-baseline", 100, 41, 7); - let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) - .unwrap(); - let now = crate::clock::unix_now_ms(); - let protocol = default_protocol_timing(); - storage - .append_ingested_safe_inputs_with_timestamp( - 1400, - now / 1000, - &[input(110, SENDER_B, vec![1])], - SENDER_A, - &protocol, - FrontierMode::Populate, - ) - .unwrap(); - storage - .close_frame_only_with_executions( - &mut head, - 110, - SafeInputRange::new(0, 1), - &[DirectInputExecution { - safe_input_index: 0, - executed_input_offset: ExecutedInputCount::new(41), - }], - ) - .unwrap(); - let current = era(&storage); - let before = storage - .history_info(Some(current), Some(RecoveryGeneration::new(0))) - .unwrap(); - assert_eq!(before.history.head, ExecutedInputCount::new(42)); - assert_eq!( - before.compatibility.unwrap().preserved_input_count, - before.history.head - ); - assert_eq!( - storage - .recover_aging_tip_for_recovery(head.batch_index, &protocol, now) - .unwrap(), - [0] - ); - - let after = storage - .history_info(Some(current), Some(RecoveryGeneration::new(0))) - .unwrap(); - assert_eq!(after.history.available_from, ExecutedInputCount::new(41)); - assert_eq!(after.history.head, ExecutedInputCount::new(42)); - assert_eq!(after.history.version.recovery_generation.get(), 1); - assert_eq!( - after.compatibility.unwrap().preserved_input_count, - ExecutedInputCount::new(41) - ); - let latest = storage - .history_info(Some(current), Some(RecoveryGeneration::new(1))) - .unwrap(); - assert_eq!( - latest.compatibility.unwrap().preserved_input_count, - latest.history.head - ); + let _ = storage.history_info(None); } diff --git a/sequencer/src/storage/fee_oracle.rs b/sequencer/src/storage/fee_oracle.rs index 9c1b1284..e8138b73 100644 --- a/sequencer/src/storage/fee_oracle.rs +++ b/sequencer/src/storage/fee_oracle.rs @@ -6,7 +6,7 @@ use rusqlite::{Result, params}; use super::Storage; -use super::convert::{i64_to_u64, now_unix_ms}; +use super::convert::now_unix_ms; impl Storage { pub fn log_gas_price(&self) -> Result { @@ -29,16 +29,6 @@ impl Storage { ) } - /// Unix-ms of the last successful `log_gas_price` write. `0` means never - /// written (migration default); completed setup guarantees a nonzero value. - pub fn log_gas_price_updated_at_ms(&self) -> Result { - self.conn.query_row( - "SELECT log_gas_price_updated_at_ms FROM batch_policy WHERE singleton_id = 0", - [], - |row| Ok(i64_to_u64(row.get::<_, i64>(0)?)), - ) - } - pub fn set_log_gas_price(&mut self, log_gas_price: i16) -> Result<()> { let changed = self.conn.execute( "UPDATE batch_policy \ @@ -55,13 +45,13 @@ impl Storage { #[cfg(test)] mod tests { - use crate::storage::Storage; + use crate::storage::test_helpers::{SENDER_A, set_up_storage}; use crate::test_support::temp_db; #[test] fn high_gas_price_clamps_recommended_fee_to_max_exponent() { let db = temp_db("clamp-fee"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); storage .set_log_gas_price(17000) .expect("set high gas price"); @@ -74,7 +64,7 @@ mod tests { #[test] fn negative_log_gas_price_lowers_the_recommended_fee_to_the_floor() { let db = temp_db("negative-fee"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let floor = storage.log_gas_price_floor().expect("floor"); assert_eq!(floor, -1356); storage @@ -93,7 +83,7 @@ mod tests { #[test] fn set_log_gas_price_stamps_updated_at() { let db = temp_db("fee-price-stamp"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); assert_eq!(storage.log_gas_price_updated_at_ms().unwrap(), 0); storage.set_log_gas_price(42).expect("set"); diff --git a/sequencer/src/storage/history.rs b/sequencer/src/storage/history.rs index 819d0021..5588353e 100644 --- a/sequencer/src/storage/history.rs +++ b/sequencer/src/storage/history.rs @@ -1,195 +1,246 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Immutable era baseline and the preserved prefix at each recovery generation. +//! The era (one immutable row): the history's whole identity, and where its +//! history starts and its live rows begin. -use rusqlite::{Connection, Result, Transaction, params, types::Type}; -use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; +use rusqlite::{Connection, OptionalExtension, Result, Transaction, params, types::Type}; +use sequencer_core::history::{EraId, L2TxCount}; -use super::Storage; use super::convert::{i64_to_u64, now_unix_ms, u64_to_i64}; mod canonical; #[cfg(test)] pub(crate) use canonical::ApplicationInputRow; +pub(super) use canonical::history_bounds_from; pub(crate) use canonical::{HistoryReadError, L2TxContext}; +/// The published era, as read. Immutable for the database's lifetime. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HistoryState { - pub version: HistoryVersion, - pub era_created_at_ms: u64, - pub base_executed_input_count: u64, - /// L1 prefix already accounted for by the era's initial application state. - pub base_safe_block: u64, +pub struct Era { + pub id: EraId, + pub published_at_ms: u64, + /// Where the era's history starts. + pub base: L2TxCount, + /// Where the live rows start: the baseline's count. + pub rebuilt_end: L2TxCount, + /// The L1 prefix the era's baseline accounts for. + pub stop_block: u64, + /// The oldest direct still queued at the stop block, or the input count + /// through it when none is: where the era's first frame drains from. + pub first_queued_input: u64, + /// The nonce the era's first batch carries (I16): the + /// scheduler's next batch nonce at the stop block. + pub anchor_batch_nonce: u64, } +/// What setup publishes: an [`Era`] without the id and time storage assigns. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NewEra { + pub base: L2TxCount, + pub rebuilt_end: L2TxCount, + pub stop_block: u64, + pub first_queued_input: u64, + pub anchor_batch_nonce: u64, +} + +/// One direct a drained range executed, at its history offset. A drained +/// range may hold inputs without one: the batch submitter's own inputs and +/// directs over the application's payload cap never execute. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DirectInputExecution { pub safe_input_index: u64, - pub executed_input_offset: ExecutedInputCount, + pub l2_tx_index: L2TxCount, } -impl Storage { - pub fn history_state(&self) -> Result { - query_history_state(&self.conn) - } +const ERA_SQL: &str = "SELECT era_id, published_at_ms, base_l2_tx_count, rebuilt_history_end, + stop_block, first_queued_input, anchor_batch_nonce + FROM era WHERE singleton_id = 0"; + +fn decode_era_id(row: &rusqlite::Row<'_>, index: usize) -> Result { + let blob = row.get::<_, Vec>(index)?; + EraId::try_from(blob.as_slice()).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(index, Type::Blob, Box::new(error)) + }) } -pub(super) fn query_history_state(conn: &Connection) -> Result { - conn.query_row( - "SELECT era_id, era_created_at_ms, recovery_generation, - base_executed_input_count, base_safe_block - FROM history_state WHERE singleton_id = 0", - [], - |row| { - let era_blob = row.get::<_, Vec>(0)?; - let era_id = EraId::try_from(era_blob.as_slice()).map_err(|error| { - rusqlite::Error::FromSqlConversionFailure(0, Type::Blob, Box::new(error)) - })?; - Ok(HistoryState { - version: HistoryVersion { - era_id, - recovery_generation: RecoveryGeneration::new(i64_to_u64(row.get(2)?)), - }, - era_created_at_ms: i64_to_u64(row.get(1)?), - base_executed_input_count: i64_to_u64(row.get(3)?), - base_safe_block: i64_to_u64(row.get(4)?), - }) - }, - ) +fn decode_era(row: &rusqlite::Row<'_>) -> Result { + Ok(Era { + id: decode_era_id(row, 0)?, + published_at_ms: i64_to_u64(row.get(1)?), + base: L2TxCount::new(i64_to_u64(row.get(2)?)), + rebuilt_end: L2TxCount::new(i64_to_u64(row.get(3)?)), + stop_block: i64_to_u64(row.get(4)?), + first_queued_input: i64_to_u64(row.get(5)?), + anchor_batch_nonce: i64_to_u64(row.get(6)?), + }) +} + +/// The published era. Fails with `QueryReturnedNoRows` before setup +/// publishes: every caller runs inside a set-up database. +pub(super) fn era_in(conn: &Connection) -> Result { + conn.query_row(ERA_SQL, [], decode_era) +} + +/// The published era, or `None` before setup publishes it. Only for setup's +/// own appends, to which "no era yet" is a real state. +pub(super) fn find_era_in(conn: &Connection) -> Result> { + conn.query_row(ERA_SQL, [], decode_era).optional() } -pub(super) fn initialize_history_in( - tx: &Transaction<'_>, - base: ExecutedInputCount, - base_safe_block: u64, -) -> Result<()> { +/// Insert the era under a fresh v4 era id, stamped now. The schema checks +/// the counts against the executed rebuilt rows, seals them, and refuses a +/// second era. Used by `publish_era` and the test fixture `open_with_era`. +pub(super) fn insert_era_in(tx: &Transaction<'_>, era: &NewEra) -> Result<()> { let mut bytes: [u8; EraId::BYTE_LEN] = tx.query_row("SELECT randomblob(16)", [], |row| row.get(0))?; bytes[6] = (bytes[6] & 0x0f) | 0x40; bytes[8] = (bytes[8] & 0x3f) | 0x80; - let era = EraId::from_bytes(bytes).expect("UUID bits were set above"); + let era_id = EraId::from_bytes(bytes).expect("UUID bits were set above"); tx.execute( - "INSERT INTO history_state - (singleton_id, era_id, era_created_at_ms, recovery_generation, - base_executed_input_count, base_safe_block) VALUES (0, ?1, ?2, 0, ?3, ?4)", + "INSERT INTO era + (singleton_id, era_id, published_at_ms, base_l2_tx_count, rebuilt_history_end, + stop_block, first_queued_input, anchor_batch_nonce) + VALUES (0, ?1, ?2, ?3, ?4, ?5, ?6, ?7)", params![ - era.as_bytes().as_slice(), + era_id.as_bytes().as_slice(), now_unix_ms(), - u64_to_i64(base.get()), - u64_to_i64(base_safe_block) + u64_to_i64(era.base.get()), + u64_to_i64(era.rebuilt_end.get()), + u64_to_i64(era.stop_block), + u64_to_i64(era.first_queued_input), + u64_to_i64(era.anchor_batch_nonce), ], )?; Ok(()) } -pub(super) fn next_executed_input_count_in(conn: &Connection) -> Result { - let base = query_history_state(conn)?.base_executed_input_count; - let greatest: Option = - conn.query_row("SELECT MAX(offset) FROM application_inputs", [], |row| { - row.get(0) - })?; - Ok(ExecutedInputCount::new(greatest.map_or(base, |offset| { - i64_to_u64(offset) - .checked_add(1) - .expect("application input count overflow") - }))) +/// The next history offset: one past the last live row, or the rebuilt +/// history's end before the era has any. +pub(super) fn next_l2_tx_count_in(conn: &Connection) -> Result { + next_l2_tx_count_of(conn, &era_in(conn)?) } -/// Called after suffix deletion and before the replacement Tip attributes directs. -pub(super) fn advance_recovery_generation_in(tx: &Transaction<'_>) -> Result { - let current = query_history_state(tx)?.version.recovery_generation.get(); - let next = current - .checked_add(1) - .expect("recovery generation exhausted"); - let preserved = next_executed_input_count_in(tx)?; - tx.execute( - "INSERT INTO history_generation_cuts (recovery_generation, preserved_input_count) \ - VALUES (?1, ?2)", - params![u64_to_i64(next), u64_to_i64(preserved.get())], - )?; - let changed = tx.execute( - "UPDATE history_state SET recovery_generation = ?1 WHERE singleton_id = 0", - [u64_to_i64(next)], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(RecoveryGeneration::new(next)) -} - -pub(super) fn preserved_input_count_in( - conn: &Connection, - from: RecoveryGeneration, - current: RecoveryGeneration, - head: ExecutedInputCount, -) -> Result { - assert!( - from <= current, - "compatibility starts after the current generation" - ); - if from == current { - return Ok(head); - } - let (count, minimum): (i64, Option) = conn.query_row( - "SELECT COUNT(*), MIN(preserved_input_count) FROM history_generation_cuts \ - WHERE recovery_generation > ?1 AND recovery_generation <= ?2", - params![u64_to_i64(from.get()), u64_to_i64(current.get())], - |row| Ok((row.get(0)?, row.get(1)?)), +/// [`next_l2_tx_count_in`] for a caller that already read the era. +pub(super) fn next_l2_tx_count_of(conn: &Connection, era: &Era) -> Result { + let rebuilt_end = era.rebuilt_end; + let greatest: Option = conn.query_row( + "SELECT MAX(l2_tx_count) FROM application_inputs", + [], + |row| row.get(0), )?; - // Unique integer generations plus the exact interval length prove that - // every intervening recovery contributed its cut, including empty batches. - assert_eq!( - i64_to_u64(count), - current.get() - from.get(), - "history generation lineage is incomplete" - ); - let minimum = minimum.expect("a nonempty complete generation interval has a minimum"); - Ok(head.min(ExecutedInputCount::new(i64_to_u64(minimum)))) + Ok(greatest.map_or(rebuilt_end, |offset| { + L2TxCount::new( + i64_to_u64(offset) + .checked_add(1) + .expect("application input count overflow"), + ) + })) } #[cfg(test)] mod tests { use super::*; - use crate::storage::LifecycleCommand; + use crate::storage::test_helpers::era; + use crate::storage::{LifecycleCommand, Storage}; use crate::test_support::temp_db; #[test] - fn complete_baseline_is_immutable_and_survives_restart() { + fn the_era_is_one_immutable_row_that_survives_restart() { let db = temp_db("history-baseline"); let mut storage = - Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); assert!(matches!( - storage.history_state(), + storage.era(), Err(rusqlite::Error::QueryReturnedNoRows) )); + assert_eq!(storage.read(|tx| find_era_in(tx)).unwrap(), None); + let mismatched = storage + .write(|tx| insert_era_in(tx, &era(41, 42, 70))) + .expect_err("an empty rebuild cannot end past its base"); + assert_eq!( + mismatched.to_string(), + "the era must span exactly its executed rebuilt rows" + ); + let past_stop = storage + .write(|tx| { + insert_era_in( + tx, + &NewEra { + first_queued_input: 1, + ..era(41, 41, 70) + }, + ) + }) + .expect_err("no input exists through the stop block"); + assert_eq!( + past_stop.to_string(), + "the first queued input lies past the stop block" + ); storage - .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 70)) + .write(|tx| { + insert_era_in( + tx, + &NewEra { + anchor_batch_nonce: 9, + ..era(41, 41, 70) + }, + ) + }) .unwrap(); - let state = storage.history_state().unwrap(); + let published = storage.era().unwrap(); + assert_eq!( + ( + published.base, + published.rebuilt_end, + published.stop_block, + published.first_queued_input, + published.anchor_batch_nonce + ), + (L2TxCount::new(41), L2TxCount::new(41), 70, 0, 9) + ); + assert_eq!(storage.read(|tx| find_era_in(tx)).unwrap(), Some(published)); let duplicate = storage - .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 70)) - .expect_err("even identical baseline values cannot initialize another era"); + .write(|tx| insert_era_in(tx, &era(41, 41, 70))) + .expect_err("even identical values cannot publish another era"); assert_eq!( duplicate.to_string(), - "history state is inserted once per database" + "the era is inserted once per database" ); for (sql, expected) in [ ( - "UPDATE history_state SET era_id = X'00000000000040008000000000000001'", - "history baseline is immutable", + "INSERT OR REPLACE INTO era (singleton_id, era_id, published_at_ms, \ + base_l2_tx_count, rebuilt_history_end, stop_block, first_queued_input, \ + anchor_batch_nonce) \ + VALUES (0, X'00000000000040008000000000000001', 0, 41, 41, 70, 0, 9)", + "the era is inserted once per database", + ), + ("UPDATE era SET singleton_id = 0", "the era is immutable"), + ( + "UPDATE era SET era_id = X'00000000000040008000000000000001'", + "the era is immutable", + ), + ("UPDATE era SET published_at_ms = 1", "the era is immutable"), + ( + "UPDATE era SET base_l2_tx_count = 42", + "the era is immutable", ), ( - "UPDATE history_state SET base_executed_input_count = 42", - "history baseline is immutable", + "UPDATE era SET rebuilt_history_end = 42", + "the era is immutable", ), + ("UPDATE era SET stop_block = 71", "the era is immutable"), ( - "UPDATE history_state SET base_safe_block = 71", - "history baseline is immutable", + "UPDATE era SET first_queued_input = 1", + "the era is immutable", ), ( - "DELETE FROM history_state", - "history state is write-once per database", + "UPDATE era SET anchor_batch_nonce = 10", + "the era is immutable", + ), + ( + "DELETE FROM era", + "the era is retained for the database lifetime", ), ] { let error = storage.conn.execute(sql, []).expect_err(sql); @@ -197,45 +248,7 @@ mod tests { } drop(storage); let mut reopened = Storage::open(&db.path).unwrap(); - assert_eq!(reopened.history_state().unwrap(), state); - assert_eq!(reopened.next_executed_input_count().unwrap().get(), 41); - } - - #[test] - fn generation_advances_exactly_once_and_transaction_rollback_preserves_it() { - let db = temp_db("history-generation"); - let mut storage = Storage::open(&db.path).unwrap(); - let original = storage.history_state().unwrap(); - let result: Result<()> = storage.write(|tx| { - advance_recovery_generation_in(tx)?; - Err(rusqlite::Error::InvalidQuery) - }); - assert!(result.is_err()); - assert_eq!(storage.history_state().unwrap(), original); - storage.write(advance_recovery_generation_in).unwrap(); - assert_eq!( - storage - .history_state() - .unwrap() - .version - .recovery_generation - .get(), - 1 - ); - assert!( - storage - .conn - .execute("UPDATE history_state SET recovery_generation = 3", []) - .is_err() - ); - assert!( - storage - .conn - .execute("UPDATE history_state SET recovery_generation = 0", []) - .is_err() - ); + assert_eq!(reopened.era().unwrap(), published); + assert_eq!(reopened.next_l2_tx_count().unwrap().get(), 41); } } - -#[cfg(test)] -mod generation_tests; diff --git a/sequencer/src/storage/history/canonical.rs b/sequencer/src/storage/history/canonical.rs index ad949994..0b6d6bd7 100644 --- a/sequencer/src/storage/history/canonical.rs +++ b/sequencer/src/storage/history/canonical.rs @@ -1,76 +1,72 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Canonical application history: version-checked pages read with their -//! boundaries in one transaction. Native restart catch-up and the WS feed -//! read it. +//! Canonical application history: era-checked pages read with their +//! boundaries in one transaction. The era's history is setup's rebuilt rows, +//! every one executed, up to the rebuilt history's end, then the live rows; a +//! page may span the seam. Native restart catch-up and the WS feed read it. use alloy_primitives::{Address, B256}; use rusqlite::{Connection, Row, Transaction, params}; -use sequencer_core::history::{ - ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, -}; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::history::{HistoryBounds, HistoryClaim, HistoryPolicyError, L2TxCount}; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use crate::storage::Storage; use crate::storage::convert::{ i64_to_u16, i64_to_u32, i64_to_u64, saturating_query_bound, u64_to_i64, }; -use crate::storage::history::{next_executed_input_count_in, query_history_state}; +use crate::storage::history::{Era, era_in, next_l2_tx_count_of}; +/// One executed history row: the record the executor ran, plus the batch +/// coordinates the feed reports beside it. #[derive(Debug, Clone)] pub(crate) enum L2TxContext { UserOp { - tx: ValidUserOp, - nonce: u32, - safe_block: u64, + tx: UserOpTx, batch_nonce: u64, }, - DirectInput { - tx: DirectInput, - input_index: u64, + Direct { + tx: Direct, batch_nonce: u64, - block_timestamp: u64, transaction_hash: B256, }, } #[derive(Debug, Clone)] pub(crate) struct ApplicationInputRow { - pub(crate) offset: ExecutedInputCount, + pub(crate) offset: L2TxCount, pub(crate) context: L2TxContext, } fn decode_application_input(row: &Row<'_>) -> rusqlite::Result { let sender = Address::from_slice(row.get::<_, Vec>(2)?.as_slice()); - let safe_block = i64_to_u64(row.get(7)?); let batch_nonce = i64_to_u64(row.get(8)?); let context = match row.get::<_, i64>(1)? { 0 => L2TxContext::UserOp { - tx: ValidUserOp { + tx: UserOpTx { sender, - data: row.get(3)?, + nonce: i64_to_u32(row.get(10)?), fee: i64_to_u16(row.get(4)?), + block: i64_to_u64(row.get(7)?), + data: row.get(3)?, }, - nonce: i64_to_u32(row.get(10)?), - safe_block, batch_nonce, }, - 1 => L2TxContext::DirectInput { - tx: DirectInput { + 1 => L2TxContext::Direct { + tx: Direct { + input_index: i64_to_u64(row.get(9)?), sender, + block: i64_to_u64(row.get(6)?), + timestamp: i64_to_u64(row.get(11)?), payload: row.get(5)?, - block_number: i64_to_u64(row.get(6)?), }, - input_index: i64_to_u64(row.get(9)?), batch_nonce, - block_timestamp: i64_to_u64(row.get(11)?), transaction_hash: B256::from_slice(row.get::<_, Vec>(12)?.as_slice()), }, kind => panic!("invalid application input kind {kind}"), }; Ok(ApplicationInputRow { - offset: ExecutedInputCount::new(i64_to_u64(row.get(0)?)), + offset: L2TxCount::new(i64_to_u64(row.get(0)?)), context, }) } @@ -79,13 +75,13 @@ fn decode_application_input(row: &Row<'_>) -> rusqlite::Result, - pub(crate) next_input: ExecutedInputCount, + pub(crate) next_input: L2TxCount, } impl CanonicalHistoryPage { pub(crate) fn next_claim(&self) -> HistoryClaim { HistoryClaim { - version: self.bounds.version, + era_id: self.bounds.era_id, next_input: self.next_input, } } @@ -112,9 +108,10 @@ impl Storage { limit: usize, ) -> Result { self.read(|tx| { - let bounds = history_bounds_in(tx)?; + let era = era_in(tx)?; + let bounds = history_bounds_from(tx, &era)?; match bounds.validate(claim) { - Ok(()) => canonical_page_in(tx, bounds, claim.next_input, limit).map(Ok), + Ok(()) => canonical_page_in(tx, &era, bounds, claim.next_input, limit).map(Ok), Err(error) => Ok(Err(error)), } })? @@ -123,71 +120,115 @@ impl Storage { } fn history_bounds_in(conn: &Connection) -> rusqlite::Result { - let state = query_history_state(conn)?; - let available_from = ExecutedInputCount::new(state.base_executed_input_count); + history_bounds_from(conn, &era_in(conn)?) +} + +pub(in crate::storage) fn history_bounds_from( + conn: &Connection, + era: &Era, +) -> rusqlite::Result { Ok(HistoryBounds { - version: state.version, - available_from, - head: next_executed_input_count_in(conn)?, + era_id: era.id, + available_from: era.base, + head: next_l2_tx_count_of(conn, era)?, }) } +const REBUILT_PAGE_SQL: &str = " + SELECT + r.l2_tx_count, + CASE WHEN r.safe_input_index IS NULL THEN 0 ELSE 1 END, + CASE WHEN r.safe_input_index IS NULL THEN r.sender ELSE d.sender END, + r.data, + r.fee, + d.payload, + d.block_number, + r.safe_block, + r.batch_nonce, + r.safe_input_index, + r.nonce, + d.block_timestamp, + d.transaction_hash + FROM rebuilt_history r + LEFT JOIN safe_inputs d ON d.safe_input_index = r.safe_input_index + WHERE r.l2_tx_count >= ?1 + ORDER BY r.l2_tx_count + LIMIT ?2 +"; + +const LIVE_PAGE_SQL: &str = " + SELECT + s.l2_tx_count, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE d.sender END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END, + f.safe_block, + s.batch_nonce, + s.safe_input_index, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END + FROM application_inputs s + LEFT JOIN user_ops u + ON u.batch_nonce = s.batch_nonce + AND u.frame_in_batch = s.frame_in_batch + AND u.pos_in_frame = s.user_op_pos_in_frame + LEFT JOIN frames f + ON f.batch_nonce = s.batch_nonce AND f.frame_in_batch = s.frame_in_batch + LEFT JOIN safe_inputs d ON d.safe_input_index = s.safe_input_index + WHERE s.l2_tx_count >= ?1 + ORDER BY s.l2_tx_count + LIMIT ?2 +"; + fn canonical_page_in( tx: &Transaction<'_>, + era: &Era, bounds: HistoryBounds, - from: ExecutedInputCount, + from: L2TxCount, limit: usize, ) -> rusqlite::Result { let limit = u64::try_from(limit).expect("page size fits u64"); let expected_len = limit.min(bounds.head.get() - from.get()); + let rebuilt_end = era.rebuilt_end; let mut rows = Vec::new(); + let mut expected = from; // H can be i64::MAX + 1, the boundary after the last representable row. // An empty-at-head request must never clamp back onto that last input. if expected_len > 0 { - const SQL: &str = " - SELECT - s.offset, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE d.sender END, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END, - f.safe_block, - b.nonce, - s.safe_input_index, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END - FROM application_inputs s - LEFT JOIN user_ops u - ON u.batch_index = s.batch_index - AND u.frame_in_batch = s.frame_in_batch - AND u.pos_in_frame = s.user_op_pos_in_frame - LEFT JOIN frames f - ON f.batch_index = s.batch_index AND f.frame_in_batch = s.frame_in_batch - LEFT JOIN safe_inputs d ON d.safe_input_index = s.safe_input_index - LEFT JOIN batches b ON b.batch_index = s.batch_index - WHERE s.offset >= ?1 - ORDER BY s.offset - LIMIT ?2 - "; - let mut stmt = tx.prepare_cached(SQL)?; - let mapped = stmt.query_map( - params![u64_to_i64(from.get()), saturating_query_bound(expected_len)], - decode_application_input, - )?; - let mut expected = from; - for row in mapped { - let row = row?; - assert_eq!( - row.offset, expected, - "canonical history page has an attribution gap" - ); - rows.push(row); - expected = expected - .checked_next() - .expect("canonical input count overflow"); + // The rebuilt rows cover [base, rebuilt_end) without gaps, + // and the live rows start at rebuilt_end, so one cursor spans both. + let rebuilt_len = expected_len.min(rebuilt_end.get().saturating_sub(from.get())); + for (sql, start, len) in [ + (REBUILT_PAGE_SQL, from, rebuilt_len), + ( + LIVE_PAGE_SQL, + from.max(rebuilt_end), + expected_len - rebuilt_len, + ), + ] { + if len == 0 { + continue; + } + let mut stmt = tx.prepare_cached(sql)?; + let mapped = stmt.query_map( + params![u64_to_i64(start.get()), saturating_query_bound(len)], + decode_application_input, + )?; + for row in mapped { + let row = row?; + assert_eq!( + row.offset, expected, + "canonical history page has an attribution gap" + ); + rows.push(row); + expected = expected + .checked_next() + .expect("canonical input count overflow"); + } } assert_eq!( rows.len() as u64, diff --git a/sequencer/src/storage/history/canonical/tests.rs b/sequencer/src/storage/history/canonical/tests.rs index e3da72cb..8c356e70 100644 --- a/sequencer/src/storage/history/canonical/tests.rs +++ b/sequencer/src/storage/history/canonical/tests.rs @@ -10,15 +10,14 @@ use tokio::sync::oneshot; use super::*; use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; use crate::storage::L2TxContext; -use crate::storage::history::initialize_history_in; -use crate::storage::test_helpers::{ - SENDER_A, SENDER_B, default_protocol_timing, local_batch_payload, -}; +use crate::storage::history::insert_era_in; +use crate::storage::test_helpers::{SENDER_A, SENDER_B, local_batch_payload, set_up_storage}; use crate::storage::{ - DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, - StoredSafeInput, + DirectInputExecution, IngestedSafeInput, LifecycleCommand, NewEra, SafeInputRange, }; use crate::test_support::{pin_test_deployment_identity, temp_db}; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::Scheduled; fn included(nonce: u32, offset: u64, payload: u8) -> IncludedUserOp { let (respond_to, _response) = oneshot::channel(); @@ -36,47 +35,29 @@ fn included(nonce: u32, offset: u64, payload: u8) -> IncludedUserOp { respond_to, received_at: SystemTime::now(), }, - executed_input_offset: ExecutedInputCount::new(offset), + l2_tx_index: L2TxCount::new(offset), } } fn claim(bounds: HistoryBounds, next: u64) -> HistoryClaim { HistoryClaim { - version: bounds.version, - next_input: ExecutedInputCount::new(next), + era_id: bounds.era_id, + next_input: L2TxCount::new(next), } } -fn seed_aging_tip(storage: &mut Storage) { - pin_test_deployment_identity(storage, SENDER_A); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - storage - .append_executed_user_ops_chunk(&mut head, &[included(0, 0, 0xaa)]) - .unwrap(); - storage.close_frame_and_batch(&mut head, 0).unwrap(); - storage - .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xbb)]) - .unwrap(); - storage - .append_safe_inputs(1_500, &[], SENDER_A, &default_protocol_timing()) - .unwrap(); -} - #[test] fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() { let db = temp_db("canonical-page-context"); - let mut storage = Storage::open(&db.path).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); - let first_fee = head.frame_fee; + let first_fee = head.frame_fee(); storage - .append_executed_user_ops_chunk(&mut head, &[included(7, 0, 0x11)]) + .append_included_user_ops(&mut head, &[included(7, 0, 0x11)]) .unwrap(); - storage.close_frame_and_batch(&mut head, 0).unwrap(); + storage.close_frame_and_batch(&mut head).unwrap(); let envelope = local_batch_payload(&mut storage, 0); let transaction_hash = B256::repeat_byte(0x42); storage @@ -99,9 +80,6 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() transaction_hash: B256::repeat_byte(0x43), }, ], - SENDER_A, - &default_protocol_timing(), - FrontierMode::Populate, ) .unwrap(); storage @@ -111,49 +89,42 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() SafeInputRange::new(0, 2), &[DirectInputExecution { safe_input_index: 0, - executed_input_offset: ExecutedInputCount::new(1), + l2_tx_index: L2TxCount::new(1), }], ) .unwrap(); storage - .append_executed_user_ops_chunk(&mut head, &[included(8, 2, 0x33)]) + .append_included_user_ops(&mut head, &[included(8, 2, 0x33)]) .unwrap(); let bounds = storage.history_bounds().unwrap(); - assert_eq!(bounds.available_from, ExecutedInputCount::ZERO); - assert_eq!(bounds.head, ExecutedInputCount::new(3)); + assert_eq!(bounds.available_from, L2TxCount::ZERO); + assert_eq!(bounds.head, L2TxCount::new(3)); let first = storage.canonical_history_page(claim(bounds, 0), 2).unwrap(); assert_eq!(first.bounds, bounds); assert_eq!(first.next_claim(), claim(bounds, 2)); assert_eq!(first.rows.len(), 2); - assert_eq!(first.rows[0].offset, ExecutedInputCount::ZERO); + assert_eq!(first.rows[0].offset, L2TxCount::ZERO); match &first.rows[0].context { - L2TxContext::UserOp { - tx, - nonce, - safe_block, - batch_nonce, - } => { + L2TxContext::UserOp { tx, batch_nonce } => { assert_eq!(tx.sender, SENDER_B); assert_eq!(tx.data, vec![0x11]); assert_eq!(tx.fee, first_fee); - assert_eq!((*nonce, *safe_block, *batch_nonce), (7, 0, 0)); + assert_eq!((tx.nonce, tx.block, *batch_nonce), (7, 0, 0)); } other => panic!("expected user op, got {other:?}"), } - assert_eq!(first.rows[1].offset, ExecutedInputCount::new(1)); + assert_eq!(first.rows[1].offset, L2TxCount::new(1)); match &first.rows[1].context { - L2TxContext::DirectInput { + L2TxContext::Direct { tx, - input_index, batch_nonce, - block_timestamp, transaction_hash: actual_hash, } => { assert_eq!(tx.sender, SENDER_B); assert_eq!(tx.payload, vec![0x22]); - assert_eq!(tx.block_number, 10); - assert_eq!((*input_index, *batch_nonce, *block_timestamp), (0, 1, 100)); + assert_eq!(tx.block, 10); + assert_eq!((tx.input_index, *batch_nonce, tx.timestamp), (0, 1, 100)); assert_eq!(*actual_hash, transaction_hash); } other => panic!("expected direct input, got {other:?}"), @@ -162,17 +133,12 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() .canonical_history_page(first.next_claim(), 2) .unwrap(); assert_eq!(last.rows.len(), 1); - assert_eq!(last.rows[0].offset, ExecutedInputCount::new(2)); + assert_eq!(last.rows[0].offset, L2TxCount::new(2)); match &last.rows[0].context { - L2TxContext::UserOp { - tx, - nonce, - safe_block, - batch_nonce, - } => { + L2TxContext::UserOp { tx, batch_nonce } => { assert_eq!(tx.data, vec![0x33]); - assert_eq!(tx.fee, head.frame_fee); - assert_eq!((*nonce, *safe_block, *batch_nonce), (8, 10, 1)); + assert_eq!(tx.fee, head.frame_fee()); + assert_eq!((tx.nonce, tx.block, *batch_nonce), (8, 10, 1)); } other => panic!("expected user op, got {other:?}"), } @@ -187,128 +153,205 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() assert_eq!(zero.next_claim(), claim(bounds, 1)); } -#[test] -fn recovery_refuses_old_claims_and_reuses_canonical_offsets_after_suffix_replacement() { - let db = temp_db("canonical-page-recovery"); - let mut storage = Storage::open(&db.path).unwrap(); - seed_aging_tip(&mut storage); - let before = storage.history_bounds().unwrap(); - assert_eq!(before.head, ExecutedInputCount::new(2)); - - assert_eq!(storage.recover_aging_tip(1_200).unwrap(), vec![1]); - let recovered = storage.history_bounds().unwrap(); - assert_eq!(recovered.version.era_id, before.version.era_id); - assert_eq!(recovered.version.recovery_generation.get(), 1); - assert_eq!(recovered.head, ExecutedInputCount::new(1)); - for limit in [0, 2] { - assert!(matches!( - storage.canonical_history_page(claim(before, 2), limit), - Err(HistoryReadError::Policy(HistoryPolicyError::StaleGeneration { current })) - if current == recovered.version - )); +fn scheduled_direct(input_index: u64, batch_nonce: u64) -> Scheduled { + Scheduled { + batch_nonce, + tx: L2Tx::Direct(Direct { + input_index, + sender: SENDER_B, + block: 0, + timestamp: 0, + payload: vec![], + }), } - assert!( - storage - .canonical_history_page(claim(recovered, 1), 2) - .unwrap() - .rows - .is_empty() - ); - let mut head = storage.open_state().unwrap().unwrap(); - storage - .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xcc)]) - .unwrap(); - let page = storage - .canonical_history_page(claim(recovered, 1), 2) - .unwrap(); - assert_eq!(page.rows.len(), 1); - assert_eq!(page.rows[0].offset, ExecutedInputCount::new(1)); - assert_eq!(page.next_input, before.head); - match &page.rows[0].context { - L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xcc]), - other => panic!("expected replacement user op, got {other:?}"), +} + +fn scheduled_op(nonce: u32, batch_nonce: u64) -> Scheduled { + Scheduled { + batch_nonce, + tx: L2Tx::UserOp(UserOpTx { + sender: SENDER_B, + nonce, + fee: 9, + block: 7, + data: vec![0xd0, nonce as u8], + }), } - let audit_rows: i64 = storage - .conn - .query_row("SELECT COUNT(*) FROM user_ops", [], |row| row.get(0)) - .unwrap(); - assert_eq!(audit_rows, 3); - assert_eq!( - storage - .canonical_history_page(claim(recovered, 0), 10) - .unwrap() - .rows - .len(), - 2 - ); } -#[test] -fn rebuilt_history_starts_at_its_absolute_base_without_padding() { - let db = temp_db("canonical-page-rebuild"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); +/// An era rebuilt from a machine at count 41: inputs 0 and 2 are directs, +/// input 1 an accepted batch with three ops, the second skipped. Counts +/// 41..45 are rebuilt; the root frame then serves live rows from 45. +fn seed_two_table_era(storage: &mut Storage) { + pin_test_deployment_identity(storage, SENDER_A); + let input = |block: u64, payload: u8| IngestedSafeInput { + sender: SENDER_B, + payload: vec![payload], + block_number: block, + block_timestamp: 1_000 + block, + transaction_hash: B256::repeat_byte(payload), + }; storage - .append_safe_inputs( + .append_ingested_safe_inputs_with_timestamp( 10, - &[ - StoredSafeInput { - sender: SENDER_B, - payload: vec![0xaa], - block_number: 10, - }, - StoredSafeInput { - sender: SENDER_B, - payload: vec![0xbb], - block_number: 10, - }, - ], - SENDER_A, - &default_protocol_timing(), + 1_010, + &[input(5, 0xa0), input(6, 0xa1), input(10, 0xa2)], ) .unwrap(); + // The op at nonce 5 is skipped, so it has no row. storage - .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 10)) + .append_rebuilt_history(&[ + (L2TxCount::new(41), scheduled_direct(0, 3)), + (L2TxCount::new(42), scheduled_op(0, 3)), + ]) .unwrap(); storage - .initialize_open_state(10, SafeInputRange::empty_at(2)) + .append_rebuilt_history(&[ + (L2TxCount::new(43), scheduled_op(1, 3)), + (L2TxCount::new(44), scheduled_direct(2, 4)), + ]) .unwrap(); + storage + .publish_era( + std::path::Path::new("/snapshot/era"), + &NewEra { + base: L2TxCount::new(41), + rebuilt_end: L2TxCount::new(45), + stop_block: 10, + first_queued_input: 3, + anchor_batch_nonce: 5, + }, + ) + .unwrap(); + storage.ensure_open_tip().unwrap(); +} + +#[test] +fn a_page_spans_the_rebuilt_history_and_the_live_rows_without_gaps() { + let db = temp_db("canonical-page-two-tables"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + seed_two_table_era(&mut storage); let bounds = storage.history_bounds().unwrap(); - assert_eq!(bounds.available_from, ExecutedInputCount::new(41)); - assert_eq!(bounds.head, bounds.available_from); + assert_eq!(bounds.available_from, L2TxCount::new(41)); + assert_eq!(bounds.head, L2TxCount::new(45)); assert!( storage - .canonical_history_page(claim(bounds, 41), 10) + .canonical_history_page(claim(bounds, 45), 10) .unwrap() .rows .is_empty() ); + let mut head = storage.open_state().unwrap().unwrap(); + // The first live row must sit at the rebuilt history's end. assert!( - matches!(storage.canonical_history_page(claim(bounds, 40), 10), - Err(HistoryReadError::Policy(HistoryPolicyError::HistoryUnavailable { available_from })) - if available_from == bounds.available_from) - ); - assert!( - matches!(storage.canonical_history_page(claim(bounds, 42), 10), - Err(HistoryReadError::Policy(HistoryPolicyError::AheadOfHead { head })) - if head == bounds.head) + storage + .append_included_user_ops(&mut head, &[included(2, 44, 0xcc)]) + .is_err() ); - let mut head = storage.open_state().unwrap().unwrap(); storage - .append_executed_user_ops_chunk(&mut head, &[included(0, 41, 0xcc)]) + .append_included_user_ops(&mut head, &[included(2, 45, 0xcc), included(3, 46, 0xdd)]) .unwrap(); - let page = storage - .canonical_history_page(claim(bounds, 41), 10) + let bounds = storage.history_bounds().unwrap(); + assert_eq!(bounds.head, L2TxCount::new(47)); + + let whole = storage + .canonical_history_page(claim(bounds, 41), 100) .unwrap(); - assert_eq!(page.rows.len(), 1); - assert_eq!(page.rows[0].offset, ExecutedInputCount::new(41)); - assert_eq!(page.next_input, ExecutedInputCount::new(42)); + assert_eq!( + whole + .rows + .iter() + .map(|row| row.offset.get()) + .collect::>(), + (41..47).collect::>() + ); + assert_eq!(whole.next_claim(), claim(bounds, 47)); + match &whole.rows[0].context { + L2TxContext::Direct { + tx, + batch_nonce, + transaction_hash, + } => { + assert_eq!( + (tx.input_index, tx.sender, tx.block, tx.timestamp), + (0, SENDER_B, 5, 1_005) + ); + assert_eq!(tx.payload, vec![0xa0]); + assert_eq!(*batch_nonce, 3); + assert_eq!(*transaction_hash, B256::repeat_byte(0xa0)); + } + other => panic!("expected the rebuilt direct, got {other:?}"), + } + // The skipped op (nonce 5) has no row, so the feed leaves it out. + let rebuilt_ops: Vec<_> = whole.rows[1..3] + .iter() + .map(|row| match &row.context { + L2TxContext::UserOp { tx, batch_nonce } => { + assert_eq!((tx.sender, tx.fee, tx.block), (SENDER_B, 9, 7)); + (tx.nonce, *batch_nonce) + } + other => panic!("expected a rebuilt user op, got {other:?}"), + }) + .collect(); + assert_eq!(rebuilt_ops, vec![(0, 3), (1, 3)]); + match &whole.rows[3].context { + L2TxContext::Direct { + tx, batch_nonce, .. + } => assert_eq!((tx.input_index, tx.timestamp, *batch_nonce), (2, 1_010, 4)), + other => panic!("expected the second rebuilt direct, got {other:?}"), + } + match &whole.rows[4].context { + L2TxContext::UserOp { tx, batch_nonce } => { + assert_eq!((tx.data.clone(), *batch_nonce), (vec![0xcc], 5)); + } + other => panic!("expected the first live op, got {other:?}"), + } + + // Pages that split at and around the seam keep one contiguous cursor. + for (from, limit) in [(43, 3), (44, 1), (44, 2), (45, 5), (41, 4)] { + let page = storage + .canonical_history_page(claim(bounds, from), limit) + .unwrap(); + let end = (from + limit as u64).min(47); + assert_eq!( + page.rows + .iter() + .map(|row| row.offset.get()) + .collect::>(), + (from..end).collect::>(), + "from {from}, limit {limit}" + ); + assert_eq!(page.next_claim(), claim(bounds, end)); + } + assert!(matches!( + storage.canonical_history_page(claim(bounds, 40), 10), + Err(HistoryReadError::Policy(HistoryPolicyError::HistoryUnavailable { available_from })) + if available_from == L2TxCount::new(41) + )); +} + +#[test] +#[should_panic(expected = "canonical history page has an attribution gap")] +fn a_missing_rebuilt_row_below_the_end_fails_loud() { + let db = temp_db("canonical-page-rebuilt-hole"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + seed_two_table_era(&mut storage); + storage + .conn + .execute_batch( + "DROP TRIGGER trg_rebuilt_history_not_deletable; \ + DELETE FROM rebuilt_history WHERE l2_tx_count = 42;", + ) + .unwrap(); + let bounds = storage.history_bounds().unwrap(); + let _ = storage.canonical_history_page(claim(bounds, 41), 10); } #[test] fn a_deep_backlog_can_be_read_in_small_bounded_pages() { let db = temp_db("canonical-page-deep-backlog"); - let mut storage = Storage::open(&db.path).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); @@ -316,10 +359,10 @@ fn a_deep_backlog_can_be_read_in_small_bounded_pages() { .map(|offset| included(offset, u64::from(offset), 0xaa)) .collect(); storage - .append_executed_user_ops_chunk(&mut head, &inputs) + .append_included_user_ops(&mut head, &inputs) .unwrap(); let bounds = storage.history_bounds().unwrap(); - assert_eq!(bounds.head, ExecutedInputCount::new(50_001)); + assert_eq!(bounds.head, L2TxCount::new(50_001)); for from in [0, 25_000, 49_999] { let page = storage .canonical_history_page(claim(bounds, from), 2) @@ -335,63 +378,28 @@ fn a_deep_backlog_can_be_read_in_small_bounded_pages() { } } -#[test] -fn one_read_transaction_keeps_history_identity_and_rows_coherent_during_recovery() { - let db = temp_db("canonical-page-read-snapshot"); - let mut writer = Storage::open(&db.path).unwrap(); - seed_aging_tip(&mut writer); - let mut reader = Storage::open_read_only(&db.path).unwrap(); - let tx = reader.conn.transaction().unwrap(); - let before = history_bounds_in(&tx).unwrap(); - - writer.recover_aging_tip(1_200).unwrap(); - let mut head = writer.open_state().unwrap().unwrap(); - writer - .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xcc)]) - .unwrap(); - let retained = canonical_page_in(&tx, before, ExecutedInputCount::new(1), 2).unwrap(); - assert_eq!(retained.bounds, before); - assert_eq!(history_bounds_in(&tx).unwrap(), before); - assert_eq!(retained.rows.len(), 1); - match &retained.rows[0].context { - L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xbb]), - other => panic!("expected old snapshot user op, got {other:?}"), - } - tx.commit().unwrap(); - - let after = reader.history_bounds().unwrap(); - assert_eq!(after.head, before.head); - assert_ne!(after.version, before.version); - let fresh = reader.canonical_history_page(claim(after, 1), 2).unwrap(); - assert_eq!(fresh.bounds, after); - match &fresh.rows[0].context { - L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xcc]), - other => panic!("expected current user op, got {other:?}"), - } -} - #[test] fn tail_after_the_largest_sqlite_offset_is_empty_without_clamping() { let db = temp_db("canonical-page-sqlite-tail"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); let last = i64::MAX as u64; storage - .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(last), 0)) + .write(|tx| insert_era_in(tx, &crate::storage::test_helpers::era(last, last, 0))) .unwrap(); storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); let mut head = storage.open_state().unwrap().unwrap(); storage - .append_executed_user_ops_chunk(&mut head, &[included(0, last, 0xaa)]) + .append_included_user_ops(&mut head, &[included(0, last, 0xaa)]) .unwrap(); let bounds = storage.history_bounds().unwrap(); - assert_eq!(bounds.head, ExecutedInputCount::new(last + 1)); + assert_eq!(bounds.head, L2TxCount::new(last + 1)); let page = storage .canonical_history_page(claim(bounds, last), 1) .unwrap(); assert_eq!(page.rows.len(), 1); - assert_eq!(page.rows[0].offset, ExecutedInputCount::new(last)); + assert_eq!(page.rows[0].offset, L2TxCount::new(last)); let tail = storage .canonical_history_page(page.next_claim(), usize::MAX) .unwrap(); @@ -403,12 +411,12 @@ fn tail_after_the_largest_sqlite_offset_is_empty_without_clamping() { #[should_panic(expected = "canonical history page has an attribution gap")] fn an_interior_history_hole_fails_loud() { let db = temp_db("canonical-page-corrupt-attribution"); - let mut storage = Storage::open(&db.path).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); storage - .append_executed_user_ops_chunk( + .append_included_user_ops( &mut head, &[ included(0, 0, 0xaa), @@ -420,8 +428,8 @@ fn an_interior_history_hole_fails_loud() { storage .conn .execute_batch( - "DROP TRIGGER trg_protect_valid_application_input_delete;\n\ - DELETE FROM application_inputs WHERE offset = 1;", + "DROP TRIGGER trg_application_inputs_not_deletable;\n\ + DELETE FROM application_inputs WHERE l2_tx_count = 1;", ) .unwrap(); let bounds = storage.history_bounds().unwrap(); diff --git a/sequencer/src/storage/history/generation_tests.rs b/sequencer/src/storage/history/generation_tests.rs deleted file mode 100644 index 18393f93..00000000 --- a/sequencer/src/storage/history/generation_tests.rs +++ /dev/null @@ -1,170 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -use super::*; -use crate::test_support::temp_db; - -fn ledger(storage: &Storage) -> Vec<(i64, i64)> { - storage - .conn - .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") - .unwrap() - .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) - .unwrap() - .collect::>() - .unwrap() -} - -// Synthetic cuts exercise the query's full interval contract independently -// of today's recovery pivot policy. Actual cascades are tested separately. -fn seed_cuts(storage: &mut Storage, cuts: &[u64]) { - storage - .write(|tx| { - for (index, cut) in cuts.iter().enumerate() { - let generation = i64::try_from(index + 1).unwrap(); - tx.execute( - "INSERT INTO history_generation_cuts (recovery_generation,preserved_input_count) VALUES (?1,?2)", - params![generation, u64_to_i64(*cut)], - )?; - tx.execute( - "UPDATE history_state SET recovery_generation=?1 WHERE singleton_id=0", - [generation], - )?; - } - Ok(()) - }) - .unwrap(); -} - -#[test] -fn generation_and_cut_are_atomic_and_immutable() { - let db = temp_db("generation-cut-atomic"); - let mut storage = Storage::open(&db.path).unwrap(); - let result: Result<()> = storage.write(|tx| { - advance_recovery_generation_in(tx)?; - Err(rusqlite::Error::InvalidQuery) - }); - assert!(result.is_err()); - assert!(ledger(&storage).is_empty()); - assert_eq!( - storage - .history_state() - .unwrap() - .version - .recovery_generation - .get(), - 0 - ); - assert!( - storage - .conn - .execute("UPDATE history_state SET recovery_generation=1", []) - .is_err() - ); - - storage.write(advance_recovery_generation_in).unwrap(); - assert_eq!(ledger(&storage), [(1, 0)]); - for sql in [ - "UPDATE history_generation_cuts SET preserved_input_count=1", - "UPDATE history_generation_cuts SET recovery_generation=2", - "DELETE FROM history_generation_cuts", - "INSERT INTO history_generation_cuts VALUES (1,1)", - "INSERT INTO history_generation_cuts VALUES (0,0)", - "INSERT INTO history_generation_cuts VALUES (2,-1)", - ] { - assert!(storage.conn.execute(sql, []).is_err(), "{sql}"); - } - drop(storage); - let reopened = Storage::open(&db.path).unwrap(); - assert_eq!(ledger(&reopened), [(1, 0)]); - assert_eq!( - reopened - .history_state() - .unwrap() - .version - .recovery_generation - .get(), - 1 - ); -} - -#[test] -fn compatibility_uses_every_intervening_cut_and_the_current_head() { - let db = temp_db("generation-cut-minimum"); - let mut storage = Storage::open(&db.path).unwrap(); - seed_cuts(&mut storage, &[3, 5, 2, 4]); - for (from, current, head, expected) in [ - (0, 1, 9, 3), - (0, 2, 9, 3), - (1, 2, 9, 5), - (0, 3, 9, 2), - (1, 3, 9, 2), - (2, 3, 9, 2), - (0, 4, 9, 2), - (3, 4, 9, 4), - (3, 4, 1, 1), - (4, 4, 9, 9), - ] { - assert_eq!( - preserved_input_count_in( - &storage.conn, - RecoveryGeneration::new(from), - RecoveryGeneration::new(current), - ExecutedInputCount::new(head) - ) - .unwrap(), - ExecutedInputCount::new(expected), - "from={from}, current={current}, head={head}", - ); - } -} - -#[test] -#[should_panic(expected = "history generation lineage is incomplete")] -fn missing_intermediate_cut_fails_instead_of_certifying_a_partial_minimum() { - let db = temp_db("generation-cut-gap"); - let mut storage = Storage::open(&db.path).unwrap(); - seed_cuts(&mut storage, &[3, 1, 5]); - storage - .conn - .execute_batch( - "DROP TRIGGER trg_history_generation_cuts_not_deletable; \ - DELETE FROM history_generation_cuts WHERE recovery_generation=2", - ) - .unwrap(); - let _ = preserved_input_count_in( - &storage.conn, - RecoveryGeneration::new(0), - RecoveryGeneration::new(3), - ExecutedInputCount::new(9), - ); -} - -#[test] -fn current_head_may_be_the_boundary_after_the_maximum_sqlite_offset() { - let db = temp_db("generation-cut-max-head"); - let mut storage = Storage::open(&db.path).unwrap(); - let max = i64::MAX as u64; - seed_cuts(&mut storage, &[max]); - let head = ExecutedInputCount::new(max + 1); - assert_eq!( - preserved_input_count_in( - &storage.conn, - RecoveryGeneration::new(1), - RecoveryGeneration::new(1), - head - ) - .unwrap(), - head, - ); - assert_eq!( - preserved_input_count_in( - &storage.conn, - RecoveryGeneration::new(0), - RecoveryGeneration::new(1), - head - ) - .unwrap(), - ExecutedInputCount::new(max), - ); -} diff --git a/sequencer/src/storage/ingress.rs b/sequencer/src/storage/ingress.rs index 20a23614..c7bd1b26 100644 --- a/sequencer/src/storage/ingress.rs +++ b/sequencer/src/storage/ingress.rs @@ -7,71 +7,101 @@ //! from the lane's rows. use std::path::Path; +use std::time::SystemTime; use alloy_primitives::Address; -use rusqlite::{OptionalExtension, Result, Transaction, params}; +use rusqlite::{OptionalExtension, Result, Transaction, named_params, params}; +use sequencer_core::l2_tx::Direct; +use sequencer_core::user_op::SignedUserOp; +use super::acceptance::{Divergence, canonical_divergence_in}; use super::convert::{ - from_unix_ms, i64_to_u32, i64_to_u64, now_unix_ms, saturating_query_bound, to_unix_ms, - u64_to_i64, + from_unix_ms, i64_to_u16, i64_to_u32, i64_to_u64, now_unix_ms, to_unix_ms, u64_to_i64, }; -use super::history::{next_executed_input_count_in, query_history_state}; +use super::history::{era_in, next_l2_tx_count_in}; use super::mutations::{ insert_new_batch, insert_open_frame, persist_frame_direct_sequence, persist_frame_direct_sequence_derived, seal_batch, }; use super::queries::{ - current_safe_block_required, load_current_write_head, query_batch_policy, - query_latest_safe_input_index_exclusive, + DIRECT_EXECUTES_SQL, DirectOperands, current_safe_block_required, load_current_write_head, + query_batch_policy, query_latest_safe_input_index_exclusive, safe_input_count_through_block_in, }; -use super::safe_accepted_batches::canonical_divergence_in; -use super::snapshot_dumps::insert_batch_snapshot_in; +use super::snapshots::insert_batch_snapshot_in; use super::{ - BatchPolicy, DirectInputExecution, ExecutedInputCount, SafeFrontierState, SafeInputFrontier, + BatchPolicy, DirectInputExecution, L2TxCount, SafeFrontierState, SafeInputFrontier, SafeInputRange, Storage, WriteHead, }; -use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; -const LAST_VALID_USER_NONCE_SQL: &str = "SELECT u.nonce FROM user_ops u - WHERE u.sender = ?1 - AND EXISTS (SELECT 1 FROM valid_batches b WHERE b.batch_index = u.batch_index) - ORDER BY u.nonce DESC +/// One executed user op as storage persists it: the signed op, when the API +/// received it, and its application-history position. Borrows the caller's op. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ExecutedUserOp<'a> { + pub signed: &'a SignedUserOp, + pub received_at: SystemTime, + pub l2_tx_index: L2TxCount, +} + +/// The `/fee` quote. Both fees are log-space exponents (base 129/128). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FeeQuote { + /// The fee committed for the Tip's open frame. + pub frame_fee: u16, + /// The fee the next frame opens at under the current policy. + pub recommended_fee: u16, +} + +const LAST_USER_NONCE_SQL: &str = "SELECT nonce FROM user_ops + WHERE sender = ?1 + ORDER BY nonce DESC LIMIT 1"; impl Storage { - /// First L1 input beyond the latest surviving frame's accounted block, - /// bounded below by the immutable era prefix. + /// First L1 input beyond the latest frame's accounted block, or the era's + /// first queued input while the era has no frame. pub fn next_undrained_safe_input_index(&mut self) -> Result { self.read(next_undrained_safe_input_index_in) } /// Resume the lane on startup. Returns `None` if storage is empty (caller - /// must establish the Tip through startup recovery). + /// must open the Tip through `run`'s startup). pub fn open_state(&mut self) -> Result> { self.read(load_current_write_head) } - /// Open-frame fee and current `recommended_fee`. `None` if there is no Tip. - pub fn current_fee_quote(&mut self) -> Result> { + /// The `/fee` quote: the fee of the Tip's latest frame and the policy's + /// current `recommended_fee`, in one read. `None` if there is no Tip. + pub fn current_fee_quote(&mut self) -> Result> { self.read(|tx| { - let Some(head) = load_current_write_head(tx)? else { + let tip: Option = tx + .query_row("SELECT nonce FROM open_batch", [], |row| row.get(0)) + .optional()?; + let Some(tip) = tip else { return Ok(None); }; - let policy = query_batch_policy(tx)?; - Ok(Some((head.frame_fee, policy.recommended_fee))) + // Every batch has a frame; a missing one fails loud. + let frame_fee: i64 = tx.query_row( + "SELECT fee FROM frames WHERE batch_nonce = ?1 \ + ORDER BY frame_in_batch DESC LIMIT 1", + params![tip], + |row| row.get(0), + )?; + Ok(Some(FeeQuote { + frame_fee: i64_to_u16(frame_fee), + recommended_fee: query_batch_policy(tx)?.recommended_fee, + })) }) } - /// The nonce `sender` must sign next: one past its latest user op in a - /// valid batch, or 0 without one. Exact under the application contract's - /// nonce rule, except that a sender idle since a rebuilt baseline reads 0. - /// - /// Invalidated batches keep their rows, and resubmission after a cascade - /// reuses their nonces, so the validity filter is load-bearing. + /// The nonce `sender` must sign next: one past its latest user op, or 0 + /// without one. Exact under the application contract's nonce rule, except + /// for a sender whose last op predates the era's live rows (in its rebuilt + /// history or below its base): that sender reads a stale value, and the + /// 422's `expected` nonce corrects the client. pub fn next_user_nonce(&mut self, sender: Address) -> Result { self.read(|tx| { let last_nonce: Option = tx - .prepare_cached(LAST_VALID_USER_NONCE_SQL)? + .prepare_cached(LAST_USER_NONCE_SQL)? .query_row(params![sender.as_slice()], |row| row.get(0)) .optional()?; Ok(last_nonce.map_or(0, |nonce| { @@ -90,7 +120,11 @@ impl Storage { /// which guarantees this in production. pub fn safe_frontier_state(&mut self) -> Result { self.read(|tx| { - if let Some((nonce, safe_input_index)) = canonical_divergence_in(tx)? { + if let Some(Divergence { + nonce, + safe_input_index, + }) = canonical_divergence_in(tx)? + { return Ok(SafeFrontierState::CanonicalDivergence { nonce, safe_input_index, @@ -103,37 +137,42 @@ impl Storage { }) } + /// The directs in `range` that execute, in input order, each with the + /// index and timestamp the application receives. A direct over the + /// pinned cap is left out, as storage's derivation of the frame's + /// executions leaves it out. pub(crate) fn fill_direct_inputs( &mut self, range: SafeInputRange, - out: &mut Vec, + out: &mut Vec, ) -> Result<()> { out.clear(); if range.is_empty() { return Ok(()); } - let identity = super::l1_inputs::query_deployment_identity(&self.conn)? - .ok_or(rusqlite::Error::QueryReturnedNoRows)?; - let mut statement = self.conn.prepare_cached( - "SELECT safe_input_index, sender, payload, block_number FROM safe_inputs - WHERE safe_input_index >= ?1 AND safe_input_index < ?2 AND sender != ?3 - ORDER BY safe_input_index", - )?; + let operands = DirectOperands::load(&self.conn)?; + let mut statement = self.conn.prepare_cached(&format!( + "SELECT safe_input_index, sender, payload, block_number, block_timestamp + FROM safe_inputs + WHERE safe_input_index >= :start AND safe_input_index < :end + AND {DIRECT_EXECUTES_SQL} + ORDER BY safe_input_index" + ))?; let rows = statement.query_map( - params![ - u64_to_i64(range.start()), - u64_to_i64(range.end()), - identity.batch_submitter_address.as_slice() - ], + operands + .bind(named_params! { + ":start": u64_to_i64(range.start()), + ":end": u64_to_i64(range.end()), + }) + .as_slice(), |row| { let sender: Vec = row.get(1)?; - Ok(super::StoredDirectInput { - safe_input_index: i64_to_u64(row.get(0)?), - input: sequencer_core::l2_tx::DirectInput { - sender: Address::from_slice(&sender), - payload: row.get(2)?, - block_number: i64_to_u64(row.get(3)?), - }, + Ok(Direct { + input_index: i64_to_u64(row.get(0)?), + sender: Address::from_slice(&sender), + block: i64_to_u64(row.get(3)?), + timestamp: i64_to_u64(row.get(4)?), + payload: row.get(2)?, }) }, )?; @@ -152,23 +191,23 @@ impl Storage { /// schema's FK + PK constraints catch the dangerous failure modes (write /// to a non-existent frame, duplicate `pos_in_frame`) by failing the /// INSERT. - pub(crate) fn append_executed_user_ops_chunk( + pub(crate) fn append_executed_user_ops_chunk<'a>( &mut self, head: &mut WriteHead, - user_ops: &[IncludedUserOp], + user_ops: impl ExactSizeIterator> + Clone, ) -> Result<()> { - if user_ops.is_empty() { + if user_ops.len() == 0 { return Ok(()); } // Validate both in-memory counter advances before the transaction. // Otherwise an overflow panic after commit would leave durable rows // that the unchanged `WriteHead` cannot describe. - let mut next_head = *head; + let mut next_head = head.duplicate(); next_head.increment_batch_user_op_count(user_ops.len()); self.write(|tx| { insert_executed_user_ops_batch( tx, - head.batch_index, + head.batch_nonce, head.frame_in_batch, head.open_frame_user_op_count, user_ops, @@ -178,7 +217,8 @@ impl Storage { Ok(()) } - /// Commit frame advancement together with its application inputs. + /// Commit frame advancement together with its application inputs, which + /// must be exactly the directs the range executes. pub fn close_frame_only_with_executions( &mut self, head: &mut WriteHead, @@ -195,86 +235,60 @@ impl Storage { /// The artifact is durable before sealing and registering its snapshot in /// one transaction. A failed commit leaves only an unreferenced artifact. + /// The successor batch's first frame opens at the closing frame's safe + /// block: a batch close never advances the frame clock. pub fn close_frame_and_batch_with_snapshot( &mut self, head: &mut WriteHead, - next_safe_block: u64, dump_dir: &Path, - executed_input_count: ExecutedInputCount, + l2_tx_count: L2TxCount, ) -> Result<()> { - let (next_batch_index, now_ms, policy) = self.write(|tx| { + let next = self.write(|tx| { assert_eq!( - next_executed_input_count_in(tx)?, - executed_input_count, + next_l2_tx_count_in(tx)?, + l2_tx_count, "application count changed between dump creation and batch close" ); - let result = seal_and_open_next_batch(tx, head.batch_index, next_safe_block)?; - insert_batch_snapshot_in(tx, dump_dir, head.batch_index, executed_input_count)?; - Ok(result) + let next = seal_and_open_next_batch(tx, head)?; + insert_batch_snapshot_in(tx, dump_dir, head.batch_nonce, l2_tx_count)?; + Ok(next) })?; - head.move_to_next_batch( - next_batch_index, - from_unix_ms(now_ms), - policy, - next_safe_block, - ); + next.apply_to(head); Ok(()) } } /// Insert the Tip and its first frame. Callers separately attach the complete -/// leading application range, except for a recovery baseline's empty root. -/// Returns its local identity; callers load `WriteHead` through the shared reader. -pub(super) fn insert_tip_rows( - tx: &Transaction<'_>, - batch_index_opt: Option, - parent: Option, - safe_block: u64, -) -> Result { +/// leading application range. Returns its nonce; callers load `WriteHead` +/// through the shared reader. +pub(super) fn insert_tip_rows(tx: &Transaction<'_>, safe_block: u64) -> Result { let now_ms = now_unix_ms(); let policy = query_batch_policy(tx)?; - let batch_index = insert_new_batch(tx, batch_index_opt, parent, now_ms)?; + let batch_nonce = insert_new_batch(tx, now_ms)?; insert_open_frame( tx, - batch_index, + batch_nonce, 0, now_ms, policy.recommended_fee, safe_block, )?; - Ok(batch_index) + Ok(batch_nonce) } -/// Extend the valid batch path with a fresh open Tip at the current safe head, -/// attributing every undrained safe input to its first frame (lane catch-up +/// Extend the batch list with a fresh open Tip at the current safe head, +/// draining every undrained safe input into its first frame: each direct that +/// executes gets its row there (lane catch-up /// executes them, I11). /// -/// One mechanism, two callers with distinct intents (each keeps its own guard): -/// startup recovery's guarded Tip creation (genesis / first startup) and -/// recovery's cascade (reopening the Tip it just invalidated, atomically — see -/// `storage/recovery.rs`). Lineage is derived from the tree: `parent` is the -/// highest-indexed valid batch — `None` when the valid path is empty (genesis, -/// or a fully-torn cascade), rooting a batch at the deployment anchor; `batch_index` is the -/// explicit genesis `0` only when the `batches` table is empty, otherwise the -/// monotonic PK (so recovery batches keep climbing and indices are never -/// reused). +/// Its production caller is `run`'s startup, through the guarded +/// `Storage::open_batch_if_none` (`storage/landing.rs`), which opens the era's +/// first batch. The new Tip follows the last batch, at the era's anchor +/// nonce when there is none. pub(super) fn open_fresh_tip_in_tx(tx: &Transaction<'_>) -> Result<()> { let safe_block = current_safe_block_required(tx)?; - let table_empty = tx - .query_row("SELECT MAX(batch_index) FROM batches", [], |row| { - row.get::<_, Option>(0) - })? - .is_none(); - let parent = tx - .query_row("SELECT MAX(batch_index) FROM valid_batches", [], |row| { - row.get::<_, Option>(0) - })? - .map(i64_to_u64); - let batch_index_opt = if table_empty { Some(0) } else { None }; insert_draining_tip_with_executions( tx, - batch_index_opt, - parent, safe_block, query_latest_safe_input_index_exclusive(tx)?, ) @@ -285,76 +299,65 @@ pub(super) fn open_fresh_tip_in_tx(tx: &Transaction<'_>) -> Result<()> { /// processing queued user operations. fn insert_draining_tip_with_executions( tx: &Transaction<'_>, - batch_index: Option, - parent: Option, safe_block: u64, drain_upper: u64, ) -> Result<()> { let leading_direct_range = SafeInputRange::new(next_undrained_safe_input_index_in(tx)?, drain_upper); - let batch_index = insert_tip_rows(tx, batch_index, parent, safe_block)?; - persist_frame_direct_sequence_derived(tx, batch_index, 0, leading_direct_range)?; - Ok(()) -} - -/// The folded prefix is represented by the baseline, so the root contains no -/// replay entries for it. Inputs beyond the stop block remain unaccounted. -pub(super) fn open_recovery_tip_in_tx(tx: &Transaction<'_>, stop_block: u64) -> Result<()> { - assert!( - load_current_write_head(tx)?.is_none(), - "recovery tip already exists" - ); - insert_tip_rows(tx, Some(0), None, stop_block)?; + let batch_nonce = insert_tip_rows(tx, safe_block)?; + persist_frame_direct_sequence_derived(tx, batch_nonce, 0, leading_direct_range)?; Ok(()) } -fn safe_input_index_exclusive_through_block_in(tx: &Transaction<'_>, block: u64) -> Result { - let last: Option = tx - .query_row( - "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 - ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", - [saturating_query_bound(block)], - |row| row.get(0), - ) - .optional()?; - Ok(last.map_or(0, |index| { - i64_to_u64(index) - .checked_add(1) - .expect("safe input index overflow") - })) -} - +/// The era's baseline accounts for the inputs through the stop block except +/// the directs still queued there, which the era's first frame drains: from +/// the first queued input, past submitter and over-cap inputs that never +/// execute. fn next_undrained_safe_input_index_in(tx: &Transaction<'_>) -> Result { - let floor = query_history_state(tx)?.base_safe_block; + let era = era_in(tx)?; let latest_frame: Option = tx .query_row( "SELECT safe_block FROM frames - WHERE batch_index = (SELECT MAX(batch_index) FROM valid_batches) + WHERE batch_nonce = (SELECT MAX(nonce) FROM batches) ORDER BY frame_in_batch DESC LIMIT 1", [], |row| row.get(0), ) .optional()?; - safe_input_index_exclusive_through_block_in(tx, floor.max(latest_frame.map_or(0, i64_to_u64))) + match latest_frame { + Some(frame_block) => { + safe_input_count_through_block_in(tx, era.stop_block.max(i64_to_u64(frame_block))) + } + None => Ok(era.first_queued_input), + } +} + +/// The successor batch a close opened, for the caller to apply to its +/// in-memory write head after the commit. +pub(super) struct NextBatch { + nonce: u64, + created_at_ms: i64, + policy: BatchPolicy, +} + +impl NextBatch { + pub(super) fn apply_to(self, head: &mut WriteHead) { + head.move_to_next_batch(self.nonce, from_unix_ms(self.created_at_ms), self.policy); + } } -/// Seal the current Tip and open the successor batch's first frame, in `tx`. +/// Seal the current Tip and open the successor batch's first frame at the +/// head's safe block, in `tx`. /// /// Shared with the test fixture `close_frame_and_batch` so the seal ordering /// lives in one place: seal first (which frees the old row from the -/// `ux_single_valid_tip` partial index), then insert the successor as the new -/// Tip. Returns the new batch index, the close timestamp, and the sampled -/// policy for the caller to apply to its in-memory write head. +/// `ux_single_open_batch` partial index), then insert the successor as the new +/// Tip. pub(super) fn seal_and_open_next_batch( tx: &Transaction<'_>, - closing_batch_index: u64, - next_safe_block: u64, -) -> Result<(u64, i64, BatchPolicy)> { - let current = load_current_write_head(tx)?.expect("a batch close requires the Tip"); - assert_eq!( - current.safe_block, next_safe_block, - "batch closure cannot advance the frame clock" - ); + head: &WriteHead, +) -> Result { + assert_head_matches_tip(tx, head)?; let now_ms = now_unix_ms(); // Batch policy is sampled here: the derived fee is committed to the newly // opened frame, and the batch size target is stored on the write head. @@ -363,21 +366,49 @@ pub(super) fn seal_and_open_next_batch( // the same path the submitter uses and stamp their keccak256 on the row, // atomically with the seal. The content-identity check later compares // accepted L1 landings against this hash. - let nonce = super::snapshot_dumps::batch_nonce_in(tx, closing_batch_index)?; - let frames = super::l1_submission::load_batch_frames_in(tx, closing_batch_index)?; + let nonce = head.batch_nonce; + let frames = super::l1_submission::load_batch_frames_in(tx, nonce)?; let encoded = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { nonce, frames }); let payload_hash = alloy_primitives::keccak256(&encoded); - seal_batch(tx, closing_batch_index, now_ms, &payload_hash.0)?; - let next_batch_index = insert_new_batch(tx, None, Some(closing_batch_index), now_ms)?; + seal_batch(tx, nonce, now_ms, &payload_hash.0)?; + let next_nonce = insert_new_batch(tx, now_ms)?; insert_open_frame( tx, - next_batch_index, + next_nonce, 0, now_ms, policy.recommended_fee, - next_safe_block, + head.safe_block, )?; - Ok((next_batch_index, now_ms, policy)) + Ok(NextBatch { + nonce: next_nonce, + created_at_ms: now_ms, + policy, + }) +} + +/// Fail loud unless the Tip reloaded inside `tx` agrees with `head` on every +/// persisted fact: batch nonce, frame, counts, fee and safe block. The size +/// target is left out: a reload samples the policy now, while the head keeps +/// the value sampled when its frame opened. Runs once per rotation or close. +fn assert_head_matches_tip(tx: &Transaction<'_>, head: &WriteHead) -> Result<()> { + let durable = + load_current_write_head(tx)?.expect("a frame rotation or batch close requires the Tip"); + let persisted = |head: &WriteHead| { + ( + head.batch_nonce, + head.frame_in_batch, + head.batch_user_op_count, + head.open_frame_user_op_count, + head.frame_fee, + head.safe_block, + ) + }; + assert!( + persisted(head) == persisted(&durable), + "write head disagrees with the durable Tip: cached={head:?}, durable={durable:?}" + ); + Ok(()) } /// Rotate to the next frame inside the current batch, in `tx`: open the @@ -397,14 +428,14 @@ fn close_frame_in( leading_direct_range, SafeInputRange::new( next_undrained_safe_input_index_in(tx)?, - safe_input_index_exclusive_through_block_in(tx, next_safe_block)?, + safe_input_count_through_block_in(tx, next_safe_block)?, ), "frame advancement must account for its complete L1 interval" ); let (policy, next_frame_in_batch) = open_successor_frame_in(tx, head, next_safe_block)?; persist_frame_direct_sequence( tx, - head.batch_index, + head.batch_nonce, next_frame_in_batch, leading_direct_range, executions, @@ -417,6 +448,7 @@ pub(super) fn open_successor_frame_in( head: &WriteHead, next_safe_block: u64, ) -> Result<(BatchPolicy, u32)> { + assert_head_matches_tip(tx, head)?; assert!( next_safe_block >= head.safe_block, "frame clock cannot regress" @@ -429,7 +461,7 @@ pub(super) fn open_successor_frame_in( .expect("frame index overflow: contract-impossible"); insert_open_frame( tx, - head.batch_index, + head.batch_nonce, next_frame_in_batch, now_ms, policy.recommended_fee, @@ -440,18 +472,18 @@ pub(super) fn open_successor_frame_in( fn insert_user_op_iter<'a>( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, frame_pos_start: u32, - user_ops: impl IntoIterator, + user_ops: impl Iterator>, ) -> Result<()> { let mut stmt = tx.prepare_cached( "INSERT INTO user_ops ( - batch_index, frame_in_batch, pos_in_frame, + batch_nonce, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, received_at_ms ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", )?; - for (offset, item) in user_ops.into_iter().enumerate() { + for (offset, item) in user_ops.enumerate() { let offset = u32::try_from(offset).expect("user-op chunk offset exceeds u32: contract-impossible"); let pos_in_frame = frame_pos_start @@ -459,7 +491,7 @@ fn insert_user_op_iter<'a>( .expect("user-op position overflow: contract-impossible"); let sig = item.signed.signature.as_bytes(); stmt.execute(params![ - u64_to_i64(batch_index), + u64_to_i64(batch_nonce), i64::from(frame_in_batch), i64::from(pos_in_frame), item.signed.sender.as_slice(), @@ -474,31 +506,31 @@ fn insert_user_op_iter<'a>( } /// User-op source and application position become durable before acknowledgement. -fn insert_executed_user_ops_batch( +fn insert_executed_user_ops_batch<'a>( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, frame_pos_start: u32, - user_ops: &[IncludedUserOp], + user_ops: impl Iterator> + Clone, ) -> Result<()> { insert_user_op_iter( tx, - batch_index, + batch_nonce, frame_in_batch, frame_pos_start, - user_ops.iter().map(|item| &item.pending), + user_ops.clone(), )?; let mut stmt = tx.prepare_cached( - "INSERT INTO application_inputs (offset, batch_index, frame_in_batch, user_op_pos_in_frame) + "INSERT INTO application_inputs (l2_tx_count, batch_nonce, frame_in_batch, user_op_pos_in_frame) VALUES (?1, ?2, ?3, ?4)", )?; - for (position, item) in user_ops.iter().enumerate() { + for (position, item) in user_ops.enumerate() { let position = frame_pos_start .checked_add(u32::try_from(position).expect("chunk fits u32")) .expect("user-op position overflow"); stmt.execute(params![ - u64_to_i64(item.executed_input_offset.get()), - u64_to_i64(batch_index), + u64_to_i64(item.l2_tx_index.get()), + u64_to_i64(batch_nonce), i64::from(frame_in_batch), i64::from(position) ])?; @@ -510,13 +542,16 @@ fn insert_executed_user_ops_batch( mod tests { use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; use crate::storage::{ - DirectInputExecution, ExecutedInputCount, LifecycleCommand, SafeFrontierState, - SafeInputFrontier, SafeInputRange, Storage, StoredSafeInput, - test_helpers::{SENDER_A, default_protocol_timing, record_canonical_divergence}, + DirectInputExecution, FeeQuote, IngestedSafeInput, L2TxCount, LifecycleCommand, + SafeFrontierState, SafeInputFrontier, SafeInputRange, Storage, StoredSafeInput, WriteHead, + test_helpers::{ + SENDER_A, TEST_MAX_DIRECT_PAYLOAD_BYTES, WriteHeadParts, record_canonical_divergence, + set_up_storage, + }, }; use crate::test_support::{pin_test_deployment_identity, temp_db}; - use alloy_primitives::{Address, Signature}; - use sequencer_core::l2_tx::SequencedL2Tx; + use alloy_primitives::{Address, B256, Signature}; + use sequencer_core::l2_tx::{Direct, L2Tx}; use sequencer_core::user_op::{SignedUserOp, UserOp}; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::time::SystemTime; @@ -542,30 +577,31 @@ mod tests { fn included_user_op(nonce: u32, offset: u64) -> IncludedUserOp { IncludedUserOp { pending: pending_user_op(nonce), - executed_input_offset: ExecutedInputCount::new(offset), + l2_tx_index: L2TxCount::new(offset), } } #[test] fn snapshot_registration_failure_rolls_back_batch_close_and_cached_head() { let db = temp_db("snapshot-close-rollback"); - let mut storage = Storage::open(&db.path).unwrap(); + // No baseline yet: the test registers its own at a known prefix. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .unwrap(); let prefix = std::path::Path::new("existing-baseline"); storage - .insert_baseline_snapshot(prefix, ExecutedInputCount::ZERO) + .insert_baseline_snapshot(prefix, L2TxCount::ZERO) .unwrap(); - let batch = head.batch_index; + let batch = head.batch_nonce(); let error = storage - .close_frame_and_batch_with_snapshot(&mut head, 10, prefix, ExecutedInputCount::ZERO) + .close_frame_and_batch_with_snapshot(&mut head, prefix, L2TxCount::ZERO) .expect_err("colliding artifact name must roll back the whole close"); assert!(error.to_string().contains("UNIQUE")); - assert_eq!(head.batch_index, batch); + assert_eq!(head.batch_nonce(), batch); let persisted = storage.open_state().unwrap().unwrap(); - assert_eq!(persisted.batch_index, batch); - assert_eq!(persisted.frame_in_batch, head.frame_in_batch); + assert_eq!(persisted.batch_nonce(), batch); + assert_eq!(persisted.frame_in_batch(), head.frame_in_batch()); assert_eq!( storage .conn @@ -577,10 +613,8 @@ mod tests { assert_eq!( storage .conn - .query_row("SELECT COUNT(*) FROM valid_closed_batches", [], |row| row - .get::<_, i64>( - 0 - )) + .query_row("SELECT COUNT(*) FROM closed_batches", [], |row| row + .get::<_, i64>(0)) .unwrap(), 0 ); @@ -594,11 +628,190 @@ mod tests { ); } + /// Heads that each disagree with the durable Tip on one persisted fact: + /// an older batch, a later one, an older or later frame, a position past + /// the frame's rows, a batch count, a fee or a safe block. + fn stale_heads(head: &WriteHead) -> Vec<(&'static str, WriteHead)> { + let live = WriteHeadParts::of(head); + let stale = |name, parts: WriteHeadParts| (name, parts.build()); + vec![ + stale( + "batch", + WriteHeadParts { + batch_nonce: live.batch_nonce - 1, + ..live + }, + ), + stale( + "nonce", + WriteHeadParts { + batch_nonce: live.batch_nonce + 1, + ..live + }, + ), + stale( + "older frame", + WriteHeadParts { + frame_in_batch: live.frame_in_batch - 1, + ..live + }, + ), + stale( + "later frame", + WriteHeadParts { + frame_in_batch: live.frame_in_batch + 1, + ..live + }, + ), + stale( + "position", + WriteHeadParts { + open_frame_user_op_count: live.open_frame_user_op_count + 1, + ..live + }, + ), + stale( + "batch count", + WriteHeadParts { + batch_user_op_count: live.batch_user_op_count + 1, + ..live + }, + ), + stale( + "fee", + WriteHeadParts { + frame_fee: live.frame_fee + 1, + ..live + }, + ), + stale( + "safe block", + WriteHeadParts { + safe_block: live.safe_block + 1, + ..live + }, + ), + ] + } + + /// A Tip at batch 1, frame 1 of the batch, with one user op in each frame. + fn tip_past_genesis(name: &str) -> (crate::test_support::TestDb, Storage, WriteHead) { + let db = temp_db(name); + let mut storage = set_up_storage(&db.path, SENDER_A); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .unwrap(); + storage.close_frame_and_batch(&mut head).unwrap(); + storage + .append_included_user_ops(&mut head, &[included_user_op(0, 0)]) + .unwrap(); + storage + .close_frame_only(&mut head, 10, SafeInputRange::empty_at(0)) + .unwrap(); + storage + .append_included_user_ops(&mut head, &[included_user_op(1, 1)]) + .unwrap(); + (db, storage, head) + } + + fn panic_message(payload: Box) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| (*s).to_owned())) + .unwrap_or_default() + } + + #[test] + fn a_head_that_disagrees_with_the_tip_fails_loud_at_frame_rotation() { + let (_db, mut storage, head) = tip_past_genesis("stale-head-rotation"); + for (name, mut stale) in stale_heads(&head) { + let panic = catch_unwind(AssertUnwindSafe(|| { + let _ = storage.close_frame_only_with_executions( + &mut stale, + 10, + SafeInputRange::empty_at(0), + &[], + ); + })) + .expect_err(name); + assert!( + panic_message(panic).contains("write head disagrees with the durable Tip"), + "{name}" + ); + } + assert_eq!(storage.open_state().unwrap().unwrap().frame_in_batch(), 1); + let mut head = head; + storage + .close_frame_only_with_executions(&mut head, 10, SafeInputRange::empty_at(0), &[]) + .expect("the live head still rotates"); + } + + #[test] + fn a_head_that_disagrees_with_the_tip_fails_loud_at_batch_close() { + let (_db, mut storage, head) = tip_past_genesis("stale-head-close"); + let count = storage.next_l2_tx_count().unwrap(); + for (name, mut stale) in stale_heads(&head) { + let panic = catch_unwind(AssertUnwindSafe(|| { + let _ = storage.close_frame_and_batch_with_snapshot( + &mut stale, + std::path::Path::new("/snapshot/stale"), + count, + ); + })) + .expect_err(name); + assert!( + panic_message(panic).contains("write head disagrees with the durable Tip"), + "{name}" + ); + } + assert_eq!(storage.latest_batch_nonce().unwrap(), Some(1)); + let mut head = head; + storage + .close_frame_and_batch_with_snapshot( + &mut head, + std::path::Path::new("/snapshot/live"), + count, + ) + .expect("the live head still closes"); + assert_eq!(head.batch_nonce(), 2); + } + + /// The head carries its batch's nonce from the loader and through every + /// close, and a reload agrees. + #[test] + fn the_head_carries_the_open_batch_nonce() { + let db = temp_db("head-batch-nonce"); + let mut storage = Storage::open_with_era( + &db.path, + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + assert_eq!( + head.batch_nonce(), + 7, + "the era's first batch is at the anchor" + ); + storage.close_frame_and_batch(&mut head).unwrap(); + assert_eq!(head.batch_nonce(), 8); + let count = storage.next_l2_tx_count().unwrap(); + storage + .close_frame_and_batch_with_snapshot(&mut head, std::path::Path::new("/s/9"), count) + .unwrap(); + assert_eq!(head.batch_nonce(), 9); + assert_eq!(storage.open_state().unwrap().unwrap().batch_nonce(), 9); + } + #[test] #[should_panic(expected = "frame clock cannot regress")] fn frame_clock_cannot_regress_even_when_the_l1_interval_has_no_inputs() { let db = temp_db("empty-frame-clock-regression"); - let mut storage = Storage::open(&db.path).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .unwrap(); @@ -610,7 +823,7 @@ mod tests { #[test] fn open_state_is_idempotent_and_rotation_is_atomic() { let db = temp_db("open-state"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); assert!( storage.open_state().expect("load open state").is_none(), @@ -625,35 +838,34 @@ mod tests { .expect("load existing open state") .expect("open state should now exist"); - assert_eq!(head_a.batch_index, head_b.batch_index); - assert_eq!(head_a.frame_in_batch, head_b.frame_in_batch); - assert_eq!(head_a.frame_fee, head_b.frame_fee); + assert_eq!(head_a.batch_nonce(), head_b.batch_nonce()); + assert_eq!(head_a.frame_in_batch(), head_b.frame_in_batch()); + assert_eq!(head_a.frame_fee(), head_b.frame_fee()); // Default log_recommended_fee = 0+296+20+419+621 = 1356 - assert_eq!(head_a.frame_fee, 1356); + assert_eq!(head_a.frame_fee(), 1356); - let mut head_c = head_b; - let next_safe_block = head_c.safe_block; + let batch_b = head_b.batch_nonce(); + let mut head = head_b; + let next_safe_block = head.safe_block(); storage - .close_frame_only(&mut head_c, next_safe_block, SafeInputRange::empty_at(0)) + .close_frame_only(&mut head, next_safe_block, SafeInputRange::empty_at(0)) .expect("rotate within same batch"); - assert_eq!(head_c.batch_index, head_b.batch_index); - assert_eq!(head_c.frame_in_batch, 1); + assert_eq!(head.batch_nonce(), batch_b); + assert_eq!(head.frame_in_batch(), 1); - let mut head_d = head_c; - let next_safe_block = head_d.safe_block; storage - .close_frame_and_batch(&mut head_d, next_safe_block) + .close_frame_and_batch(&mut head) .expect("close batch and rotate"); - assert!(head_d.batch_index > head_c.batch_index); - assert_eq!(head_d.frame_in_batch, 0); + assert!(head.batch_nonce() > batch_b); + assert_eq!(head.frame_in_batch(), 0); } #[test] fn safe_frontier_state_withholds_poisoned_projection() { let db = temp_db("safe-frontier-state-poison"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); storage - .append_safe_inputs(12, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(12, &[], SENDER_A) .expect("record safe head"); assert_eq!( @@ -680,15 +892,19 @@ mod tests { #[test] fn append_counter_overflow_happens_before_the_transaction() { let db = temp_db("append-counter-overflow-before-write"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage + let mut storage = set_up_storage(&db.path, SENDER_A); + let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - head.open_frame_user_op_count = u32::MAX; + let mut head = WriteHeadParts { + open_frame_user_op_count: u32::MAX, + ..WriteHeadParts::of(&head) + } + .build(); let included = included_user_op(0, 0); let panic = catch_unwind(AssertUnwindSafe(|| { - let _ = storage.append_executed_user_ops_chunk(&mut head, &[included]); + let _ = storage.append_included_user_ops(&mut head, &[included]); })); assert!(panic.is_err(), "counter overflow must fail loud"); @@ -698,7 +914,7 @@ mod tests { .expect("count user ops"); assert_eq!(persisted, 0, "overflow must occur before any durable write"); assert_eq!( - head.open_frame_user_op_count, + head.open_frame_user_op_count(), u32::MAX, "the authoritative head must remain unchanged" ); @@ -707,14 +923,14 @@ mod tests { #[test] fn mismatched_execution_offset_rolls_back_source_and_history() { let db = temp_db("user-execution-offset-atomicity"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); let included = included_user_op(0, 1); let error = storage - .append_executed_user_ops_chunk(&mut head, &[included]) + .append_included_user_ops(&mut head, &[included]) .expect_err("non-canonical execution offset must fail loud"); assert!( error.to_string().contains("must equal next count"), @@ -731,16 +947,15 @@ mod tests { assert_eq!(persisted, 0, "{table} must roll back atomically"); } assert_eq!( - storage.next_executed_input_count().expect("next count"), - ExecutedInputCount::ZERO + storage.next_l2_tx_count().expect("next count"), + L2TxCount::ZERO ); } #[test] fn live_direct_rotation_requires_complete_classified_attribution() { let db = temp_db("direct-execution-attribution-complete"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); @@ -757,12 +972,12 @@ mod tests { }, ]; storage - .append_safe_inputs(10, &directs, SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, &directs, SENDER_A) .expect("persist direct inputs"); let incomplete = [DirectInputExecution { safe_input_index: 0, - executed_input_offset: ExecutedInputCount::ZERO, + l2_tx_index: L2TxCount::ZERO, }]; let panic = catch_unwind(AssertUnwindSafe(|| { let _ = storage.close_frame_only_with_executions( @@ -773,7 +988,7 @@ mod tests { ); })); assert!(panic.is_err(), "omitted executable direct must fail loud"); - assert_eq!(head.frame_in_batch, 0); + assert_eq!(head.frame_in_batch(), 0); let frames: i64 = storage .conn .query_row("SELECT COUNT(*) FROM frames", [], |row| row.get(0)) @@ -783,88 +998,23 @@ mod tests { let complete = [ DirectInputExecution { safe_input_index: 0, - executed_input_offset: ExecutedInputCount::ZERO, + l2_tx_index: L2TxCount::ZERO, }, DirectInputExecution { safe_input_index: 1, - executed_input_offset: ExecutedInputCount::new(1), + l2_tx_index: L2TxCount::new(1), }, ]; storage .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 2), &complete) .expect("commit complete direct attribution"); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::new(2) - ); - } - - #[test] - fn invalidation_rewinds_and_replacement_reuses_logical_offset() { - let db = temp_db("execution-offset-invalidation-reuse"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) - .expect("append gold input"); - storage - .close_frame_and_batch(&mut head, 0) - .expect("seal gold batch"); - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(1, 1)]) - .expect("append doomed input"); - assert_eq!( - storage - .next_executed_input_count() - .expect("pre-recovery count"), - ExecutedInputCount::new(2) - ); - - storage - .append_safe_inputs(1_500, &[], SENDER_A, &default_protocol_timing()) - .expect("advance safe head"); - assert_eq!( - storage - .recover_aging_tip(1_200) - .expect("invalidate stale Tip"), - vec![1] - ); - assert_eq!( - storage.next_executed_input_count().expect("rewound count"), - ExecutedInputCount::new(1) - ); - - let mut replacement = storage.open_state().unwrap().unwrap(); - storage - .append_executed_user_ops_chunk(&mut replacement, &[included_user_op(1, 1)]) - .expect("reuse invalidated logical offset"); - - let current: Vec = storage - .conn - .prepare("SELECT offset FROM application_inputs ORDER BY offset") - .unwrap() - .query_map([], |row| row.get(0)) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!(current, vec![0, 1]); - let source_rows: i64 = storage - .conn - .query_row("SELECT COUNT(*) FROM user_ops", [], |row| row.get(0)) - .unwrap(); - assert_eq!( - source_rows, 3, - "invalidation preserves original signed operations" - ); + assert_eq!(storage.next_l2_tx_count().unwrap(), L2TxCount::new(2)); } #[test] fn saturated_block_bound_counts_every_representable_block() { let db = temp_db("saturated-block-bound"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); storage .append_safe_inputs( 10, @@ -874,12 +1024,11 @@ mod tests { block_number: 10, }], SENDER_A, - &default_protocol_timing(), ) .expect("append safe input"); let boundary = storage - .read(|tx| super::safe_input_index_exclusive_through_block_in(tx, u64::MAX)) + .read(|tx| super::safe_input_count_through_block_in(tx, u64::MAX)) .expect("query upper bound above SQLite range"); assert_eq!( boundary, 1, @@ -890,7 +1039,7 @@ mod tests { #[test] fn next_frame_fee_comes_from_batch_policy() { let db = temp_db("batch-policy-fee"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let policy = storage.batch_policy().expect("default policy"); // Default: log_gas_price=0, log_recommended_fee = 0+296+20+419+621 = 1356 assert_eq!(policy.recommended_fee, 1356); @@ -900,17 +1049,16 @@ mod tests { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - let next_safe_block = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe_block) + .close_frame_and_batch(&mut head) .expect("rotate batch"); let policy = storage.batch_policy().expect("read policy"); // log_recommended_fee = 100+296+20+419+621 = 1456 - assert_eq!(head.frame_fee, 1456); - assert_eq!(head.frame_fee, policy.recommended_fee); + assert_eq!(head.frame_fee(), 1456); + assert_eq!(head.frame_fee(), policy.recommended_fee); assert!( - head.max_batch_user_op_bytes > 0, + head.max_batch_user_op_bytes() > 0, "batch size target should be set" ); } @@ -924,15 +1072,15 @@ mod tests { // frame know the fee they're paying, regardless of upstream policy // drift during their round-trip. let db = temp_db("frame-fee-immutable"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); - let original_batch_index = head.batch_index; - let original_frame_in_batch = head.frame_in_batch; + let original_batch_nonce = head.batch_nonce(); + let original_frame_in_batch = head.frame_in_batch(); // Default: log_gas_price=0 → log_recommended_fee = 0+296+20+419+621 = 1356 - assert_eq!(head.frame_fee, 1356); + assert_eq!(head.frame_fee(), 1356); // Simulate an operator policy update mid-frame: fee oracle reports a // higher gas price. The derived view reflects the new fee immediately. @@ -949,8 +1097,8 @@ mod tests { let persisted_frame_fee: i64 = storage .conn .query_row( - "SELECT fee FROM frames WHERE batch_index = ?1 AND frame_in_batch = ?2", - rusqlite::params![original_batch_index as i64, original_frame_in_batch as i64,], + "SELECT fee FROM frames WHERE batch_nonce = ?1 AND frame_in_batch = ?2", + rusqlite::params![original_batch_nonce as i64, original_frame_in_batch as i64,], |row| row.get(0), ) .expect("query open frame fee"); @@ -962,28 +1110,36 @@ mod tests { // And the in-memory WriteHead mirror must also be stable — the lane // submitting against this head should see a consistent fee. assert_eq!( - head.frame_fee, 1356, + head.frame_fee(), + 1356, "WriteHead.frame_fee must stay stable until advance_frame runs", ); assert_eq!( storage.current_fee_quote().expect("quote"), - Some((1356, 1456)), + Some(FeeQuote { + frame_fee: 1356, + recommended_fee: 1456 + }), "quote splits the frozen open-frame fee from the advanced recommended_fee", ); // Closing the frame picks up the new policy — the *next* frame opens // at 1456. This is the expected policy-flow boundary. - let next_safe_block = head.safe_block; + let next_safe_block = head.safe_block(); storage .close_frame_only(&mut head, next_safe_block, SafeInputRange::empty_at(0)) .expect("rotate within same batch"); assert_eq!( - head.frame_fee, 1456, + head.frame_fee(), + 1456, "the next frame must use the updated policy's fee (policy flows in at close)", ); assert_eq!( storage.current_fee_quote().expect("quote"), - Some((1456, 1456)), + Some(FeeQuote { + frame_fee: 1456, + recommended_fee: 1456 + }), "after rotation the quote's two fees agree again", ); } @@ -991,8 +1147,7 @@ mod tests { #[test] fn next_undrained_input_is_derived_from_accounted_frame_block() { let db = temp_db("safe-cursor"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); assert_eq!( storage .next_undrained_safe_input_index() @@ -1016,7 +1171,7 @@ mod tests { }, ]; storage - .append_safe_inputs(10, drained.as_slice(), SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, drained.as_slice(), SENDER_A) .expect("insert direct inputs"); let mut head = head; storage @@ -1031,33 +1186,98 @@ mod tests { ); } + /// The era's root frame drains the directs still queued at the stop + /// block, from the first queued input. + #[test] + fn the_era_root_drains_from_its_first_queued_input() { + use crate::storage::test_helpers::{all_ordered_l2_txs, era}; + let db = temp_db("era-root-drain"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let input = |sender, payload: Vec, block_number| StoredSafeInput { + sender, + payload, + block_number, + }; + let direct = Address::repeat_byte(0x22); + storage + .append_safe_inputs_with_timestamp( + 12, + 12, + &[ + // Executed before the base. + input(direct, vec![0], 5), + // Queued at the stop block 10, from input 1. + input(direct, vec![1], 8), + input(SENDER_A, vec![0xff], 8), + input( + direct, + vec![2; TEST_MAX_DIRECT_PAYLOAD_BYTES as usize + 1], + 9, + ), + input(direct, vec![4], 10), + // Past the stop block. + input(direct, vec![5], 12), + ], + SENDER_A, + ) + .unwrap(); + storage + .publish_era( + std::path::Path::new("/snapshot/era"), + &crate::storage::NewEra { + first_queued_input: 1, + anchor_batch_nonce: 6, + ..era(1, 1, 10) + }, + ) + .unwrap(); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 1); + + let drained_directs = |storage: &mut Storage| -> Vec { + all_ordered_l2_txs(storage) + .into_iter() + .map(|tx| match tx { + L2Tx::Direct(direct) => direct.input_index, + other => panic!("expected a direct, got {other:?}"), + }) + .collect() + }; + storage.ensure_open_tip().unwrap(); + let root = storage.open_state().unwrap().unwrap(); + assert_eq!(root.safe_block(), 12); + assert_eq!(drained_directs(&mut storage), vec![1, 4, 5]); + assert_eq!(storage.next_l2_tx_count().unwrap().get(), 4); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 6); + } + #[test] fn initialize_open_state_creates_first_real_batch_and_frame() { let db = temp_db("initialize-open-state"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let head = storage .initialize_open_state(12, SafeInputRange::empty_at(0)) .expect("initialize open state"); - assert_eq!(head.batch_index, 0); - assert_eq!(head.frame_in_batch, 0); - assert_eq!(head.safe_block, 12); + assert_eq!(head.batch_nonce(), 0); + assert_eq!(head.frame_in_batch(), 0); + assert_eq!(head.safe_block(), 12); let loaded = storage .open_state() .expect("load open state") .expect("open state should exist"); - assert_eq!(loaded.batch_index, 0); - assert_eq!(loaded.frame_in_batch, 0); - assert_eq!(loaded.safe_block, 12); + assert_eq!(loaded.batch_nonce(), 0); + assert_eq!(loaded.frame_in_batch(), 0); + assert_eq!(loaded.safe_block(), 12); } #[test] fn replay_returns_direct_inputs_in_drain_order() { let db = temp_db("replay-order"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); @@ -1075,7 +1295,7 @@ mod tests { }, ]; storage - .append_safe_inputs(10, drained.as_slice(), SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, drained.as_slice(), SENDER_A) .expect("insert direct inputs"); let mut head = head; storage @@ -1085,11 +1305,17 @@ mod tests { let replay = crate::storage::test_helpers::all_ordered_l2_txs(&mut storage); assert_eq!(replay.len(), 2); match &replay[0] { - SequencedL2Tx::Direct(value) => assert_eq!(value.payload.as_slice(), &[0xaa]), + L2Tx::Direct(value) => { + assert_eq!(value.payload.as_slice(), &[0xaa]); + assert_eq!((value.input_index, value.block), (0, 10)); + } _ => panic!("expected direct input at position 0"), } match &replay[1] { - SequencedL2Tx::Direct(value) => assert_eq!(value.payload.as_slice(), &[0xbb]), + L2Tx::Direct(value) => { + assert_eq!(value.payload.as_slice(), &[0xbb]); + assert_eq!((value.input_index, value.block), (1, 10)); + } _ => panic!("expected direct input at position 1"), } } @@ -1097,11 +1323,10 @@ mod tests { #[test] fn ensure_open_tip_opens_genesis_and_sequences_leading_range() { let db = temp_db("ensure-tip-genesis"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); // Pre-existing L1 history: two safe inputs at block 10, and an - // observed safe head of 10 (as the startup recovery sync would leave). + // observed safe head of 10 (as run's startup sync would leave). let leading = vec![ StoredSafeInput { sender: Address::ZERO, @@ -1115,7 +1340,7 @@ mod tests { }, ]; storage - .append_safe_inputs(10, leading.as_slice(), SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, leading.as_slice(), SENDER_A) .expect("seed safe inputs + head"); assert!( @@ -1131,10 +1356,15 @@ mod tests { .open_state() .expect("load open state") .expect("genesis tip exists"); - assert_eq!(head.batch_index, 0, "genesis batch is index 0"); - assert_eq!(head.frame_in_batch, 0); assert_eq!( - head.safe_block, 10, + head.batch_nonce(), + 0, + "the genesis era's first batch carries nonce 0" + ); + assert_eq!(head.frame_in_batch(), 0); + assert_eq!( + head.safe_block(), + 10, "genesis frame opens at the synced safe head" ); @@ -1155,7 +1385,7 @@ mod tests { let replay = storage .canonical_history_page( sequencer_core::history::HistoryClaim { - version: bounds.version, + era_id: bounds.era_id, next_input: bounds.available_from, }, 100, @@ -1167,14 +1397,14 @@ mod tests { 2, "leading directs are in the replay stream for catch-up" ); - assert_eq!(replay[0].offset, ExecutedInputCount::ZERO); - assert_eq!(replay[1].offset, ExecutedInputCount::new(1)); + assert_eq!(replay[0].offset, L2TxCount::ZERO); + assert_eq!(replay[1].offset, L2TxCount::new(1)); } #[test] fn ensure_open_tip_is_noop_when_tip_already_exists() { let db = temp_db("ensure-tip-noop"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // Seed an explicit open Tip at safe_block 5. Note: no safe-head // observation is recorded, so a genesis-branch `current_safe_block` @@ -1189,11 +1419,13 @@ mod tests { .expect("load open state") .expect("tip still exists"); assert_eq!( - head.batch_index, seeded.batch_index, + head.batch_nonce(), + seeded.batch_nonce(), "warm path leaves the existing Tip, does not reopen" ); assert_eq!( - head.safe_block, 5, + head.safe_block(), + 5, "existing frame's safe_block is preserved, not refreshed" ); } @@ -1201,30 +1433,40 @@ mod tests { fn application_rows_require_initialized_baseline_and_a_live_tip() { let db = temp_db("application-history-baseline-required"); let mut storage = - Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - assert!( - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) - .is_err() - ); + Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + // Before the era: no Tip can root, and no application row lands. + assert!(matches!( + storage.initialize_open_state(0, SafeInputRange::empty_at(0)), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + let err = storage + .conn + .execute( + "INSERT INTO application_inputs \ + (l2_tx_count, batch_nonce, frame_in_batch, user_op_pos_in_frame) \ + VALUES (0, 0, 0, 0)", + [], + ) + .unwrap_err(); + assert_eq!(err.to_string(), "application history baseline is missing"); storage .write(|tx| { - super::super::history::initialize_history_in(tx, ExecutedInputCount::ZERO, 0) + super::super::history::insert_era_in(tx, &crate::storage::NewEra::EMPTY_GENESIS) }) .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) + .append_included_user_ops(&mut head, &[included_user_op(0, 0)]) .unwrap(); - storage.close_frame_and_batch(&mut head, 0).unwrap(); - let err = storage.conn.execute("INSERT INTO application_inputs (offset,batch_index,frame_in_batch,user_op_pos_in_frame) VALUES (1,0,0,0)", []).unwrap_err(); - assert!(err.to_string().contains("current valid Tip")); + storage.close_frame_and_batch(&mut head).unwrap(); + let err = storage.conn.execute("INSERT INTO application_inputs (l2_tx_count,batch_nonce,frame_in_batch,user_op_pos_in_frame) VALUES (1,0,0,0)", []).unwrap_err(); + assert!(err.to_string().contains("must target the open batch")); assert!( storage .conn - .execute("UPDATE application_inputs SET offset = 1", []) + .execute("UPDATE application_inputs SET l2_tx_count = 1", []) .is_err() ); assert!( @@ -1238,8 +1480,7 @@ mod tests { #[test] fn envelopes_advance_accounting_without_becoming_application_inputs() { let db = temp_db("envelope-only-accounting"); - let mut storage = Storage::open(&db.path).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); @@ -1252,19 +1493,15 @@ mod tests { block_number: 10, }], SENDER_A, - &default_protocol_timing(), ) .unwrap(); storage .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 1), &[]) .unwrap(); assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 1); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::ZERO - ); + assert_eq!(storage.next_l2_tx_count().unwrap(), L2TxCount::ZERO); assert!(crate::storage::test_helpers::all_ordered_l2_txs(&mut storage).is_empty()); - storage.close_frame_and_batch(&mut head, 10).unwrap(); + storage.close_frame_and_batch(&mut head).unwrap(); assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 1); } @@ -1275,18 +1512,18 @@ mod tests { } #[test] - fn next_user_nonce_follows_the_latest_valid_op_across_a_cascade() { + fn next_user_nonce_follows_the_latest_op_across_batches() { let alice = Address::repeat_byte(0xa1); let bob = Address::repeat_byte(0xb0); let db = temp_db("next-user-nonce"); - let mut storage = Storage::open(&db.path).unwrap(); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); assert_eq!(storage.next_user_nonce(alice).unwrap(), 0, "unseen sender"); storage - .append_executed_user_ops_chunk( + .append_included_user_ops( &mut head, &[ included_user_op_from(alice, 0, 0), @@ -1295,9 +1532,9 @@ mod tests { ], ) .unwrap(); - storage.close_frame_and_batch(&mut head, 0).unwrap(); + storage.close_frame_and_batch(&mut head).unwrap(); storage - .append_executed_user_ops_chunk( + .append_included_user_ops( &mut head, &[ included_user_op_from(alice, 2, 3), @@ -1307,47 +1544,17 @@ mod tests { .unwrap(); assert_eq!(storage.next_user_nonce(alice).unwrap(), 4); assert_eq!(storage.next_user_nonce(bob).unwrap(), 1); - - // A cascade rolls Alice back; her invalidated rows stay for audit. - storage.insert_invalid_batch(head.batch_index).unwrap(); - assert_eq!(storage.next_user_nonce(alice).unwrap(), 2); - assert_eq!(storage.next_user_nonce(bob).unwrap(), 1); - - // Resubmitting nonce 2 leaves a duplicate (sender, nonce) pair across - // the invalid and valid batches; only the valid one counts. - storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) - .unwrap(); - storage.ensure_open_tip().unwrap(); - let mut head = storage.open_state().unwrap().unwrap(); - let offset = storage.next_executed_input_count().unwrap().get(); - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op_from(alice, 2, offset)]) - .unwrap(); - assert_eq!(storage.next_user_nonce(alice).unwrap(), 3); - let raw_max: i64 = storage - .conn - .query_row( - "SELECT MAX(nonce) FROM user_ops WHERE sender = ?1", - [alice.as_slice()], - |row| row.get(0), - ) - .unwrap(); - assert_eq!( - raw_max, 3, - "an unfiltered MAX would still report the rolled-back op" - ); } #[test] fn next_user_nonce_seeks_the_sender_index_without_sorting() { let db = temp_db("next-user-nonce-plan"); - let storage = Storage::open(&db.path).unwrap(); + let storage = set_up_storage(&db.path, SENDER_A); let plan: Vec = storage .conn .prepare(&format!( "EXPLAIN QUERY PLAN {}", - super::LAST_VALID_USER_NONCE_SQL + super::LAST_USER_NONCE_SQL )) .unwrap() .query_map([Address::ZERO.as_slice()], |row| row.get(3)) @@ -1364,4 +1571,148 @@ mod tests { "index order must serve ORDER BY:\n{plan}" ); } + + /// Four directs at block 10, timestamp 120: one at the payload cap, one a + /// byte over it, then the submitter's own input, then one more direct. + fn seed_directs_around_the_payload_cap(storage: &mut Storage) { + let cap = usize::try_from(TEST_MAX_DIRECT_PAYLOAD_BYTES).unwrap(); + let row = |sender: Address, payload: Vec, hash: u8| IngestedSafeInput { + sender, + payload, + block_number: 10, + block_timestamp: 120, + transaction_hash: B256::repeat_byte(hash), + }; + storage + .append_ingested_safe_inputs_with_timestamp( + 10, + 120, + &[ + row(Address::ZERO, vec![0xaa; cap], 1), + row(Address::ZERO, vec![0xbb; cap + 1], 2), + row(SENDER_A, vec![0xff], 3), + row(Address::repeat_byte(0x44), vec![0xcc], 4), + ], + ) + .expect("persist direct inputs"); + } + + #[test] + fn an_over_cap_direct_never_executes_and_later_directs_keep_contiguous_offsets() { + let db = temp_db("direct-payload-cap"); + let mut storage = set_up_storage(&db.path, SENDER_A); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize open state"); + seed_directs_around_the_payload_cap(&mut storage); + + // The lane reads exactly the directs that execute, each with the + // index and timestamp the application receives. + let range = SafeInputRange::new(0, 4); + let mut directs = Vec::new(); + storage + .fill_direct_inputs(range, &mut directs) + .expect("read executable directs"); + let cap = usize::try_from(TEST_MAX_DIRECT_PAYLOAD_BYTES).unwrap(); + let at_cap = Direct { + input_index: 0, + sender: Address::ZERO, + block: 10, + timestamp: 120, + payload: vec![0xaa; cap], + }; + let after_skip = Direct { + input_index: 3, + sender: Address::repeat_byte(0x44), + block: 10, + timestamp: 120, + payload: vec![0xcc], + }; + assert_eq!(directs, vec![at_cap.clone(), after_skip.clone()]); + + // Offering an execution for the over-cap direct breaks the frame's + // agreement with storage's derivation and rolls the close back. + let with_over_cap = [ + DirectInputExecution { + safe_input_index: 0, + l2_tx_index: L2TxCount::ZERO, + }, + DirectInputExecution { + safe_input_index: 1, + l2_tx_index: L2TxCount::new(1), + }, + DirectInputExecution { + safe_input_index: 3, + l2_tx_index: L2TxCount::new(2), + }, + ]; + let panic = catch_unwind(AssertUnwindSafe(|| { + let _ = storage.close_frame_only_with_executions(&mut head, 10, range, &with_over_cap); + })); + assert!(panic.is_err(), "an over-cap execution must fail loud"); + assert_eq!(head.frame_in_batch(), 0); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 0); + + let executions = [ + DirectInputExecution { + safe_input_index: 0, + l2_tx_index: L2TxCount::ZERO, + }, + DirectInputExecution { + safe_input_index: 3, + l2_tx_index: L2TxCount::new(1), + }, + ]; + storage + .close_frame_only_with_executions(&mut head, 10, range, &executions) + .expect("close the frame over the executable directs"); + + assert_eq!( + storage.next_undrained_safe_input_index().unwrap(), + 4, + "the whole range drains, the over-cap direct included" + ); + assert_eq!(storage.next_l2_tx_count().unwrap(), L2TxCount::new(2)); + let rows: Vec<(i64, i64)> = storage + .conn + .prepare( + "SELECT l2_tx_count, safe_input_index FROM application_inputs ORDER BY l2_tx_count", + ) + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(rows, vec![(0, 0), (1, 3)]); + assert_eq!( + crate::storage::test_helpers::all_ordered_l2_txs(&mut storage), + vec![L2Tx::Direct(at_cap), L2Tx::Direct(after_skip)] + ); + + storage + .close_frame_and_batch(&mut head) + .expect("the batch closes over the drained frame"); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 4); + } + + #[test] + fn a_derived_drain_skips_the_over_cap_direct_too() { + let db = temp_db("direct-payload-cap-derived"); + let mut storage = set_up_storage(&db.path, SENDER_A); + seed_directs_around_the_payload_cap(&mut storage); + + storage.ensure_open_tip().expect("open genesis tip"); + + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 4); + let replay = crate::storage::test_helpers::all_ordered_l2_txs(&mut storage); + let indexes: Vec = replay + .iter() + .map(|tx| match tx { + L2Tx::Direct(direct) => direct.input_index, + L2Tx::UserOp(op) => panic!("unexpected user op {op:?}"), + }) + .collect(); + assert_eq!(indexes, vec![0, 3]); + assert_eq!(storage.next_l2_tx_count().unwrap(), L2TxCount::new(2)); + } } diff --git a/sequencer/src/storage/l1_inputs.rs b/sequencer/src/storage/l1_inputs.rs index 76e2679d..594e7e4c 100644 --- a/sequencer/src/storage/l1_inputs.rs +++ b/sequencer/src/storage/l1_inputs.rs @@ -5,21 +5,33 @@ //! `l1_safe_head`, pin the deployment identity, and serve the safe-head reads. use alloy_primitives::Address; -use rusqlite::{OptionalExtension, Result, Transaction, params}; +use rusqlite::{OptionalExtension, Result, Transaction, named_params, params}; +use sequencer_core::scheduler::L1Input; use super::Storage; -use super::convert::{ - external_u64_to_i64, i64_to_u64, now_unix_ms, saturating_query_bound, u64_to_i64, -}; +use super::acceptance::record_accepted_batches; +use super::convert::{external_u64_to_i64, i64_to_u64, saturating_query_bound, u64_to_i64}; use super::queries::{ - current_safe_block, current_safe_block_timestamp, last_safe_progress_ms, - query_latest_safe_input_index_exclusive, -}; -use super::safe_accepted_batches::populate_safe_accepted_batches; -use super::{ - DeploymentIdentity, FeeOracleIdentity, FrontierMode, IngestedSafeInput, StoredSafeInput, + DIRECT_EXECUTES_SQL, DirectOperands, current_safe_block, current_safe_block_timestamp, + query_latest_safe_input_index_exclusive, safe_input_count_through_block_in, }; -use sequencer_core::protocol::ProtocolTiming; +use super::{DeploymentIdentity, FeeOracleIdentity, IngestedSafeInput, SafeInputRange}; + +/// What a base's export is checked against: the admitted directs above its +/// clock among the inputs it consumed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AdmittedDirects { + pub count: u64, + /// The lowest index among them; `None` when there are none. + pub first_index: Option, +} + +/// A batch-submitter input on the InputBox: its index and inclusion block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SubmitterInput { + pub safe_input_index: u64, + pub block_number: u64, +} type FeeOracleIdentitySql = ( &'static str, @@ -41,14 +53,16 @@ impl Storage { current_safe_block(&self.conn) } + /// L1 timestamp (Unix seconds) of the persisted safe block, or `None` + /// before the first sync: the safe head's age. pub fn current_safe_block_timestamp(&mut self) -> Result> { current_safe_block_timestamp(&self.conn) } /// First batch-submitter `safe_inputs` row strictly past `after_block`, by - /// ascending `safe_input_index` — returns `(safe_input_index, block_number)` - /// or `None`. Read-only; `setup`'s detection gate uses it - /// to find any previous-instance batch past the checkpoint block. + /// ascending `safe_input_index`, or `None`. Read-only; `setup`'s detection + /// gate uses it to find any previous-instance batch past the checkpoint + /// block. /// /// Queries the reader-synced `safe_inputs` table rather than issuing its own /// `get_logs`, so it inherits the reader's input-completeness witness (a @@ -65,7 +79,7 @@ impl Storage { &mut self, batch_submitter: Address, after_block: u64, - ) -> Result> { + ) -> Result> { self.conn .query_row( "SELECT safe_input_index, block_number FROM safe_inputs \ @@ -76,76 +90,146 @@ impl Storage { saturating_query_bound(after_block) ], |row| { - let index: i64 = row.get(0)?; - let block: i64 = row.get(1)?; - Ok((i64_to_u64(index), i64_to_u64(block))) + Ok(SubmitterInput { + safe_input_index: i64_to_u64(row.get(0)?), + block_number: i64_to_u64(row.get(1)?), + }) }, ) .optional() } - /// All `safe_inputs` rows whose `block_number` is in `(after_block, - /// through_block]`, ordered by `safe_input_index` ascending — i.e. L1 - /// inclusion order. The recovery fold sources - /// its seeds from `(A, B]` and its replay stream from `(B, C]` through this: - /// half-open on the lower bound (directs at block `B` belong to the fridge, - /// batches at `B` are already in `S` — open-question 1 boundary convention). - /// - /// Returns both senders and directs; the caller classifies (a batch iff - /// `sender == batch_submitter`) and drops batches from the `(A, B]` seed set. - /// Read-only; queries the reader-synced table rather than a fresh log scan, - /// inheriting the reader's input-completeness witness (see - /// [`Storage::first_batch_submitter_input_after_block`]). - pub fn safe_inputs_in_block_range( + /// Up to `limit` safe inputs with index in `[start, end_exclusive)`, in + /// order, as the scheduler takes them: the chain id and application come + /// from the pinned deployment identity. Fewer come back only when the + /// table holds no more, so a caller that needs the whole range checks the + /// length. Pages keep a rebuild's memory bounded. + pub fn l1_inputs_page( &mut self, - after_block: u64, - through_block: u64, - ) -> Result> { - const SQL: &str = "SELECT sender, payload, block_number FROM safe_inputs \ - WHERE block_number > ?1 AND block_number <= ?2 \ - ORDER BY safe_input_index ASC"; - let mut stmt = self.conn.prepare_cached(SQL)?; - let rows = stmt.query_map( - params![ - saturating_query_bound(after_block), - saturating_query_bound(through_block) - ], - |row| { - Ok(( - row.get::<_, Vec>(0)?, - row.get::<_, Vec>(1)?, - row.get::<_, i64>(2)?, - )) - }, - )?; - let mut out = Vec::new(); - for row in rows { - let (sender, payload, block_number) = row?; - out.push(StoredSafeInput { - sender: Address::from_slice(sender.as_slice()), - payload, - block_number: i64_to_u64(block_number), - }); + start: u64, + end_exclusive: u64, + limit: usize, + ) -> Result> { + let len = end_exclusive + .saturating_sub(start) + .min(u64::try_from(limit).expect("limit fits u64")); + if len == 0 { + return Ok(Vec::new()); } - Ok(out) + self.read(|tx| { + let identity = + query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; + let mut statement = tx.prepare_cached( + "SELECT safe_input_index, sender, payload, block_number, block_timestamp \ + FROM safe_inputs WHERE safe_input_index >= ?1 \ + ORDER BY safe_input_index LIMIT ?2", + )?; + let rows = statement.query_map( + params![saturating_query_bound(start), saturating_query_bound(len)], + |row| { + Ok(L1Input { + index: i64_to_u64(row.get(0)?), + sender: Address::from_slice(&row.get::<_, Vec>(1)?), + block: i64_to_u64(row.get(3)?), + timestamp: i64_to_u64(row.get(4)?), + chain_id: identity.chain_id, + app_contract: identity.app_address, + payload: row.get(2)?, + }) + }, + )?; + let mut inputs = Vec::new(); + for row in rows { + let input = row?; + let expected = start + .checked_add(inputs.len() as u64) + .expect("safe input index overflow"); + assert_eq!(input.index, expected, "safe inputs have an index gap"); + inputs.push(input); + } + Ok(inputs) + }) + } + + /// How many safe inputs lie at or below `block`: the L1 input count + /// through it. + pub fn safe_input_count_through_block(&mut self, block: u64) -> Result { + self.read(|tx| safe_input_count_through_block_in(tx, block)) + } + + /// The admitted directs among the first `l1_input_count` inputs whose + /// block is above `clock`, by the predicate every drain uses: not from + /// the batch submitter, payload within the pinned cap. A + /// canonical export queues exactly these, from the first of them. + pub fn admitted_directs_above_clock( + &mut self, + l1_input_count: u64, + clock: u64, + ) -> Result { + self.read(|tx| { + let operands = DirectOperands::load(tx)?; + tx.prepare_cached(&format!( + "SELECT COUNT(*), MIN(safe_input_index) FROM safe_inputs \ + WHERE safe_input_index < :count AND block_number > :clock \ + AND {DIRECT_EXECUTES_SQL}" + ))? + .query_row( + operands + .bind(named_params! { + ":count": saturating_query_bound(l1_input_count), + ":clock": saturating_query_bound(clock), + }) + .as_slice(), + |row| { + Ok(AdmittedDirects { + count: i64_to_u64(row.get(0)?), + first_index: row.get::<_, Option>(1)?.map(i64_to_u64), + }) + }, + ) + }) + } + + /// How many inputs with an index in `range` execute as directs, by the + /// predicate every drain uses. Setup checks its executed directs against it. + pub fn admitted_directs_in(&mut self, range: SafeInputRange) -> Result { + self.read(|tx| { + let operands = DirectOperands::load(tx)?; + tx.prepare_cached(&format!( + "SELECT COUNT(*) FROM safe_inputs \ + WHERE safe_input_index >= :start AND safe_input_index < :end \ + AND {DIRECT_EXECUTES_SQL}" + ))? + .query_row( + operands + .bind(named_params! { + ":start": saturating_query_bound(range.start()), + ":end": saturating_query_bound(range.end()), + }) + .as_slice(), + |row| Ok(i64_to_u64(row.get(0)?)), + ) + }) } /// Production input-reader path — the one write of the safe-input stream. /// /// Atomically: insert `inputs` (assigned contiguous indexes starting from /// the current MAX+1) with their per-input L1 provenance, advance - /// `l1_safe_head.block_number` to `safe_block`, stamp `synced_at_ms` as - /// the wall-clock time when the safe frontier advanced, and update - /// `safe_accepted_batches` via `timing` so the scheduler-accepted - /// frontier view stays consistent with the safe head. + /// `l1_safe_head` to `safe_block` and its timestamp, and record + /// `batches.accepted_input` so the scheduler-accepted frontier stays + /// consistent with the safe head. /// - /// The materialized `safe_accepted_batches` view is an invariant of this + /// Once the era exists, the accepted frontier is an invariant of this /// operation while no divergence exists: every safe input up to /// `safe_block` has been evaluated against the scheduler's acceptance - /// rules. A foreign/mismatched accepted landing instead commits the - /// `canonical_divergence` fact with this head and freezes the projection; - /// later head advances remain paired with that terminal fact. Readers - /// (submitter, recovery, danger checks) never populate separately. + /// rules for the pinned batch submitter and window. Before the era + /// (setup's syncs) nothing is accepted; the first append after it judges + /// every input past the stop block. A foreign/mismatched accepted landing + /// instead commits the `canonical_divergence` fact with this head and + /// freezes the projection; later head advances remain paired with that + /// terminal fact. Readers (submitter, landing watch, startup) never + /// populate separately. /// /// Asserts `safe_block` is monotonic and that it strictly advances when /// `inputs` is non-empty. @@ -154,9 +238,6 @@ impl Storage { safe_block: u64, safe_block_timestamp: u64, inputs: &[IngestedSafeInput], - batch_submitter: Address, - timing: &ProtocolTiming, - frontier: FrontierMode, ) -> Result<()> { self.write(|tx| { if let Some(current) = current_safe_block(tx)? { @@ -174,69 +255,34 @@ impl Storage { insert_safe_inputs_batch(tx, next_index, inputs)?; let changed = tx.execute( - "INSERT INTO l1_safe_head \ - (singleton_id, block_number, block_timestamp, synced_at_ms) \ - VALUES (0, ?1, ?2, ?3) \ + "INSERT INTO l1_safe_head (singleton_id, block_number, block_timestamp) \ + VALUES (0, ?1, ?2) \ ON CONFLICT(singleton_id) DO UPDATE SET \ block_number = excluded.block_number, \ - block_timestamp = excluded.block_timestamp, \ - synced_at_ms = excluded.synced_at_ms", + block_timestamp = excluded.block_timestamp", params![ external_u64_to_i64(safe_block, "L1 safe block")?, external_u64_to_i64(safe_block_timestamp, "L1 safe block timestamp")?, - now_unix_ms() ], )?; if changed != 1 { return Err(rusqlite::Error::StatementChangedRows(changed)); } - if matches!(frontier, FrontierMode::Populate) { - populate_safe_accepted_batches(tx, batch_submitter, timing)?; - } - Ok(()) + record_accepted_batches(tx) }) } - /// Wall-clock timestamp (Unix ms) of the last observed safe-head advance, - /// or `None` if no real safe-head observation has occurred yet. - pub fn last_safe_progress_ms(&self) -> Result> { - last_safe_progress_ms(&self.conn) - } - /// Read the deployment identity this DB is pinned to. Returns `None` on /// first startup, before L1 bootstrap has discovered the InputBox stream. pub fn deployment_identity(&self) -> Result> { query_deployment_identity(&self.conn) } - /// Whether this DB already contains deployment-bound state. Used to avoid - /// silently pinning an old, non-empty DB that predates `deployment_identity` - /// to whatever config happens to start it next. - pub fn has_persisted_deployment_state(&self) -> Result { - let present: i64 = self.conn.query_row( - "SELECT \ - EXISTS(SELECT 1 FROM batches) OR \ - EXISTS(SELECT 1 FROM safe_inputs) OR \ - EXISTS(SELECT 1 FROM l1_safe_head)", - [], - |row| row.get(0), - )?; - Ok(present != 0) - } - - /// Insert `identity` on first startup, or return the already-persisted - /// identity on later startups. The caller compares the returned value with - /// its configured/discovered identity and refuses on mismatch. - pub fn load_or_insert_deployment_identity( - &mut self, - identity: DeploymentIdentity, - ) -> Result { + /// Pin the deployment identity. Setup calls it once, on the database it + /// just created; the row's key refuses a second identity. + pub fn pin_deployment_identity(&mut self, identity: DeploymentIdentity) -> Result<()> { self.write(|tx| { - if let Some(existing) = query_deployment_identity(tx)? { - return Ok(existing); - } - let (mode, fixed, weth, fee_token, pool, window): FeeOracleIdentitySql = match identity.fee_oracle { FeeOracleIdentity::Fixed { log_gas_price } => ( @@ -264,10 +310,10 @@ impl Storage { let changed = tx.execute( "INSERT INTO deployment_identity \ (singleton_id, chain_id, app_address, input_box_address, \ - app_deployment_block, batch_submitter_address, fee_oracle_mode, \ - fixed_log_gas_price, fee_oracle_weth, fee_oracle_fee_token, \ - fee_oracle_pool, fee_oracle_twap_window_secs) \ - VALUES (0, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + app_deployment_block, batch_submitter_address, max_direct_payload_bytes, \ + max_wait_blocks, fee_oracle_mode, fixed_log_gas_price, fee_oracle_weth, \ + fee_oracle_fee_token, fee_oracle_pool, fee_oracle_twap_window_secs) \ + VALUES (0, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", params![ external_u64_to_i64(identity.chain_id, "deployment chain ID")?, identity.app_address.as_slice(), @@ -277,6 +323,11 @@ impl Storage { "application deployment block" )?, identity.batch_submitter_address.as_slice(), + external_u64_to_i64( + identity.max_direct_payload_bytes, + "maximum direct payload bytes" + )?, + external_u64_to_i64(identity.max_wait_blocks, "staleness window")?, mode, fixed, weth, @@ -288,20 +339,15 @@ impl Storage { if changed != 1 { return Err(rusqlite::Error::StatementChangedRows(changed)); } - Ok(identity) + Ok(()) }) } - /// Whether `setup` has completed on this DB. `run` refuses to boot when - /// this is `false`: setup either never ran or crashed midway, both of - /// which require `setup` (re-)run, not `run`. + /// Whether `setup` has completed on this DB: whether its era exists. + /// `run` refuses to boot when this is `false`: setup either never ran or + /// crashed midway, both of which require `setup` (re-)run, not `run`. pub fn is_setup_complete(&self) -> Result { - let present: i64 = self.conn.query_row( - "SELECT EXISTS(SELECT 1 FROM setup_complete WHERE singleton_id = 0)", - [], - |row| row.get(0), - )?; - Ok(present != 0) + super::lifecycle::is_set_up_in(&self.conn) } } @@ -312,7 +358,8 @@ pub(super) fn query_deployment_identity( "SELECT chain_id, app_address, input_box_address, \ app_deployment_block, batch_submitter_address, fee_oracle_mode, \ fixed_log_gas_price, fee_oracle_weth, fee_oracle_fee_token, \ - fee_oracle_pool, fee_oracle_twap_window_secs \ + fee_oracle_pool, fee_oracle_twap_window_secs, max_direct_payload_bytes, \ + max_wait_blocks \ FROM deployment_identity WHERE singleton_id = 0", [], |row| { @@ -335,6 +382,8 @@ pub(super) fn query_deployment_identity( input_box_address: Address::from_slice(&row.get::<_, Vec>(2)?), app_deployment_block: i64_to_u64(row.get::<_, i64>(3)?), batch_submitter_address: Address::from_slice(&row.get::<_, Vec>(4)?), + max_direct_payload_bytes: i64_to_u64(row.get::<_, i64>(11)?), + max_wait_blocks: i64_to_u64(row.get::<_, i64>(12)?), fee_oracle, }) }, @@ -376,12 +425,13 @@ fn insert_safe_inputs_batch( #[cfg(test)] mod tests { + use super::{AdmittedDirects, SubmitterInput}; use crate::storage::{ - DeploymentIdentity, FeeOracleIdentity, FrontierMode, IngestedSafeInput, Storage, + DeploymentIdentity, FeeOracleIdentity, IngestedSafeInput, SafeInputRange, Storage, StoredSafeInput, - test_helpers::{SENDER_A, SENDER_B, default_protocol_timing}, + test_helpers::{SENDER_A, SENDER_B, TEST_MAX_DIRECT_PAYLOAD_BYTES, set_up_storage}, }; - use crate::test_support::temp_db; + use crate::test_support::{pin_test_deployment_identity, temp_db}; use alloy_primitives::{Address, B256}; fn identity() -> DeploymentIdentity { @@ -391,6 +441,8 @@ mod tests { input_box_address: Address::repeat_byte(0x22), app_deployment_block: 42, batch_submitter_address: SENDER_A, + max_direct_payload_bytes: 4096, + max_wait_blocks: 1200, fee_oracle: FeeOracleIdentity::Fixed { log_gas_price: 0 }, } } @@ -398,8 +450,7 @@ mod tests { #[test] fn safe_input_end_exclusive_counts_appended_inputs() { let db = temp_db("safe-input-api"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); assert_eq!(storage.safe_input_end_exclusive().expect("safe head"), 0); @@ -416,23 +467,21 @@ mod tests { }, ]; storage - .append_safe_inputs(10, inserted.as_slice(), SENDER_A, &protocol) + .append_safe_inputs(10, inserted.as_slice(), SENDER_A) .expect("insert safe directs"); assert_eq!(storage.safe_input_end_exclusive().expect("safe head"), 2); + assert_eq!(storage.safe_input_count_through_block(9).expect("count"), 0); assert_eq!( - storage - .safe_inputs_in_block_range(9, 10) - .expect("read back"), - inserted + storage.safe_input_count_through_block(10).expect("count"), + 2 ); } #[test] fn first_batch_submitter_input_after_block_scans_strictly_past() { let db = temp_db("first-submitter-input-after"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); // index 0: submitter @5; index 1: non-submitter @12; index 2: submitter @18. // Payloads are junk (scheduler no-ops on decode) — detection scans the @@ -456,7 +505,7 @@ mod tests { }, ]; storage - .append_safe_inputs(20, inputs.as_slice(), SENDER_A, &protocol) + .append_safe_inputs(20, inputs.as_slice(), SENDER_A) .expect("seed safe inputs"); // From genesis (0): the earliest submitter input is index 0 @5. @@ -464,7 +513,10 @@ mod tests { storage .first_batch_submitter_input_after_block(SENDER_A, 0) .expect("scan"), - Some((0, 5)) + Some(SubmitterInput { + safe_input_index: 0, + block_number: 5 + }) ); // Strictly past: block 5 is excluded; the non-submitter @12 is skipped; // next submitter is index 2 @18. @@ -472,7 +524,10 @@ mod tests { storage .first_batch_submitter_input_after_block(SENDER_A, 5) .expect("scan"), - Some((2, 18)) + Some(SubmitterInput { + safe_input_index: 2, + block_number: 18 + }) ); // Past the last submitter input: nothing. assert_eq!( @@ -498,117 +553,183 @@ mod tests { ); } - #[test] - fn safe_inputs_in_block_range_is_half_open_lower_and_ordered() { - let db = temp_db("safe-inputs-block-range"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + /// Blocks 5 (direct), 10 (batch from SENDER_A, over-cap direct, direct) + /// and 15 (direct). SENDER_A is the batch submitter. + fn seed_classified_inputs(storage: &mut Storage) { + pin_test_deployment_identity(storage, SENDER_A); + seed_inputs(storage); + } - // Blocks 5 (direct), 10 (batch from SENDER_A + direct from SENDER_B), - // 15 (direct). SENDER_A is the batch submitter (identity above). - let inputs = vec![ - StoredSafeInput { - sender: SENDER_B, - payload: vec![0x05], - block_number: 5, - }, - StoredSafeInput { - sender: SENDER_A, - payload: vec![0x10], - block_number: 10, - }, - StoredSafeInput { - sender: SENDER_B, - payload: vec![0x11], - block_number: 10, - }, - StoredSafeInput { - sender: SENDER_B, - payload: vec![0x15], - block_number: 15, - }, - ]; + fn seed_inputs(storage: &mut Storage) { + let input = |sender, payload: Vec, block_number| StoredSafeInput { + sender, + payload, + block_number, + }; storage - .append_safe_inputs(15, inputs.as_slice(), SENDER_A, &protocol) + .append_safe_inputs( + 15, + &[ + input(SENDER_B, vec![0x05], 5), + input(SENDER_A, vec![0x10], 10), + input( + SENDER_B, + vec![0; TEST_MAX_DIRECT_PAYLOAD_BYTES as usize + 1], + 10, + ), + input(SENDER_B, vec![0x11], 10), + input(SENDER_B, vec![0x15], 15), + ], + SENDER_A, + ) .expect("seed safe inputs"); + } - // (5, 15] excludes block 5, includes block 15 — ordered by index. - let mid = storage - .safe_inputs_in_block_range(5, 15) - .expect("range (5,15]"); - let blocks: Vec = mid.iter().map(|i| i.block_number).collect(); - assert_eq!(blocks, vec![10, 10, 15], "half-open lower bound; ascending"); - - // Caller classifies: drop sender == batch_submitter to get the directs - // of the seed range (the (A,B] fridge reconstruction). - let directs: Vec<&[u8]> = mid - .iter() - .filter(|i| i.sender != SENDER_A) - .map(|i| i.payload.as_slice()) - .collect(); + #[test] + fn l1_inputs_page_reads_an_index_range_in_bounded_pages() { + let db = temp_db("l1-inputs-page"); + // No identity yet: the test pins its own. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + seed_classified_inputs(&mut storage); + let identity = storage + .deployment_identity() + .expect("read") + .expect("pinned"); + + let page = storage.l1_inputs_page(1, 5, 2).expect("page"); assert_eq!( - directs, - vec![&[0x11u8][..], &[0x15u8][..]], - "batch at block 10 dropped" + page.iter() + .map(|input| (input.index, input.sender, input.block)) + .collect::>(), + vec![(1, SENDER_A, 10), (2, SENDER_B, 10)] ); - - // Empty when the lower bound covers everything. + assert!(page.iter().all(|input| input.chain_id == identity.chain_id + && input.app_contract == identity.app_address)); + assert_eq!(page[0].payload, vec![0x10]); + let rest = storage.l1_inputs_page(3, 5, 10).expect("rest"); + assert_eq!( + rest.iter().map(|input| input.index).collect::>(), + vec![3, 4] + ); + // Past the table's end it returns what exists, and nothing for an empty range. + assert_eq!(storage.l1_inputs_page(4, 9, 10).expect("tail").len(), 1); + assert!(storage.l1_inputs_page(3, 3, 10).expect("empty").is_empty()); assert!( storage - .safe_inputs_in_block_range(15, 15) - .expect("empty") + .l1_inputs_page(4, 2, 10) + .expect("inverted") .is_empty() ); - // Full span from genesis. - assert_eq!( + } + + #[test] + fn export_check_counts_admitted_directs_above_the_clock() { + let db = temp_db("export-check"); + // No identity yet: the test pins its own. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + seed_classified_inputs(&mut storage); + let check = |storage: &mut Storage, count, clock| { storage - .safe_inputs_in_block_range(0, 100) - .expect("all") - .len(), + .admitted_directs_above_clock(count, clock) + .expect("check") + }; + // Above clock 5 among all five: the submitter input and the over-cap + // direct are not admitted, so inputs 3 and 4 remain. + assert_eq!( + check(&mut storage, 5, 5), + AdmittedDirects { + count: 2, + first_index: Some(3) + } + ); + assert_eq!( + check(&mut storage, 5, 0), + AdmittedDirects { + count: 3, + first_index: Some(0) + } + ); + // The base consumed only four inputs. + assert_eq!( + check(&mut storage, 4, 5), + AdmittedDirects { + count: 1, + first_index: Some(3) + } + ); + assert_eq!( + check(&mut storage, 5, 15), + AdmittedDirects { + count: 0, + first_index: None + } + ); + assert_eq!(check(&mut storage, u64::MAX, 0).count, 3); + assert_eq!(storage.safe_input_count_through_block(5).expect("count"), 1); + assert_eq!( + storage.safe_input_count_through_block(14).expect("count"), 4 ); assert_eq!( storage - .safe_inputs_in_block_range(0, u64::MAX) - .expect("all through saturated upper bound") - .len(), - 4, - "an upper bound above SQLite's range includes every stored row" + .safe_input_count_through_block(u64::MAX) + .expect("count"), + 5 ); - assert!( + } + + #[test] + fn counts_the_admitted_directs_in_an_index_range() { + let db = temp_db("admitted-in-range"); + // No identity yet: the test pins its own. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + seed_classified_inputs(&mut storage); + let count = |storage: &mut Storage, start, end| { storage - .safe_inputs_in_block_range(u64::MAX, u64::MAX) - .expect("empty past SQLite range") - .is_empty(), - "a lower bound above SQLite's range excludes every stored row" - ); + .admitted_directs_in(SafeInputRange::new(start, end)) + .expect("count") + }; + // Input 1 is the submitter's and input 2 is over the cap. + assert_eq!(count(&mut storage, 0, 5), 3); + assert_eq!(count(&mut storage, 1, 3), 0); + assert_eq!(count(&mut storage, 1, 4), 1); + assert_eq!(count(&mut storage, 4, 4), 0); + assert_eq!(count(&mut storage, 0, u64::MAX), 3); } + /// The predicate's cap is the pinned one: under a cap of zero no + /// one-byte direct executes, so nothing in the seeded inputs is admitted. #[test] - fn batch_tree_anchor_roundtrips_and_freezes_after_setup() { - let db = temp_db("anchor-roundtrip"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - assert_eq!(storage.batch_tree_anchor().expect("default"), 0); - storage.set_batch_tree_anchor(1200).expect("set anchor"); - assert_eq!(storage.batch_tree_anchor().expect("read back"), 1200); + fn the_pinned_cap_decides_which_directs_execute() { + let db = temp_db("pinned-cap"); + // No identity yet: the test pins its own. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); storage - .conn - .execute( - "INSERT INTO setup_complete (singleton_id, completed_at_ms) VALUES (0, 1)", - [], - ) - .expect("seed setup completion fact"); - assert!( - storage.set_batch_tree_anchor(1300).is_err(), - "anchor must be frozen after setup_complete" + .pin_deployment_identity(DeploymentIdentity { + max_direct_payload_bytes: 0, + ..identity() + }) + .expect("pin identity"); + seed_inputs(&mut storage); + assert_eq!( + storage + .admitted_directs_in(SafeInputRange::new(0, 5)) + .expect("count"), + 0 + ); + assert_eq!( + storage + .admitted_directs_above_clock(5, 0) + .expect("check") + .count, + 0 ); - assert_eq!(storage.batch_tree_anchor().expect("unchanged"), 1200); } #[test] fn new_db_has_no_observed_safe_head() { let db = temp_db("new-db-no-safe-head"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); assert_eq!( storage.current_safe_block().expect("read safe block"), @@ -622,44 +743,55 @@ mod tests { None, "fresh storage should not have a safe block timestamp" ); - assert_eq!( - storage - .last_safe_progress_ms() - .expect("read sync timestamp"), - None, - "fresh storage should not have a safe-progress timestamp" - ); } #[test] fn external_values_outside_sqlite_integer_range_are_typed_refusals() { let db = temp_db("external-integer-range"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + // No identity yet: the test's refused pins must leave none. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + + let identity_err = storage + .pin_deployment_identity(DeploymentIdentity { + chain_id: u64::MAX, + ..identity() + }) + .expect_err("unrepresentable configured chain ID must be refused"); + assert!(matches!( + identity_err, + rusqlite::Error::ToSqlConversionFailure(_) + )); + let deployment_block_err = storage + .pin_deployment_identity(DeploymentIdentity { + app_deployment_block: u64::MAX, + ..identity() + }) + .expect_err("unrepresentable configured application deployment block must be refused"); + assert!(matches!( + deployment_block_err, + rusqlite::Error::ToSqlConversionFailure(_) + )); + let cap_err = storage + .pin_deployment_identity(DeploymentIdentity { + max_direct_payload_bytes: u64::MAX, + ..identity() + }) + .expect_err("an unrepresentable direct payload cap must be refused"); + assert!(matches!( + cap_err, + rusqlite::Error::ToSqlConversionFailure(_) + )); + assert_eq!(storage.deployment_identity().expect("read identity"), None); let safe_head_err = storage - .append_safe_inputs_with_timestamp( - u64::MAX, - 1, - &[], - SENDER_A, - &protocol, - FrontierMode::Populate, - ) + .append_safe_inputs_with_timestamp(u64::MAX, 1, &[], SENDER_A) .expect_err("unrepresentable provider safe block must be refused"); assert!(matches!( safe_head_err, rusqlite::Error::ToSqlConversionFailure(_) )); let safe_timestamp_err = storage - .append_safe_inputs_with_timestamp( - 10, - u64::MAX, - &[], - SENDER_A, - &protocol, - FrontierMode::Populate, - ) + .append_safe_inputs_with_timestamp(10, u64::MAX, &[], SENDER_A) .expect_err("unrepresentable provider safe timestamp must be refused"); assert!(matches!( safe_timestamp_err, @@ -675,7 +807,6 @@ mod tests { block_number: u64::MAX, }], SENDER_A, - &protocol, ) .expect_err("unrepresentable provider input block must be refused"); assert!(matches!( @@ -693,9 +824,6 @@ mod tests { block_timestamp: u64::MAX, transaction_hash: B256::ZERO, }], - SENDER_A, - &protocol, - FrontierMode::Populate, ) .expect_err("unrepresentable provider input timestamp must be refused"); assert!(matches!( @@ -711,46 +839,22 @@ mod tests { storage.safe_input_end_exclusive().expect("read input head"), 0 ); - - let identity_err = storage - .load_or_insert_deployment_identity(DeploymentIdentity { - chain_id: u64::MAX, - ..identity() - }) - .expect_err("unrepresentable configured chain ID must be refused"); - assert!(matches!( - identity_err, - rusqlite::Error::ToSqlConversionFailure(_) - )); - let deployment_block_err = storage - .load_or_insert_deployment_identity(DeploymentIdentity { - app_deployment_block: u64::MAX, - ..identity() - }) - .expect_err("unrepresentable configured application deployment block must be refused"); - assert!(matches!( - deployment_block_err, - rusqlite::Error::ToSqlConversionFailure(_) - )); - assert_eq!(storage.deployment_identity().expect("read identity"), None); } #[test] fn deployment_identity_is_inserted_once() { let db = temp_db("deployment-identity-insert-once"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + // No identity yet: the test pins the first one. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); let first = identity(); assert_eq!( storage.deployment_identity().expect("read empty identity"), None ); - assert_eq!( - storage - .load_or_insert_deployment_identity(first) - .expect("insert identity"), - first - ); + storage + .pin_deployment_identity(first) + .expect("insert identity"); assert_eq!( storage .deployment_identity() @@ -762,12 +866,12 @@ mod tests { batch_submitter_address: SENDER_B, ..first }; - assert_eq!( - storage - .load_or_insert_deployment_identity(changed) - .expect("load existing identity"), - first, - "identity must be pinned after the first insert" + let second = storage + .pin_deployment_identity(changed) + .expect_err("a second identity must be refused"); + assert!( + crate::storage::is_persistent_storage_error(&second), + "a second identity is a constraint violation, got {second:?}" ); assert_eq!( storage @@ -780,28 +884,16 @@ mod tests { #[test] fn append_safe_inputs_creates_and_advances_safe_head() { let db = temp_db("append-safe-inputs-creates-safe-head"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); + let mut storage = set_up_storage(&db.path, SENDER_A); storage - .append_safe_inputs_with_timestamp( - 7, - 1234, - &[], - SENDER_A, - &protocol, - FrontierMode::Populate, - ) + .append_safe_inputs_with_timestamp(7, 1234, &[], SENDER_A) .expect("record first real safe-head observation"); assert_eq!( storage.current_safe_block().expect("read safe block"), Some(7), "append should create the safe-head row" ); - let recorded_sync = storage - .last_safe_progress_ms() - .expect("read sync timestamp") - .expect("first observation should record wall-clock time"); assert_eq!( storage .current_safe_block_timestamp() @@ -811,14 +903,7 @@ mod tests { ); storage - .append_safe_inputs_with_timestamp( - 9, - 5678, - &[], - SENDER_A, - &protocol, - FrontierMode::Populate, - ) + .append_safe_inputs_with_timestamp(9, 5678, &[], SENDER_A) .expect("advance safe head"); assert_eq!( storage.current_safe_block().expect("read safe block"), @@ -832,13 +917,5 @@ mod tests { Some(5678), "append should record the observed L1 block timestamp" ); - assert!( - storage - .last_safe_progress_ms() - .expect("read sync timestamp") - .expect("advanced observation should record wall-clock time") - >= recorded_sync, - "safe-progress timestamp should stay monotonic across appends" - ); } } diff --git a/sequencer/src/storage/l1_submission.rs b/sequencer/src/storage/l1_submission.rs index 1ef7e04c..86f2d690 100644 --- a/sequencer/src/storage/l1_submission.rs +++ b/sequencer/src/storage/l1_submission.rs @@ -9,24 +9,25 @@ use rusqlite::{Result, params}; use super::Storage; +use super::acceptance::frontier_nonce; use super::convert::{external_u64_to_i64, i64_to_u16, i64_to_u32, i64_to_u64, u64_to_i64}; use super::queries::current_safe_block_required; -use super::safe_accepted_batches::frontier_nonce; use super::{FrameHeader, PendingBatch, SubmitterFrontier}; use sequencer_core::batch::{Batch, Frame as BatchFrame, WireUserOp}; impl Storage { /// Read-only frontier view used by the submitter each tick to derive the /// next batch nonce. `accepted_next_nonce` is the next nonce the scheduler - /// is expected to accept, derived from `safe_accepted_batches`. + /// is expected to accept, derived from `batches.accepted_input`. /// /// The scheduler-accepted frontier is maintained by - /// `Storage::append_ingested_safe_inputs_with_timestamp`, so this is a pure read. + /// `Storage::append_ingested_safe_inputs_with_timestamp` once the era + /// exists, so this is a pure read. /// /// **Precondition:** at least one safe-head observation must have been /// recorded (by `Storage::append_ingested_safe_inputs_with_timestamp`). In production this is - /// always true because the startup recovery procedure completes InitialSync and - /// refuses admission unless its persisted view is usable. Tests must seed + /// always true because setup's sync records one before `run` starts. + /// Tests must seed /// an observation explicitly; a fresh DB returns `QueryReturnedNoRows`. pub fn submitter_frontier(&mut self) -> Result { self.read(|tx| { @@ -76,54 +77,47 @@ impl Storage { }) } - /// Load all valid closed batches with nonce >= `min_nonce`, in nonce order, + /// Load all closed batches with nonce >= `min_nonce`, in nonce order, /// each one fully assembled and SSZ-encoded with its authoritative nonce. /// /// Authoritative because the nonce stamped into the wire payload is the - /// one the DB persists on the batch row (via the `parent.nonce + 1` - /// structural invariant). The caller never sees an unstamped batch — + /// one the DB persists on the batch row (the last nonce + 1 at + /// insert, I16). The caller never sees an unstamped batch — /// there is no way to accidentally encode with the wrong nonce. pub fn pending_batches(&mut self, min_nonce: u64) -> Result> { - const SQL: &str = "SELECT batch_index, nonce FROM valid_closed_batches \ + const SQL: &str = "SELECT nonce FROM closed_batches \ WHERE nonce >= ?1 ORDER BY nonce ASC"; - let pending_refs: Vec<(u64, u64)> = { + let nonces: Vec = { let mut stmt = self.conn.prepare_cached(SQL)?; let rows = stmt.query_map(params![u64_to_i64(min_nonce)], |row| { - let bi: i64 = row.get(0)?; - let nonce: i64 = row.get(1)?; - Ok((i64_to_u64(bi), i64_to_u64(nonce))) + row.get::<_, i64>(0).map(i64_to_u64) })?; rows.collect::>>()? }; - let mut batches = Vec::with_capacity(pending_refs.len()); - for (batch_index, nonce) in pending_refs { - let frames = self.load_batch_frames(batch_index)?; + let mut batches = Vec::with_capacity(nonces.len()); + for nonce in nonces { + let frames = self.load_batch_frames(nonce)?; let batch = Batch { nonce, frames }; let encoded = ssz::Encode::as_ssz_bytes(&batch); - batches.push(PendingBatch { - batch_index, - nonce, - encoded, - }); + batches.push(PendingBatch { nonce, encoded }); } Ok(batches) } - /// Load every frame (header + user ops) of `batch_index` in frame order. - /// Internal helper for [`Self::pending_batches`]; does NOT filter on - /// validity — callers only reach this after they know the batch is valid. - fn load_batch_frames(&mut self, batch_index: u64) -> Result> { - load_batch_frames_in(&self.conn, batch_index) + /// Load every frame (header + user ops) of `batch_nonce` in frame order. + /// Internal helper for [`Self::pending_batches`]. + fn load_batch_frames(&mut self, batch_nonce: u64) -> Result> { + load_batch_frames_in(&self.conn, batch_nonce) } } -fn frames_for_batch_in(conn: &rusqlite::Connection, batch_index: u64) -> Result> { +fn frames_for_batch_in(conn: &rusqlite::Connection, batch_nonce: u64) -> Result> { let mut stmt = conn.prepare_cached( "SELECT frame_in_batch, fee, safe_block FROM frames \ - WHERE batch_index = ?1 ORDER BY frame_in_batch ASC", + WHERE batch_nonce = ?1 ORDER BY frame_in_batch ASC", )?; - let rows = stmt.query_map(params![u64_to_i64(batch_index)], |row| { + let rows = stmt.query_map(params![u64_to_i64(batch_nonce)], |row| { Ok(FrameHeader { frame_in_batch: i64_to_u32(row.get(0)?), fee: i64_to_u16(row.get(1)?), @@ -139,18 +133,18 @@ fn frames_for_batch_in(conn: &rusqlite::Connection, batch_index: u64) -> Result< /// function being that single path is load-bearing. pub(super) fn load_batch_frames_in( conn: &rusqlite::Connection, - batch_index: u64, + batch_nonce: u64, ) -> Result> { - let frame_headers = frames_for_batch_in(conn, batch_index)?; + let frame_headers = frames_for_batch_in(conn, batch_nonce)?; let mut frames = Vec::with_capacity(frame_headers.len()); for header in frame_headers { let mut stmt = conn.prepare_cached( "SELECT nonce, max_fee, data, sig FROM user_ops \ - WHERE batch_index = ?1 AND frame_in_batch = ?2 \ + WHERE batch_nonce = ?1 AND frame_in_batch = ?2 \ ORDER BY pos_in_frame ASC", )?; let rows = stmt.query_map( - params![u64_to_i64(batch_index), i64::from(header.frame_in_batch)], + params![u64_to_i64(batch_nonce), i64::from(header.frame_in_batch)], |row| { Ok(WireUserOp { nonce: i64_to_u32(row.get(0)?), @@ -173,7 +167,7 @@ pub(super) fn load_batch_frames_in( #[cfg(test)] mod tests { use super::super::test_helpers::{ - SENDER_A, default_protocol_timing, seed_closed_batches, seed_safe_inputs_with_batch_nonces, + SENDER_A, seed_closed_batches, seed_safe_inputs_with_batch_nonces, set_up_storage, }; use crate::storage::{SafeInputRange, Storage}; use crate::test_support::temp_db; @@ -185,24 +179,23 @@ mod tests { // `Batch { nonce: 0, … }` placeholder, and callers had to remember to // stamp the real nonce via `encode_for_scheduler_with_nonce`. The new // `pending_batches` reads the DB-authoritative nonce from - // `valid_closed_batches` and bakes it straight into the SSZ bytes — so + // `closed_batches` and bakes it straight into the SSZ bytes — so // decoding the payload must round-trip back to that nonce, and the // frame body must match what storage persisted. let db = temp_db("pending-batches-nonce-baked-in"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(12, SafeInputRange::empty_at(0)) .expect("initialize open state"); // Close batch 0 so it becomes eligible for submission. storage - .close_frame_and_batch(&mut head, 12) + .close_frame_and_batch(&mut head) .expect("close batch 0"); let pending = storage.pending_batches(0).expect("load pending batches"); assert_eq!(pending.len(), 1); let entry = &pending[0]; - assert_eq!(entry.batch_index, 0); assert_eq!(entry.nonce, 0, "genesis batch has nonce 0"); // The wire bytes must decode back to the authoritative nonce AND the @@ -221,12 +214,12 @@ mod tests { #[test] fn batch_level_helpers_expose_latest_index_frames_and_txs() { let db = temp_db("batch-level-helpers"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // Before initialization there should be no batches. assert!( storage - .latest_batch_index() + .latest_batch_nonce() .expect("query latest batch nonce on empty db") .is_none() ); @@ -237,14 +230,13 @@ mod tests { .expect("initialize open state"); // Close current batch and move to next so batch 0 becomes closed. - let next_safe_block = head.safe_block; storage - .close_frame_and_batch(&mut head, next_safe_block) + .close_frame_and_batch(&mut head) .expect("close batch and rotate"); // Latest batch nonce should now be 1 (open), with batch 0 closed. let latest = storage - .latest_batch_index() + .latest_batch_nonce() .expect("query latest batch nonce") .expect("latest batch should exist"); assert_eq!(latest, 1); @@ -266,12 +258,12 @@ mod tests { #[test] fn closed_batch_becomes_eligible_for_submission_with_assigned_nonce() { // : closing a batch transitions it from "open Tip" to "eligible - // for L1 submission" — it appears in `valid_closed_batches` with a - // nonce derived from its parent pointer. Pins the submitter's + // for L1 submission" — it appears in `closed_batches` with a + // nonce derived at insert. Pins the submitter's // contract: open batches are NOT pulled into the submission pipeline, // and closed batches ARE, at the schema-guaranteed nonce. let db = temp_db("closed-batch-eligible"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -287,13 +279,12 @@ mod tests { ); // Close batch 0 — this rotates the Tip to batch 1 and seals batch 0. - let safe_block = head.safe_block; storage - .close_frame_and_batch(&mut head, safe_block) + .close_frame_and_batch(&mut head) .expect("close batch 0"); - // After close: batch 0 is eligible with nonce 0 (genesis, parent - // NULL → trigger assigns nonce 0). + // After close: batch 0 is eligible with nonce 0 (genesis: no earlier + // batch, so it carries the era's anchor, 0). let pending_after = storage .pending_batches(0) .expect("load pending batches (post-close)"); @@ -302,15 +293,14 @@ mod tests { 1, "exactly one batch should be eligible after the first close", ); - assert_eq!(pending_after[0].batch_index, 0); assert_eq!( pending_after[0].nonce, 0, - "closed batch 0 must carry nonce 0 (genesis, no parent)", + "closed batch 0 must carry nonce 0 (genesis, the era's anchor)", ); // The new open Tip (batch 1) must NOT be eligible even though it // exists — eligibility requires sealed_at_ms NOT NULL. assert!( - pending_after.iter().all(|b| b.batch_index != 1), + pending_after.iter().all(|b| b.nonce != 1), "open batch 1 (the new Tip) must not be eligible: {pending_after:?}", ); } @@ -318,7 +308,7 @@ mod tests { #[test] fn wallet_nonce_watermark_is_monotonic_and_absent_at_genesis() { let db = temp_db("wallet-nonce-watermark"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); assert_eq!( storage.wallet_nonce_watermark().expect("read"), @@ -350,9 +340,9 @@ mod tests { #[test] fn submitter_frontier_returns_zero_when_no_batches_were_accepted() { let db = temp_db("submitter-frontier-empty"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("record observed safe head"); let frontier = storage.submitter_frontier().expect("submitter frontier"); assert_eq!(frontier.safe_block, 0); @@ -361,18 +351,22 @@ mod tests { #[test] fn submitter_frontier_uses_anchor_when_no_batches_accepted() { - // A cockroach-recovered deployment boots with an empty - // safe_accepted_batches and a non-zero batch-tree anchor (N'). The - // submitter's start frontier must be N', not 0 — otherwise it would - // re-submit its first post-recovery batch (nonce N') on every tick - // until the first accepted row lands. Regression test for the - // frontier_nonce/anchor asymmetry (frontier_nonce defaulted to 0 while - // populate_safe_accepted_batches seeds `expected` from the anchor). + // A cockroach-recovered deployment boots with no accepted batch and + // a non-zero era anchor (N'). The submitter's start frontier must be + // N', not 0 — otherwise it would re-submit its first post-recovery + // batch (nonce N') on every tick until the first acceptance. + // `frontier_nonce` and `record_accepted_batches` both start from the + // anchor. let db = temp_db("submitter-frontier-anchor"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); + let mut storage = Storage::open_with_era( + db.path.as_str(), + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); storage - .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) + .append_safe_inputs(0, &[], SENDER_A) .expect("record observed safe head"); let frontier = storage.submitter_frontier().expect("submitter frontier"); assert_eq!(frontier.accepted_next_nonce, 7); @@ -381,21 +375,17 @@ mod tests { #[test] fn submitter_frontier_tracks_accepted_prefix() { let db = temp_db("submitter-frontier-prefix"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // Local closed batches 0 and 1 — their landings carry the real wire // bytes (content-identity check); 3..5 stay synthetic, which is fine // because the nonce gap means the fold never accepts them. let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close 0"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close 1"); + storage.close_frame_and_batch(&mut head).expect("close 0"); + storage.close_frame_and_batch(&mut head).expect("close 1"); // seed_safe_inputs_with_batch_nonces already calls append_safe_inputs, - // which auto-populates safe_accepted_batches. + // which records the accepted batches. seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0, 1, 3, 4, 5]); let frontier = storage.submitter_frontier().expect("submitter frontier"); @@ -404,25 +394,24 @@ mod tests { } #[test] - fn pending_batches_skips_invalidated_and_respects_min_nonce() { + fn pending_batches_respects_min_nonce() { let db = temp_db("load-pending-batches-filter"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); seed_closed_batches(&mut storage, 3); - storage.insert_invalid_batch(1).expect("invalidate batch 1"); - // From nonce 0: batches 0 and 2 remain valid. + // From nonce 0: every closed batch, and not the open one. let from_zero = storage .pending_batches(0) .expect("load pending batches from 0"); let nonces: Vec = from_zero.iter().map(|b| b.nonce).collect(); - assert_eq!(nonces, vec![0, 2], "batch 1 must be filtered out"); + assert_eq!(nonces, vec![0, 1, 2]); - // From nonce 1: only batch 2 remains (batch 0 is below min_nonce). + // From nonce 1: batch 0 is below min_nonce. let from_one = storage .pending_batches(1) .expect("load pending batches from 1"); let nonces: Vec = from_one.iter().map(|b| b.nonce).collect(); - assert_eq!(nonces, vec![2]); + assert_eq!(nonces, vec![1, 2]); // Past the suffix: empty. let from_three = storage @@ -433,48 +422,4 @@ mod tests { "no batch should remain at nonce >= 3" ); } - - #[test] - fn nonce_is_reused_after_torn_cascade() { - // After a torn cascade invalidates every batch (including genesis), - // the recovery batch has no valid ancestor. Its parent is NULL, - // so its nonce resets to 0 — effectively reusing the nonce of the - // original genesis. The scheduler's "expected next nonce" also - // resets to 0, since no accepted batches were ever submitted. - let db = temp_db("nonce-reuse-after-torn-cascade"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - - storage.insert_invalid_batch(0).expect("invalidate batch 0"); - storage.insert_invalid_batch(1).expect("invalidate batch 1"); - storage - .append_safe_inputs(10, &[], SENDER_A, &default_protocol_timing()) - .expect("record observed safe head"); - storage - .recover_post_flush(1200) - .expect("open recovery batch after torn invalidation"); - - let head = storage - .open_state() - .expect("load open state") - .expect("recovery batch"); - assert_eq!(head.batch_index, 2); - - // Recovery Tip has no valid ancestor → parent NULL → nonce 0. - let recovery_nonce: i64 = storage - .conn - .query_row( - "SELECT nonce FROM batches WHERE batch_index = 2", - [], - |row| row.get(0), - ) - .expect("query recovery nonce"); - assert_eq!(recovery_nonce, 0, "recovery Tip reuses nonce 0"); - } } diff --git a/sequencer/src/storage/landing.rs b/sequencer/src/storage/landing.rs new file mode 100644 index 00000000..9cdf0201 --- /dev/null +++ b/sequencer/src/storage/landing.rs @@ -0,0 +1,216 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Batch landing: the oldest batch L1 has not accepted, the facts that end the +//! era (a divergence or a void history), and the one batch `run` opens at +//! start. Every read derives from the database as it is; nothing here is a +//! stored flag. + +use rusqlite::{Connection, OptionalExtension, Result, TransactionBehavior, params}; +use sequencer_core::protocol::age_exceeds; + +use super::Storage; +use super::acceptance::{Divergence, canonical_divergence_in, frontier_nonce}; +use super::convert::{i64_to_u64, u64_to_i64}; +use super::ingress::open_fresh_tip_in_tx; +use super::l1_inputs::query_deployment_identity; +use super::queries::current_safe_block; + +/// The oldest batch L1 has not accepted, closed or open, at the safe head +/// `H`, with its first frame's safe block `s`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LandingLag { + pub batch_nonce: u64, + pub first_frame_safe_block: u64, + pub safe_block: u64, +} + +impl LandingLag { + /// `H + 1 - s`, in L1 blocks: the batch's age at `H + 1`, the earliest + /// block it can still land in. The history is void exactly when this + /// reaches `max_wait_blocks`, so the two compare directly. + pub fn blocks(&self) -> u64 { + (self.safe_block + 1).saturating_sub(self.first_frame_safe_block) + } + + /// No block after the safe head can accept this batch. Any landing still + /// to come is in block `H + 1` or later, and the scheduler skips a batch + /// that lands `max_wait_blocks` or more after its first frame's safe + /// block, keeping its nonce, so no later batch can be accepted either. + pub fn is_void(&self, max_wait_blocks: u64) -> bool { + age_exceeds( + self.safe_block + 1, + self.first_frame_safe_block, + max_wait_blocks, + ) + } +} + +/// A fact that ends the era: `run` refuses while it holds, and it never stops +/// holding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum TerminalFact { + /// A foreign or different batch was accepted at one of our nonces. + #[error( + "canonical divergence at batch nonce {}; only cockroach recovery (fresh-directory rebuild) can proceed", + .0.nonce + )] + Divergence(Divergence), + /// The oldest unaccepted batch can no longer be accepted. + #[error( + "the history is void: batch {}, opened at safe block {}, was not accepted by safe block {}, \ + and no later block can accept it (max_wait_blocks {max_wait_blocks}); recover with \ + setup --base in a new data directory", + lag.batch_nonce, + lag.first_frame_safe_block, + lag.safe_block + )] + VoidHistory { + lag: LandingLag, + max_wait_blocks: u64, + }, +} + +/// Opening the start batch was refused. +#[derive(Debug, thiserror::Error)] +pub(crate) enum OpenBatchError { + #[error(transparent)] + Storage(#[from] rusqlite::Error), + #[error(transparent)] + Terminal(TerminalFact), + /// The open transaction left no open batch: a broken postcondition, + /// rolled back. + #[error("opening a batch left no open batch in its own transaction")] + OpenBatchMissing, +} + +impl Storage { + /// The canonical-divergence marker (I15), if present. + #[cfg(test)] + pub fn canonical_divergence(&mut self) -> Result> { + self.read(|tx| canonical_divergence_in(tx)) + } + + /// The oldest batch L1 has not accepted, at the persisted safe head. + /// `None` when there is no safe head yet or every batch is accepted. + pub fn landing_lag(&mut self) -> Result> { + self.read(|tx| landing_lag_in(tx)) + } + + /// The fact that ends the era, if one holds: a recorded divergence first, + /// then a void history under the pinned window. Reads only the database, + /// so it answers with L1 unreachable. + pub fn terminal_fact(&mut self) -> Result> { + self.read(|tx| terminal_fact_in(tx)) + } + + /// Open a batch at the safe head if none is open, in one transaction that + /// refuses while a terminal fact holds. Returns whether it opened one. + /// The new batch drains the undrained directs. The first `run` of an era + /// opens its first batch here; setup opens none. + pub(crate) fn open_batch_if_none(&mut self) -> std::result::Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(fact) = terminal_fact_in(&tx)? { + return Err(OpenBatchError::Terminal(fact)); + } + if open_batch(&tx)?.is_some() { + return Ok(false); + } + open_fresh_tip_in_tx(&tx)?; + // A broken postcondition must roll back this transaction and refuse + // startup, including in release builds. + if open_batch(&tx)?.is_none() { + return Err(OpenBatchError::OpenBatchMissing); + } + tx.commit()?; + Ok(true) + } +} + +/// The staleness window `setup` pinned with the deployment identity. Storage +/// reads it from the pin, as it reads the submitter. +fn pinned_max_wait_blocks_in(conn: &Connection) -> Result { + Ok(query_deployment_identity(conn)? + .ok_or(rusqlite::Error::QueryReturnedNoRows)? + .max_wait_blocks) +} + +pub(super) fn terminal_fact_in(conn: &Connection) -> Result> { + if let Some(divergence) = canonical_divergence_in(conn)? { + return Ok(Some(TerminalFact::Divergence(divergence))); + } + let Some(lag) = landing_lag_in(conn)? else { + return Ok(None); + }; + let max_wait_blocks = pinned_max_wait_blocks_in(conn)?; + Ok(lag + .is_void(max_wait_blocks) + .then_some(TerminalFact::VoidHistory { + lag, + max_wait_blocks, + })) +} + +/// The batch at the accepted frontier's next nonce: the first unaccepted +/// closed batch, else the open batch. First-frame safe blocks do not +/// decrease along the batches (I3), so it is the oldest unaccepted one. +fn landing_lag_in(conn: &Connection) -> Result> { + let Some(safe_block) = current_safe_block(conn)? else { + return Ok(None); + }; + let batch_nonce = match first_unaccepted_closed_batch(conn)? { + Some(nonce) => nonce, + None => match open_batch(conn)? { + Some(nonce) => nonce, + None => return Ok(None), + }, + }; + Ok(Some(LandingLag { + batch_nonce, + first_frame_safe_block: first_frame_safe_block_of(conn, batch_nonce)?, + safe_block, + })) +} + +/// First closed batch with `nonce >= frontier_nonce`, the next nonce +/// the scheduler is expected to accept. `>=`, not `>`: the batch at the +/// frontier carries `nonce == frontier_nonce`. The nonce seek, not +/// `accepted_input IS NULL`, keeps this off the accepted prefix: the landing +/// watch calls it every tick. Returns `None` if every closed batch is +/// accepted. +fn first_unaccepted_closed_batch(conn: &Connection) -> Result> { + let frontier = frontier_nonce(conn)?; + let nonce: Option = conn + .query_row( + "SELECT nonce FROM closed_batches \ + WHERE nonce >= ?1 ORDER BY nonce ASC LIMIT 1", + params![u64_to_i64(frontier)], + |row| row.get(0), + ) + .optional()?; + Ok(nonce.map(i64_to_u64)) +} + +fn open_batch(conn: &Connection) -> Result> { + let nonce: Option = conn + .query_row("SELECT nonce FROM open_batch", [], |row| row.get(0)) + .optional()?; + Ok(nonce.map(i64_to_u64)) +} + +/// `frames.safe_block` of the lowest `frame_in_batch` in `batch_nonce`. +/// Every committed batch has a first frame; missing one is an invariant +/// violation and propagates as `QueryReturnedNoRows`. +fn first_frame_safe_block_of(conn: &Connection, batch_nonce: u64) -> Result { + conn.query_row( + "SELECT safe_block FROM frames \ + WHERE batch_nonce = ?1 ORDER BY frame_in_batch ASC LIMIT 1", + params![u64_to_i64(batch_nonce)], + |row| row.get::<_, i64>(0).map(i64_to_u64), + ) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/landing/tests.rs b/sequencer/src/storage/landing/tests.rs new file mode 100644 index 00000000..799624d8 --- /dev/null +++ b/sequencer/src/storage/landing/tests.rs @@ -0,0 +1,353 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::super::test_helpers::{ + LandingFixture, SENDER_A, local_batch_payload, open_temp_storage, record_canonical_divergence, + set_up_storage_with_window, +}; +use super::*; +use crate::storage::{SafeInputRange, Storage}; + +/// The window the test identity pins (the wallet's). +const W: u64 = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; + +fn void_at(storage: &mut Storage, safe_block: u64) -> Option { + storage.advance_safe_head(safe_block); + storage.terminal_fact().expect("terminal fact") +} + +/// The batches as `(nonce, sealed)`, in nonce order. +fn batch_rows(storage: &mut Storage) -> Vec<(i64, bool)> { + storage + .read(|tx| { + tx.prepare("SELECT nonce, sealed_at_ms IS NOT NULL FROM batches ORDER BY nonce")? + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect() + }) + .expect("read batches") +} + +mod void_history { + use super::*; + + #[test] + fn nothing_is_void_before_the_first_sync() { + let (_db, mut storage) = open_temp_storage("landing-before-sync"); + storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch"); + assert_eq!(storage.landing_lag().expect("lag"), None); + assert_eq!(storage.terminal_fact().expect("terminal fact"), None); + } + + #[test] + fn nothing_is_void_without_a_batch() { + let (_db, mut storage) = open_temp_storage("landing-no-batch"); + assert_eq!(void_at(&mut storage, 10 * W), None); + assert_eq!(storage.landing_lag().expect("lag"), None); + } + + /// `H + 1 - s == W` is void: block `H + 1` is the first a landing can + /// use, and the scheduler skips a batch `W` blocks past its first frame. + /// One block less is not. + #[test] + fn the_exact_boundary_is_void_and_one_block_less_is_not() { + let (_db, mut storage) = open_temp_storage("landing-boundary"); + let mut head = storage + .initialize_open_state(100, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + storage + .close_batch_at(&mut head, 100) + .expect("close batch 0"); + + let last_live = 100 + W - 2; + assert_eq!(void_at(&mut storage, last_live), None); + let lag = storage + .landing_lag() + .expect("lag") + .expect("an unaccepted batch"); + assert_eq!(lag.batch_nonce, 0); + assert_eq!(lag.blocks(), W - 1); + assert!(!lag.is_void(W)); + + let fact = void_at(&mut storage, last_live + 1).expect("void at the boundary"); + let lag = LandingLag { + batch_nonce: 0, + first_frame_safe_block: 100, + safe_block: 100 + W - 1, + }; + assert_eq!( + fact, + TerminalFact::VoidHistory { + lag, + max_wait_blocks: W, + } + ); + assert!(lag.is_void(W)); + assert_eq!( + lag.blocks(), + W, + "void exactly when the lag reaches the window" + ); + assert!(!lag.is_void(W + 1), "the window decides the boundary"); + } + + /// The window is the pinned identity's, not the wallet's: under a pinned + /// window of 50, the lag is void at 50 and not at 49. + #[test] + fn the_boundary_follows_the_pinned_window() { + let db = crate::test_support::temp_db("landing-pinned-window"); + let mut storage = set_up_storage_with_window(&db.path, SENDER_A, 50); + storage + .initialize_open_state(100, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + assert_eq!(void_at(&mut storage, 100 + 48), None); + assert_eq!( + storage.landing_lag().expect("lag").map(|lag| lag.blocks()), + Some(49) + ); + assert!(matches!( + void_at(&mut storage, 100 + 49), + Some(TerminalFact::VoidHistory { + lag, + max_wait_blocks: 50, + }) if lag.blocks() == 50 + )); + } + + /// The open batch has no L1 footprint, but its first frame fixes `s` all + /// the same: sealed late, it lands stale. + #[test] + fn the_open_batch_counts() { + let (_db, mut storage) = open_temp_storage("landing-open-batch"); + storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + assert_eq!(void_at(&mut storage, 10 + W - 2), None); + assert_eq!( + storage.landing_lag().expect("lag"), + Some(LandingLag { + batch_nonce: 0, + first_frame_safe_block: 10, + safe_block: 10 + W - 2, + }) + ); + assert!(matches!( + void_at(&mut storage, 10 + W - 1), + Some(TerminalFact::VoidHistory { lag, .. }) if lag.batch_nonce == 0 + )); + } + + /// An accepted batch is judged no more, however old its first frame: the + /// open batch after it is the oldest unaccepted one. + #[test] + fn an_accepted_batch_never_makes_the_history_void() { + let (_db, mut storage) = open_temp_storage("landing-accepted"); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + // Batch 0 opens at 10; batch 1 opens at 1000. + storage + .close_batch_at(&mut head, 1000) + .expect("close batch 0"); + let landed = local_batch_payload(&mut storage, 0); + storage.land(1100, &[(SENDER_A, landed)]); + assert_eq!( + storage + .submitter_frontier() + .expect("frontier") + .accepted_next_nonce, + 1, + "batch 0 is accepted" + ); + + // Far past batch 0's window, inside batch 1's. + let safe_block = 1000 + W - 2; + assert!( + age_exceeds(safe_block + 1, 10, W), + "batch 0 would be void were it unaccepted" + ); + assert_eq!(void_at(&mut storage, safe_block), None); + assert_eq!( + storage.landing_lag().expect("lag"), + Some(LandingLag { + batch_nonce: 1, + first_frame_safe_block: 1000, + safe_block, + }) + ); + } + + /// The batch at the accepted frontier's next nonce is judged: an old + /// unaccepted closed batch voids the history while younger batches, + /// the open one included, could still land on their own. + #[test] + fn the_oldest_unaccepted_batch_is_the_one_judged() { + let (_db, mut storage) = open_temp_storage("landing-oldest"); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + // Batch 0 and batch 1 open at 10; the open batch 2 opens at 500. + storage + .close_batch_at(&mut head, 10) + .expect("close batch 0"); + storage + .close_batch_at(&mut head, 500) + .expect("close batch 1"); + assert_eq!(head.batch_nonce(), 2); + + assert_eq!(void_at(&mut storage, 20), None); + assert_eq!( + storage + .landing_lag() + .expect("lag") + .map(|lag| lag.batch_nonce), + Some(0) + ); + + // Accepting batch 0 moves the judgement to batch 1, not to the open + // batch. + let landed = local_batch_payload(&mut storage, 0); + storage.land(30, &[(SENDER_A, landed)]); + assert_eq!( + storage + .landing_lag() + .expect("lag") + .map(|lag| lag.batch_nonce), + Some(1) + ); + assert!(matches!( + void_at(&mut storage, 10 + W - 1), + Some(TerminalFact::VoidHistory { lag, .. }) + if lag.batch_nonce == 1 && lag.first_frame_safe_block == 10 + )); + } + + /// A divergence outranks a void history: it is the first terminal fact. + #[test] + fn a_divergence_is_reported_before_a_void_history() { + let (_db, mut storage) = open_temp_storage("landing-divergence-first"); + storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + assert!(void_at(&mut storage, 10 + W).is_some()); + record_canonical_divergence(&mut storage, 7, 3); + assert_eq!( + storage.terminal_fact().expect("terminal fact"), + Some(TerminalFact::Divergence(Divergence { + nonce: 7, + safe_input_index: 3, + })) + ); + } + + #[test] + fn a_batch_without_a_first_frame_fails_loud() { + let (_db, mut storage) = open_temp_storage("landing-missing-first-frame"); + storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + storage.advance_safe_head(10); + storage + .conn + .execute_batch( + "DROP TRIGGER trg_frames_not_deletable; \ + DELETE FROM frames WHERE batch_nonce = 0;", + ) + .expect("inject missing-frame corruption"); + let err = storage + .landing_lag() + .expect_err("a batch without a first frame must fail loud"); + assert!(matches!(err, rusqlite::Error::QueryReturnedNoRows)); + } +} + +mod open_batch_if_none { + use super::*; + + #[test] + fn opens_one_batch_at_the_safe_head_and_only_once() { + let (_db, mut storage) = open_temp_storage("open-batch-once"); + storage.advance_safe_head(42); + assert!(storage.open_batch_if_none().expect("open")); + let head = storage.open_state().expect("load").expect("an open batch"); + assert_eq!(head.batch_nonce(), 0); + assert_eq!(head.safe_block(), 42); + let opened = batch_rows(&mut storage); + assert_eq!(opened, vec![(0, false)]); + + assert!(!storage.open_batch_if_none().expect("no-op")); + assert_eq!(batch_rows(&mut storage), opened); + } + + #[test] + fn refuses_a_divergence_without_mutation() { + let (_db, mut storage) = open_temp_storage("open-batch-divergence"); + storage.advance_safe_head(42); + record_canonical_divergence(&mut storage, 7, 0); + let error = storage + .open_batch_if_none() + .expect_err("a divergence refuses the opening"); + assert!(matches!( + error, + OpenBatchError::Terminal(TerminalFact::Divergence(Divergence { nonce: 7, .. })) + )); + assert!(batch_rows(&mut storage).is_empty()); + } + + #[test] + fn refuses_a_void_history_without_mutation() { + let (_db, mut storage) = open_temp_storage("open-batch-void"); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("open batch 0"); + storage + .close_batch_at(&mut head, 10) + .expect("close batch 0"); + storage.advance_safe_head(10 + W - 1); + let before = batch_rows(&mut storage); + let error = storage + .open_batch_if_none() + .expect_err("a void history refuses the opening"); + assert!(matches!( + error, + OpenBatchError::Terminal(TerminalFact::VoidHistory { lag, .. }) + if lag.batch_nonce == 0 + )); + assert_eq!(batch_rows(&mut storage), before); + } + + #[test] + fn refuses_and_rolls_back_if_its_transaction_leaves_no_open_batch() { + let (db, mut storage) = open_temp_storage("open-batch-postcondition"); + storage.advance_safe_head(42); + drop(storage); + // Fault injection: seal the new batch the moment its first frame + // lands, so `open_fresh_tip_in_tx` returns `Ok` having left no open + // batch. (`open.rs` documents failure triggers as the sanctioned way + // past the typed API in tests.) + Storage::open_connection(db.path.as_str()) + .expect("raw connection") + .execute_batch( + "CREATE TRIGGER test_seal_new_tip AFTER INSERT ON frames \ + BEGIN \ + UPDATE batches SET sealed_at_ms = 1, payload_hash = zeroblob(32) \ + WHERE nonce = NEW.batch_nonce; \ + END", + ) + .expect("install failure trigger"); + let mut storage = Storage::open(db.path.as_str()).expect("reopen"); + + let error = storage + .open_batch_if_none() + .expect_err("an opening that leaves no batch must refuse, not commit"); + assert!( + matches!(error, OpenBatchError::OpenBatchMissing), + "expected the postcondition refusal, got {error:?}" + ); + assert!( + batch_rows(&mut storage).is_empty(), + "the refused opening must roll back its insert" + ); + } +} diff --git a/sequencer/src/storage/lifecycle.rs b/sequencer/src/storage/lifecycle.rs index 2d43973e..603749bb 100644 --- a/sequencer/src/storage/lifecycle.rs +++ b/sequencer/src/storage/lifecycle.rs @@ -14,7 +14,6 @@ use super::convert::{i64_to_u64, now_unix_ms}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LifecycleCommand { Setup, - Rebuild, Run, MaintenanceFlush, } @@ -23,7 +22,6 @@ impl LifecycleCommand { pub(crate) const fn as_str(self) -> &'static str { match self { Self::Setup => "setup", - Self::Rebuild => "rebuild", Self::Run => "run", Self::MaintenanceFlush => "maintenance_flush", } @@ -32,7 +30,6 @@ impl LifecycleCommand { fn parse(value: &str) -> Result { match value { "setup" => Ok(Self::Setup), - "rebuild" => Ok(Self::Rebuild), "run" => Ok(Self::Run), "maintenance_flush" => Ok(Self::MaintenanceFlush), other => Err(LifecycleError::Malformed(format!( @@ -77,37 +74,30 @@ pub enum LifecycleError { } impl Storage { - /// Publish a durable setup dump and its complete baseline atomically. - pub(crate) fn complete_baseline_setup( + /// Publish the new era with its durable baseline dump, in one transaction: + /// the era row (which also makes the database set up) and the baseline + /// snapshot at the rebuilt history's end. Setup opens no Tip; `run`'s + /// startup opens the era's first batch at its anchor nonce. + pub(crate) fn publish_era( &mut self, - prefix: &std::path::Path, - count: sequencer_core::history::ExecutedInputCount, - safe_block: u64, - next_batch_nonce: u64, - recovery: bool, + baseline_prefix: &std::path::Path, + era: &super::NewEra, ) -> Result<(), LifecycleError> { let tx = self .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; refuse_on_canonical_divergence(&tx)?; - require_command_fits_completion( - &tx, - if recovery { - LifecycleCommand::Rebuild - } else { - LifecycleCommand::Setup - }, - )?; - super::history::initialize_history_in(&tx, count, safe_block)?; - super::mutations::set_batch_tree_anchor_in(&tx, next_batch_nonce)?; - if recovery { - super::ingress::open_recovery_tip_in_tx(&tx, safe_block)?; + require_command_fits_completion(&tx, LifecycleCommand::Setup)?; + let has_batch: bool = tx.query_row("SELECT EXISTS (SELECT 1 FROM batches)", [], |row| { + row.get(0) + })?; + if has_batch { + return Err(LifecycleError::Malformed( + "an era is published before any batch exists".to_string(), + )); } - super::snapshot_dumps::insert_baseline_snapshot_in(&tx, prefix, count)?; - tx.execute( - "INSERT INTO setup_complete (singleton_id, completed_at_ms) VALUES (0, ?1)", - [now_unix_ms()], - )?; + super::history::insert_era_in(&tx, era)?; + super::snapshots::insert_baseline_snapshot_in(&tx, baseline_prefix, era.rebuilt_end)?; tx.commit()?; Ok(()) } @@ -182,49 +172,23 @@ impl Storage { })?, })) } - - /// When setup completed, or `None` before it has. - pub fn setup_completed_at_ms(&self) -> Result, LifecycleError> { - let completed: Option = self - .conn - .query_row( - "SELECT completed_at_ms FROM setup_complete WHERE singleton_id = 0", - [], - |row| row.get(0), - ) - .optional()?; - completed - .map(|ms| { - u64::try_from(ms).map_err(|_| { - LifecycleError::Malformed(format!( - "setup_complete completed_at_ms {ms} is negative" - )) - }) - }) - .transpose() - } } -/// The two-sided completion rule — the one command-ordering fact: -/// setup/rebuild never restart over a completed setup (completion is -/// once-per-database), run and maintenance never start before one exists. -/// Run and maintenance reach it through `preflight_lifecycle_command`; -/// setup and rebuild inside `complete_baseline_setup`'s transaction, after -/// `crate::commands::setup` has checked the same fact up front. +/// The two-sided completion rule — the one command-ordering fact: setup +/// never restarts once the era exists (an era is published once per +/// database), run and maintenance never start before it does. Run and +/// maintenance reach it through `preflight_lifecycle_command`; setup inside +/// `publish_era`'s transaction. fn require_command_fits_completion( conn: &rusqlite::Connection, command: LifecycleCommand, ) -> Result<(), LifecycleError> { - let setup_complete = setup_complete_exists(conn)?; + let set_up = is_set_up_in(conn)?; let (fits, reason) = match command { - LifecycleCommand::Setup | LifecycleCommand::Rebuild => ( - !setup_complete, - "setup is already complete for this data directory", - ), - LifecycleCommand::Run | LifecycleCommand::MaintenanceFlush => ( - setup_complete, - "setup has not completed for this data directory", - ), + LifecycleCommand::Setup => (!set_up, "setup is already complete for this data directory"), + LifecycleCommand::Run | LifecycleCommand::MaintenanceFlush => { + (set_up, "setup has not completed for this data directory") + } }; if !fits { return Err(LifecycleError::NotAdmissible { @@ -235,12 +199,10 @@ fn require_command_fits_completion( Ok(()) } -fn setup_complete_exists(conn: &rusqlite::Connection) -> Result { - Ok(conn.query_row( - "SELECT EXISTS (SELECT 1 FROM setup_complete WHERE singleton_id = 0)", - [], - |row| row.get::<_, bool>(0), - )?) +/// Whether setup has completed: exactly when the era row exists. The one +/// home of "set up" for admission and `Storage::is_setup_complete`. +pub(super) fn is_set_up_in(conn: &rusqlite::Connection) -> rusqlite::Result { + conn.query_row("SELECT EXISTS (SELECT 1 FROM era)", [], |row| row.get(0)) } fn refuse_on_canonical_divergence(conn: &rusqlite::Connection) -> Result<(), LifecycleError> { @@ -273,12 +235,9 @@ mod tests { fn complete_seeded_setup(storage: &mut Storage) { storage - .complete_baseline_setup( + .publish_era( std::path::Path::new("/tmp/facts-genesis"), - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, + &crate::storage::NewEra::EMPTY_GENESIS, ) .expect("complete setup"); } @@ -295,23 +254,10 @@ mod tests { .expect("seed divergence"); } - #[test] - fn setup_completed_at_ms_reads_the_completion_fact() { - let (_db, mut storage) = seeded(LifecycleCommand::Setup); - assert_eq!(storage.setup_completed_at_ms().expect("read"), None); - complete_seeded_setup(&mut storage); - assert!( - storage - .setup_completed_at_ms() - .expect("read") - .is_some_and(|ms| ms > 0) - ); - } - #[test] fn completion_rule_is_two_sided() { - let (_db, mut storage) = seeded(LifecycleCommand::Rebuild); - // Before completion: run and maintenance refuse; rebuild may retry. + let (_db, mut storage) = seeded(LifecycleCommand::Setup); + // Before completion: run and maintenance refuse; setup may proceed. assert!(matches!( storage.preflight_lifecycle_command(LifecycleCommand::Run), Err(LifecycleError::NotAdmissible { .. }) @@ -321,19 +267,15 @@ mod tests { Err(LifecycleError::NotAdmissible { .. }) )); storage - .preflight_lifecycle_command(LifecycleCommand::Rebuild) - .expect("rebuild retry over its incomplete database"); + .preflight_lifecycle_command(LifecycleCommand::Setup) + .expect("setup over its incomplete database"); complete_seeded_setup(&mut storage); - // After completion: setup/rebuild refuse; run and maintenance pass. + // After completion: setup refuses; run and maintenance pass. assert!(matches!( storage.preflight_lifecycle_command(LifecycleCommand::Setup), Err(LifecycleError::NotAdmissible { .. }) )); - assert!(matches!( - storage.preflight_lifecycle_command(LifecycleCommand::Rebuild), - Err(LifecycleError::NotAdmissible { .. }) - )); storage .preflight_lifecycle_command(LifecycleCommand::Run) .expect("run passes over a completed setup"); @@ -350,7 +292,6 @@ mod tests { for command in [ LifecycleCommand::Setup, - LifecycleCommand::Rebuild, LifecycleCommand::Run, LifecycleCommand::MaintenanceFlush, ] { @@ -360,17 +301,97 @@ mod tests { )); } assert!(matches!( - storage.complete_baseline_setup( + storage.publish_era( std::path::Path::new("/tmp/facts-genesis"), - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - 0, - false, + &crate::storage::NewEra::EMPTY_GENESIS, ), Err(LifecycleError::CanonicalDivergence { nonce: 7 }) )); } + #[test] + fn publish_era_is_atomic_and_one_shot() { + use crate::storage::NewEra; + use crate::storage::test_helpers::era; + let (_db, mut storage) = seeded(LifecycleCommand::Setup); + let prefix = std::path::Path::new("/tmp/era-baseline"); + let unspanned = NewEra { + anchor_batch_nonce: 5, + ..era(3, 4, 0) + }; + let refused = storage + .publish_era(prefix, &unspanned) + .expect_err("trg_era_fits refuses an era past its rebuilt rows"); + assert!( + matches!(&refused, LifecycleError::Storage(error) + if error.to_string() == "the era must span exactly its executed rebuilt rows"), + "{refused}" + ); + // Nothing of the refused era survives: no era and no snapshot. + assert!(matches!( + storage.era(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(storage.list_snapshots().unwrap().is_empty()); + assert!(!storage.is_setup_complete().unwrap()); + + let published = NewEra { + anchor_batch_nonce: 5, + ..era(3, 3, 0) + }; + storage.publish_era(prefix, &published).expect("publish"); + assert!(storage.is_setup_complete().unwrap()); + let state = storage.era().unwrap(); + assert_eq!( + ( + state.base.get(), + state.rebuilt_end.get(), + state.stop_block, + state.first_queued_input, + state.anchor_batch_nonce + ), + (3, 3, 0, 0, 5) + ); + assert_eq!( + storage.latest_snapshot().unwrap().unwrap().l2_tx_count, + sequencer_core::history::L2TxCount::new(3) + ); + assert!( + storage.open_state().unwrap().is_none(), + "setup opens no Tip" + ); + assert!(matches!( + storage.publish_era(prefix, &published), + Err(LifecycleError::NotAdmissible { + requested: LifecycleCommand::Setup, + .. + }) + )); + assert_eq!(storage.era().unwrap(), state); + } + + #[test] + fn publish_era_refuses_a_directory_with_a_batch() { + let (_db, mut storage) = seeded(LifecycleCommand::Setup); + // The schema refuses any batch before the era exists, so the seed + // drops that trigger: the Rust guard is what this test exercises. + storage + .conn + .execute_batch( + "DROP TRIGGER trg_batch_insert; \ + INSERT INTO batches (nonce, created_at_ms) VALUES (0, 0);", + ) + .expect("seed a batch"); + assert!(matches!( + storage.publish_era( + std::path::Path::new("/tmp/era-baseline"), + &crate::storage::NewEra::EMPTY_GENESIS + ), + Err(LifecycleError::Malformed(_)) + )); + assert!(!storage.is_setup_complete().unwrap()); + } + #[test] fn terminal_cause_recording_is_gated_on_nothing() { let (_db, mut storage) = seeded(LifecycleCommand::Setup); diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index f471c19c..28c5d655 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -1,84 +1,70 @@ -- --------------------------------------------------------------------------- -- Batch lifecycle -- --- A batch has two monotonic events in its lifetime, each stored as a nullable --- write-once timestamp on the row: +-- The batches of an era are an append-only list in nonce order: each +-- batch is created once, sealed once, accepted once, and never removed. A +-- batch has two monotonic events in its lifetime, each stored as a nullable +-- write-once column on the row: -- --- * `sealed_at_ms` — inclusion lane closed the batch (no more ops). --- * `invalidated_at_ms` — recovery cascade-invalidated the batch. +-- * `sealed_at_ms` — inclusion lane closed the batch (no more ops). +-- * `accepted_input` — input reader saw the scheduler accept the batch's +-- landing at that safe input (I9). -- --- NULL means the event hasn't happened. Once set, triggers below make the --- column write-once. The only "mutable" state on the row is these two NULL→value --- transitions, each owned by exactly one writer (inclusion lane vs recovery). +-- NULL means the event hasn't happened. `trg_batch_lifecycle` below makes each +-- write-once, and each has exactly one writer (inclusion lane, input reader). +-- A history that can no longer land ends the era; the operator rebuilds in a +-- new data directory, so no batch is ever repaired in place. -- --- The **Tip** is the one batch currently accepting ops: sealed_at_ms IS NULL --- AND invalidated_at_ms IS NULL. A partial unique index enforces at-most-one. +-- The **open batch** is the one batch still taking ops: sealed_at_ms IS NULL. +-- A partial unique index enforces at-most-one. -- --- `nonce` is structural: equal to `parent.nonce + 1`, or the batch-tree anchor's --- nonce for the parentless root (0 at genesis). Enforced by trigger on INSERT. The scheduler's view of a batch's --- identity; reused across recovery cascades (new Tip forks from last valid --- ancestor, inheriting nonce via the +1 rule). +-- `nonce` is the key, and the scheduler's view of a batch's identity: the last +-- batch's nonce + 1, or the era's `anchor_batch_nonce` (0 at genesis) for the +-- era's first batch. Enforced by trigger on INSERT, so a nonce names one batch +-- for the life of the database and no insert lands on an existing one. -- --------------------------------------------------------------------------- --- `sealed_at_ms` / `invalidated_at_ms` are write-once observability stamps from --- the wall clock, deliberately not cross-checked against `created_at_ms`: --- wall-clock order is environmental, not an invariant (NTP steps, VM resume), and --- such a CHECK would wedge batch close and the recovery cascade on a clock --- regression. Every reader is an IS NULL / IS NOT NULL predicate. +-- `sealed_at_ms` is a write-once observability stamp from the wall clock, +-- deliberately not cross-checked against `created_at_ms`: wall-clock order is +-- environmental, not an invariant (NTP steps, VM resume), and such a CHECK +-- would wedge batch close on a clock regression. Every reader is an IS NULL / +-- IS NOT NULL predicate. -- `payload_hash` is the keccak256 of the batch's SSZ wire bytes, stamped at -- seal time by the encode path the submitter uses. The content-identity check -- compares an accepted L1 landing against it; because the sealing code computed --- it, it survives wire-format upgrades. NULL only on a never-sealed batch (the --- open Tip, or a Tip a cascade invalidated). +-- it, it survives wire-format upgrades. NULL exactly on the open batch. +-- `accepted_input` is the safe input whose landing of this batch the scheduler +-- accepted; the accepted batches are a prefix of the list, in input order, and +-- the greatest one is the accepted frontier. Its UNIQUE index serves the +-- frontier reads. The key serves the submitter's hot path ("closed batches +-- with nonce >= N", ordered). `nonce` is a rowid alias: an insert without one +-- would be numbered by SQLite, and a BEFORE trigger sees that nonce as -1, +-- which `trg_batch_insert` refuses. CREATE TABLE IF NOT EXISTS batches ( - batch_index INTEGER PRIMARY KEY, - parent_batch_index INTEGER REFERENCES batches(batch_index), -- NULL only for genesis - nonce INTEGER NOT NULL CHECK (typeof(nonce) = 'integer' AND nonce >= 0), + nonce INTEGER PRIMARY KEY CHECK (nonce >= 0), created_at_ms INTEGER NOT NULL, sealed_at_ms INTEGER CHECK (sealed_at_ms IS NULL OR sealed_at_ms >= 0), - invalidated_at_ms INTEGER CHECK (invalidated_at_ms IS NULL OR invalidated_at_ms >= 0), - payload_hash BLOB CHECK (payload_hash IS NULL OR length(payload_hash) = 32) -); - --- "At most one valid Tip" — structural via partial unique index. The predicate --- references only local columns of `batches`, so SQLite accepts it. --- --- We index on COALESCE(sealed_at_ms, 0) instead of sealed_at_ms directly --- because SQLite UNIQUE indexes treat NULLs as distinct — so indexing directly --- on `sealed_at_ms` would allow many NULL rows. COALESCE maps all matching --- rows to the same non-NULL value (0), forcing real uniqueness. -CREATE UNIQUE INDEX IF NOT EXISTS ux_single_valid_tip + payload_hash BLOB CHECK (payload_hash IS NULL OR length(payload_hash) = 32), + accepted_input INTEGER UNIQUE REFERENCES safe_inputs(safe_input_index), + CHECK ((sealed_at_ms IS NULL) = (payload_hash IS NULL)), + CHECK (accepted_input IS NULL OR sealed_at_ms IS NOT NULL) +) STRICT; + +-- "At most one open batch" — structural via partial unique index. We index on +-- COALESCE(sealed_at_ms, 0) instead of sealed_at_ms directly because SQLite +-- UNIQUE indexes treat NULLs as distinct — so indexing directly on +-- `sealed_at_ms` would allow many NULL rows. COALESCE maps all matching rows +-- to the same non-NULL value (0), forcing real uniqueness. +CREATE UNIQUE INDEX IF NOT EXISTS ux_single_open_batch ON batches(COALESCE(sealed_at_ms, 0)) - WHERE sealed_at_ms IS NULL AND invalidated_at_ms IS NULL; - --- Submitter hot path: "give me valid closed batches with nonce >= N", ordered. -CREATE INDEX IF NOT EXISTS idx_batches_valid_closed_by_nonce - ON batches(nonce) - WHERE invalidated_at_ms IS NULL AND sealed_at_ms IS NOT NULL; + WHERE sealed_at_ms IS NULL; -- ── Views ────────────────────────────────────────────────────────────────── --- Readers over batch data go through these `valid_*` views, which exclude --- invalidated rows; writers always target the base tables. -CREATE VIEW IF NOT EXISTS valid_batches AS - SELECT * FROM batches WHERE invalidated_at_ms IS NULL; - -CREATE VIEW IF NOT EXISTS valid_closed_batches AS - SELECT * FROM valid_batches WHERE sealed_at_ms IS NOT NULL; +CREATE VIEW IF NOT EXISTS closed_batches AS + SELECT * FROM batches WHERE sealed_at_ms IS NOT NULL; -- At most one row by the partial unique index above. -CREATE VIEW IF NOT EXISTS valid_open_batch AS - SELECT * FROM valid_batches WHERE sealed_at_ms IS NULL; - --- Batch-tree anchor (I16): the nonce the single valid parentless root carries, --- 0 for genesis and N' (the resume nonce) after a cockroach rebuild, so `run`'s --- first Tip roots at N' without replaying history. Read by --- `trg_enforce_nonce_contiguity` and `compute_next_nonce`. Setup writes it once, --- before `setup_complete`; `trg_batch_tree_anchor_write_once` then freezes it, --- since re-anchoring a live deployment would strand its spine. -CREATE TABLE IF NOT EXISTS batch_tree_anchor ( - singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - nonce INTEGER NOT NULL CHECK (nonce >= 0) -); -INSERT OR IGNORE INTO batch_tree_anchor(singleton_id, nonce) VALUES (0, 0); +CREATE VIEW IF NOT EXISTS open_batch AS + SELECT * FROM batches WHERE sealed_at_ms IS NULL; -- ── Triggers ─────────────────────────────────────────────────────────────── -- @@ -88,125 +74,111 @@ INSERT OR IGNORE INTO batch_tree_anchor(singleton_id, nonce) VALUES (0, 0); -- transition sequence — triggers just ensure the DB never reaches an -- inconsistent state if the writer misbehaves. -- --- typeof() guards: INTEGER affinity does not reject an unconvertible value, --- and TEXT/BLOB sort above INTEGER, so a bare `x >= 0` CHECK passes 'abc'. --- Columns that feed SQL-level arithmetic or comparisons (trigger math, --- frontier folds, lease counts) therefore carry an explicit --- typeof(x) = 'integer' guard: a mis-bound positional parameter must refuse, --- not coerce to 0 inside a trigger. - --- Nonce contiguity: `nonce = parent.nonce + 1`, or the batch-tree anchor nonce --- (0 for a genesis deployment, N' for a recovered one) for the parentless root. -CREATE TRIGGER IF NOT EXISTS trg_enforce_nonce_contiguity -AFTER INSERT ON batches -FOR EACH ROW +-- Every table is STRICT: a value of the wrong storage class is refused for +-- every column before any trigger runs, so trigger arithmetic, frontier folds +-- and comparisons only ever see the declared class (integers where they count). + +-- A batch is inserted unaccepted (acceptance is an UPDATE, which the triggers +-- below order), carrying the last batch's nonce + 1, or the era's +-- `anchor_batch_nonce` (its next batch nonce at the stop block) when there is +-- no batch. With neither the right side is NULL, `IS NOT` holds and the insert +-- aborts: no batch exists before the era. SQLite fires the most recently +-- created BEFORE trigger first, so the divergence freeze (created below) +-- answers before this one while its marker exists. +CREATE TRIGGER IF NOT EXISTS trg_batch_insert +BEFORE INSERT ON batches FOR EACH ROW BEGIN SELECT CASE - -- A parentless root must carry exactly the deployment's anchor nonce, - -- so a buggy root at any other nonce ABORTs, including 0 on a recovered one. - WHEN NEW.parent_batch_index IS NULL - AND NEW.nonce != (SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0) - THEN RAISE(ABORT, 'parentless root must carry the batch-tree anchor nonce') - -- At most one *valid* parentless root. A fully-torn cascade invalidates - -- the old root, then re-roots parentless at the anchor (the documented - -- `open_fresh_tip_in_tx` parent=None path) — leaving exactly one valid - -- root again, with the invalidated old root(s) coexisting. (Counts the - -- just-inserted row, hence `> 1`.) - WHEN NEW.parent_batch_index IS NULL - AND (SELECT COUNT(*) FROM batches - WHERE parent_batch_index IS NULL AND invalidated_at_ms IS NULL) > 1 - THEN RAISE(ABORT, 'at most one valid parentless root per deployment') - WHEN NEW.parent_batch_index IS NOT NULL - AND NEW.nonce != (SELECT nonce + 1 FROM batches WHERE batch_index = NEW.parent_batch_index) - THEN RAISE(ABORT, 'batch nonce must equal parent.nonce + 1') + WHEN NEW.accepted_input IS NOT NULL + THEN RAISE(ABORT, 'a batch is inserted unaccepted') + WHEN NEW.nonce IS NOT COALESCE( + (SELECT MAX(nonce) + 1 FROM batches), + (SELECT anchor_batch_nonce FROM era WHERE singleton_id = 0)) + THEN RAISE(ABORT, 'a batch nonce is the last nonce + 1, or the era anchor') END; END; --- Write-once: sealed_at_ms transitions only NULL → non-NULL. -CREATE TRIGGER IF NOT EXISTS trg_sealed_at_ms_write_once -BEFORE UPDATE OF sealed_at_ms ON batches -FOR EACH ROW -WHEN OLD.sealed_at_ms IS NOT NULL -BEGIN - SELECT RAISE(ABORT, 'sealed_at_ms is write-once'); -END; - --- Write-once: invalidated_at_ms transitions only NULL → non-NULL. -CREATE TRIGGER IF NOT EXISTS trg_invalidated_at_ms_write_once -BEFORE UPDATE OF invalidated_at_ms ON batches -FOR EACH ROW -WHEN OLD.invalidated_at_ms IS NOT NULL -BEGIN - SELECT RAISE(ABORT, 'invalidated_at_ms is write-once'); -END; - --- Write-once: payload_hash transitions only NULL → non-NULL (stamped in the --- same UPDATE that seals the batch). -CREATE TRIGGER IF NOT EXISTS trg_payload_hash_write_once -BEFORE UPDATE OF payload_hash ON batches -FOR EACH ROW -WHEN OLD.payload_hash IS NOT NULL -BEGIN - SELECT RAISE(ABORT, 'payload_hash is write-once'); -END; - --- parent_batch_index is immutable after insert. -CREATE TRIGGER IF NOT EXISTS trg_parent_batch_index_immutable -BEFORE UPDATE OF parent_batch_index ON batches -FOR EACH ROW -WHEN (OLD.parent_batch_index IS NULL) != (NEW.parent_batch_index IS NULL) - OR OLD.parent_batch_index IS NOT NULL AND NEW.parent_batch_index IS NOT NULL - AND OLD.parent_batch_index != NEW.parent_batch_index -BEGIN - SELECT RAISE(ABORT, 'parent_batch_index is immutable'); -END; - --- nonce is immutable after insert. -CREATE TRIGGER IF NOT EXISTS trg_nonce_immutable -BEFORE UPDATE OF nonce ON batches -FOR EACH ROW -WHEN OLD.nonce != NEW.nonce -BEGIN - SELECT RAISE(ABORT, 'nonce is immutable'); -END; +-- A batch row changes only by sealing once, with its hash, while unaccepted; +-- or by being accepted once while sealed. SQLite fires the most recently +-- created BEFORE trigger first: the divergence freeze (created below) answers +-- first, then the acceptance order, then this one. +CREATE TRIGGER IF NOT EXISTS trg_batch_lifecycle +BEFORE UPDATE ON batches FOR EACH ROW +WHEN NOT (NEW.nonce IS OLD.nonce + AND NEW.created_at_ms IS OLD.created_at_ms + AND ( (OLD.sealed_at_ms IS NULL AND NEW.sealed_at_ms IS NOT NULL + AND OLD.payload_hash IS NULL AND NEW.payload_hash IS NOT NULL + AND OLD.accepted_input IS NULL AND NEW.accepted_input IS NULL) + OR (OLD.sealed_at_ms IS NOT NULL + AND NEW.sealed_at_ms IS OLD.sealed_at_ms AND NEW.payload_hash IS OLD.payload_hash + AND OLD.accepted_input IS NULL AND NEW.accepted_input IS NOT NULL))) +BEGIN SELECT RAISE(ABORT, 'a batch changes only by sealing once with its hash, or by being accepted once while sealed'); END; + +-- The batches are append-only: no batch is ever deleted. +CREATE TRIGGER IF NOT EXISTS trg_batches_not_deletable +BEFORE DELETE ON batches FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'batches are retained for the database lifetime'); END; + +-- Acceptance extends the accepted prefix of the list: the accepted batch +-- carries the greatest accepted batch's nonce + 1 (the era anchor before any +-- acceptance) and a later input than every earlier acceptance. With no +-- accepted batch and no era the right side is NULL, `IS NOT` holds and the +-- update aborts. +CREATE TRIGGER IF NOT EXISTS trg_batch_acceptance_extends_the_prefix +BEFORE UPDATE OF accepted_input ON batches FOR EACH ROW +WHEN OLD.accepted_input IS NULL AND NEW.accepted_input IS NOT NULL AND ( + NEW.nonce IS NOT COALESCE( + (SELECT nonce + 1 FROM batches WHERE accepted_input = (SELECT MAX(accepted_input) FROM batches)), + (SELECT anchor_batch_nonce FROM era WHERE singleton_id = 0)) + OR NEW.accepted_input <= COALESCE((SELECT MAX(accepted_input) FROM batches), -1)) +BEGIN SELECT RAISE(ABORT, 'an acceptance extends the accepted prefix of the batches'); END; -- --------------------------------------------------------------------------- --- Frames and user ops: must target the current Tip. +-- Frames and user ops: must target the open batch. -- -- These catch "stale WriteHead" bugs — where a writer holds an in-memory --- batch_index that's no longer the Tip (sealed or invalidated between reads). +-- batch nonce that's no longer the open batch (sealed between reads). -- A PK lookup per row: microseconds, negligible overhead even on hot paths. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS frames ( - batch_index INTEGER NOT NULL REFERENCES batches(batch_index), + batch_nonce INTEGER NOT NULL REFERENCES batches(nonce), frame_in_batch INTEGER NOT NULL CHECK (frame_in_batch >= 0), created_at_ms INTEGER NOT NULL, -- Fee committed by the sequencer for this whole frame. fee INTEGER NOT NULL CHECK (fee >= 0), -- Claimed safe L1 block frontier for this frame. safe_block INTEGER NOT NULL CHECK (safe_block >= 0), - PRIMARY KEY(batch_index, frame_in_batch) -); + PRIMARY KEY(batch_nonce, frame_in_batch) +) STRICT; CREATE TRIGGER IF NOT EXISTS trg_frames_target_must_be_tip BEFORE INSERT ON frames FOR EACH ROW WHEN NOT EXISTS ( SELECT 1 FROM batches - WHERE batch_index = NEW.batch_index + WHERE nonce = NEW.batch_nonce AND sealed_at_ms IS NULL - AND invalidated_at_ms IS NULL ) BEGIN SELECT RAISE(ABORT, 'frames can only be inserted into the current Tip'); END; +-- A frame is written once: a sealed batch's `payload_hash` covers its frames +-- and user ops, so a changed or missing row would re-encode the batch to other +-- bytes. A REPLACE's delete fires the DELETE trigger (`recursive_triggers`). +CREATE TRIGGER IF NOT EXISTS trg_frames_immutable +BEFORE UPDATE ON frames FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'frames are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_frames_not_deletable +BEFORE DELETE ON frames FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'frames are retained for the database lifetime'); END; + -- No (sender, nonce) uniqueness is enforced here: included user-op identity --- is tracked by application nonce logic, and a DB constraint would block --- legitimate resubmission after a recovery cascade. +-- is tracked by application nonce logic, which admits an op only at its +-- sender's next nonce and advances that nonce on every included op. CREATE TABLE IF NOT EXISTS user_ops ( - batch_index INTEGER NOT NULL, + batch_nonce INTEGER NOT NULL, frame_in_batch INTEGER NOT NULL, pos_in_frame INTEGER NOT NULL CHECK (pos_in_frame >= 0), sender BLOB NOT NULL CHECK (length(sender) = 20), @@ -215,11 +187,11 @@ CREATE TABLE IF NOT EXISTS user_ops ( data BLOB NOT NULL, sig BLOB NOT NULL CHECK (length(sig) = 65), received_at_ms INTEGER NOT NULL, - PRIMARY KEY(batch_index, frame_in_batch, pos_in_frame), - FOREIGN KEY(batch_index, frame_in_batch) REFERENCES frames(batch_index, frame_in_batch) -); + PRIMARY KEY(batch_nonce, frame_in_batch, pos_in_frame), + FOREIGN KEY(batch_nonce, frame_in_batch) REFERENCES frames(batch_nonce, frame_in_batch) +) STRICT; --- `GET /nonce` reads a sender's latest op in a valid batch. Rows are never +-- `GET /nonce` reads a sender's latest op. Rows are never -- pruned, so without this index every public read would scan all history. -- It is the chunk commit's costliest index: sender-keyed entries land on -- scattered leaves, dirtying about one WAL page per distinct sender per chunk. @@ -232,26 +204,32 @@ BEFORE INSERT ON user_ops FOR EACH ROW WHEN NOT EXISTS ( SELECT 1 FROM batches - WHERE batch_index = NEW.batch_index + WHERE nonce = NEW.batch_nonce AND sealed_at_ms IS NULL - AND invalidated_at_ms IS NULL ) BEGIN SELECT RAISE(ABORT, 'user_ops can only be inserted into the current Tip'); END; +-- A user op is written once, like its frame. +CREATE TRIGGER IF NOT EXISTS trg_user_ops_immutable +BEFORE UPDATE ON user_ops FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'user ops are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_user_ops_not_deletable +BEFORE DELETE ON user_ops FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'user ops are retained for the database lifetime'); END; + CREATE TABLE IF NOT EXISTS safe_inputs ( safe_input_index INTEGER PRIMARY KEY, sender BLOB NOT NULL CHECK (length(sender) = 20), payload BLOB NOT NULL, -- Block number of the chain block where this direct input was included (e.g. InputAdded event block). - block_number INTEGER NOT NULL - CHECK (typeof(block_number) = 'integer' AND block_number >= 0), + block_number INTEGER NOT NULL CHECK (block_number >= 0), -- Timestamp of the carrying L1 block. block_timestamp INTEGER NOT NULL CHECK (block_timestamp >= 0), -- Hash of the L1 transaction that carried this input. transaction_hash BLOB NOT NULL CHECK (length(transaction_hash) = 32) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_safe_inputs_sender ON safe_inputs(sender); @@ -266,54 +244,35 @@ CREATE TRIGGER IF NOT EXISTS trg_safe_inputs_not_deletable BEFORE DELETE ON safe_inputs FOR EACH ROW BEGIN SELECT RAISE(ABORT, 'safe inputs are retained for the database lifetime'); END; --- Current application order. Recovery replaces only an invalidated suffix; --- immutable source payloads remain in their owning tables. +-- The era's live application order, append-only; immutable source payloads +-- remain in their owning tables. CREATE TABLE IF NOT EXISTS application_inputs ( - offset INTEGER PRIMARY KEY CHECK (typeof(offset) = 'integer' AND offset >= 0), - batch_index INTEGER NOT NULL, + l2_tx_count INTEGER PRIMARY KEY CHECK (l2_tx_count >= 0), + batch_nonce INTEGER NOT NULL, frame_in_batch INTEGER NOT NULL, user_op_pos_in_frame INTEGER, safe_input_index INTEGER, - FOREIGN KEY(batch_index, frame_in_batch) - REFERENCES frames(batch_index, frame_in_batch), - FOREIGN KEY(batch_index, frame_in_batch, user_op_pos_in_frame) - REFERENCES user_ops(batch_index, frame_in_batch, pos_in_frame), + FOREIGN KEY(batch_nonce, frame_in_batch) + REFERENCES frames(batch_nonce, frame_in_batch), + FOREIGN KEY(batch_nonce, frame_in_batch, user_op_pos_in_frame) + REFERENCES user_ops(batch_nonce, frame_in_batch, pos_in_frame), FOREIGN KEY(safe_input_index) REFERENCES safe_inputs(safe_input_index), CHECK ((user_op_pos_in_frame IS NOT NULL AND safe_input_index IS NULL) OR (user_op_pos_in_frame IS NULL AND safe_input_index IS NOT NULL)), - UNIQUE(batch_index, frame_in_batch, user_op_pos_in_frame), + -- Its index also serves lookups by batch and by frame. + UNIQUE(batch_nonce, frame_in_batch, user_op_pos_in_frame), UNIQUE(safe_input_index) -); -CREATE INDEX IF NOT EXISTS idx_application_inputs_frame - ON application_inputs(batch_index, frame_in_batch); +) STRICT; CREATE INDEX IF NOT EXISTS idx_safe_inputs_block ON safe_inputs(block_number, safe_input_index); --- Derived log of batch submissions the scheduler would actually execute. --- Unlike a raw log of all safe submissions, this only contains the accepted --- prefix: batches whose nonce matched the expected sequence and were not stale. --- Maintained atomically by Storage::append_ingested_safe_inputs_with_timestamp --- (via populate_safe_accepted_batches), which simulates the scheduler's --- acceptance logic over new safe_inputs rows. -CREATE TABLE IF NOT EXISTS safe_accepted_batches ( - safe_input_index INTEGER PRIMARY KEY REFERENCES safe_inputs(safe_input_index), - -- CHECK aligns this column with its siblings (batches.nonce, anchor nonce); - -- the writer is u64-sourced, so a negative value is corruption. - nonce INTEGER NOT NULL CHECK (typeof(nonce) = 'integer' AND nonce >= 0), - first_frame_safe_block INTEGER NOT NULL, - inclusion_block INTEGER NOT NULL -); - CREATE TABLE IF NOT EXISTS l1_safe_head ( singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), -- Highest L1 safe block the input reader has observed and atomically synced into storage. block_number INTEGER NOT NULL CHECK (block_number >= 0), - -- L1 timestamp (Unix seconds) of block_number. - block_timestamp INTEGER NOT NULL CHECK (block_timestamp >= 0), - -- Wall-clock time (Unix ms) of the last successful L1 sync. - -- Used for wall-clock danger estimation when L1 is unreachable. - synced_at_ms INTEGER NOT NULL CHECK (synced_at_ms >= 0) -); + -- L1 timestamp (Unix seconds) of block_number: the safe head's age. + block_timestamp INTEGER NOT NULL CHECK (block_timestamp >= 0) +) STRICT; -- Highest wallet nonce ever broadcast by this deployment's batch-submitter -- key (the durable realization of the TLA+ spec's `walletNonce`). Write-before-broadcast: any component about to send a tx @@ -326,127 +285,185 @@ CREATE TABLE IF NOT EXISTS l1_safe_head ( CREATE TABLE IF NOT EXISTS wallet_nonce_watermark ( singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), watermark INTEGER NOT NULL CHECK (watermark >= 0) -); +) STRICT; -- Canonical-divergence poison marker. Written by the input -- reader's acceptance simulation — atomically with the sync that detected -- it — when a fully-accepted L1 landing fails the content-identity check: --- either no valid closed local batch exists at the accepted nonce +-- either no closed local batch exists at the accepted nonce -- (kind = 'foreign': a zombie or foreign batch from our key) or the landed -- bytes hash differently from ours (kind = 'mismatch'). Once present, the --- acceptance frontier freezes, `check_danger` reports `CanonicalDivergence` --- ahead of every other arm, startup refuses, and the runtime detector exits. --- The remedy is cockroach recovery (wipe + rebuild from L1), never standard --- recovery — canonical state contains executed effects with no reliable --- local source. Keep-first: only the earliest detection is recorded. +-- acceptance frontier freezes, `terminal_fact` reports the divergence first, +-- `run` refuses at start, and the landing watch stops a running process. +-- The remedy is cockroach recovery (a new era in a new data directory): +-- canonical state contains executed effects with no reliable local source. +-- Keep-first: only the earliest detection is recorded. CREATE TABLE IF NOT EXISTS canonical_divergence ( singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), nonce INTEGER NOT NULL CHECK (nonce >= 0), safe_input_index INTEGER NOT NULL CHECK (safe_input_index >= 0), kind TEXT NOT NULL CHECK (kind IN ('foreign', 'mismatch')), detected_at_ms INTEGER NOT NULL CHECK (detected_at_ms >= 0) -); +) STRICT; -- I15 structural enforcement: while the divergence marker exists, the batch --- tree and snapshot collection are frozen in the engine --- itself. Standard recovery is forbidden on a diverged frontier; the typed --- Rust refusals (the local-first startup recovery procedure plus guarded Tip/Cascade --- mutations and atomic runtime admission) remain the friendly error surface, but these --- triggers are the enforcement a forgotten call site cannot bypass. The accepted --- frontier (`safe_accepted_batches`) has no trigger: its single writer refuses --- past the marker in Rust. +-- list, its accepted frontier and snapshot collection are frozen in the engine +-- itself. The typed Rust refusals (`run`'s terminal-fact check at start, the +-- landing watch, and the open-a-batch transaction's re-check) remain the +-- friendly error surface, but these triggers are the enforcement a forgotten +-- call site cannot bypass. Created +-- after every other BEFORE trigger on `batches`, so they answer first. CREATE TRIGGER IF NOT EXISTS trg_batches_frozen_on_divergence_insert BEFORE INSERT ON batches FOR EACH ROW WHEN EXISTS (SELECT 1 FROM canonical_divergence WHERE singleton_id = 0) -BEGIN SELECT RAISE(ABORT, 'batch tree frozen: canonical divergence marker present'); END; +BEGIN SELECT RAISE(ABORT, 'batch list frozen: canonical divergence marker present'); END; CREATE TRIGGER IF NOT EXISTS trg_batches_frozen_on_divergence_update BEFORE UPDATE ON batches FOR EACH ROW WHEN EXISTS (SELECT 1 FROM canonical_divergence WHERE singleton_id = 0) -BEGIN SELECT RAISE(ABORT, 'batch tree frozen: canonical divergence marker present'); END; - --- A complete immutable baseline is registered with its durable artifact. --- Only the recovery generation changes during this database's lifetime. -CREATE TABLE IF NOT EXISTS history_state ( - singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - era_id BLOB NOT NULL CHECK ( - typeof(era_id) = 'blob' AND length(era_id) = 16 +BEGIN SELECT RAISE(ABORT, 'batch list frozen: canonical divergence marker present'); END; + +-- The history setup rebuilds from L1, from its base through the stop block: +-- one row per transaction that executed, keyed by its L2 tx count. A direct +-- keeps only its input index (safe_inputs holds the rest); a user op keeps what +-- executed. `batch_nonce` is the scheduler's next batch nonce when it handed +-- the row out. A user op the executor skipped has no row. Only setup writes +-- here, and only before it publishes the era. +CREATE TABLE IF NOT EXISTS rebuilt_history ( + l2_tx_count INTEGER PRIMARY KEY CHECK (l2_tx_count >= 0), + batch_nonce INTEGER NOT NULL CHECK (batch_nonce >= 0), + safe_input_index INTEGER UNIQUE REFERENCES safe_inputs(safe_input_index), + sender BLOB CHECK (sender IS NULL OR length(sender) = 20), + nonce INTEGER CHECK (nonce IS NULL OR nonce BETWEEN 0 AND 4294967295), + fee INTEGER CHECK (fee IS NULL OR fee BETWEEN 0 AND 65535), + safe_block INTEGER CHECK (safe_block IS NULL OR safe_block >= 0), + data BLOB, + CHECK ((safe_input_index IS NOT NULL AND sender IS NULL AND nonce IS NULL AND fee IS NULL + AND safe_block IS NULL AND data IS NULL) + OR (safe_input_index IS NULL AND sender IS NOT NULL AND nonce IS NOT NULL AND fee IS NOT NULL + AND safe_block IS NOT NULL AND data IS NOT NULL)) +) STRICT; + +-- Rows are inserted in execution order until the era is published. The key +-- is a rowid alias: an insert without a count would be numbered by SQLite, +-- and a BEFORE trigger sees that count as -1. The first row may carry any +-- count; `trg_era_fits` ties it to the base count. A direct's input index +-- rises row over row because the scheduler's queue is FIFO; with no earlier +-- direct the MAX is NULL, the comparison is NULL and the row passes. +CREATE TRIGGER IF NOT EXISTS trg_rebuilt_history_insert +BEFORE INSERT ON rebuilt_history FOR EACH ROW +BEGIN + SELECT CASE + WHEN EXISTS (SELECT 1 FROM era) + THEN RAISE(ABORT, 'rebuilt history is sealed once its era is published') + WHEN NEW.l2_tx_count IS NULL OR NEW.l2_tx_count < 0 + THEN RAISE(ABORT, 'a rebuilt row carries its L2 tx count') + WHEN EXISTS (SELECT 1 FROM rebuilt_history) + AND NEW.l2_tx_count IS NOT (SELECT MAX(l2_tx_count) + 1 FROM rebuilt_history) + THEN RAISE(ABORT, 'rebuilt counts follow one another without gaps') + WHEN NEW.safe_input_index IS NOT NULL + AND (SELECT sender FROM safe_inputs WHERE safe_input_index = NEW.safe_input_index) + = (SELECT batch_submitter_address FROM deployment_identity WHERE singleton_id = 0) + THEN RAISE(ABORT, 'a batch submitter input never executes as a direct') + WHEN NEW.safe_input_index IS NOT NULL + AND NEW.safe_input_index <= (SELECT MAX(safe_input_index) FROM rebuilt_history) + THEN RAISE(ABORT, 'rebuilt directs drain in input order') + END; +END; +CREATE TRIGGER IF NOT EXISTS trg_rebuilt_history_immutable +BEFORE UPDATE ON rebuilt_history FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'rebuilt history rows are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_rebuilt_history_not_deletable +BEFORE DELETE ON rebuilt_history FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'rebuilt history is retained for the database lifetime'); END; + +-- The era: one immutable row, written by setup's publish. Its history starts +-- at the base count; setup's rebuild covers [base_l2_tx_count, +-- rebuilt_history_end) and the live rows follow. The lane drains directs from +-- `first_queued_input` until a frame of the era accounts for more. +-- `anchor_batch_nonce` is the nonce the era's first batch carries (I16). A database is set up exactly when this row exists. +CREATE TABLE IF NOT EXISTS era ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + era_id BLOB NOT NULL CHECK ( + length(era_id) = 16 AND substr(hex(era_id), 13, 1) = '4' AND substr(hex(era_id), 17, 1) IN ('8', '9', 'A', 'B')), - era_created_at_ms INTEGER NOT NULL CHECK ( - typeof(era_created_at_ms) = 'integer' AND era_created_at_ms >= 0), - recovery_generation INTEGER NOT NULL CHECK ( - typeof(recovery_generation) = 'integer' AND recovery_generation >= 0), - base_executed_input_count INTEGER NOT NULL CHECK ( - typeof(base_executed_input_count) = 'integer' AND base_executed_input_count >= 0), - base_safe_block INTEGER NOT NULL CHECK ( - typeof(base_safe_block) = 'integer' AND base_safe_block >= 0) -); --- One cut for each transition, measured after invalidation removes the old --- suffix and before reopening attributes replacement direct inputs. -CREATE TABLE IF NOT EXISTS history_generation_cuts ( - recovery_generation INTEGER PRIMARY KEY CHECK (recovery_generation > 0), - preserved_input_count INTEGER NOT NULL CHECK ( - typeof(preserved_input_count) = 'integer' AND preserved_input_count >= 0) -); -CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_immutable -BEFORE UPDATE ON history_generation_cuts FOR EACH ROW -BEGIN SELECT RAISE(ABORT, 'history generation cuts are immutable'); END; -CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_not_deletable -BEFORE DELETE ON history_generation_cuts FOR EACH ROW -BEGIN SELECT RAISE(ABORT, 'history generation cuts are retained throughout the era'); END; -CREATE TRIGGER IF NOT EXISTS trg_history_state_single_insert -BEFORE INSERT ON history_state FOR EACH ROW -WHEN EXISTS (SELECT 1 FROM history_state) -BEGIN SELECT RAISE(ABORT, 'history state is inserted once per database'); END; -CREATE TRIGGER IF NOT EXISTS trg_history_identity_write_once -BEFORE UPDATE OF singleton_id, era_id, era_created_at_ms, - base_executed_input_count, base_safe_block ON history_state FOR EACH ROW -BEGIN SELECT RAISE(ABORT, 'history baseline is immutable'); END; -CREATE TRIGGER IF NOT EXISTS trg_history_generation_monotonic -BEFORE UPDATE OF recovery_generation ON history_state FOR EACH ROW -WHEN OLD.recovery_generation = 9223372036854775807 - OR NEW.recovery_generation != OLD.recovery_generation + 1 - OR NOT EXISTS (SELECT 1 FROM history_generation_cuts - WHERE recovery_generation = NEW.recovery_generation) -BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one with a recorded cut'); END; -CREATE TRIGGER IF NOT EXISTS trg_history_state_not_deletable -BEFORE DELETE ON history_state FOR EACH ROW -BEGIN SELECT RAISE(ABORT, 'history state is write-once per database'); END; + published_at_ms INTEGER NOT NULL CHECK (published_at_ms >= 0), + base_l2_tx_count INTEGER NOT NULL CHECK (base_l2_tx_count >= 0), + rebuilt_history_end INTEGER NOT NULL CHECK (rebuilt_history_end >= base_l2_tx_count), + stop_block INTEGER NOT NULL CHECK (stop_block >= 0), + first_queued_input INTEGER NOT NULL CHECK (first_queued_input >= 0), + anchor_batch_nonce INTEGER NOT NULL CHECK (anchor_batch_nonce >= 0) +) STRICT; +-- The era's counts are exactly the span of the rebuilt rows, and its first +-- queued input lies among the inputs through the stop block. The scheduler +-- drains its queue in input order, so every rebuilt direct lies before the +-- first one still queued: a queued one would run twice. +CREATE TRIGGER IF NOT EXISTS trg_era_fits +BEFORE INSERT ON era FOR EACH ROW +BEGIN + SELECT CASE + WHEN NEW.base_l2_tx_count != COALESCE( + (SELECT MIN(l2_tx_count) FROM rebuilt_history), NEW.base_l2_tx_count) + OR NEW.rebuilt_history_end != COALESCE( + (SELECT MAX(l2_tx_count) + 1 FROM rebuilt_history), NEW.base_l2_tx_count) + THEN RAISE(ABORT, 'the era must span exactly its executed rebuilt rows') + WHEN EXISTS (SELECT 1 FROM rebuilt_history + WHERE safe_input_index IS NOT NULL + AND safe_input_index >= NEW.first_queued_input) + THEN RAISE(ABORT, 'a rebuilt direct is never still queued') + WHEN NEW.first_queued_input > COALESCE( + (SELECT safe_input_index + 1 FROM safe_inputs + WHERE block_number <= NEW.stop_block + ORDER BY block_number DESC, safe_input_index DESC LIMIT 1), 0) + THEN RAISE(ABORT, 'the first queued input lies past the stop block') + END; +END; +-- `INSERT OR REPLACE` would delete the old row; every connection turns +-- `recursive_triggers` on, so that delete fires `trg_era_not_deletable`, but +-- this BEFORE INSERT trigger answers first, with the clearer message. +CREATE TRIGGER IF NOT EXISTS trg_era_single_insert +BEFORE INSERT ON era FOR EACH ROW +WHEN EXISTS (SELECT 1 FROM era) +BEGIN SELECT RAISE(ABORT, 'the era is inserted once per database'); END; +CREATE TRIGGER IF NOT EXISTS trg_era_immutable +BEFORE UPDATE ON era FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'the era is immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_era_not_deletable +BEFORE DELETE ON era FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'the era is retained for the database lifetime'); END; CREATE TRIGGER IF NOT EXISTS trg_application_inputs_target_must_be_tip BEFORE INSERT ON application_inputs FOR EACH ROW -WHEN NOT EXISTS (SELECT 1 FROM batches WHERE batch_index = NEW.batch_index - AND sealed_at_ms IS NULL AND invalidated_at_ms IS NULL) -BEGIN SELECT RAISE(ABORT, 'application input must target the current valid Tip'); END; -CREATE TRIGGER IF NOT EXISTS trg_application_inputs_requires_baseline -BEFORE INSERT ON application_inputs FOR EACH ROW -WHEN NOT EXISTS (SELECT 1 FROM history_state) -BEGIN SELECT RAISE(ABORT, 'application history baseline is missing'); END; +WHEN NOT EXISTS (SELECT 1 FROM batches WHERE nonce = NEW.batch_nonce + AND sealed_at_ms IS NULL) +BEGIN SELECT RAISE(ABORT, 'application input must target the open batch'); END; +-- Live rows continue the era's rebuilt history. With no era the subquery is +-- NULL, `IS NOT` holds and the insert aborts. CREATE TRIGGER IF NOT EXISTS trg_application_inputs_contiguous BEFORE INSERT ON application_inputs FOR EACH ROW -WHEN NEW.offset != (SELECT MAX(base_executed_input_count, - COALESCE((SELECT MAX(offset) + 1 FROM application_inputs), 0)) - FROM history_state WHERE singleton_id = 0) +WHEN NEW.l2_tx_count IS NOT (SELECT MAX(rebuilt_history_end, + COALESCE((SELECT MAX(l2_tx_count) + 1 FROM application_inputs), 0)) + FROM era WHERE singleton_id = 0) BEGIN SELECT RAISE(ABORT, 'application input offset must equal next count'); END; +-- The same refusal with a clearer message. SQLite fires the most recently +-- created trigger first, so this one, created after the arm above, speaks. +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_requires_baseline +BEFORE INSERT ON application_inputs FOR EACH ROW +WHEN NOT EXISTS (SELECT 1 FROM era) +BEGIN SELECT RAISE(ABORT, 'application history baseline is missing'); END; CREATE TRIGGER IF NOT EXISTS trg_application_inputs_immutable BEFORE UPDATE ON application_inputs FOR EACH ROW BEGIN SELECT RAISE(ABORT, 'application inputs are immutable'); END; -CREATE TRIGGER IF NOT EXISTS trg_protect_valid_application_input_delete +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_not_deletable BEFORE DELETE ON application_inputs FOR EACH ROW -WHEN EXISTS (SELECT 1 FROM batches WHERE batch_index = OLD.batch_index - AND invalidated_at_ms IS NULL) -BEGIN SELECT RAISE(ABORT, 'valid application input cannot be deleted'); END; -CREATE TRIGGER IF NOT EXISTS trg_drop_invalidated_application_inputs -AFTER UPDATE OF invalidated_at_ms ON batches FOR EACH ROW -WHEN OLD.invalidated_at_ms IS NULL AND NEW.invalidated_at_ms IS NOT NULL -BEGIN DELETE FROM application_inputs WHERE batch_index = NEW.batch_index; END; +BEGIN SELECT RAISE(ABORT, 'application inputs are retained for the database lifetime'); END; -- Terminal-fault black box: an append-only trail of terminal causes, -- best-effort recorded before death. DELIBERATELY NOT AN ADMISSION GATE: -- admission -- is governed by facts — the kernel process lock excludes concurrent --- owners, `setup_complete` orders commands (two-sided), and +-- owners, the era's existence orders commands (two-sided), and -- `canonical_divergence` is the one absorbing refusal (cockroach rebuild is -- the only exit). Restart policy after a terminal fault is the exit-code -- contract (30 = do not restart, page), not a database gate. Nothing reads @@ -455,15 +472,11 @@ BEGIN DELETE FROM application_inputs WHERE batch_index = NEW.batch_index; END; CREATE TABLE IF NOT EXISTS terminal_faults ( fault_id INTEGER PRIMARY KEY AUTOINCREMENT, command TEXT NOT NULL CHECK (command IN ( - 'setup', 'rebuild', 'run', 'maintenance_flush' + 'setup', 'run', 'maintenance_flush' )), - cause TEXT NOT NULL CHECK ( - typeof(cause) = 'text' AND length(cause) > 0 - ), - recorded_at_ms INTEGER NOT NULL CHECK ( - typeof(recorded_at_ms) = 'integer' AND recorded_at_ms >= 0 - ) -); + cause TEXT NOT NULL CHECK (length(cause) > 0), + recorded_at_ms INTEGER NOT NULL CHECK (recorded_at_ms >= 0) +) STRICT; CREATE TRIGGER IF NOT EXISTS trg_terminal_faults_append_only_update BEFORE UPDATE ON terminal_faults @@ -489,6 +502,12 @@ CREATE TABLE IF NOT EXISTS deployment_identity ( input_box_address BLOB NOT NULL CHECK (length(input_box_address) = 20), app_deployment_block INTEGER NOT NULL CHECK (app_deployment_block >= 0), batch_submitter_address BLOB NOT NULL CHECK (length(batch_submitter_address) = 20), + -- The longest direct payload the application executes; with the batch + -- submitter address it decides which safe inputs execute as directs. + max_direct_payload_bytes INTEGER NOT NULL CHECK (max_direct_payload_bytes >= 0), + -- The application's staleness window in L1 blocks: a batch whose first + -- frame is this old when it lands is skipped, so acceptance is judged by it. + max_wait_blocks INTEGER NOT NULL CHECK (max_wait_blocks >= 1), -- Immutable setup-pinned fee source. Fixed has only fixed_log_gas_price; -- Uniswap has the complete reviewed token/pool tuple. fee_oracle_mode TEXT NOT NULL CHECK (fee_oracle_mode IN ('fixed', 'uniswap')), @@ -506,29 +525,7 @@ CREATE TABLE IF NOT EXISTS deployment_identity ( AND fee_oracle_weth IS NOT NULL AND fee_oracle_fee_token IS NOT NULL AND fee_oracle_pool IS NOT NULL AND fee_oracle_twap_window_secs IS NOT NULL) ) -); - --- Setup publishes the complete baseline, snapshot, anchor, and this marker in --- one final transaction after artifact durability. Earlier identity/sync work --- can be retried; run cannot admit an incomplete setup. -CREATE TABLE IF NOT EXISTS setup_complete ( - singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - completed_at_ms INTEGER NOT NULL CHECK (completed_at_ms >= 0) -); - --- The batch-tree anchor is frozen once setup completes. `setup` writes it (0 --- by default, N' on recovery) before inserting the `setup_complete` marker; --- after the marker exists, re-anchoring a live deployment would strand the --- existing batch spine, so any further UPDATE aborts. Defense-in-depth: the --- `setup --recovery` path is already a strict one-shot on a fresh DB. -CREATE TRIGGER IF NOT EXISTS trg_batch_tree_anchor_write_once -BEFORE UPDATE OF nonce ON batch_tree_anchor -FOR EACH ROW -WHEN EXISTS (SELECT 1 FROM setup_complete WHERE singleton_id = 0) -BEGIN - SELECT RAISE(ABORT, 'batch-tree anchor is frozen after setup completes'); -END; - +) STRICT; -- --------------------------------------------------------------------------- -- Batch policy singleton @@ -591,7 +588,7 @@ CREATE TABLE IF NOT EXISTS batch_policy ( CHECK (log_base_gas - log_alpha - log_delta < log_max_batch_bytes), -- Fee floor: the derived per-op fee never drops below exponent 0. CHECK (log_gas_price + log_slack + log_one_plus_alpha + log_delta + log_user_op_bytes >= 0) -); +) STRICT; -- Default values. All log-space exponents with base 129/128: -- log_alpha = log_{129/128}(0.168) = -229 @@ -631,30 +628,24 @@ SELECT *, FROM batch_policy; -- --------------------------------------------------------------------------- --- Snapshot dumps +-- Snapshots -- --- Immutable snapshots belong to one local batch, or to the era baseline. --- Acceptance is derived from safe_accepted_batches; it never mutates snapshots. --- Lease acquisition and garbage collection serialize in SQLite. Files are --- durable before insertion; database rows are deleted before filesystem cleanup. +-- A snapshot: an immutable artifact directory and the boundary it holds — one +-- local batch at its close, or the era baseline (batch_nonce NULL). Files are +-- durable before the row is inserted; rows are deleted before files. +-- Acceptance is read from `batches.accepted_input`; it never mutates snapshots. +-- Download leases live in the serving process, not here +-- (docs/snapshots/lifecycle.md). -- --------------------------------------------------------------------------- -CREATE TABLE IF NOT EXISTS dumps ( - id INTEGER PRIMARY KEY, - prefix TEXT NOT NULL UNIQUE, - lease_count INTEGER NOT NULL DEFAULT 0 - CHECK (typeof(lease_count) = 'integer' AND lease_count >= 0) -); - CREATE TABLE IF NOT EXISTS snapshots ( - dump_id INTEGER PRIMARY KEY REFERENCES dumps(id) ON DELETE RESTRICT, - batch_index INTEGER UNIQUE REFERENCES batches(batch_index) ON DELETE RESTRICT, - executed_input_count INTEGER NOT NULL CHECK ( - typeof(executed_input_count) = 'integer' AND executed_input_count >= 0 - ) -); + snapshot_id INTEGER PRIMARY KEY, + batch_nonce INTEGER UNIQUE REFERENCES batches(nonce) ON DELETE RESTRICT, + l2_tx_count INTEGER NOT NULL CHECK (l2_tx_count >= 0), + prefix TEXT NOT NULL UNIQUE +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS snapshots_single_baseline -ON snapshots ((1)) WHERE batch_index IS NULL; +ON snapshots ((1)) WHERE batch_nonce IS NULL; CREATE TRIGGER IF NOT EXISTS trg_snapshot_immutable BEFORE UPDATE ON snapshots diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index b866a88e..cfcd6ea8 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -4,8 +4,9 @@ //! SQLite storage. [`Storage`] is the single entry point; each writer role's //! methods live in its own sibling file (writer table: `docs/invariants.md`), //! shared reads in `queries` and shared writes in `mutations`. The schema and -//! `valid_*` views live in `migrations/0001_schema.sql`. +//! its views live in `migrations/0001_schema.sql`. +mod acceptance; mod convert; mod egress; mod fee_oracle; @@ -13,54 +14,42 @@ mod history; mod ingress; mod l1_inputs; mod l1_submission; +mod landing; mod lifecycle; mod mutations; mod open; mod queries; -mod recovery; -mod safe_accepted_batches; -mod snapshot_dumps; +mod rebuilt_history; +mod snapshots; #[cfg(test)] pub(crate) mod test_helpers; +#[cfg(test)] +pub(crate) use test_helpers::StoredSafeInput; pub(crate) use convert::is_persistent_storage_error; use std::time::SystemTime; use thiserror::Error; +#[cfg(test)] +pub(crate) use acceptance::Divergence; pub(crate) use egress::HistoricalReadError; #[cfg(test)] pub(crate) use history::ApplicationInputRow; -pub use history::{DirectInputExecution, HistoryState}; +pub use history::{DirectInputExecution, NewEra}; pub(crate) use history::{HistoryReadError, L2TxContext}; -pub use lifecycle::{LifecycleCommand, LifecycleError, TerminalFault}; +pub(crate) use ingress::ExecutedUserOp; +pub use ingress::FeeQuote; +pub(crate) use landing::OpenBatchError; +pub use landing::{LandingLag, TerminalFact}; +pub use lifecycle::{LifecycleCommand, LifecycleError}; pub use open::Storage; -pub use recovery::DangerStatus; -pub(crate) use recovery::{RecoveryInspection, RecoveryMutationError}; -pub use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; -pub use snapshot_dumps::{ - DumpRow, FinalizedDump, FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, - PersistentReleaseFailureReporter, ReleaseScheduler, Snapshot, +pub use sequencer_core::history::L2TxCount; +pub use snapshots::{ + FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, Snapshot, SnapshotLeases, }; -/// One safe input as stored on the L1 InputBox: sender, opaque payload, and -/// the L1 block where it was included. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StoredSafeInput { - pub sender: alloy_primitives::Address, - pub payload: Vec, - /// Chain block number where this input was included (e.g. InputAdded event block). - pub block_number: u64, -} - -/// A sender-classified L1 direct input ready for application execution. -#[derive(Debug, Clone)] -pub(crate) struct StoredDirectInput { - pub safe_input_index: u64, - pub input: sequencer_core::l2_tx::DirectInput, -} - /// One InputBox event with the L1 provenance persisted for feed consumers. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct IngestedSafeInput { @@ -71,24 +60,6 @@ pub(crate) struct IngestedSafeInput { pub transaction_hash: alloy_primitives::B256, } -/// Whether a sync also maintains the scheduler-accepted gold frontier -/// (`safe_accepted_batches`). -/// -/// Every normal sync uses [`FrontierMode::Populate`]. `setup --recovery` uses -/// [`FrontierMode::DeferUntilAnchorSet`] for its interim syncs: the local batch -/// tree is rebuilt from the checkpoint *after* the sync, so populating the -/// frontier against the empty tree would mark every L1 batch `Foreign` and -/// falsely freeze it. `run`'s first sync populates it once the anchor is `N'`. -/// See `docs/recovery/cockroach.md` and the anchor-aware-frontier note on I15. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FrontierMode { - /// Maintain `safe_accepted_batches` as part of the sync (the default). - Populate, - /// Skip frontier maintenance — the tree is rebuilt and the anchor set - /// afterwards, and the next normal sync will populate it correctly. - DeferUntilAnchorSet, -} - /// Half-open range `[start, end)` over `safe_input_index` values. Used to /// describe which safe inputs a frame drained. /// @@ -113,11 +84,6 @@ impl SafeInputRange { } } - #[cfg(test)] - pub(crate) fn empty_at(index: u64) -> Self { - Self::new(index, index) - } - pub fn start(self) -> u64 { self.start_inclusive } @@ -198,10 +164,9 @@ pub struct FrameHeader { pub safe_block: u64, } -/// A batch ready for L1 submission: its local index, assigned nonce, and SSZ-encoded payload. +/// A batch ready for L1 submission: its nonce and SSZ-encoded payload. #[derive(Debug)] pub struct PendingBatch { - pub batch_index: u64, pub nonce: u64, pub encoded: Vec, } @@ -209,7 +174,8 @@ pub struct PendingBatch { /// Deployment identity the database is bound to. /// /// The persisted sequencer state is not portable across these fields: -/// historical safe inputs are classified using the batch-submitter address, +/// historical safe inputs are classified using the batch-submitter address +/// and the direct-payload cap, /// user-op signatures use the app/domain identity, and recovery/frontier logic /// depends on the app's InputBox stream, which is empty before /// `app_deployment_block` (the v3 InputBox refuses inputs for apps that do @@ -221,6 +187,12 @@ pub struct DeploymentIdentity { pub input_box_address: alloy_primitives::Address, pub app_deployment_block: u64, pub batch_submitter_address: alloy_primitives::Address, + /// The application's `scheduler_limits().max_direct_payload_bytes`: with + /// the batch submitter, it decides which safe inputs execute as directs. + pub max_direct_payload_bytes: u64, + /// The application's `scheduler_limits().max_wait_blocks`: the staleness + /// window that decides whether a landed batch was accepted. + pub max_wait_blocks: u64, pub fee_oracle: FeeOracleIdentity, } @@ -247,9 +219,9 @@ pub enum StorageOpenError { Sqlite(#[from] rusqlite::Error), #[error(transparent)] Migration(#[from] rusqlite_migration::Error), - /// No database file at the given path. Databases are only created by an - /// owning command (`setup` / `rebuild`); a missing file at `open` time is - /// a deployment mistake, never a cue to create one. + /// No database file at the given path. Databases are only created by the + /// owning command, `setup`; a missing file at `open` time is a deployment + /// mistake, never a cue to create one. #[error( "no database at {path} — this data directory was never initialized; run `setup` (or check --data-dir)" )] @@ -282,26 +254,59 @@ pub struct BatchPolicy { } /// Trusted, reconstructible in-memory cache of the latest durable open batch + -/// frame. Mutated by `Storage` methods that change the open state -/// (`append_executed_user_ops_chunk`, attributed `close_*`). The sole-writer -/// lane loads one from SQLite and threads it through every call; failed -/// writes/restarts discard it. Physical-only siblings are test fixtures. -#[derive(Debug, Clone, Copy)] +/// frame. Opaque outside storage: only the loader (`queries::load_current_write_head`) +/// builds one, and only the `Storage` methods that change the open state +/// (`append_executed_user_ops_chunk`, attributed `close_*`) advance it, after +/// their commit. Storage reloads the Tip and compares it at every frame +/// rotation and batch close. Failed writes and restarts discard it. +#[derive(Debug)] pub struct WriteHead { - pub batch_index: u64, - pub batch_created_at: SystemTime, + /// The open batch's nonce, its key. + batch_nonce: u64, + batch_created_at: SystemTime, /// Log-space fee exponent (base 129/128) committed for this open frame. - pub frame_fee: u16, - pub safe_block: u64, - pub batch_user_op_count: u64, - pub open_frame_user_op_count: u32, - pub frame_in_batch: u32, + frame_fee: u16, + safe_block: u64, + batch_user_op_count: u64, + open_frame_user_op_count: u32, + frame_in_batch: u32, // Soft batch size threshold read from batch_policy at each frame/batch transition. - pub max_batch_user_op_bytes: u64, + max_batch_user_op_bytes: u64, } impl WriteHead { - pub fn increment_batch_user_op_count(&mut self, count: usize) { + pub(crate) fn batch_nonce(&self) -> u64 { + self.batch_nonce + } + + pub(crate) fn batch_created_at(&self) -> SystemTime { + self.batch_created_at + } + + /// Log-space fee exponent (base 129/128) committed for the open frame. + pub(crate) fn frame_fee(&self) -> u16 { + self.frame_fee + } + + pub(crate) fn safe_block(&self) -> u64 { + self.safe_block + } + + pub(crate) fn batch_user_op_count(&self) -> u64 { + self.batch_user_op_count + } + + pub(crate) fn max_batch_user_op_bytes(&self) -> u64 { + self.max_batch_user_op_bytes + } + + /// A private copy, so a counter advance is validated before the + /// transaction that commits it. + fn duplicate(&self) -> Self { + Self { ..*self } + } + + fn increment_batch_user_op_count(&mut self, count: usize) { let count_u64 = u64::try_from(count).expect("user-op chunk length exceeds u64: contract-impossible"); let count_u32 = @@ -318,7 +323,7 @@ impl WriteHead { self.open_frame_user_op_count = next_frame_count; } - pub fn advance_frame(&mut self, policy: BatchPolicy, safe_block: u64) { + fn advance_frame(&mut self, policy: BatchPolicy, safe_block: u64) { self.frame_in_batch = self .frame_in_batch .checked_add(1) @@ -329,17 +334,15 @@ impl WriteHead { self.max_batch_user_op_bytes = batch_size_target_bytes(policy); } - pub fn move_to_next_batch( + fn move_to_next_batch( &mut self, - batch_index: u64, + batch_nonce: u64, batch_created_at: SystemTime, policy: BatchPolicy, - safe_block: u64, ) { - self.batch_index = batch_index; + self.batch_nonce = batch_nonce; self.batch_created_at = batch_created_at; self.frame_fee = policy.recommended_fee; - self.safe_block = safe_block; self.batch_user_op_count = 0; self.open_frame_user_op_count = 0; self.frame_in_batch = 0; @@ -360,44 +363,40 @@ fn batch_size_target_bytes(policy: BatchPolicy) -> u64 { #[cfg(test)] mod tests { + use super::test_helpers::WriteHeadParts; use super::*; use alloy_primitives::U256; - use std::time::UNIX_EPOCH; - - fn write_head() -> WriteHead { - WriteHead { - batch_index: 0, - batch_created_at: UNIX_EPOCH, - frame_fee: 0, - safe_block: 0, - batch_user_op_count: 0, - open_frame_user_op_count: 0, - frame_in_batch: 0, - max_batch_user_op_bytes: 1, - } - } #[test] #[should_panic(expected = "batch user-op count overflow: contract-impossible")] fn write_head_batch_count_fails_loud_on_overflow() { - let mut head = write_head(); - head.batch_user_op_count = u64::MAX; + let mut head = WriteHeadParts { + batch_user_op_count: u64::MAX, + ..WriteHeadParts::ZERO + } + .build(); head.increment_batch_user_op_count(1); } #[test] #[should_panic(expected = "frame user-op count overflow: contract-impossible")] fn write_head_frame_count_fails_loud_on_overflow() { - let mut head = write_head(); - head.open_frame_user_op_count = u32::MAX; + let mut head = WriteHeadParts { + open_frame_user_op_count: u32::MAX, + ..WriteHeadParts::ZERO + } + .build(); head.increment_batch_user_op_count(1); } #[test] #[should_panic(expected = "frame index overflow: contract-impossible")] fn write_head_frame_index_fails_loud_on_overflow() { - let mut head = write_head(); - head.frame_in_batch = u32::MAX; + let mut head = WriteHeadParts { + frame_in_batch: u32::MAX, + ..WriteHeadParts::ZERO + } + .build(); head.advance_frame( BatchPolicy { recommended_fee: 0, diff --git a/sequencer/src/storage/mutations.rs b/sequencer/src/storage/mutations.rs index 1c82e540..e7f8b985 100644 --- a/sequencer/src/storage/mutations.rs +++ b/sequencer/src/storage/mutations.rs @@ -4,96 +4,54 @@ //! Write helpers shared across writer roles. Like [`super::queries`], they take a //! `&Transaction` so they compose inside a larger atomic unit. -use alloy_primitives::Address; -use rusqlite::{Connection, Result, Transaction, params}; +use rusqlite::{Result, Transaction, named_params, params}; use super::convert::{i64_to_u64, u64_to_i64}; -use super::history::next_executed_input_count_in; -use super::l1_inputs::query_deployment_identity; +use super::history::{era_in, next_l2_tx_count_in}; +use super::queries::{DIRECT_EXECUTES_SQL, DirectOperands}; use super::{DirectInputExecution, SafeInputRange}; -/// Insert a new batch. Nonce is derived from `parent_batch_index`: -/// `parent.nonce + 1`, or the deployment's anchor for a parentless root -/// (genesis, cockroach recovery, or a fully invalidated branch). -/// -/// If `batch_index_opt` is None, SQLite auto-assigns (highest existing +1). -/// The explicit form is used only by `initialize_open_state` to pin the -/// very first genesis batch at `batch_index = 0`. -/// -/// The `trg_enforce_nonce_contiguity` trigger verifies the nonce matches -/// `parent.nonce + 1`, so caller and schema agree. -pub(super) fn insert_new_batch( - tx: &Transaction<'_>, - batch_index_opt: Option, - parent_batch_index: Option, - created_at_ms: i64, -) -> Result { - let nonce = compute_next_nonce(tx, parent_batch_index)?; - match batch_index_opt { - Some(bi) => { - tx.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms) \ - VALUES (?1, ?2, ?3, ?4)", - params![ - u64_to_i64(bi), - parent_batch_index.map(u64_to_i64), - u64_to_i64(nonce), - created_at_ms - ], - )?; - Ok(bi) - } - None => { - tx.execute( - "INSERT INTO batches (parent_batch_index, nonce, created_at_ms) \ - VALUES (?1, ?2, ?3)", - params![ - parent_batch_index.map(u64_to_i64), - u64_to_i64(nonce), - created_at_ms - ], - )?; - Ok(i64_to_u64(tx.last_insert_rowid())) - } - } +/// Insert a new batch after the last one and return its nonce: one past the +/// last nonce, or the era's anchor for the era's first batch. +/// `trg_batch_insert` checks the same rule, so caller and schema agree. +pub(super) fn insert_new_batch(tx: &Transaction<'_>, created_at_ms: i64) -> Result { + let nonce = compute_next_nonce(tx)?; + tx.execute( + "INSERT INTO batches (nonce, created_at_ms) VALUES (?1, ?2)", + params![u64_to_i64(nonce), created_at_ms], + )?; + Ok(nonce) } -fn compute_next_nonce(tx: &Transaction<'_>, parent_batch_index: Option) -> Result { - match parent_batch_index { - // A parentless root carries the deployment's batch-tree anchor nonce: - // 0 for a genesis deployment, N' for a cockroach-recovered one. Mirrored by - // `trg_enforce_nonce_contiguity`'s parentless arm. - None => batch_tree_anchor_in(tx), - Some(parent_bi) => { - let parent_nonce: i64 = tx.query_row( - "SELECT nonce FROM batches WHERE batch_index = ?1", - params![u64_to_i64(parent_bi)], - |row| row.get(0), - )?; - Ok(i64_to_u64(parent_nonce) - .checked_add(1) - .expect("batch nonce overflow: contract-impossible")) - } +/// The last nonce + 1, or the era's anchor when there is no batch: the rule of +/// `trg_batch_insert`. With neither, reading the era fails. +fn compute_next_nonce(tx: &Transaction<'_>) -> Result { + let last: Option = tx.query_row("SELECT MAX(nonce) FROM batches", [], |row| row.get(0))?; + match last { + Some(nonce) => Ok(i64_to_u64(nonce) + .checked_add(1) + .expect("batch nonce overflow: contract-impossible")), + None => Ok(era_in(tx)?.anchor_batch_nonce), } } -/// Mark a batch as sealed (inclusion lane closed it). Write-once per the -/// `trg_sealed_at_ms_write_once` / `trg_payload_hash_write_once` triggers. +/// Mark a batch as sealed (inclusion lane closed it). Write-once per +/// `trg_batch_lifecycle`. /// The payload hash is stamped in the same UPDATE: a sealed batch always /// carries the hash the content-identity check compares accepted L1 /// landings against (hash-at-seal). pub(super) fn seal_batch( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, sealed_at_ms: i64, payload_hash: &[u8; 32], ) -> Result<()> { let changed = tx.execute( - "UPDATE batches SET sealed_at_ms = ?1, payload_hash = ?2 WHERE batch_index = ?3", + "UPDATE batches SET sealed_at_ms = ?1, payload_hash = ?2 WHERE nonce = ?3", params![ sealed_at_ms, payload_hash.as_slice(), - u64_to_i64(batch_index) + u64_to_i64(batch_nonce) ], )?; if changed != 1 { @@ -104,17 +62,17 @@ pub(super) fn seal_batch( pub(super) fn insert_open_frame( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, created_at_ms: i64, frame_fee: u16, safe_block: u64, ) -> Result<()> { tx.execute( - "INSERT INTO frames (batch_index, frame_in_batch, created_at_ms, fee, safe_block) \ + "INSERT INTO frames (batch_nonce, frame_in_batch, created_at_ms, fee, safe_block) \ VALUES (?1, ?2, ?3, ?4, ?5)", params![ - u64_to_i64(batch_index), + u64_to_i64(batch_nonce), i64::from(frame_in_batch), created_at_ms, i64::from(frame_fee), @@ -124,41 +82,10 @@ pub(super) fn insert_open_frame( Ok(()) } -/// Set the batch-tree anchor nonce — the nonce the single parentless root -/// carries (0 for genesis, `N'` for a cockroach-recovered deployment). Composes -/// inside the recovery-fill transaction. `trg_batch_tree_anchor_write_once` -/// rejects this once `setup_complete` exists, so it is callable only during -/// setup, before its atomic completion. -pub(super) fn set_batch_tree_anchor_in(tx: &Transaction<'_>, nonce: u64) -> Result<()> { - let changed = tx.execute( - "UPDATE batch_tree_anchor SET nonce = ?1 WHERE singleton_id = 0", - params![u64_to_i64(nonce)], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(()) -} - -/// Read the batch-tree anchor nonce (default 0). Mirrors -/// [`set_batch_tree_anchor_in`]. The single home for the anchor-read query: -/// takes `&Connection` (not `&Transaction`) so both the in-transaction writer -/// ([`compute_next_nonce`]) and the connection-level frontier readers -/// (`frontier_nonce`, `populate_safe_accepted_batches`) share it. A -/// `&Transaction` coerces to `&Connection` at the call site. -pub(super) fn batch_tree_anchor_in(conn: &Connection) -> Result { - let anchor: i64 = conn.query_row( - "SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0", - [], - |row| row.get(0), - )?; - Ok(i64_to_u64(anchor)) -} - /// Commit the exact direct inputs executed while opening a frame. pub(super) fn persist_frame_direct_sequence( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, range: SafeInputRange, executions: &[DirectInputExecution], @@ -168,35 +95,35 @@ pub(super) fn persist_frame_direct_sequence( derive_direct_input_executions_in(tx, range)?, "direct executions must cover the complete external-input range at the next application count" ); - persist_direct_executions(tx, batch_index, frame_in_batch, executions) + persist_direct_executions(tx, batch_nonce, frame_in_batch, executions) } /// Startup fixes the replay sequence before the engine is restored. Admission /// succeeds only after catch-up applies every committed application input. pub(super) fn persist_frame_direct_sequence_derived( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, range: SafeInputRange, ) -> Result<()> { let executions = derive_direct_input_executions_in(tx, range)?; - persist_direct_executions(tx, batch_index, frame_in_batch, &executions) + persist_direct_executions(tx, batch_nonce, frame_in_batch, &executions) } fn persist_direct_executions( tx: &Transaction<'_>, - batch_index: u64, + batch_nonce: u64, frame_in_batch: u32, executions: &[DirectInputExecution], ) -> Result<()> { let mut stmt = tx.prepare_cached( - "INSERT INTO application_inputs (offset, batch_index, frame_in_batch, safe_input_index) + "INSERT INTO application_inputs (l2_tx_count, batch_nonce, frame_in_batch, safe_input_index) VALUES (?1, ?2, ?3, ?4)", )?; for execution in executions { stmt.execute(params![ - u64_to_i64(execution.executed_input_offset.get()), - u64_to_i64(batch_index), + u64_to_i64(execution.l2_tx_index.get()), + u64_to_i64(batch_nonce), i64::from(frame_in_batch), u64_to_i64(execution.safe_input_index) ])?; @@ -204,6 +131,9 @@ fn persist_direct_executions( Ok(()) } +/// The executions a drained range owes: one per direct that +/// [`DIRECT_EXECUTES_SQL`] admits, at consecutive counts. A submitter input +/// or an over-cap direct takes no count and gets no row. fn derive_direct_input_executions_in( tx: &Transaction<'_>, range: SafeInputRange, @@ -212,23 +142,30 @@ fn derive_direct_input_executions_in( return Ok(Vec::new()); } - let identity = query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; - let mut next = next_executed_input_count_in(tx)?; - let mut stmt = tx.prepare_cached( - "SELECT safe_input_index, sender \ + let operands = DirectOperands::load(tx)?; + let mut next = next_l2_tx_count_in(tx)?; + // Every input in the range is read, so the range's completeness is + // checked even where none of its inputs executes. + let mut stmt = tx.prepare_cached(&format!( + "SELECT safe_input_index, {DIRECT_EXECUTES_SQL} \ FROM safe_inputs \ - WHERE safe_input_index >= ?1 AND safe_input_index < ?2 \ - ORDER BY safe_input_index ASC", - )?; + WHERE safe_input_index >= :start AND safe_input_index < :end \ + ORDER BY safe_input_index ASC" + ))?; let rows = stmt.query_map( - params![u64_to_i64(range.start()), u64_to_i64(range.end())], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)), + operands + .bind(named_params! { + ":start": u64_to_i64(range.start()), + ":end": u64_to_i64(range.end()), + }) + .as_slice(), + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, bool>(1)?)), )?; let mut executions = Vec::new(); let mut fetched = 0_u64; for row in rows { - let (safe_input_index, sender) = row?; + let (safe_input_index, executes) = row?; let safe_input_index = i64_to_u64(safe_input_index); let expected_index = range .start() @@ -238,14 +175,14 @@ fn derive_direct_input_executions_in( safe_input_index, expected_index, "non-contiguous safe-input range while assigning execution offsets" ); - if Address::from_slice(sender.as_slice()) != identity.batch_submitter_address { + if executes { executions.push(DirectInputExecution { safe_input_index, - executed_input_offset: next, + l2_tx_index: next, }); next = next .checked_next() - .expect("executed input count overflow: contract-impossible"); + .expect("L2 tx count overflow: contract-impossible"); } fetched = fetched .checked_add(1) diff --git a/sequencer/src/storage/open.rs b/sequencer/src/storage/open.rs index 03f0b9ef..d94e1f4e 100644 --- a/sequencer/src/storage/open.rs +++ b/sequencer/src/storage/open.rs @@ -7,7 +7,7 @@ //! `l1_submission`, `recovery`) — each adds its own `impl Storage`. use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior}; -use rusqlite_migration::{HookResult, M, Migrations}; +use rusqlite_migration::{M, Migrations}; use super::{LifecycleCommand, StorageOpenError}; @@ -38,15 +38,12 @@ const SYNCHRONOUS_PRAGMA: &str = "FULL"; /// `Storage` instance opened via [`Storage::open_read_only`]. pub struct Storage { pub(super) conn: Connection, - /// The path this connection was opened from. Carried so a lease guard can - /// re-open a brief writer connection to release on drop (see - /// `snapshot_dumps::LeaseGuard`); the egress snapshot handlers open - /// per-op, so the guard can't borrow this `Storage`. - pub(super) path: String, } impl Storage { /// Production open: runs migrations, uses the canonical synchronous pragma. + /// `run` calls it once, before its startup sync; every later open of the + /// run is [`Storage::open_writer`] or [`Storage::open_read_only`]. /// /// Refuses a path with no database file (production builds): every /// database is created by an owning command through @@ -54,8 +51,11 @@ impl Storage { /// deployment mistake (mistyped `--data-dir`, wrong mount). Creating one /// on the fly would create an ownerless schema with no creating command — /// database absence means uninitialized, never create-and-proceed. - /// Crate tests keep create-on-open as their fixture idiom; the - /// command-less baseline in [`baseline_migration`] exists for them. + /// On an existing file that was never migrated it creates the schema + /// with no era, so the database is not set up. In crate tests it also + /// creates a missing file, with the genesis era that + /// [`baseline_migration`] inserts only in test builds; a set-up test + /// database comes from `test_helpers::set_up_storage`. pub(crate) fn open(path: &str) -> Result { #[cfg(not(test))] if !std::path::Path::new(path).exists() { @@ -65,14 +65,11 @@ impl Storage { } let mut conn = open_writer_connection(path)?; run_migrations(&mut conn, None)?; - Ok(Self { - conn, - path: path.to_string(), - }) + Ok(Self { conn }) } - /// Create the schema for setup or rebuild in one migration transaction. - /// The complete history baseline is published later. On + /// Create the schema for setup in one migration transaction. The era is + /// published later, and its row is setup's completion. On /// an already-migrated database the hook does not run; callers must /// inspect the existing facts. pub(crate) fn initialize_for_command( @@ -80,39 +77,37 @@ impl Storage { command: LifecycleCommand, ) -> Result { assert!( - matches!(command, LifecycleCommand::Setup | LifecycleCommand::Rebuild), - "an uninitialized lifecycle may begin only with setup or rebuild" + matches!(command, LifecycleCommand::Setup), + "an uninitialized lifecycle may begin only with setup" ); let mut conn = open_writer_connection(path)?; run_migrations(&mut conn, Some(command))?; - Ok(Self { - conn, - path: path.to_string(), - }) + Ok(Self { conn }) + } + + /// A private in-memory database with setup's schema, for questions the + /// schema answers before setup may create its database file. + pub(crate) fn schema_only() -> Result { + let mut conn = Connection::open_in_memory()?; + set_recursive_triggers(&conn)?; + run_migrations(&mut conn, Some(LifecycleCommand::Setup))?; + Ok(Self { conn }) } /// Read-only handle. Uses a 50ms `busy_timeout` (vs. 5s for writers) so /// readers fail fast under write pressure and don't block on hot paths. pub fn open_read_only(path: &str) -> Result { let conn = open_reader_connection(path)?; - Ok(Self { - conn, - path: path.to_string(), - }) + Ok(Self { conn }) } /// Read-write handle that does NOT run migrations — for components that - /// open the DB *after* startup has already migrated it (e.g. egress - /// snapshot handlers doing brief lease writes, and tests that pre-seed - /// the schema). Same pragmas as [`Storage::open`] (WAL, `foreign_keys`, - /// 5s `busy_timeout`); running migrations is the caller's responsibility - /// (the runtime does it once via [`Storage::open`] at startup). + /// open the DB *after* `run` has migrated it (recovery, the lane, the + /// input reader, and tests that pre-seed the schema). Same pragmas as + /// [`Storage::open`] (WAL, `foreign_keys`, 5s `busy_timeout`). pub fn open_writer(path: &str) -> Result { let conn = open_writer_connection(path)?; - Ok(Self { - conn, - path: path.to_string(), - }) + Ok(Self { conn }) } /// Run `f` inside a Deferred transaction, commit on success. For pure reads. @@ -121,7 +116,7 @@ impl Storage { /// don't hold a write lock and don't block writers. If `f` returns `Err` /// the transaction is dropped unsent (auto-rollback); on success the /// commit is issued before returning `Ok`. - pub fn read(&mut self, f: F) -> Result + pub(super) fn read(&mut self, f: F) -> Result where F: FnOnce(&Transaction<'_>) -> Result, { @@ -140,7 +135,7 @@ impl Storage { /// see `SQLITE_BUSY` immediately rather than mid-transaction — this is /// the right cadence under WAL + single-writer discipline. Same commit / /// auto-rollback semantics as [`Storage::read`]. - pub fn write(&mut self, f: F) -> Result + pub(super) fn write(&mut self, f: F) -> Result where F: FnOnce(&Transaction<'_>) -> Result, { @@ -158,6 +153,7 @@ impl Storage { pub(super) fn open_writer_connection(path: &str) -> Result { let conn = Connection::open(path)?; conn.pragma_update(None, "foreign_keys", "ON")?; + set_recursive_triggers(&conn)?; conn.pragma_update(None, "journal_mode", "WAL")?; conn.pragma_update(None, "synchronous", SYNCHRONOUS_PRAGMA)?; conn.pragma_update(None, "busy_timeout", 5000)?; @@ -169,44 +165,46 @@ fn open_reader_connection(path: &str) -> Result { let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; conn.pragma_update(None, "query_only", "ON")?; conn.pragma_update(None, "busy_timeout", 50)?; + set_recursive_triggers(&conn)?; Ok(conn) } +/// Without `recursive_triggers`, `INSERT OR REPLACE` deletes a conflicting row +/// without firing its BEFORE DELETE trigger, which bypasses every "never +/// deleted" trigger. No schema trigger writes a table (each only RAISEs), so +/// nothing can recurse. Harmless on a read-only connection; set on every one. +fn set_recursive_triggers(conn: &Connection) -> rusqlite::Result<()> { + conn.pragma_update(None, "recursive_triggers", "ON") +} + /// Apply all migrations. Package-private — callers use [`Storage::open`] /// which runs this automatically. pub(super) fn run_migrations( conn: &mut Connection, initial_command: Option, ) -> Result<(), StorageOpenError> { - let migration = baseline_migration(initial_command, None); + let migration = baseline_migration(initial_command); Migrations::from_slice(&[migration]).to_latest(conn)?; Ok(()) } -type PostInitialMetadataHook = fn(&Transaction<'_>) -> HookResult; +/// The schema migration: the schema alone, so only setup's publish ever +/// writes an era. +#[cfg(not(test))] +fn baseline_migration(_initial_command: Option) -> M<'static> { + M::up(MIGRATION_0001_SCHEMA) +} -/// Build the exact baseline migration used in production. The optional hook -/// exists only so the atomicity test can fail *after* observing the -/// production history insert, without maintaining a shadow migration. -/// -/// `initial_command: None` is the crate-test fixture path (create-on-open -/// with genesis history bases); production cannot reach it because -/// [`Storage::open`] refuses paths with no database file. -fn baseline_migration( - initial_command: Option, - post_initial_metadata: Option, -) -> M<'static> { - M::up_with_hook(MIGRATION_0001_SCHEMA, move |tx: &Transaction<'_>| { - if initial_command.is_none() { - super::history::initialize_history_in( - tx, - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - )?; - } - if let Some(hook) = post_initial_metadata { - hook(tx)?; - } +/// The schema migration in crate tests: [`Storage::open`]'s path +/// (`initial_command: None`) also inserts the genesis era, in the +/// migration's transaction, for the create-on-open fixtures. +#[cfg(test)] +fn baseline_migration(initial_command: Option) -> M<'static> { + if initial_command.is_some() { + return M::up(MIGRATION_0001_SCHEMA); + } + M::up_with_hook(MIGRATION_0001_SCHEMA, |tx: &Transaction<'_>| { + super::history::insert_era_in(tx, &super::NewEra::EMPTY_GENESIS)?; Ok(()) }) } @@ -215,38 +213,19 @@ fn baseline_migration( mod tests { use super::*; - fn fail_after_observing_initial_metadata(tx: &Transaction<'_>) -> HookResult { - let history_count: i64 = tx.query_row( - "SELECT COUNT(*) FROM history_state \ - WHERE singleton_id = 0 AND recovery_generation = 0 \ - AND base_executed_input_count = 0 \ - AND base_safe_block = 0", - [], - |row| row.get(0), - )?; - if history_count != 1 { - return Err(rusqlite_migration::HookError::Hook( - "production initial history insert was not observed".to_string(), - )); - } - Err(rusqlite_migration::HookError::Hook( - "injected failure after initial history insert".to_string(), - )) - } - + /// The schema migration is all or nothing: a failure after its last + /// statement leaves no table and the version unset. #[test] - fn failing_initial_hook_rolls_back_schema_and_history_together() { + fn failing_schema_migration_leaves_no_schema() { let dir = tempfile::tempdir().expect("tempdir"); let path = dir.path().join("atomic-init.sqlite"); let mut conn = open_writer_connection(path.to_str().expect("utf8")).expect("open"); - let definitions = [baseline_migration( - None, - Some(fail_after_observing_initial_metadata), - )]; + let failing = format!("{MIGRATION_0001_SCHEMA}\nCREATE TABLE era (duplicate INTEGER);"); + let definitions = [M::up(failing.as_str())]; let migrations = Migrations::from_slice(&definitions); migrations .to_latest(&mut conn) - .expect_err("the injected hook failure must abort migration"); + .expect_err("the failing last statement must abort the migration"); let table_count: i64 = conn .query_row( @@ -264,49 +243,31 @@ mod tests { } #[test] - fn setup_registers_history_only_with_complete_baseline() { + fn setup_schema_has_no_era_until_one_is_inserted() { let setup_dir = tempfile::tempdir().unwrap(); - let setup_path = setup_dir.path().join("setup.sqlite"); - let mut setup = - Storage::initialize_for_command(setup_path.to_str().unwrap(), LifecycleCommand::Setup) + let mut eras = Vec::new(); + for (name, era) in [ + ("genesis.sqlite", super::super::NewEra::EMPTY_GENESIS), + ("machine.sqlite", super::super::test_helpers::era(8, 8, 100)), + ] { + let path = setup_dir.path().join(name); + let mut setup = + Storage::initialize_for_command(path.to_str().unwrap(), LifecycleCommand::Setup) + .unwrap(); + assert!(matches!( + setup.era(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(!setup.is_setup_complete().unwrap()); + setup + .write(|tx| super::super::history::insert_era_in(tx, &era)) .unwrap(); - assert!(matches!( - setup.history_state(), - Err(rusqlite::Error::QueryReturnedNoRows) - )); - setup - .write(|tx| { - super::super::history::initialize_history_in( - tx, - sequencer_core::history::ExecutedInputCount::ZERO, - 0, - ) - }) - .unwrap(); - let a = setup.history_state().unwrap(); - let rebuild_path = setup_dir.path().join("rebuild.sqlite"); - let mut rebuild = Storage::initialize_for_command( - rebuild_path.to_str().unwrap(), - LifecycleCommand::Rebuild, - ) - .unwrap(); - assert!(matches!( - rebuild.history_state(), - Err(rusqlite::Error::QueryReturnedNoRows) - )); - rebuild - .write(|tx| { - super::super::history::initialize_history_in( - tx, - sequencer_core::history::ExecutedInputCount::new(8), - 100, - ) - }) - .unwrap(); - let b = rebuild.history_state().unwrap(); - assert_ne!(a.version.era_id, b.version.era_id); - assert_eq!(b.base_executed_input_count, 8); - assert_eq!(b.base_safe_block, 100); + eras.push(setup.era().unwrap()); + } + assert_ne!(eras[0].id, eras[1].id); + assert_eq!(eras[1].base.get(), 8); + assert_eq!(eras[1].stop_block, 100); + assert_eq!(eras[1].rebuilt_end.get(), 8); } } diff --git a/sequencer/src/storage/open/schema_tests.rs b/sequencer/src/storage/open/schema_tests.rs index 7e16fd1c..ac6f1a6e 100644 --- a/sequencer/src/storage/open/schema_tests.rs +++ b/sequencer/src/storage/open/schema_tests.rs @@ -2,44 +2,58 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! Schema-invariant regression tests: the triggers, CHECKs and partial unique -//! indexes in `migrations/0001_schema.sql` enforce the batch-tree invariants, and +//! indexes in `migrations/0001_schema.sql` enforce the batch invariants, and //! each test pins one tripwire. use rusqlite::params; use super::super::test_helpers::{ - RecoveryFixture, SENDER_A, SENDER_B, default_protocol_timing, local_batch_payload, - open_temp_storage, temp_db_with_default_deployment_identity as temp_db, + LandingFixture, SENDER_A, SENDER_B, open_temp_storage, set_up_storage, }; use crate::storage::{SafeInputRange, Storage, StoredSafeInput}; +const NONCE_RULE: &str = "a batch nonce is the last nonce + 1, or the era anchor"; +const LIFECYCLE_RULE: &str = + "a batch changes only by sealing once with its hash, or by being accepted once while sealed"; +const ACCEPTANCE_RULE: &str = "an acceptance extends the accepted prefix of the batches"; +const NOT_DELETABLE_RULE: &str = "batches are retained for the database lifetime"; +const FROZEN_RULE: &str = "batch list frozen: canonical divergence marker present"; + +fn assert_refused(storage: &Storage, sql: &str, message: &str) { + let err = storage.conn.execute(sql, []); + assert!( + format!("{err:?}").contains(message), + "`{sql}` must abort with {message:?}, got: {err:?}" + ); +} + #[test] -fn schema_rejects_second_valid_tip() { - // The partial unique index `ux_single_valid_tip` catches a writer that +fn schema_rejects_a_second_open_batch() { + // The partial unique index `ux_single_open_batch` catches a writer that // opens a new Tip without sealing the old one first. let (_db, mut storage) = open_temp_storage("schema-second-tip"); storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); - // Try to bypass the lane and insert a second valid Tip directly. + // Try to bypass the lane and insert a second open batch directly, at the + // right nonce so only the Tip index can refuse it. let err = storage.conn.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms) \ - VALUES (99, 0, 1, 1000)", + "INSERT INTO batches (nonce, created_at_ms) VALUES (1, 1000)", [], ); let msg = format!("{err:?}"); assert!( - msg.contains("UNIQUE constraint failed") && msg.contains("ux_single_valid_tip"), - "expected ux_single_valid_tip violation, got: {msg}" + msg.contains("UNIQUE constraint failed") && msg.contains("ux_single_open_batch"), + "expected ux_single_open_batch violation, got: {msg}" ); } #[test] -fn schema_rejects_bad_nonce_contiguity() { - // Nonce must equal parent.nonce + 1 — trigger enforces it. - // Insert the bad-nonce batch as already-sealed so it doesn't collide - // with the existing Tip on `ux_single_valid_tip`. +fn schema_rejects_a_wrong_batch_nonce() { + // A new batch carries the last nonce + 1. Insert the candidate batches + // as already sealed so they don't collide with the Tip on + // `ux_single_open_batch`. let (_db, mut storage) = open_temp_storage("schema-bad-nonce"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -47,182 +61,457 @@ fn schema_rejects_bad_nonce_contiguity() { storage .close_batch_at(&mut head, 0) .expect("close batch 0; batch 1 is now Tip"); - // Batch 1 has nonce 1 (0 + 1). Insert child with nonce 99 (should be 2). - let err = storage.conn.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (999, 1, 99, \ - (SELECT created_at_ms FROM batches WHERE batch_index = 1), \ - (SELECT created_at_ms FROM batches WHERE batch_index = 1))", - [], - ); - assert!( - format!("{err:?}").contains("batch nonce must equal parent.nonce + 1"), - "expected nonce trigger, got: {err:?}" - ); + // Batch 1 has nonce 1, so the next is 2: a gap, a repeat and an earlier + // nonce are refused. + for nonce in [99, 3, 1, 0] { + assert_refused( + &storage, + &format!( + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, \ + payload_hash) VALUES ({nonce}, 100, 100, zeroblob(32))" + ), + NONCE_RULE, + ); + } + storage + .conn + .execute( + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (2, 100, 100, zeroblob(32))", + [], + ) + .expect("the batch after nonce 1 carries nonce 2"); +} + +#[test] +fn schema_rejects_a_batch_before_the_era_exists() { + // With no batch and no era both sides of the COALESCE are NULL, + // `IS NOT` holds, and every insert ABORTs, whatever its nonce. + let db = crate::test_support::temp_db("schema-batch-without-era"); + let storage = + Storage::initialize_for_command(db.path.as_str(), crate::storage::LifecycleCommand::Setup) + .expect("setup schema, no era"); + for nonce in [0, 7] { + assert_refused( + &storage, + &format!("INSERT INTO batches (nonce, created_at_ms) VALUES ({nonce}, 100)"), + NONCE_RULE, + ); + } } #[test] fn schema_rejects_genesis_with_nonzero_nonce() { - // Default anchor is 0, so a parentless root at a non-zero nonce ABORTs - // (genesis must be 0) — the exact-match form of the contiguity rule. + // The genesis era's anchor is 0, so a first batch at a non-zero nonce + // ABORTs. let (_db, storage) = open_temp_storage("schema-genesis-nonzero"); - let err = storage.conn.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms) \ - VALUES (0, NULL, 7, 100)", - [], + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms) VALUES (7, 100)", + NONCE_RULE, ); - assert!( - format!("{err:?}").contains("parentless root must carry the batch-tree anchor nonce"), - "expected anchor-nonce trigger, got: {err:?}" +} + +#[test] +fn schema_rejects_a_batch_without_its_nonce() { + // `nonce` is a rowid alias: SQLite would number a row inserted without + // one, and the insert trigger sees that nonce as -1 and refuses it. + let (_db, storage) = open_temp_storage("schema-batch-without-nonce"); + assert_refused( + &storage, + "INSERT INTO batches (created_at_ms) VALUES (100)", + NONCE_RULE, ); } #[test] -fn schema_allows_parentless_root_at_anchor_nonce() { - // With the anchor set to N' (a recovered deployment), a parentless root - // at N' is permitted — the recovery generalization of "genesis at 0". - // (Rows are inserted sealed so the single-Tip partial index, which only - // covers open rows, doesn't mask the contiguity trigger under test.) - let db = temp_db("schema-anchor-root"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); +fn schema_first_batch_carries_the_era_anchor() { + // With an era anchored at N' (a recovered deployment), the first batch + // carries exactly N', and the next N' + 1. (Rows are inserted sealed so + // the single-Tip index, which only covers open rows, doesn't mask the + // trigger under test.) + let db = crate::test_support::temp_db("schema-anchor-first"); + let storage = Storage::open_with_era( + db.path.as_str(), + &crate::storage::NewEra { + anchor_batch_nonce: 42, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (0, 100, 100, zeroblob(32))", + NONCE_RULE, + ); storage .conn .execute( - "UPDATE batch_tree_anchor SET nonce = 42 WHERE singleton_id = 0", + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (42, 100, 100, zeroblob(32))", [], ) - .expect("set anchor"); - // A root at the anchor nonce (42) is accepted... + .expect("the first batch at the anchor nonce is allowed"); + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (42, 100, 100, zeroblob(32))", + NONCE_RULE, + ); storage .conn .execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (0, NULL, 42, 100, 100)", + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (43, 100, 100, zeroblob(32))", [], ) - .expect("parentless root at the anchor nonce is allowed"); - // ...but a root at 0 ABORTs: the rule is an exact match against the anchor. - let err = storage.conn.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (1, NULL, 0, 100, 100)", - [], - ); - assert!( - format!("{err:?}").contains("parentless root must carry the batch-tree anchor nonce"), - "a root at the wrong nonce must abort, got: {err:?}" - ); + .expect("the next batch carries the anchor + 1"); } #[test] -fn schema_rejects_second_valid_parentless_root() { - // At most one *valid* parentless root: a second parentless insert (both - // valid, both at the anchor nonce 0) ABORTs on the single-root guard. - // (Sealed inserts isolate this from the single-Tip index; a fully-torn - // cascade re-roots only after invalidating the old root, covered below.) - let db = temp_db("schema-two-roots"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); +fn schema_rejects_two_batches_at_one_nonce() { + // The nonce trigger refuses a duplicate nonce first (it is never the last + // nonce + 1); the primary key on `nonce` refuses it on its own. + let (_db, storage) = open_temp_storage("schema-duplicate-nonce"); storage .conn .execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (0, NULL, 0, 100, 100)", + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (0, 100, 100, zeroblob(32))", [], ) - .expect("first root ok"); - let err = storage.conn.execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (1, NULL, 0, 100, 100)", - [], - ); - assert!( - format!("{err:?}").contains("at most one valid parentless root"), - "expected single-root guard, got: {err:?}" - ); - // Invalidate the first root, then a second parentless root is allowed — - // exactly the fully-torn-cascade re-root shape. + .expect("first batch"); + let duplicate = "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, \ + payload_hash) VALUES (0, 100, 100, zeroblob(32))"; + assert_refused(&storage, duplicate, NONCE_RULE); storage .conn - .execute( - "UPDATE batches SET invalidated_at_ms = 200 WHERE batch_index = 0", - [], - ) - .expect("invalidate first root"); + .execute_batch("DROP TRIGGER trg_batch_insert") + .expect("isolate the key"); + let msg = format!("{:?}", storage.conn.execute(duplicate, [])); + assert!( + msg.contains("UNIQUE constraint failed") && msg.contains("batches.nonce"), + "expected a key violation on batches.nonce, got: {msg}" + ); +} + +#[test] +fn schema_rejects_a_sealed_batch_without_its_hash() { + // The CHECK pairs `sealed_at_ms` with `payload_hash` on every row. On an + // UPDATE the lifecycle trigger (a BEFORE trigger) answers before it. + let (_db, mut storage) = open_temp_storage("schema-seal-without-hash"); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage.close_batch_at(&mut head, 0).expect("close batch 0"); + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms) \ + VALUES (2, 100, 100)", + "CHECK constraint failed", + ); + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, payload_hash) \ + VALUES (2, 100, zeroblob(32))", + "CHECK constraint failed", + ); + assert_refused( + &storage, + "UPDATE batches SET sealed_at_ms = 100 WHERE nonce = 1", + LIFECYCLE_RULE, + ); +} + +// ── The batch lifecycle (`trg_batch_lifecycle`) ────────────────────────── +// +// The fixture holds batch 0 sealed and batch 1 the open Tip. + +fn lifecycle_fixture(name: &str) -> (crate::test_support::TestDb, Storage) { + let (db, mut storage) = open_temp_storage(name); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage.close_batch_at(&mut head, 0).expect("close batch 0"); + (db, storage) +} + +#[test] +fn lifecycle_allows_sealing_the_tip_with_its_hash() { + let (_db, storage) = lifecycle_fixture("lifecycle-seal"); storage .conn .execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ - VALUES (1, NULL, 0, 100, 100)", + "UPDATE batches SET sealed_at_ms = 100, payload_hash = zeroblob(32) \ + WHERE nonce = 1", [], ) - .expect("re-root parentless after the old root is invalidated"); + .expect("the seal arm admits the open Tip's seal"); +} + +#[test] +fn lifecycle_rejects_un_seal() { + let (_db, storage) = lifecycle_fixture("lifecycle-un-seal"); + assert_refused( + &storage, + "UPDATE batches SET sealed_at_ms = NULL, payload_hash = NULL WHERE nonce = 0", + LIFECYCLE_RULE, + ); } #[test] -fn schema_freezes_anchor_after_setup_complete() { - // The anchor is write-once: once `setup_complete` exists, UPDATEs abort. - let db = temp_db("schema-anchor-frozen"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); - // Pre-marker: UPDATE is allowed (this is what recovery setup does). +fn lifecycle_rejects_re_seal() { + let (_db, storage) = lifecycle_fixture("lifecycle-re-seal"); + assert_refused( + &storage, + "UPDATE batches SET sealed_at_ms = sealed_at_ms + 1 WHERE nonce = 0", + LIFECYCLE_RULE, + ); +} + +#[test] +fn lifecycle_rejects_a_payload_hash_rewrite() { + // payload_hash is the content-identity anchor for the canonical- + // divergence check; a rewrite would let a foreign/zombie L1 landing + // false-match a local batch. + let (_db, storage) = lifecycle_fixture("lifecycle-hash-rewrite"); + assert_refused( + &storage, + "UPDATE batches SET payload_hash = randomblob(32) WHERE nonce = 0", + LIFECYCLE_RULE, + ); +} + +#[test] +fn lifecycle_rejects_a_nonce_change() { + // Nonce immutability underpins the frontier and contiguity invariants. + let (_db, storage) = lifecycle_fixture("lifecycle-nonce"); + assert_refused( + &storage, + "UPDATE batches SET nonce = nonce + 1 WHERE nonce = 0", + LIFECYCLE_RULE, + ); +} + +#[test] +fn lifecycle_rejects_a_created_at_change() { + let (_db, storage) = lifecycle_fixture("lifecycle-created-at"); + assert_refused( + &storage, + "UPDATE batches SET created_at_ms = created_at_ms + 1 WHERE nonce = 0", + LIFECYCLE_RULE, + ); +} + +#[test] +fn lifecycle_rejects_an_update_that_changes_nothing() { + // A no-op update is neither a seal nor an acceptance. + let (_db, storage) = lifecycle_fixture("lifecycle-no-op"); + for nonce in [0, 1] { + assert_refused( + &storage, + &format!("UPDATE batches SET nonce = nonce WHERE nonce = {nonce}"), + LIFECYCLE_RULE, + ); + } +} + +// ── Deletion (`trg_batches_not_deletable`) ─────────────────────────────── + +#[test] +fn a_batch_is_never_deleted() { + // The rows are inserted raw, with no frame, so no foreign key refers to + // them and only the trigger can refuse. + let db = crate::test_support::temp_db("schema-batch-not-deletable"); + let storage = Storage::open_with_era(db.path.as_str(), &crate::storage::NewEra::EMPTY_GENESIS); storage .conn - .execute( - "UPDATE batch_tree_anchor SET nonce = 7 WHERE singleton_id = 0", - [], + .execute_batch( + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (0, 100, 100, zeroblob(32)); + INSERT INTO batches (nonce, created_at_ms) VALUES (1, 100);", ) - .expect("anchor settable before setup_complete"); - storage + .expect("a sealed batch and an open one"); + for sql in [ + "DELETE FROM batches WHERE nonce = 0", + "DELETE FROM batches WHERE nonce = 1", + "DELETE FROM batches", + ] { + assert_refused(&storage, sql, NOT_DELETABLE_RULE); + } + let count: i64 = storage .conn - .execute( - "INSERT INTO setup_complete (singleton_id, completed_at_ms) VALUES (0, 1)", - [], - ) - .expect("mark setup complete"); - let err = storage.conn.execute( - "UPDATE batch_tree_anchor SET nonce = 8 WHERE singleton_id = 0", - [], - ); - assert!( - format!("{err:?}").contains("batch-tree anchor is frozen after setup completes"), - "expected anchor-freeze trigger, got: {err:?}" - ); + .query_row("SELECT COUNT(*) FROM batches", [], |row| row.get(0)) + .expect("count batches"); + assert_eq!(count, 2); } -#[test] -fn schema_rejects_re_seal() { - let (_db, mut storage) = open_temp_storage("schema-re-seal"); +// ── Acceptance (`trg_batch_acceptance_extends_the_prefix`, `trg_batch_lifecycle`) ── +// +// The fixture's era is anchored at 7: batches 7, 8, 9 are sealed, batch 10 is +// the open Tip, and safe inputs 0..=5 exist. + +fn acceptance_fixture(name: &str) -> (crate::test_support::TestDb, Storage) { + let db = crate::test_support::temp_db(name); + let mut storage = Storage::open_with_era( + db.path.as_str(), + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..crate::storage::NewEra::EMPTY_GENESIS + }, + ); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); + for _ in 0..3 { + storage.close_batch_at(&mut head, 0).expect("close a batch"); + } + for index in 0..=5 { + storage + .conn + .execute( + "INSERT INTO safe_inputs \ + (safe_input_index, sender, payload, block_number, block_timestamp, transaction_hash) \ + VALUES (?1, zeroblob(20), X'', 10, 0, zeroblob(32))", + [index], + ) + .expect("insert a safe input"); + } + (db, storage) +} + +fn accept(storage: &Storage, nonce: i64, input: i64) -> rusqlite::Result { + storage.conn.execute( + "UPDATE batches SET accepted_input = ?1 WHERE nonce = ?2", + [input, nonce], + ) +} + +#[test] +fn acceptance_extends_the_prefix_from_the_anchor() { + let (_db, storage) = acceptance_fixture("acceptance-prefix"); + assert_eq!(accept(&storage, 7, 1).expect("the batch at the anchor"), 1); + assert_eq!(accept(&storage, 8, 3).expect("the next batch, later"), 1); +} + +#[test] +fn a_batch_is_never_inserted_accepted() { + let (_db, storage) = acceptance_fixture("acceptance-insert"); + // Nonce 11 follows the Tip's 10, so only the acceptance arm can refuse. + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash, accepted_input) \ + VALUES (11, 0, 1, zeroblob(32), 1)", + "a batch is inserted unaccepted", + ); + // Without the trigger, the table still refuses an acceptance on a batch + // that is not sealed. storage - .close_batch_at(&mut head, 0) - .expect("close batch 0 (seals it)"); - // Batch 0 is sealed. Attempt to re-seal with a different timestamp. - let err = storage.conn.execute( - "UPDATE batches SET sealed_at_ms = sealed_at_ms + 1 WHERE batch_index = 0", - [], + .conn + .execute_batch("DROP TRIGGER trg_batch_insert") + .expect("drop the insert trigger"); + assert_refused( + &storage, + "INSERT INTO batches (nonce, created_at_ms, accepted_input) VALUES (11, 0, 1)", + "CHECK constraint failed", ); - assert!( - format!("{err:?}").contains("sealed_at_ms is write-once"), - "expected write-once trigger, got: {err:?}" +} + +#[test] +fn acceptance_rejects_skipping_a_nonce() { + let (_db, storage) = acceptance_fixture("acceptance-skip"); + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 1 WHERE nonce = 8", + ACCEPTANCE_RULE, + ); + accept(&storage, 7, 1).expect("the batch at the anchor"); + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 2 WHERE nonce = 9", + ACCEPTANCE_RULE, ); } #[test] -fn schema_rejects_re_invalidate() { - let (_db, mut storage) = open_temp_storage("schema-re-invalidate"); - storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - // Seed via test helper (uses now_unix_ms internally). - storage.insert_invalid_batch(0).expect("first invalidate"); - let err = storage.conn.execute( - "UPDATE batches SET invalidated_at_ms = invalidated_at_ms + 1 \ - WHERE batch_index = 0", - [], +fn acceptance_rejects_an_input_not_after_the_last_acceptance() { + let (_db, storage) = acceptance_fixture("acceptance-earlier-input"); + accept(&storage, 7, 3).expect("the batch at the anchor"); + for input in [2, 3] { + assert_refused( + &storage, + &format!("UPDATE batches SET accepted_input = {input} WHERE nonce = 8"), + ACCEPTANCE_RULE, + ); + } +} + +#[test] +fn acceptance_rejects_an_unsealed_batch() { + let (_db, storage) = acceptance_fixture("acceptance-unsealed"); + accept(&storage, 7, 1).expect("accept nonce 7"); + accept(&storage, 8, 2).expect("accept nonce 8"); + accept(&storage, 9, 3).expect("accept nonce 9"); + // The Tip carries the frontier nonce 10, so only the lifecycle refuses. + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 4 WHERE nonce = 10", + LIFECYCLE_RULE, ); - assert!( - format!("{err:?}").contains("invalidated_at_ms is write-once"), - "expected write-once trigger, got: {err:?}" +} + +#[test] +fn acceptance_rejects_sealing_and_accepting_in_one_statement() { + let (_db, storage) = acceptance_fixture("acceptance-seal-and-accept"); + accept(&storage, 7, 1).expect("accept nonce 7"); + accept(&storage, 8, 2).expect("accept nonce 8"); + accept(&storage, 9, 3).expect("accept nonce 9"); + // The Tip carries the frontier nonce 10, so only the lifecycle refuses. + assert_refused( + &storage, + "UPDATE batches SET sealed_at_ms = 100, payload_hash = zeroblob(32), \ + accepted_input = 4 WHERE nonce = 10", + LIFECYCLE_RULE, + ); +} + +#[test] +fn acceptance_is_written_once() { + let (_db, storage) = acceptance_fixture("acceptance-twice"); + accept(&storage, 7, 1).expect("the batch at the anchor"); + for sql in [ + "UPDATE batches SET accepted_input = 2 WHERE nonce = 7", + "UPDATE batches SET accepted_input = NULL WHERE nonce = 7", + ] { + assert_refused(&storage, sql, LIFECYCLE_RULE); + } +} + +#[test] +fn acceptance_changes_nothing_else() { + let (_db, storage) = acceptance_fixture("acceptance-alone"); + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 1, payload_hash = randomblob(32) \ + WHERE nonce = 7", + LIFECYCLE_RULE, + ); +} + +#[test] +fn the_divergence_freeze_answers_an_acceptance_first() { + let (_db, mut storage) = acceptance_fixture("acceptance-frozen"); + crate::storage::test_helpers::record_canonical_divergence(&mut storage, 7, 1); + // A well-formed acceptance, then one that breaks both other rules. + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 1 WHERE nonce = 7", + FROZEN_RULE, + ); + assert_refused( + &storage, + "UPDATE batches SET accepted_input = 1 WHERE nonce = 10", + FROZEN_RULE, ); } @@ -239,7 +528,7 @@ fn schema_rejects_frame_insert_into_sealed_batch() { .expect("close batch 0; batch 0 is now sealed"); // Batch 0 is sealed. Any direct insert into its frames must fail. let err = storage.conn.execute( - "INSERT INTO frames (batch_index, frame_in_batch, created_at_ms, fee, safe_block) \ + "INSERT INTO frames (batch_nonce, frame_in_batch, created_at_ms, fee, safe_block) \ VALUES (0, 1, 100, 1356, 0)", [], ); @@ -250,48 +539,140 @@ fn schema_rejects_frame_insert_into_sealed_batch() { } #[test] -fn schema_rejects_frame_insert_into_invalidated_batch() { - let (_db, mut storage) = open_temp_storage("schema-frame-into-invalid"); - storage +fn schema_rejects_user_op_insert_into_sealed_batch() { + let (_db, mut storage) = open_temp_storage("schema-user-op-into-sealed"); + let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); - // Invalidate (without sealing) — Tip that never closed, now dead. - storage.insert_invalid_batch(0).expect("invalidate tip"); - let err = storage.conn.execute( - "INSERT INTO frames (batch_index, frame_in_batch, created_at_ms, fee, safe_block) \ - VALUES (0, 1, 100, 1356, 0)", - [], - ); - assert!( - format!("{err:?}").contains("frames can only be inserted into the current Tip"), - "expected tip-only-frames trigger, got: {err:?}" + storage + .close_batch_at(&mut head, 0) + .expect("close batch 0; batch 0 is now sealed"); + // Frame (0, 0) exists, so only the trigger refuses. + assert_refused( + &storage, + &raw_user_op_insert(0, 0, 0), + "user_ops can only be inserted into the current Tip", ); } +/// A well-formed user-op row at `(batch_nonce, frame_in_batch, pos_in_frame)`. +fn raw_user_op_insert(batch_nonce: u64, frame_in_batch: u64, pos_in_frame: u64) -> String { + format!( + "INSERT INTO user_ops \ + (batch_nonce, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, \ + received_at_ms) \ + VALUES ({batch_nonce}, {frame_in_batch}, {pos_in_frame}, zeroblob(20), 0, 0, X'', \ + zeroblob(65), 0)" + ) +} + #[test] -fn schema_rejects_parent_batch_index_mutation() { - let (_db, mut storage) = open_temp_storage("schema-parent-immutable"); +fn frames_and_user_ops_are_written_once() { + // Batch 0 is sealed with one frame holding one user op; batch 1's frame + // has none, so no foreign key refuses its delete. + let (_db, mut storage) = open_temp_storage("schema-frames-user-ops-written-once"); + storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .conn + .execute(&raw_user_op_insert(0, 0, 0), []) + .expect("a user op in the open frame"); + let mut head = storage + .open_state() + .expect("reload") + .expect("the open batch"); + storage.close_batch_at(&mut head, 0).expect("seal batch 0"); + for (sql, message) in [ + ( + "UPDATE user_ops SET data = X'01' WHERE batch_nonce = 0", + "user ops are immutable", + ), + ( + "DELETE FROM user_ops WHERE batch_nonce = 0", + "user ops are retained for the database lifetime", + ), + ( + "UPDATE frames SET fee = fee + 1 WHERE batch_nonce = 0", + "frames are immutable", + ), + ( + "DELETE FROM frames WHERE batch_nonce = 1", + "frames are retained for the database lifetime", + ), + ] { + assert_refused(&storage, sql, message); + } +} + +// ── REPLACE (`recursive_triggers`) ─────────────────────────────────────── +// +// `INSERT OR REPLACE` deletes the row it conflicts with. The connection's +// `recursive_triggers` makes that delete fire the table's BEFORE DELETE +// trigger, so a "never deleted" table refuses it. + +#[test] +fn a_replace_never_deletes_a_batch() { + let db = crate::test_support::temp_db("schema-replace-batch"); + let storage = Storage::open_with_era(db.path.as_str(), &crate::storage::NewEra::EMPTY_GENESIS); + storage + .conn + .execute( + "INSERT INTO batches (nonce, created_at_ms, sealed_at_ms, payload_hash) \ + VALUES (0, 100, 100, zeroblob(32))", + [], + ) + .expect("a sealed batch"); + // An insert over an existing nonce is never the last nonce + 1, so the + // nonce rule refuses it first; without that rule the REPLACE's delete + // still fires `trg_batches_not_deletable`. + let replace = "INSERT OR REPLACE INTO batches (nonce, created_at_ms) VALUES (0, 100)"; + assert_refused(&storage, replace, NONCE_RULE); + storage + .conn + .execute_batch("DROP TRIGGER trg_batch_insert") + .expect("isolate the delete trigger"); + assert_refused(&storage, replace, NOT_DELETABLE_RULE); +} + +#[test] +fn a_replace_never_deletes_an_application_input() { + let (_db, mut storage) = open_temp_storage("schema-replace-application-input"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); - storage.close_batch_at(&mut head, 0).expect("close batch 0"); - // Try to change parent of batch 1 — should be rejected. - let err = storage.conn.execute( - "UPDATE batches SET parent_batch_index = NULL WHERE batch_index = 1", - [], + storage.land(10, &[(alloy_primitives::Address::ZERO, vec![0xaa])]); + storage + .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) + .expect("close a frame over one direct"); + // The next count, into the open frame (0, 1), conflicting on input 0. + assert_refused( + &storage, + "INSERT OR REPLACE INTO application_inputs \ + (l2_tx_count, batch_nonce, frame_in_batch, safe_input_index) VALUES (1, 0, 1, 0)", + "application inputs are retained for the database lifetime", ); - assert!( - format!("{err:?}").contains("parent_batch_index is immutable"), - "expected parent-immutable trigger, got: {err:?}" +} + +#[test] +fn a_replace_never_deletes_a_safe_input() { + // `safe_inputs` has no insert trigger, so only the delete trigger, fired + // by the pragma, can refuse. + let (_db, mut storage) = open_temp_storage("schema-replace-safe-input"); + storage.land(10, &[(alloy_primitives::Address::ZERO, vec![0xaa])]); + assert_refused( + &storage, + "INSERT OR REPLACE INTO safe_inputs \ + (safe_input_index, sender, payload, block_number, block_timestamp, transaction_hash) \ + VALUES (0, zeroblob(20), X'bb', 10, 0, zeroblob(32))", + "safe inputs are retained for the database lifetime", ); } #[test] fn schema_rejects_application_input_into_non_tip() { - // The third sibling of the tip-only triggers (frames + user_ops already - // have negative tests; application_inputs did not). The global replay order - // is the source for recovery re-drain and catch-up; a stale-WriteHead row - // into a sealed batch would corrupt it. + // The application order is the source for catch-up and the history feed; + // a stale-WriteHead row into a sealed batch would corrupt it. let (_db, mut storage) = open_temp_storage("schema-sequenced-into-sealed"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) @@ -303,109 +684,42 @@ fn schema_rejects_application_input_into_non_tip() { // trigger fires before the safe_input_index FK is evaluated. let err = storage.conn.execute( "INSERT INTO application_inputs \ - (offset, batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ + (l2_tx_count, batch_nonce, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ VALUES (0, 0, 0, NULL, 0)", [], ); assert!( - format!("{err:?}").contains("application input must target the current valid Tip"), + format!("{err:?}").contains("application input must target the open batch"), "expected tip-only-sequenced trigger, got: {err:?}" ); } #[test] -fn schema_rejects_nonce_mutation() { - // Nonce immutability underpins the frontier + contiguity invariants. - let (_db, mut storage) = open_temp_storage("schema-nonce-immutable"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.close_batch_at(&mut head, 0).expect("close batch 0"); - let err = storage.conn.execute( - "UPDATE batches SET nonce = nonce + 1 WHERE batch_index = 0", - [], - ); - assert!( - format!("{err:?}").contains("nonce is immutable"), - "expected nonce-immutable trigger, got: {err:?}" - ); -} - -#[test] -fn schema_rejects_payload_hash_rewrite() { - // payload_hash is the content-identity anchor for the canonical- - // divergence check; a rewrite would let a foreign/zombie L1 landing - // false-match a local batch. Write-once once stamped at seal. - let (_db, mut storage) = open_temp_storage("schema-payload-hash-write-once"); +fn an_application_input_is_never_deleted() { + // Nothing refers to an application input, so only the trigger refuses. + let (_db, mut storage) = open_temp_storage("schema-application-input-not-deletable"); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); + storage.land(10, &[(alloy_primitives::Address::ZERO, vec![0xaa])]); storage - .close_batch_at(&mut head, 0) - .expect("close batch 0; payload_hash is stamped at seal"); - let err = storage.conn.execute( - "UPDATE batches SET payload_hash = \ - X'0000000000000000000000000000000000000000000000000000000000000000' \ - WHERE batch_index = 0", - [], - ); - assert!( - format!("{err:?}").contains("payload_hash is write-once"), - "expected payload-hash-write-once trigger, got: {err:?}" + .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) + .expect("close a frame over one direct"); + let rows = || -> i64 { + storage + .conn + .query_row("SELECT COUNT(*) FROM application_inputs", [], |row| { + row.get(0) + }) + .expect("count application inputs") + }; + assert_eq!(rows(), 1); + assert_refused( + &storage, + "DELETE FROM application_inputs", + "application inputs are retained for the database lifetime", ); -} - -#[test] -fn nonce_reuse_after_cascade_with_valid_ancestor() { - // Beautiful part of parent-pointer + structural nonce: after a cascade - // that invalidates only the suffix (keeping an ancestor valid), the - // new Tip's parent is the last valid ancestor, so its nonce is - // `ancestor.nonce + 1` — the same nonce the invalidated suffix's - // first batch had. Nonce reuse is automatic. - // - // Scenario: batch 0 is accepted (safe_accepted_batches advances past - // nonce 0). Batch 1 is stale and triggers cascade. Batches 1, 2, 3 - // invalidated; batch 0 remains valid. - let (_db, mut storage) = open_temp_storage("nonce-reuse-with-ancestor"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize at safe_block=10"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 0 (nonce 0)"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 1 (nonce 1)"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 2 (nonce 2)"); - // Head is now batch 3 (nonce 3, first_frame_safe_block=100). - - // Batch 0 lands on L1 (accepted): safe_input at block 20 with nonce 0. - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); // append batch 0 submission - // Advance safe head so batches 1, 2, 3 (first_frame=100) are stale. - // current_safe=1400 → 1400-100=1300 >= 1200. - storage.advance_safe_head(1400); // advance past threshold - - let inv = storage.recover_post_flush(1200).expect("recover"); - // Batches 1, 2, 3 invalidated; batch 0 (accepted) stays valid. - assert_eq!(inv, vec![1, 2, 3], "only the suffix cascades, got {inv:?}"); - - // The NEW Tip has parent=0 (the last valid ancestor), nonce=1. - // This is what nonce reuse looks like: the invalidated batch 1 had - // nonce 1; the recovery batch gets the same nonce via +1-from-parent. - let (tip_nonce, tip_parent): (i64, i64) = storage - .conn - .query_row( - "SELECT nonce, parent_batch_index FROM valid_open_batch", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .expect("query recovery tip"); - assert_eq!(tip_nonce, 1, "recovery Tip reuses nonce 1"); - assert_eq!(tip_parent, 0, "recovery Tip's parent is batch 0"); + assert_eq!(rows(), 1); } // ── CHECK-constraint regressions ────────────────────────── @@ -445,7 +759,7 @@ fn schema_rejects_user_op_with_wrong_sender_length() { .expect("initialize"); let err = storage.conn.execute( "INSERT INTO user_ops \ - (batch_index, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, received_at_ms) \ + (batch_nonce, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, received_at_ms) \ VALUES (0, 0, 0, X'010203', 0, 0, X'', ?1, 0)", params![vec![0u8; 65]], ); @@ -468,7 +782,7 @@ fn schema_rejects_user_op_with_wrong_signature_length() { let short_sig = vec![0u8; 32]; // Should be 65. let err = storage.conn.execute( "INSERT INTO user_ops \ - (batch_index, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, received_at_ms) \ + (batch_nonce, frame_in_batch, pos_in_frame, sender, nonce, max_fee, data, sig, received_at_ms) \ VALUES (0, 0, 0, ?1, 0, 0, X'', ?2, 0)", params![valid_sender, short_sig], ); @@ -492,7 +806,7 @@ fn schema_rejects_application_input_with_neither_xor_branch() { .expect("initialize"); let err = storage.conn.execute( "INSERT INTO application_inputs \ - (offset, batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ + (l2_tx_count, batch_nonce, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ VALUES (0, 0, 0, NULL, NULL)", [], ); @@ -509,24 +823,71 @@ fn schema_rejects_deployment_identity_with_zero_chain_id() { // and break signature recovery; the CHECK refuses to persist it // in the first place. let db = crate::test_support::temp_db("schema-deployment-chain-id-zero"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); + // No identity yet: the test inserts the row itself. + let storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); let address = vec![0u8; 20]; let err = storage.conn.execute( "INSERT INTO deployment_identity \ (singleton_id, chain_id, app_address, input_box_address, \ - app_deployment_block, batch_submitter_address, \ - fee_oracle_mode, fixed_log_gas_price, \ + app_deployment_block, batch_submitter_address, max_direct_payload_bytes, \ + max_wait_blocks, fee_oracle_mode, fixed_log_gas_price, \ fee_oracle_weth, fee_oracle_fee_token, fee_oracle_pool, \ fee_oracle_twap_window_secs) \ - VALUES (0, 0, ?1, ?1, 0, ?1, 'fixed', 0, NULL, NULL, NULL, NULL)", + VALUES (0, 0, ?1, ?1, 0, ?1, 1024, 1200, 'fixed', 0, NULL, NULL, NULL, NULL)", params![address], ); assert!( - format!("{err:?}").contains("CHECK constraint failed"), + format!("{err:?}").contains("CHECK constraint failed: chain_id > 0"), "expected CHECK constraint error on chain_id > 0, got: {err:?}", ); } +#[test] +fn schema_rejects_deployment_identity_with_negative_direct_payload_cap() { + let db = crate::test_support::temp_db("schema-deployment-negative-cap"); + // No identity yet: the test inserts the row itself. + let storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + let address = vec![0u8; 20]; + let err = storage.conn.execute( + "INSERT INTO deployment_identity \ + (singleton_id, chain_id, app_address, input_box_address, \ + app_deployment_block, batch_submitter_address, max_direct_payload_bytes, \ + max_wait_blocks, fee_oracle_mode, fixed_log_gas_price, \ + fee_oracle_weth, fee_oracle_fee_token, fee_oracle_pool, \ + fee_oracle_twap_window_secs) \ + VALUES (0, 1, ?1, ?1, 0, ?1, -1, 1200, 'fixed', 0, NULL, NULL, NULL, NULL)", + params![address], + ); + assert!( + format!("{err:?}").contains("CHECK constraint failed: max_direct_payload_bytes >= 0"), + "expected CHECK constraint error on max_direct_payload_bytes >= 0, got: {err:?}", + ); +} + +#[test] +fn schema_rejects_deployment_identity_with_a_zero_window() { + // A window of zero would make every batch stale on arrival, and the + // scheduler refuses to start with one. + let db = crate::test_support::temp_db("schema-deployment-zero-window"); + // No identity yet: the test inserts the row itself. + let storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + let address = vec![0u8; 20]; + let err = storage.conn.execute( + "INSERT INTO deployment_identity \ + (singleton_id, chain_id, app_address, input_box_address, \ + app_deployment_block, batch_submitter_address, max_direct_payload_bytes, \ + max_wait_blocks, fee_oracle_mode, fixed_log_gas_price, \ + fee_oracle_weth, fee_oracle_fee_token, fee_oracle_pool, \ + fee_oracle_twap_window_secs) \ + VALUES (0, 1, ?1, ?1, 0, ?1, 1024, 0, 'fixed', 0, NULL, NULL, NULL, NULL)", + params![address], + ); + assert!( + format!("{err:?}").contains("CHECK constraint failed: max_wait_blocks >= 1"), + "expected CHECK constraint error on max_wait_blocks >= 1, got: {err:?}", + ); +} + #[test] fn schema_rejects_safe_input_with_negative_block_number() { // `safe_inputs.block_number >= 0`. Catches a regression @@ -547,7 +908,7 @@ fn schema_rejects_safe_input_with_negative_block_number() { } #[test] -fn divergence_marker_freezes_batch_tree_promotions_and_pending_clears() { +fn divergence_marker_freezes_batch_list_promotions_and_pending_clears() { // I15 structural enforcement: with the canonical-divergence marker // present, the trg_*_frozen_on_divergence family must RAISE on batch // inserts/updates, promotion inserts, and pending-snapshot deletes — @@ -572,24 +933,23 @@ fn divergence_marker_freezes_batch_tree_promotions_and_pending_clears() { ); }; + // Nonce 0 breaks the nonce rule too (the next is 2): the freeze answers + // first, as the most recently created BEFORE INSERT trigger. frozen( storage .conn .execute( - "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms) \ - VALUES (99, NULL, 0, 0)", + "INSERT INTO batches (nonce, created_at_ms) VALUES (0, 0)", [], ) .expect_err("batch INSERT must be frozen"), ); + // Likewise before the lifecycle trigger: this nonce change breaks both. frozen( storage .conn - .execute( - "UPDATE batches SET invalidated_at_ms = 1 WHERE invalidated_at_ms IS NULL", - [], - ) - .expect_err("batch UPDATE (cascade shape) must be frozen"), + .execute("UPDATE batches SET nonce = 7 WHERE nonce = 0", []) + .expect_err("batch UPDATE must be frozen"), ); frozen( storage @@ -598,13 +958,14 @@ fn divergence_marker_freezes_batch_tree_promotions_and_pending_clears() { .expect_err("pending-snapshot DELETE must be frozen"), ); - // Reads stay usable on a frozen DB: the danger check must still - // classify the divergence (the boot-refusal path depends on it). - let protocol = default_protocol_timing(); - let status = storage - .check_danger(&protocol, 1_000_000) - .expect("check danger on a frozen DB"); - assert_eq!(status, crate::storage::DangerStatus::CanonicalDivergence(0)); + // Reads stay usable on a frozen DB: the terminal-fact read must still + // report the divergence (the boot refusal depends on it). + assert!(matches!( + storage + .terminal_fact() + .expect("terminal fact on a frozen DB"), + Some(crate::storage::TerminalFact::Divergence(divergence)) if divergence.nonce == 0 + )); } #[test] @@ -644,7 +1005,7 @@ fn schema_rejects_safe_input_with_wrong_transaction_hash_length() { #[test] fn batch_policy_check_rejects_unsafe_alpha() { let db = crate::test_support::temp_db("unsafe-alpha"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); + let storage = set_up_storage(&db.path, SENDER_A); // log_alpha=-350 → log_batch_size_target = 1403-(-350)-419 = 1334 >= log_max_batch_bytes=1333 let err = storage.conn.execute( "UPDATE batch_policy SET log_alpha = ?1, log_one_plus_alpha = ?2 WHERE singleton_id = 0", @@ -664,7 +1025,7 @@ fn seeded_log_slack_is_the_exponent_nearest_tenfold() { use sequencer_core::fee::fee_to_linear; let db = crate::test_support::temp_db("seeded-log-slack"); - let storage = Storage::open(db.path.as_str()).expect("open storage"); + let storage = set_up_storage(&db.path, SENDER_A); let slack: u16 = storage .conn .query_row( @@ -687,7 +1048,7 @@ fn seeded_log_slack_is_the_exponent_nearest_tenfold() { #[test] fn schema_rejects_safe_input_update_and_delete() { let db = crate::test_support::temp_db("safe-inputs-append-only"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); let inputs = vec![ StoredSafeInput { sender: SENDER_B, @@ -701,7 +1062,7 @@ fn schema_rejects_safe_input_update_and_delete() { }, ]; storage - .append_safe_inputs(10, inputs.as_slice(), SENDER_A, &default_protocol_timing()) + .append_safe_inputs(10, inputs.as_slice(), SENDER_A) .expect("seed safe inputs"); let update = storage @@ -724,9 +1085,53 @@ fn schema_rejects_safe_input_update_and_delete() { assert_eq!(storage.safe_input_end_exclusive().expect("end"), 2); assert_eq!( - storage - .safe_inputs_in_block_range(9, 10) - .expect("read back"), - inputs + storage.safe_input_count_through_block(10).expect("count"), + 2 + ); +} + +#[test] +fn every_schema_table_is_strict() { + // STRICT is what refuses a mis-classed value before any trigger runs, so a + // table added without it would reopen that hole. + let (_db, storage) = open_temp_storage("schema-every-table-strict"); + let mut stmt = storage + .conn + .prepare( + "SELECT name FROM pragma_table_list \ + WHERE type = 'table' AND strict = 0 AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .expect("prepare table list"); + let lax: Vec = stmt + .query_map([], |row| row.get(0)) + .expect("list tables") + .collect::>() + .expect("read table names"); + assert!(lax.is_empty(), "tables without STRICT: {lax:?}"); +} + +#[test] +fn strict_type_refusal_is_a_persistent_storage_error() { + let (_db, storage) = open_temp_storage("schema-strict-type-refusal"); + let err = storage + .conn + .execute( + "INSERT INTO wallet_nonce_watermark (singleton_id, watermark) VALUES (0, ?1)", + ["not-an-integer"], + ) + .expect_err("STRICT must refuse text in an INTEGER column"); + let rusqlite::Error::SqliteFailure(source, _) = &err else { + panic!("expected a SQLite failure, got {err:?}"); + }; + assert_eq!( + source.extended_code, + rusqlite::ffi::SQLITE_CONSTRAINT_DATATYPE, + "{err}" + ); + assert!( + err.to_string() + .contains("cannot store TEXT value in INTEGER column"), + "{err}" ); + assert!(crate::storage::convert::is_persistent_storage_error(&err)); } diff --git a/sequencer/src/storage/queries.rs b/sequencer/src/storage/queries.rs index 5e3a0c14..6619bf43 100644 --- a/sequencer/src/storage/queries.rs +++ b/sequencer/src/storage/queries.rs @@ -5,28 +5,70 @@ //! they compose inside any writer's transaction; single-caller reads stay with the //! writer that owns them. -use rusqlite::{Connection, OptionalExtension, Result, Transaction, params}; +use rusqlite::{Connection, OptionalExtension, Result, ToSql, Transaction, params}; -use super::convert::{from_unix_ms, i64_to_u16, i64_to_u32, i64_to_u64}; +use super::convert::{from_unix_ms, i64_to_u16, i64_to_u32, i64_to_u64, u64_to_i64}; use super::{BatchPolicy, WriteHead}; +/// Whether a safe input executes as a direct when its range drains: any input +/// but the batch submitter's, unless its payload is over the application's +/// cap. Every query that picks the directs to execute uses this one predicate, +/// so the lane's executions and storage's derivation of them always agree. +/// It binds `:batch_submitter` and `:max_direct_payload_bytes`, both from the +/// pinned deployment identity through [`DirectOperands`]. `length` takes +/// the bare column, so SQLite answers it from the record header without +/// reading the payload; every writer stores the payload as a blob. +pub(super) const DIRECT_EXECUTES_SQL: &str = "(sender != :batch_submitter \ + AND length(payload) <= :max_direct_payload_bytes)"; + +/// Both operands of [`DIRECT_EXECUTES_SQL`], read from the one pinned +/// deployment identity. A statement that uses the predicate binds them +/// together through [`DirectOperands::bind`], so neither is ever bound alone. +pub(super) struct DirectOperands { + batch_submitter: Vec, + max_direct_payload_bytes: i64, +} + +impl DirectOperands { + /// Fails with `QueryReturnedNoRows` when no identity is pinned. + pub(super) fn load(conn: &Connection) -> Result { + let identity = super::l1_inputs::query_deployment_identity(conn)? + .ok_or(rusqlite::Error::QueryReturnedNoRows)?; + Ok(Self { + batch_submitter: identity.batch_submitter_address.to_vec(), + max_direct_payload_bytes: u64_to_i64(identity.max_direct_payload_bytes), + }) + } + + /// `params` followed by the predicate's two bindings. + pub(super) fn bind<'a>( + &'a self, + params: &[(&'a str, &'a dyn ToSql)], + ) -> Vec<(&'a str, &'a dyn ToSql)> { + let mut bound = params.to_vec(); + bound.push((":batch_submitter", &self.batch_submitter)); + bound.push((":max_direct_payload_bytes", &self.max_direct_payload_bytes)); + bound + } +} + // ── Write-head loading ─────────────────────────────────────────────────── // -// Used by ingress (initialize/resume open state) and recovery (open recovery -// batch after cascade). The WriteHead is the in-memory mirror of the latest -// open batch/frame and must always match what's persisted in `batches` and -// `frames`. +// The only constructor of a `WriteHead` outside tests. Ingress loads the lane's +// head with it and reloads it to compare at every frame rotation and batch +// close. The head is the in-memory mirror of the latest open batch/frame and +// must always match what's persisted in `batches` and `frames`. pub(super) fn load_current_write_head(tx: &Transaction<'_>) -> Result> { - // The Tip is the single row in `valid_open_batch` (enforced by - // `ux_single_valid_tip`). Returns None if there's no Tip (fresh DB, - // or torn state between cascade and recovery-batch open). + // The Tip is the single row in `open_batch` (enforced by + // `ux_single_open_batch`). Returns None if there's no Tip (a fresh era, + // before `run` opens its first batch). let latest_batch = match tx.query_row( "SELECT - b.batch_index, + b.nonce, b.created_at_ms, - (SELECT COUNT(*) FROM user_ops u WHERE u.batch_index = b.batch_index) AS user_op_count - FROM valid_open_batch b", + (SELECT COUNT(*) FROM user_ops u WHERE u.batch_nonce = b.nonce) AS user_op_count + FROM open_batch b", [], |row| { Ok(( @@ -40,24 +82,24 @@ pub(super) fn load_current_write_head(tx: &Transaction<'_>) -> Result return Ok(None), Err(other) => return Err(other), }; - let (batch_index_i64, batch_created_at_ms, batch_user_op_count_i64) = latest_batch; + let (batch_nonce_i64, batch_created_at_ms, batch_user_op_count_i64) = latest_batch; let (frame_in_batch_i64, frame_fee_i64, safe_block_i64): (i64, i64, i64) = tx.query_row( "SELECT frame_in_batch, fee, safe_block FROM frames \ - WHERE batch_index = ?1 ORDER BY frame_in_batch DESC LIMIT 1", - params![batch_index_i64], + WHERE batch_nonce = ?1 ORDER BY frame_in_batch DESC LIMIT 1", + params![batch_nonce_i64], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), )?; let open_frame_user_op_count: i64 = tx.query_row( - "SELECT COUNT(*) FROM user_ops WHERE batch_index = ?1 AND frame_in_batch = ?2", - params![batch_index_i64, frame_in_batch_i64], + "SELECT COUNT(*) FROM user_ops WHERE batch_nonce = ?1 AND frame_in_batch = ?2", + params![batch_nonce_i64, frame_in_batch_i64], |row| row.get(0), )?; let policy = query_batch_policy(tx)?; Ok(Some(WriteHead { - batch_index: i64_to_u64(batch_index_i64), + batch_nonce: i64_to_u64(batch_nonce_i64), batch_created_at: from_unix_ms(batch_created_at_ms), frame_fee: i64_to_u16(frame_fee_i64), safe_block: i64_to_u64(safe_block_i64), @@ -83,6 +125,24 @@ pub(super) fn query_latest_safe_input_index_exclusive(conn: &Connection) -> Resu }) } +/// How many safe inputs lie at or below `block`: the exclusive index bound +/// through it, since InputBox order never lowers the block. +pub(super) fn safe_input_count_through_block_in(conn: &Connection, block: u64) -> Result { + let last: Option = conn + .query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 \ + ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", + [super::convert::saturating_query_bound(block)], + |row| row.get(0), + ) + .optional()?; + Ok(last.map_or(0, |index| { + i64_to_u64(index) + .checked_add(1) + .expect("safe input index overflow") + })) +} + pub(super) fn current_safe_block(conn: &Connection) -> Result> { let value: Option = conn .query_row( @@ -95,9 +155,9 @@ pub(super) fn current_safe_block(conn: &Connection) -> Result> { } /// Current safe block, or `QueryReturnedNoRows` if no observation has been -/// recorded yet. Use from code paths that only run after preemptive recovery -/// has produced a safe-head observation (submitter, lane, post-recovery -/// open-batch helpers); callers that legitimately handle the "never synced" +/// recorded yet. Use from code paths that only run once setup's sync has +/// recorded a safe head (submitter, lane, the start batch's opening); +/// callers that legitimately handle the "never synced" /// case should use [`current_safe_block`] instead. pub(super) fn current_safe_block_required(conn: &Connection) -> Result { current_safe_block(conn)?.ok_or(rusqlite::Error::QueryReturnedNoRows) @@ -116,19 +176,6 @@ pub(super) fn current_safe_block_timestamp(conn: &Connection) -> Result Result> { - let value: Option = conn - .query_row( - "SELECT synced_at_ms FROM l1_safe_head WHERE singleton_id = 0", - [], - |row| row.get(0), - ) - .optional()?; - Ok(value.map(i64_to_u64)) -} - pub(super) fn query_batch_policy(conn: &Connection) -> Result { let (log_recommended_fee, log_batch_size_target): (i64, i64) = conn.query_row( "SELECT log_recommended_fee, log_batch_size_target FROM batch_policy_derived \ @@ -170,13 +217,51 @@ pub(super) fn query_batch_policy(conn: &Connection) -> Result { #[cfg(test)] mod tests { - use crate::storage::Storage; + use crate::storage::test_helpers::{SENDER_A, set_up_storage}; use crate::test_support::temp_db; + /// The frame close derives its executions inside the write transaction; + /// measuring a payload must not load it, overflow pages included. + #[test] + fn the_direct_predicate_measures_payloads_without_reading_them() { + // SQLite's OP_Column flag for a column read only by length(). + const OPFLAG_LENGTHARG: i64 = 0x40; + let db = temp_db("direct-predicate-plan"); + let storage = set_up_storage(&db.path, SENDER_A); + let mut statement = storage + .conn + .prepare(&format!( + "EXPLAIN SELECT {} FROM safe_inputs", + super::DIRECT_EXECUTES_SQL + )) + .expect("explain the predicate"); + let program: Vec<(String, i64)> = statement + .query_map( + rusqlite::named_params! { + ":batch_submitter": [0_u8; 20].as_slice(), + ":max_direct_payload_bytes": 1024_i64, + }, + |row| Ok((row.get("opcode")?, row.get("p5")?)), + ) + .expect("read the program") + .collect::>() + .expect("program rows"); + assert!( + program.iter().all(|(opcode, _)| opcode != "Cast"), + "{program:?}" + ); + assert!( + program + .iter() + .any(|(opcode, p5)| opcode == "Column" && p5 & OPFLAG_LENGTHARG != 0), + "{program:?}" + ); + } + #[test] fn negative_batch_size_target_floors_to_zero() { let db = temp_db("floor-batch-target"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // This row satisfies every schema constraint while deriving a negative // target: 1403 - 2000 - 419 = -1016. @@ -199,7 +284,7 @@ mod tests { #[should_panic(expected = "batch policy derived recommended fee")] fn negative_recommended_fee_fails_loud() { let db = temp_db("negative-recommended-fee"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // A negative derived fee is ruled out by the column constraints. Model // corruption/tampering that bypassed them and ensure the read cannot @@ -219,7 +304,7 @@ mod tests { #[should_panic(expected = "batch policy derived batch size target")] fn oversized_batch_size_target_fails_loud() { let db = temp_db("oversized-batch-target"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut storage = set_up_storage(&db.path, SENDER_A); // This is schema-valid: the derived target is 17984, which remains // below the row's configured log_max_batch_bytes of 20000. It is still diff --git a/sequencer/src/storage/rebuilt_history.rs b/sequencer/src/storage/rebuilt_history.rs new file mode 100644 index 00000000..c7fffd25 --- /dev/null +++ b/sequencer/src/storage/rebuilt_history.rs @@ -0,0 +1,107 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Setup's rebuilt history: the transactions that executed from the base +//! through the stop block, each written once with its L2 tx count. Setup is +//! the only writer, and only before it publishes the era; the feed reads the +//! rows through `history::canonical`. + +use rusqlite::{Result, named_params, params}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::L2Tx; +use sequencer_core::scheduler::Scheduled; + +use super::convert::{i64_to_u64, saturating_query_bound, u64_to_i64}; +use super::queries::{DIRECT_EXECUTES_SQL, DirectOperands}; +use super::{SafeInputRange, Storage}; + +/// The rebuilt directs, counted against the input table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RebuiltDirects { + /// Every direct the rebuilt history holds. + pub stored: u64, + /// Those that are inputs of the range and execute as directs. + pub admitted_in_range: u64, +} + +impl Storage { + /// Insert executed transactions with their counts, in execution order, + /// in one transaction. A direct stores only its input index. + pub fn append_rebuilt_history(&mut self, rows: &[(L2TxCount, Scheduled)]) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + self.write(|tx| { + let mut insert = tx.prepare_cached( + "INSERT INTO rebuilt_history \ + (l2_tx_count, batch_nonce, safe_input_index, sender, nonce, fee, safe_block, data) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + )?; + for (count, scheduled) in rows { + let count = u64_to_i64(count.get()); + let batch_nonce = u64_to_i64(scheduled.batch_nonce); + match &scheduled.tx { + L2Tx::Direct(direct) => insert.execute(params![ + count, + batch_nonce, + u64_to_i64(direct.input_index), + None::>, + None::, + None::, + None::, + None::>, + ])?, + L2Tx::UserOp(op) => insert.execute(params![ + count, + batch_nonce, + None::, + op.sender.as_slice(), + i64::from(op.nonce), + i64::from(op.fee), + u64_to_i64(op.block), + op.data.as_slice(), + ])?, + }; + } + Ok(()) + }) + } + + /// The directs the rebuilt history holds, and how many of them are inputs + /// of `range` that execute as directs by the predicate every drain uses. + /// Setup checks both against the admitted inputs of the range it drained. + pub fn rebuilt_directs(&mut self, range: SafeInputRange) -> Result { + self.read(|tx| { + let operands = DirectOperands::load(tx)?; + let stored: i64 = tx.query_row( + "SELECT COUNT(*) FROM rebuilt_history WHERE safe_input_index IS NOT NULL", + [], + |row| row.get(0), + )?; + let admitted_in_range: i64 = tx + .prepare_cached(&format!( + "SELECT COUNT(*) FROM safe_inputs \ + WHERE safe_input_index >= :start AND safe_input_index < :end \ + AND {DIRECT_EXECUTES_SQL} \ + AND safe_input_index IN (SELECT safe_input_index FROM rebuilt_history \ + WHERE safe_input_index IS NOT NULL)" + ))? + .query_row( + operands + .bind(named_params! { + ":start": saturating_query_bound(range.start()), + ":end": saturating_query_bound(range.end()), + }) + .as_slice(), + |row| row.get(0), + )?; + Ok(RebuiltDirects { + stored: i64_to_u64(stored), + admitted_in_range: i64_to_u64(admitted_in_range), + }) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/rebuilt_history/tests.rs b/sequencer/src/storage/rebuilt_history/tests.rs new file mode 100644 index 00000000..2f8926fc --- /dev/null +++ b/sequencer/src/storage/rebuilt_history/tests.rs @@ -0,0 +1,315 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use alloy_primitives::B256; +use sequencer_core::l2_tx::{Direct, UserOpTx}; + +use super::*; +use crate::storage::test_helpers::{SENDER_A, SENDER_B, era}; +use crate::storage::{IngestedSafeInput, LifecycleCommand, NewEra}; +use crate::test_support::{TestDb, pin_test_deployment_identity, temp_db}; + +fn count(value: u64) -> L2TxCount { + L2TxCount::new(value) +} + +fn direct(input_index: u64, batch_nonce: u64) -> Scheduled { + Scheduled { + batch_nonce, + tx: L2Tx::Direct(Direct { + input_index, + sender: SENDER_B, + block: 0, + timestamp: 0, + payload: vec![], + }), + } +} + +fn op(nonce: u32, batch_nonce: u64) -> Scheduled { + Scheduled { + batch_nonce, + tx: L2Tx::UserOp(UserOpTx { + sender: SENDER_B, + nonce, + fee: 3, + block: 4, + data: vec![nonce as u8], + }), + } +} + +/// A setup directory with three safe inputs: two directs around a batch +/// submitter input. +fn setup_storage(name: &str) -> (TestDb, Storage) { + let db = temp_db(name); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let input = |sender, block: u64| IngestedSafeInput { + sender, + payload: vec![block as u8], + block_number: block, + block_timestamp: 100 + block, + transaction_hash: B256::repeat_byte(block as u8), + }; + storage + .append_ingested_safe_inputs_with_timestamp( + 9, + 109, + &[input(SENDER_B, 5), input(SENDER_A, 6), input(SENDER_B, 9)], + ) + .unwrap(); + (db, storage) +} + +/// Every stored row as `(count, batch nonce, input index, sender, nonce)`, +/// in count order. +type StoredRow = (i64, i64, Option, Option>, Option); + +fn stored(storage: &mut Storage) -> Vec { + storage + .conn + .prepare( + "SELECT l2_tx_count, batch_nonce, safe_input_index, sender, nonce \ + FROM rebuilt_history ORDER BY l2_tx_count", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .unwrap() + .collect::>() + .unwrap() +} + +#[test] +fn rows_are_stored_once_with_their_counts() { + let (_db, mut storage) = setup_storage("rebuilt-append"); + // The first row may carry any count; the era ties it to the base. + storage + .append_rebuilt_history(&[(count(41), direct(0, 2)), (count(42), op(0, 2))]) + .unwrap(); + storage + .append_rebuilt_history(&[(count(43), op(1, 2)), (count(44), direct(2, 3))]) + .unwrap(); + storage.append_rebuilt_history(&[]).unwrap(); + let sender = Some(SENDER_B.to_vec()); + assert_eq!( + stored(&mut storage), + vec![ + (41, 2, Some(0), None, None), + (42, 2, None, sender.clone(), Some(0)), + (43, 2, None, sender, Some(1)), + (44, 3, Some(2), None, None), + ] + ); +} + +#[test] +fn a_row_without_a_count_is_refused() { + let (_db, mut storage) = setup_storage("rebuilt-no-count"); + // The key is a rowid alias: without the trigger SQLite would number it. + let error = storage + .conn + .execute( + "INSERT INTO rebuilt_history (batch_nonce, safe_input_index) VALUES (0, 0)", + [], + ) + .expect_err("a row needs its count"); + assert_eq!(error.to_string(), "a rebuilt row carries its L2 tx count"); + let error = storage + .conn + .execute( + "INSERT INTO rebuilt_history (l2_tx_count, batch_nonce, safe_input_index) \ + VALUES (NULL, 0, 0)", + [], + ) + .expect_err("a NULL count is numbered by SQLite"); + assert_eq!(error.to_string(), "a rebuilt row carries its L2 tx count"); + assert!(stored(&mut storage).is_empty()); +} + +#[test] +fn counts_follow_one_another_without_gaps() { + let (_db, mut storage) = setup_storage("rebuilt-gaps"); + storage + .append_rebuilt_history(&[(count(7), direct(0, 0))]) + .unwrap(); + for value in [9, 7, 6] { + let error = storage + .append_rebuilt_history(&[(count(value), op(0, 0))]) + .expect_err("a count other than the next"); + assert_eq!( + error.to_string(), + "rebuilt counts follow one another without gaps", + "{value}" + ); + } + // A page applies all its rows or none. + let error = storage + .append_rebuilt_history(&[(count(8), op(0, 0)), (count(10), op(1, 0))]) + .expect_err("the second row leaves a gap"); + assert_eq!( + error.to_string(), + "rebuilt counts follow one another without gaps" + ); + assert_eq!(stored(&mut storage).len(), 1); +} + +#[test] +fn a_submitter_input_is_refused_as_a_direct() { + let (_db, mut storage) = setup_storage("rebuilt-submitter"); + let error = storage + .append_rebuilt_history(&[(count(0), direct(1, 0))]) + .expect_err("input 1 is the batch submitter's"); + assert_eq!( + error.to_string(), + "a batch submitter input never executes as a direct" + ); +} + +#[test] +fn directs_drain_in_input_order() { + let (_db, mut storage) = setup_storage("rebuilt-direct-order"); + storage + .append_rebuilt_history(&[(count(0), direct(2, 0)), (count(1), op(0, 0))]) + .unwrap(); + for input_index in [0, 2] { + let error = storage + .append_rebuilt_history(&[(count(2), direct(input_index, 0))]) + .expect_err("a direct at or before an earlier one"); + assert_eq!( + error.to_string(), + "rebuilt directs drain in input order", + "{input_index}" + ); + } +} + +/// Appending a row reads the last count and the last direct through their +/// keys: a full scan per row would make the rebuild quadratic in the history. +#[test] +fn appending_a_row_never_scans_the_table() { + let (_db, mut storage) = setup_storage("rebuilt-append-plan"); + let rows: Vec<_> = (0..64_u32) + .map(|nonce| (count(u64::from(nonce)), op(nonce, 0))) + .collect(); + storage.append_rebuilt_history(&rows).unwrap(); + + let mut insert = storage + .conn + .prepare( + "INSERT INTO rebuilt_history (l2_tx_count, batch_nonce, safe_input_index) \ + VALUES (64, 0, 0)", + ) + .unwrap(); + assert_eq!(insert.execute([]).unwrap(), 1); + let scanned = insert.get_status(rusqlite::StatementStatus::FullscanStep); + assert!(scanned < 8, "{scanned} full-scan steps for one row"); +} + +#[test] +fn rows_are_immutable_and_never_deleted() { + let (_db, mut storage) = setup_storage("rebuilt-immutable"); + storage + .append_rebuilt_history(&[(count(0), direct(0, 0)), (count(1), op(0, 0))]) + .unwrap(); + for sql in [ + "UPDATE rebuilt_history SET batch_nonce = 1 WHERE l2_tx_count = 1", + "UPDATE rebuilt_history SET l2_tx_count = 5 WHERE l2_tx_count = 1", + ] { + let error = storage.conn.execute(sql, []).expect_err(sql); + assert_eq!(error.to_string(), "rebuilt history rows are immutable"); + } + let error = storage + .conn + .execute("DELETE FROM rebuilt_history WHERE l2_tx_count = 1", []) + .expect_err("rows are never deleted"); + assert_eq!( + error.to_string(), + "rebuilt history is retained for the database lifetime" + ); + assert_eq!(stored(&mut storage).len(), 2); +} + +#[test] +fn the_published_era_seals_its_rebuilt_rows() { + let (_db, mut storage) = setup_storage("rebuilt-sealed"); + storage + .append_rebuilt_history(&[(count(41), direct(0, 0)), (count(42), op(0, 1))]) + .unwrap(); + let prefix = std::path::Path::new("/snapshot/era"); + // The era must span exactly the rows: base 41, end 43. + for (base, end) in [(41, 42), (40, 43), (41, 44)] { + let error = storage + .publish_era(prefix, &era(base, end, 9)) + .expect_err("the counts do not fit"); + assert!( + error + .to_string() + .contains("the era must span exactly its executed rebuilt rows"), + "{error}" + ); + } + storage + .publish_era( + prefix, + &NewEra { + first_queued_input: 3, + anchor_batch_nonce: 2, + ..era(41, 43, 9) + }, + ) + .unwrap(); + assert_eq!(storage.era().unwrap().rebuilt_end.get(), 43); + + let error = storage + .append_rebuilt_history(&[(count(43), op(1, 1))]) + .expect_err("no row after the era is published"); + assert_eq!( + error.to_string(), + "rebuilt history is sealed once its era is published" + ); + assert_eq!(stored(&mut storage).len(), 2); +} + +/// An era that still queues a direct the rebuild executed would make run's +/// root frame execute it again. +#[test] +fn an_era_queueing_a_rebuilt_direct_is_refused() { + let (_db, mut storage) = setup_storage("rebuilt-direct-queued"); + storage + .append_rebuilt_history(&[(count(41), direct(0, 0)), (count(42), direct(2, 1))]) + .unwrap(); + let prefix = std::path::Path::new("/snapshot/era"); + let error = storage + .publish_era( + prefix, + &NewEra { + first_queued_input: 2, + ..era(41, 43, 9) + }, + ) + .expect_err("input 2 was drained by the rebuild"); + assert!( + error + .to_string() + .contains("a rebuilt direct is never still queued"), + "{error}" + ); + storage + .publish_era( + prefix, + &NewEra { + first_queued_input: 3, + ..era(41, 43, 9) + }, + ) + .unwrap(); +} diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs deleted file mode 100644 index c8589299..00000000 --- a/sequencer/src/storage/recovery.rs +++ /dev/null @@ -1,517 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! Recovery storage: danger inspection, guarded suffix invalidation, and Tip -//! creation. `docs/recovery/README.md` owns the design; read it before editing. -//! Helpers take `&Connection`/`&Transaction` so one startup transaction can -//! compose inspection and repair. - -use rusqlite::{Connection, OptionalExtension, Result, Transaction, TransactionBehavior, params}; -use sequencer_core::protocol::{ProtocolTiming, age_exceeds}; - -use super::Storage; -use super::convert::{i64_to_u64, now_unix_ms, u64_to_i64}; -use super::history::advance_recovery_generation_in; -use super::ingress::open_fresh_tip_in_tx; -use super::queries::{ - current_safe_block_required, current_safe_block_timestamp, last_safe_progress_ms, -}; -use super::safe_accepted_batches::{canonical_divergence_in, frontier_nonce}; -use super::snapshot_dumps::has_rollback_safe_snapshot_in; - -/// Outcome of [`Storage::check_danger`]. The runtime detector exits on any -/// non-`Safe` variant; startup maps each variant to one action -/// (`docs/recovery/README.md`, "Lifecycle and startup dispatch"). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DangerStatus { - /// No check tripped. - Safe, - /// An accepted landing failed the content-identity check (I15); carries the - /// diverged batch nonce. Only cockroach recovery clears it. - CanonicalDivergence(u64), - /// The safe-block timestamp is missing or too old, or the local clock is a - /// block-time or more behind a persisted baseline. - L1ViewStale, - /// A closed batch past the accepted frontier is past the danger threshold at - /// the observed safe block. - ClosedBatchInDanger(u64), - /// The open Tip is past the danger threshold and no closed batch is. - TipInDanger(u64), - /// A batch is past the threshold only by wall-clock estimate. - EstimatedBatchInDanger(u64), -} - -/// One transactionally consistent local view consumed by the startup -/// recovery procedure. -/// -/// Keeping these facts together is load-bearing: admission and repair -/// selection must not combine a danger verdict from one SQLite snapshot with -/// Tip/snapshot/head facts from another. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct RecoveryInspection { - pub(crate) danger: DangerStatus, - pub(crate) has_recovery_checkpoint: bool, - pub(crate) has_open_tip: bool, - pub(crate) current_safe_block: Option, -} - -/// A recovery mutation was refused because the transaction no longer -/// satisfies the selected startup action's preconditions. -#[derive(Debug, thiserror::Error)] -pub(crate) enum RecoveryMutationError { - #[error(transparent)] - Storage(#[from] rusqlite::Error), - #[error("canonical divergence at batch nonce {nonce} forbids standard recovery")] - CanonicalDivergence { nonce: u64 }, - #[error("recovery decision is stale: expected {expected:?}, found {actual:?}")] - StaleDecision { - expected: DangerStatus, - actual: DangerStatus, - }, - #[error("cannot open the Tip without a recovery checkpoint")] - MissingRecoveryCheckpoint, - /// `EnsureOpenTip` found a valid open Tip already present. Unreachable - /// under the process lock, so it refuses startup. - #[error("the Tip was already open when the EnsureOpenTip phase ran")] - TipAlreadyOpen, - /// `EnsureOpenTip` left no valid open Tip after opening one. This broken - /// postcondition must roll back and refuse startup. - #[error("the EnsureOpenTip phase left no valid open Tip in its own transaction")] - TipMissingAfterOpen, - #[error( - "post-flush re-sync reached safe block {resynced_safe_block}, behind the flush observation at {flush_observed_safe_block}" - )] - ResyncBehindFlushView { - resynced_safe_block: u64, - flush_observed_safe_block: u64, - }, - #[error("post-flush re-sync did not persist a safe head")] - MissingSafeHead, -} - -impl DangerStatus { - /// Stable label for logs/metrics. An inherent method (not a free - /// projection) so a new variant must add its label right here. - pub(crate) fn label(self) -> &'static str { - match self { - DangerStatus::Safe => "safe", - DangerStatus::CanonicalDivergence(_) => "canonical_divergence", - DangerStatus::L1ViewStale => "l1_view_stale", - DangerStatus::ClosedBatchInDanger(_) => "closed_batch_in_danger", - DangerStatus::TipInDanger(_) => "tip_in_danger", - DangerStatus::EstimatedBatchInDanger(_) => "estimated_batch_in_danger", - } - } - - /// The batch nonce a danger arm points at, if any (log context). The - /// `CanonicalDivergence` nonce is deliberately not reported here — it is a - /// diverged-state nonce, not a batch in the danger pipeline. - pub(crate) fn batch_index(self) -> Option { - match self { - DangerStatus::ClosedBatchInDanger(batch_index) - | DangerStatus::TipInDanger(batch_index) - | DangerStatus::EstimatedBatchInDanger(batch_index) => Some(batch_index), - DangerStatus::Safe - | DangerStatus::L1ViewStale - | DangerStatus::CanonicalDivergence(_) => None, - } - } -} - -impl Storage { - /// Whether the canonical-divergence marker (I15) is present, and the - /// recorded `(nonce, safe_input_index)` if so. Standard - /// recovery is forbidden while the marker exists; callers on the recovery - /// path must check this before any batch-tree mutation. - pub fn canonical_divergence(&mut self) -> Result> { - self.read(|tx| canonical_divergence_in(tx)) - } - - /// Classify danger in one read transaction. Arms fire in order: divergence, - /// stale view, closed batch, Tip, clock fault, estimate; the order is the - /// startup dispatch policy (I4; `docs/recovery/README.md`, "When safe-head - /// progress stops"). `now_ms` is injected so storage tests need no clock mock. - pub fn check_danger(&mut self, protocol: &ProtocolTiming, now_ms: u64) -> Result { - self.read(|tx| check_danger_in(tx, protocol, now_ms)) - } - - /// Read every local fact used by the startup recovery procedure in one transaction. - pub(crate) fn inspect_recovery( - &mut self, - protocol: &ProtocolTiming, - now_ms: u64, - ) -> Result { - self.read(|tx| inspect_recovery_in(tx, protocol, now_ms)) - } - - /// Execute `EnsureOpenTip` only if its local decision - /// still holds in the write transaction. - pub(crate) fn ensure_open_tip_for_recovery( - &mut self, - protocol: &ProtocolTiming, - now_ms: u64, - ) -> std::result::Result<(), RecoveryMutationError> { - let tx = self - .conn - .transaction_with_behavior(TransactionBehavior::Immediate)?; - let facts = inspect_recovery_in(&tx, protocol, now_ms)?; - refuse_divergence(facts.danger)?; - if !facts.has_recovery_checkpoint { - return Err(RecoveryMutationError::MissingRecoveryCheckpoint); - } - if facts.danger != DangerStatus::Safe { - return Err(RecoveryMutationError::StaleDecision { - expected: DangerStatus::Safe, - actual: facts.danger, - }); - } - if facts.has_open_tip { - return Err(RecoveryMutationError::TipAlreadyOpen); - } - open_fresh_tip_in_tx(&tx)?; - // A broken postcondition must roll back this transaction and refuse - // startup, including in release builds. - if !has_valid_open_batch(&tx)? { - return Err(RecoveryMutationError::TipMissingAfterOpen); - } - tx.commit()?; - Ok(()) - } - - /// Execute `RecoverTip` only while the same Tip is - /// still the observed-danger arm in the write transaction. - pub(crate) fn recover_aging_tip_for_recovery( - &mut self, - expected_batch_index: u64, - protocol: &ProtocolTiming, - now_ms: u64, - ) -> std::result::Result, RecoveryMutationError> { - let tx = self - .conn - .transaction_with_behavior(TransactionBehavior::Immediate)?; - let facts = inspect_recovery_in(&tx, protocol, now_ms)?; - refuse_divergence(facts.danger)?; - if !facts.has_recovery_checkpoint { - return Err(RecoveryMutationError::MissingRecoveryCheckpoint); - } - let expected = DangerStatus::TipInDanger(expected_batch_index); - if facts.danger != expected { - return Err(RecoveryMutationError::StaleDecision { - expected, - actual: facts.danger, - }); - } - let invalidated = recover_aging_tip_inner(&tx, protocol.danger_threshold())?; - tx.commit()?; - Ok(invalidated) - } - - /// Execute post-flush Cascade. The boot-local flush observation is - /// represented by its observed safe-block floor; this transaction - /// reasserts both I15 and the post-flush resync coherence check - /// immediately before changing the batch tree. - pub(crate) fn recover_post_flush_for_recovery( - &mut self, - flush_observed_safe_block: u64, - protocol: &ProtocolTiming, - now_ms: u64, - ) -> std::result::Result, RecoveryMutationError> { - let tx = self - .conn - .transaction_with_behavior(TransactionBehavior::Immediate)?; - let facts = inspect_recovery_in(&tx, protocol, now_ms)?; - refuse_divergence(facts.danger)?; - if !facts.has_recovery_checkpoint { - return Err(RecoveryMutationError::MissingRecoveryCheckpoint); - } - let resynced_safe_block = facts - .current_safe_block - .ok_or(RecoveryMutationError::MissingSafeHead)?; - if resynced_safe_block < flush_observed_safe_block { - return Err(RecoveryMutationError::ResyncBehindFlushView { - resynced_safe_block, - flush_observed_safe_block, - }); - } - let invalidated = recover_post_flush_inner(&tx, protocol.danger_threshold())?; - tx.commit()?; - Ok(invalidated) - } -} - -pub(super) fn inspect_recovery_in( - conn: &Connection, - protocol: &ProtocolTiming, - now_ms: u64, -) -> Result { - let danger = check_danger_in(conn, protocol, now_ms)?; - let has_recovery_checkpoint = has_rollback_safe_snapshot_in(conn)?; - Ok(RecoveryInspection { - danger, - has_recovery_checkpoint, - has_open_tip: has_valid_open_batch(conn)?, - current_safe_block: super::queries::current_safe_block(conn)?, - }) -} - -fn check_danger_in( - conn: &Connection, - protocol: &ProtocolTiming, - now_ms: u64, -) -> Result { - // The divergence marker outranks everything — including the L1-staleness - // gate. It records an already-confirmed canonical fact. - if let Some((nonce, _)) = canonical_divergence_in(conn)? { - return Ok(DangerStatus::CanonicalDivergence(nonce)); - } - - let safe_block_timestamp = current_safe_block_timestamp(conn)?; - let last_progress = last_safe_progress_ms(conn)?; - if protocol.l1_view_is_stale(safe_block_timestamp, now_ms) { - return Ok(DangerStatus::L1ViewStale); - } - - let danger_threshold = protocol.danger_threshold(); - if let Some(idx) = find_closed_frontier_batch_in_danger(conn, danger_threshold)? { - return Ok(DangerStatus::ClosedBatchInDanger(idx)); - } - if let Some(idx) = find_tip_batch_in_danger(conn, danger_threshold)? { - return Ok(DangerStatus::TipInDanger(idx)); - } - - if protocol.clock_cannot_age_l1_view(safe_block_timestamp, now_ms) { - return Ok(DangerStatus::L1ViewStale); - } - let adjusted_danger_threshold = - match protocol.wall_clock_adjusted_danger_threshold(last_progress, now_ms) { - Ok(adjusted) => adjusted, - Err(_) => return Ok(DangerStatus::L1ViewStale), - }; - if let Some(adjusted) = adjusted_danger_threshold - && let Some(idx) = find_first_batch_in_danger(conn, adjusted)? - { - return Ok(DangerStatus::EstimatedBatchInDanger(idx)); - } - Ok(DangerStatus::Safe) -} - -fn refuse_divergence(danger: DangerStatus) -> std::result::Result<(), RecoveryMutationError> { - if let DangerStatus::CanonicalDivergence(nonce) = danger { - return Err(RecoveryMutationError::CanonicalDivergence { nonce }); - } - Ok(()) -} - -// ── Repair bodies, pub(super) for storage::test_helpers ─────────────────── - -/// Discard the entire non-accepted closed suffix after flush and caught-up Sync. -/// This is a convergence policy: replaced or never-submitted work could be -/// submitted fresh, but preserving it can re-enter the same danger/recovery cycle. -/// The caller must settle wallet slots and refresh acceptance through the flush -/// observation; the production guard checks that the local view caught up. -/// -/// If no closed pivot remains, only an aging Tip is invalidated. A closed batch -/// can have landed fresh while the Tip, even with the same first-frame clock, -/// has since crossed the danger threshold. Cascade and reopening share `tx`. -/// -/// Returns the newly-invalidated batch indices (empty if none). -pub(super) fn recover_post_flush_inner( - tx: &Transaction<'_>, - danger_threshold: u64, -) -> Result> { - // Path 1: any closed batch past gold cascades unconditionally. - let pivot = match first_non_gold_closed_batch(tx)? { - Some(batch_index) => Some(batch_index), - // All closed batches are accepted; the Tip can still have aged. - None => find_tip_batch_in_danger(tx, danger_threshold)?, - }; - cascade_and_reopen(tx, pivot) -} - -/// Discard only the aging Tip. It has no L1 footprint, so no flush is required. -/// Using `danger_threshold` avoids restarting with the same age that triggered -/// recovery; it is a policy threshold, not proof of canonical staleness. -/// The production caller rechecks the exact `TipInDanger` decision against the -/// current local inspection before entering this shared body. -/// -/// Returns the newly-invalidated batch indices (empty if Tip is fresh, -/// `[tip_index]` when the Tip was cascaded). -pub(super) fn recover_aging_tip_inner( - tx: &Transaction<'_>, - danger_threshold: u64, -) -> Result> { - let pivot = find_tip_batch_in_danger(tx, danger_threshold)?; - cascade_and_reopen(tx, pivot) -} - -/// Shared tail of both recovery paths — the pivot selection above is the -/// only thing that varies. In the caller's transaction: -/// -/// 1. **Cascade** from `pivot` (no-op when `None`): invalidate it and every -/// successor, including the open Tip. -/// 2. Snapshot selection excludes invalidated batches in the same committed -/// state. Accepted batch snapshots survive; before any acceptance the -/// baseline supplies the rollback-safe restore point. -/// 3. **Advance `RecoveryGeneration`** exactly once when the cascade -/// invalidated any valid batch, recording the surviving application count -/// before replacement directs can reuse its offsets. Cut, generation, and -/// invalidation remain inseparable across crashes. -/// 4. **Reopen the Tip** the cascade just invalidated (or one a torn crash -/// left missing), atomically with the cascade. Same mechanism the -/// runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. -fn cascade_and_reopen(tx: &Transaction<'_>, pivot: Option) -> Result> { - let invalidated = match pivot { - Some(batch_index) => cascade_invalidate_from(tx, batch_index)?, - None => Vec::new(), - }; - if !invalidated.is_empty() { - advance_recovery_generation_in(tx)?; - } - if !invalidated.is_empty() || !has_valid_open_batch(tx)? { - open_fresh_tip_in_tx(tx)?; - } - Ok(invalidated) -} - -/// First valid closed batch sitting at the gold frontier — i.e., with -/// `nonce >= frontier_nonce` (the next nonce the scheduler is expected to -/// accept). Used by [`recover_post_flush_inner`] as the cascade pivot, and -/// by [`find_closed_frontier_batch_in_danger`] as the candidate to age-check. -/// -/// `>=`, not `>`: `frontier_nonce` is the *next-expected* nonce -/// (`latest_accepted.nonce + 1`, or the anchor before any acceptance), so the -/// actual cascade-pivot batch carries `nonce == frontier_nonce`. Using `>` -/// would skip it. -/// -/// Valid-path nonce contiguity (I16) makes the first match exactly -/// `frontier_nonce`. Returns `None` if all closed batches are accepted. -fn first_non_gold_closed_batch(conn: &Connection) -> Result> { - let frontier = frontier_nonce(conn)?; - let batch_index: Option = conn - .query_row( - "SELECT batch_index FROM valid_closed_batches \ - WHERE nonce >= ?1 ORDER BY nonce ASC LIMIT 1", - rusqlite::params![u64_to_i64(frontier)], - |row| row.get(0), - ) - .optional()?; - Ok(batch_index.map(i64_to_u64)) -} - -/// Either the closed-frontier batch or the Tip, whichever (if either) has -/// aged past `threshold` against `current_safe_block`. Used by -/// [`Storage::check_danger`]'s wall-clock-adjusted arm, where the dispatch -/// is the same (`Retry`) regardless of which one fired. -/// -/// Closed-frontier wins: frame `safe_block`s are non-decreasing along the -/// spine, so an existing closed frontier is at least as old as the Tip. -/// -/// Reads `safe_accepted_batches`, which is maintained atomically with each -/// `Storage::append_ingested_safe_inputs_with_timestamp` call. -pub(super) fn find_first_batch_in_danger(conn: &Connection, threshold: u64) -> Result> { - if let Some(batch_index) = find_closed_frontier_batch_in_danger(conn, threshold)? { - return Ok(Some(batch_index)); - } - find_tip_batch_in_danger(conn, threshold) -} - -/// First valid closed batch past the gold frontier whose first frame is older -/// than `current_safe_block - threshold`. Returns `None` if no such batch -/// exists. -/// -/// First-frame clocks are non-decreasing along the valid path (I3), so the -/// earliest non-accepted closed batch is at least as old as its successors. -/// Checking younger batches cannot reveal danger that this check missed. -/// -/// Does NOT consider the Tip — the Tip has no L1 transaction, so it's not -/// part of the closed-frontier-staleness category. -/// [`find_first_batch_in_danger`] composes with [`find_tip_batch_in_danger`] -/// when callers want both. -pub(super) fn find_closed_frontier_batch_in_danger( - conn: &Connection, - threshold: u64, -) -> Result> { - match first_non_gold_closed_batch(conn)? { - Some(batch_index) => batch_in_danger(conn, batch_index, threshold), - None => Ok(None), - } -} - -/// The Tip (if any) whose first frame is older than -/// `current_safe_block - threshold`. Returns `None` if no Tip exists or it -/// isn't in danger yet. -fn find_tip_batch_in_danger(conn: &Connection, threshold: u64) -> Result> { - let tip_batch_index: Option = conn - .query_row("SELECT batch_index FROM valid_open_batch", [], |row| { - row.get(0) - }) - .optional()?; - match tip_batch_index { - Some(tip_batch_index) => batch_in_danger(conn, i64_to_u64(tip_batch_index), threshold), - None => Ok(None), - } -} - -/// Shared age-check used by the closed-frontier and Tip helpers. Returns -/// `Some(batch_index)` if `current_safe_block - first_frame.safe_block >= threshold`. -fn batch_in_danger(conn: &Connection, batch_index: u64, threshold: u64) -> Result> { - let first_frame_safe_block = first_frame_safe_block_of(conn, u64_to_i64(batch_index))?; - let safe_block = current_safe_block_required(conn)?; - Ok(age_exceeds(safe_block, first_frame_safe_block, threshold).then_some(batch_index)) -} - -/// `frames.safe_block` of the lowest `frame_in_batch` in `batch_index`. -/// -/// Every committed valid batch has a first frame. Missing one is an -/// invariant violation and propagates as `QueryReturnedNoRows`. -fn first_frame_safe_block_of(conn: &Connection, batch_index: i64) -> Result { - conn.query_row( - "SELECT safe_block FROM frames \ - WHERE batch_index = ?1 ORDER BY frame_in_batch ASC LIMIT 1", - params![batch_index], - |row| row.get::<_, i64>(0).map(i64_to_u64), - ) -} - -/// Cascade-invalidate all valid batches with `batch_index >= from_batch_index`. -/// -/// Reads the list BEFORE mutating — the SELECT must see the rows the UPDATE -/// will then mark invalid. The `invalidated_at_ms IS NULL` guard on the UPDATE -/// keeps this idempotent: rows already invalid are untouched. -fn cascade_invalidate_from(tx: &Transaction<'_>, from_batch_index: u64) -> Result> { - let from_i64 = u64_to_i64(from_batch_index); - - let invalidated: Vec = { - let mut stmt = tx.prepare( - "SELECT batch_index FROM valid_batches \ - WHERE batch_index >= ?1 ORDER BY batch_index ASC", - )?; - stmt.query_map(params![from_i64], |row| { - row.get::<_, i64>(0).map(i64_to_u64) - })? - .collect::>()? - }; - - if !invalidated.is_empty() { - let now_ms = now_unix_ms(); - tx.execute( - "UPDATE batches SET invalidated_at_ms = ?1 \ - WHERE batch_index >= ?2 AND invalidated_at_ms IS NULL", - params![now_ms, from_i64], - )?; - } - - Ok(invalidated) -} - -/// Check whether the DB has a valid Tip (`sealed_at_ms IS NULL AND -/// `invalidated_at_ms IS NULL`). -fn has_valid_open_batch(tx: &Connection) -> Result { - let count: i64 = tx.query_row("SELECT COUNT(*) FROM valid_open_batch", [], |row| { - row.get(0) - })?; - Ok(count > 0) -} - -#[cfg(test)] -mod tests; diff --git a/sequencer/src/storage/recovery/tests.rs b/sequencer/src/storage/recovery/tests.rs deleted file mode 100644 index a09f14c0..00000000 --- a/sequencer/src/storage/recovery/tests.rs +++ /dev/null @@ -1,2398 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -use super::super::test_helpers::{ - RecoveryFixture, SENDER_A, all_ordered_l2_txs, default_protocol_timing, local_batch_payload, - make_stale_batch_payload, open_temp_storage, seed_closed_batches, - seed_safe_inputs_with_batch_nonces, -}; -use super::{find_closed_frontier_batch_in_danger, find_first_batch_in_danger}; -use crate::storage::{DirectInputExecution, ExecutedInputCount, SafeInputRange, Storage}; -use alloy_primitives::Address; -use sequencer_core::l2_tx::SequencedL2Tx; -use sequencer_core::protocol::ProtocolTiming; - -fn generation_cuts(storage: &Storage) -> Vec<(i64, i64)> { - storage - .conn - .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") - .unwrap() - .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) - .unwrap() - .collect::>() - .unwrap() -} - -mod guarded_phases { - use super::*; - use crate::storage::RecoveryMutationError; - - fn invalidated_count(storage: &Storage) -> u64 { - storage - .conn - .query_row( - "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", - [], - |row| row.get::<_, i64>(0), - ) - .expect("count invalidated batches") as u64 - } - - fn open_tip_count(storage: &Storage) -> u64 { - storage - .conn - .query_row("SELECT COUNT(*) FROM valid_open_batch", [], |row| { - row.get::<_, i64>(0) - }) - .expect("count open Tips") as u64 - } - - fn ensure_tip_fixture( - name: &str, - with_recovery_checkpoint: bool, - ) -> (crate::test_support::TestDb, Storage, ProtocolTiming) { - let (db, mut storage) = open_temp_storage(name); - let protocol = default_protocol_timing(); - let now_ms = crate::clock::unix_now_ms(); - storage - .append_safe_inputs_with_timestamp( - 0, - now_ms / 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("seed fresh safe head"); - if with_recovery_checkpoint { - let prefix = db._dir.path().join("finalized"); - storage - .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) - .expect("seed finalized snapshot"); - } - (db, storage, protocol) - } - - #[test] - fn ensure_tip_guard_opens_one_tip_only_from_clean_facts() { - let (_db, mut storage, protocol) = ensure_tip_fixture("guarded-ensure-tip", true); - assert_eq!(open_tip_count(&storage), 0); - - storage - .ensure_open_tip_for_recovery(&protocol, crate::clock::unix_now_ms()) - .expect("clean no-Tip facts authorize one Tip"); - assert_eq!(open_tip_count(&storage), 1); - } - - #[test] - fn ensure_tip_guard_refuses_missing_snapshot_without_mutation() { - let (_db, mut storage, protocol) = - ensure_tip_fixture("guarded-ensure-tip-no-snapshot", false); - storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("a stale phase opened the Tip"); - - let error = storage - .ensure_open_tip_for_recovery(&protocol, crate::clock::unix_now_ms()) - .expect_err("missing finalized state must outrank the stale Tip decision"); - assert!(matches!( - error, - RecoveryMutationError::MissingRecoveryCheckpoint - )); - assert_eq!(open_tip_count(&storage), 1, "the existing Tip is untouched"); - } - - #[test] - fn ensure_tip_guard_refuses_divergence_without_mutation() { - let (_db, mut storage, protocol) = - ensure_tip_fixture("guarded-ensure-tip-divergence", true); - crate::storage::test_helpers::record_canonical_divergence(&mut storage, 7, 0); - - let error = storage - .ensure_open_tip_for_recovery(&protocol, crate::clock::unix_now_ms()) - .expect_err("divergence must outrank Tip creation"); - assert!(matches!( - error, - RecoveryMutationError::CanonicalDivergence { nonce: 7 } - )); - assert_eq!(open_tip_count(&storage), 0); - } - - #[test] - fn ensure_tip_guard_rejects_an_already_open_tip_without_duplication() { - let (_db, mut storage, protocol) = ensure_tip_fixture("guarded-ensure-tip-existing", true); - storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("a competing phase opened the Tip"); - - let error = storage - .ensure_open_tip_for_recovery(&protocol, crate::clock::unix_now_ms()) - .expect_err("an unexpected open Tip must be rejected"); - assert!(matches!(error, RecoveryMutationError::TipAlreadyOpen)); - assert_eq!(open_tip_count(&storage), 1); - } - - #[test] - fn ensure_tip_refuses_and_rolls_back_if_its_transaction_leaves_no_tip() { - // The fixture seeds no safe inputs on purpose: with a non-empty drain - // range the invalidated Tip would trip the sequenced-rows trigger - // first and the opener would fail with a storage error instead of - // returning `Ok` — which is the exact shape this test needs. - let (db, storage, protocol) = ensure_tip_fixture("guarded-ensure-tip-postcondition", true); - // Fault injection: invalidate the new Tip the moment its first frame - // lands, so `open_fresh_tip_in_tx` returns `Ok` having left no valid - // open Tip. (Invalidating on the batch insert itself is caught one - // step earlier by the schema — "frames can only be inserted into the - // current Tip" — so the injection must fire after the frame. - // `open.rs` documents failure triggers as the sanctioned way past the - // typed API in tests.) - drop(storage); - Storage::open_connection(db.path.as_str()) - .expect("raw connection") - .execute_batch( - "CREATE TRIGGER test_invalidate_new_tip AFTER INSERT ON frames \ - BEGIN \ - UPDATE batches SET invalidated_at_ms = 1 \ - WHERE batch_index = NEW.batch_index; \ - END", - ) - .expect("install failure trigger"); - let mut storage = Storage::open(db.path.as_str()).expect("reopen"); - - let error = storage - .ensure_open_tip_for_recovery(&protocol, crate::clock::unix_now_ms()) - .expect_err("a phase that leaves no Tip must refuse, not commit"); - assert!( - matches!(error, RecoveryMutationError::TipMissingAfterOpen), - "expected the postcondition refuse, got {error:?}" - ); - assert_eq!(open_tip_count(&storage), 0); - // Refused inside the transaction: nothing of the attempt persisted. - let batches: i64 = storage - .read(|tx| tx.query_row("SELECT COUNT(*) FROM batches", [], |row| row.get(0))) - .expect("count batches"); - assert_eq!(batches, 0, "the refused phase must roll back its insert"); - } - - fn cascade_fixture(name: &str) -> (crate::test_support::TestDb, Storage) { - let (db, mut storage) = open_temp_storage(name); - let protocol = default_protocol_timing(); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize Tip"); - storage - .close_batch_at(&mut head, 10) - .expect("close cascade candidate"); - storage - .append_safe_inputs_with_timestamp( - 1_200, - 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("advance safe head"); - let prefix = db._dir.path().join("finalized"); - storage - .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) - .expect("seed finalized snapshot"); - (db, storage) - } - - #[test] - fn cascade_guard_ranks_divergence_before_f2_and_mutates_nothing() { - let (_db, mut storage) = cascade_fixture("guarded-cascade-divergence"); - let protocol = default_protocol_timing(); - crate::storage::test_helpers::record_canonical_divergence(&mut storage, 0, 0); - - let error = storage - .recover_post_flush_for_recovery(1_201, &protocol, crate::clock::unix_now_ms()) - .expect_err("divergence must refuse before lag"); - assert!(matches!( - error, - RecoveryMutationError::CanonicalDivergence { nonce: 0 } - )); - assert_eq!(invalidated_count(&storage), 0); - } - - #[test] - fn cascade_guard_rejects_lagging_sync_without_tree_mutation() { - let (_db, mut storage) = cascade_fixture("guarded-cascade-f2"); - let protocol = default_protocol_timing(); - - let error = storage - .recover_post_flush_for_recovery(1_201, &protocol, crate::clock::unix_now_ms()) - .expect_err("lagging post-flush view must retry"); - assert!(matches!( - error, - RecoveryMutationError::ResyncBehindFlushView { - resynced_safe_block: 1_200, - flush_observed_safe_block: 1_201, - } - )); - assert_eq!(invalidated_count(&storage), 0); - } - - #[test] - fn cascade_guard_ranks_missing_snapshot_before_f2_without_mutation() { - let (_db, mut storage) = cascade_fixture("guarded-cascade-no-snapshot"); - let protocol = default_protocol_timing(); - storage - .conn - .execute("DELETE FROM snapshots WHERE batch_index IS NULL", []) - .expect("remove finalized snapshot fact"); - - let error = storage - .recover_post_flush_for_recovery(1_201, &protocol, crate::clock::unix_now_ms()) - .expect_err("missing finalized state must outrank a lagging view"); - assert!(matches!( - error, - RecoveryMutationError::MissingRecoveryCheckpoint - )); - assert_eq!(invalidated_count(&storage), 0); - } - - #[test] - fn tip_guard_reasserts_the_exact_reducer_decision() { - let (db, mut storage) = open_temp_storage("guarded-tip-decision"); - let protocol = default_protocol_timing(); - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize Tip"); - storage - .append_safe_inputs_with_timestamp( - 1_200, - 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("advance safe head"); - let prefix = db._dir.path().join("finalized"); - storage - .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) - .expect("seed finalized snapshot"); - - let error = storage - .recover_aging_tip_for_recovery(1, &protocol, 1_000_000) - .expect_err("a stale expected Tip must not mutate"); - assert!(matches!( - error, - RecoveryMutationError::StaleDecision { - expected: crate::storage::DangerStatus::TipInDanger(1), - actual: crate::storage::DangerStatus::TipInDanger(0), - } - )); - assert_eq!(invalidated_count(&storage), 0); - } - - #[test] - fn tip_guard_refuses_missing_snapshot_without_tree_mutation() { - let (_db, mut storage) = open_temp_storage("guarded-tip-no-snapshot"); - let protocol = default_protocol_timing(); - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize Tip"); - storage - .append_safe_inputs_with_timestamp( - 1_200, - 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("advance safe head"); - - let error = storage - .recover_aging_tip_for_recovery(0, &protocol, 1_000_000) - .expect_err("missing finalized state must refuse Tip recovery"); - assert!(matches!( - error, - RecoveryMutationError::MissingRecoveryCheckpoint - )); - assert_eq!(invalidated_count(&storage), 0); - } -} - -mod invalid_batches { - use super::*; - - // ── invalid_batches filtering ────────────────────────────────────── - - #[test] - fn invalid_batches_excluded_from_latest_batch_index() { - let (_db, mut storage) = open_temp_storage("invalid-latest-batch"); - seed_closed_batches(&mut storage, 3); - assert_eq!( - storage.latest_batch_index().expect("latest").unwrap(), - 3, - "open batch should be 3" - ); - - storage.insert_invalid_batch(3).expect("mark invalid"); - assert_eq!(storage.latest_batch_index().expect("latest").unwrap(), 2,); - - storage.insert_invalid_batch(2).expect("mark invalid"); - assert_eq!(storage.latest_batch_index().expect("latest").unwrap(), 1,); - } - - #[test] - fn invalid_batches_excluded_from_ordered_l2_txs() { - let (_db, mut storage) = open_temp_storage("invalid-ordered-txs"); - - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.land(10, &[(Address::ZERO, vec![0xaa])]); - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) - .expect("close frame"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - storage.land(20, &[(Address::ZERO, vec![0xbb])]); - storage - .close_frame_only(&mut head, 20, SafeInputRange::new(1, 2)) - .expect("close frame"); - - let all = all_ordered_l2_txs(&mut storage); - assert_eq!(all.len(), 2); - - storage - .insert_invalid_batch(1) - .expect("invalidate the suffix"); - - let filtered = all_ordered_l2_txs(&mut storage); - assert_eq!(filtered.len(), 1); - match &filtered[0] { - SequencedL2Tx::Direct(d) => assert_eq!(d.payload.as_slice(), &[0xaa]), - _ => panic!("expected direct input"), - } - } - - #[test] - fn invalid_batches_excluded_from_ordered_l2_txs_for_batch() { - let (_db, mut storage) = open_temp_storage("invalid-ordered-for-batch"); - - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.land(10, &[(Address::ZERO, vec![0xaa])]); - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) - .expect("close frame"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - let txs = storage.ordered_l2_txs_for_batch(0).expect("load batch 0"); - assert_eq!(txs.len(), 1); - - storage.insert_invalid_batch(0).expect("mark invalid"); - let txs = storage - .ordered_l2_txs_for_batch(0) - .expect("load batch 0 after invalidation"); - assert!(txs.is_empty(), "invalid batch should return no txs"); - } - - #[test] - fn invalid_batches_excluded_from_drained_direct_count() { - let (_db, mut storage) = open_temp_storage("invalid-drained-count"); - - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.land( - 10, - &[(Address::ZERO, vec![0xaa]), (Address::ZERO, vec![0xbb])], - ); - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 2)) - .expect("close frame"); - assert_eq!( - storage.next_undrained_safe_input_index().expect("cursor"), - 2 - ); - - storage.insert_invalid_batch(0).expect("mark invalid"); - assert_eq!( - storage - .next_undrained_safe_input_index() - .expect("cursor after invalidation"), - 0 - ); - } -} - -mod recover_post_flush { - use super::*; - - // ── recover_post_flush: cascade from first non-gold ──────────────── - // - // These tests simulate the post-flush state directly: append safe-inputs - // to drive `populate_safe_accepted_batches`, then call - // `recover_post_flush()`. The cascade is unconditional (no threshold) — - // any closed batch past the gold frontier is doomed and gets cascaded - // along with everything after it. - - #[test] - fn cascades_from_stale() { - let (_db, mut storage) = open_temp_storage("detect-stale"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - for _ in 0..3 { - storage.close_batch_at(&mut head, 10).expect("close batch"); - } - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let invalidated = storage - .recover_post_flush(1200) - .expect("detect and recover"); - assert_eq!(invalidated, vec![0, 1, 2, 3]); - - let head = storage.open_state().expect("load open state"); - assert!(head.is_some(), "recovery should have opened a fresh batch"); - assert_eq!(head.unwrap().batch_index, 4); - } - - #[test] - fn is_idempotent() { - let (_db, mut storage) = open_temp_storage("detect-idempotent"); - let initial_history = storage.history_state().expect("initial history"); - assert_eq!(initial_history.version.recovery_generation.get(), 0); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.close_batch_at(&mut head, 10).expect("close batch"); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let first = storage.recover_post_flush(1200).expect("first detect"); - assert_eq!(first, vec![0, 1]); - assert_eq!( - generation_cuts(&storage), - [(1, 0)], - "empty invalidated batches still record the old head" - ); - assert_eq!( - storage - .history_state() - .expect("history after recovery") - .version - .recovery_generation - .get(), - 1, - "one invalidating recovery advances exactly once" - ); - assert_eq!( - storage - .history_state() - .expect("history after recovery") - .version - .era_id, - initial_history.version.era_id, - "standard recovery stays within the same era" - ); - - let second = storage.recover_post_flush(1200).expect("second detect"); - assert!(second.is_empty()); - assert_eq!( - generation_cuts(&storage), - [(1, 0)], - "a no-op must not add a cut" - ); - assert_eq!( - storage - .history_state() - .expect("history after no-op") - .version - .recovery_generation - .get(), - 1, - "a recovery no-op must not invent a new history reality" - ); - } - - #[test] - fn no_op_when_recovery_batch_landed_fresh_in_new_generation() { - // Post-flush contract: `recover_post_flush` is called only after the - // gold frontier has been re-synced. If the recovery batch from a - // previous cycle landed on L1 fresh (within MAX_WAIT) and was - // accepted by `populate_safe_accepted_batches`, the new generation - // is gold and a fresh `recover_post_flush` call should be a no-op — - // the stale ancestor's safe-input must not false-match the - // nonce-reused gen-2 batch. - let (_db, mut storage) = open_temp_storage("post-flush-no-op-after-fresh-gen2"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - // Gen 1 cascade. Stale batch 0 + open Tip 1 invalidated; recovery - // batch 2 opened with nonce reused (= 0). - let first = storage.recover_post_flush(1200).expect("gen1 recovery"); - assert_eq!(first, vec![0, 1]); - assert_eq!( - storage - .history_state() - .expect("generation one") - .version - .recovery_generation - .get(), - 1 - ); - - // Submitter posts the recovery batch; it lands fresh on L1. - let mut head = storage.open_state().expect("load").unwrap(); - storage - .close_batch_at(&mut head, 1300) - .expect("close recovery batch"); - let landed = local_batch_payload(&mut storage, 0); - storage.land(1310, &[(SENDER_A, landed)]); // append fresh safe input for recovery batch - - // populate_safe_accepted_batches accepts the gen-2 batch (nonce 0 at - // first_frame=1300, inclusion=1310 → inclusion-staleness 10 < MAX_WAIT). - // Gold advances. A fresh recover_post_flush is a no-op: the only - // closed-non-gold candidate would have to be past the new frontier, - // but everything is gold. - let second = storage.recover_post_flush(1200).expect("post-flush no-op"); - assert!( - second.is_empty(), - "fresh gen-2 batch must be gold; old stale row must not false-match \ - via reused nonce, got: {second:?}" - ); - } - - #[test] - fn detects_stale_reused_nonce_in_new_generation() { - let (_db, mut storage) = open_temp_storage("detect-reused-stale"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - // Append gen1 stale safe input. - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let first = storage.recover_post_flush(1200).expect("gen1 recovery"); - assert_eq!(first, vec![0, 1]); - - let mut head = storage.open_state().expect("load").unwrap(); - storage - .close_batch_at(&mut head, 1210) - .expect("close gen2 batch"); - - // Append gen2 stale safe input. - storage.land(2410, &[(SENDER_A, make_stale_batch_payload(0, 1210))]); - let second = storage.recover_post_flush(1200).expect("gen2 recovery"); - assert_eq!( - second, - vec![2, 3], - "stale reused nonce in gen2 must still be detected" - ); - assert_eq!( - storage - .history_state() - .expect("generation two") - .version - .recovery_generation - .get(), - 2 - ); - } - - #[test] - fn cascades_aging_tip_when_no_closed_pivot_exists() { - // Regression for the no-pivot Tip case: the closed batch landed fresh - // (becomes gold) but the Tip's `first_frame.safe_block` matches the - // closed batch's, and after the flush wait the current safe block has - // pushed the Tip's age into the danger zone. Pure monotonicity - // (`S_tip >= S_closed`) doesn't rule this out — equality is allowed - // when the lane rotates without a safe-block advance between frames. - // - // Earlier `recover_post_flush` only checked the first non-gold closed - // batch; on no closed pivot it became a no-op, leaving the stale Tip - // for the next danger trip to catch. Fix: fall through to a Tip - // check against `danger_threshold` and cascade the Tip directly. - let (_db, mut storage) = open_temp_storage("post-flush-tip-no-pivot"); - - // S_closed = S_tip = 10 (lane rotated without a safe-block advance). - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0; open Tip with same safe_block=10"); - - // Closed batch 0 landed fresh: inclusion_block=200 → inclusion-staleness 190 - // is well below MAX_WAIT, so populate accepts it as gold. - // Then advance safe head to 1100: batch 0 age (gold) is irrelevant, - // but Tip's age = 1100 - 10 = 1090 > danger_threshold (let's pass 1000). - let landed = local_batch_payload(&mut storage, 0); - storage.land(200, &[(SENDER_A, landed)]); // append fresh safe input — batch 0 becomes gold - storage.advance_safe_head(1100); // advance safe head past Tip's danger threshold - - // Sanity: no closed batch is past gold (all gold). - assert_eq!( - find_closed_frontier_batch_in_danger(&storage.conn, 900).expect("strict check"), - None, - "all closed batches gold; closed-frontier check returns None" - ); - - // recover_post_flush(danger_threshold=1000): Tip's age (1090) > 1000. - // Fall-through Tip check fires; Tip cascaded; recovery batch opened. - let invalidated = storage - .recover_post_flush(1000) - .expect("recover_post_flush with danger_threshold=1000"); - assert_eq!( - invalidated, - vec![1], - "Tip (batch 1) cascaded via danger_threshold fall-through" - ); - - // A fresh recovery Tip exists with a higher batch_index. - let head = storage.open_state().expect("load").expect("recovery Tip"); - assert_eq!(head.batch_index, 2); - } - - #[test] - fn no_op_when_tip_age_below_danger_threshold() { - // Negative companion to the above: closed batch is gold, Tip exists, - // but Tip's age is below `danger_threshold`. Cascade must not fire. - let (_db, mut storage) = open_temp_storage("post-flush-tip-below-danger"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - let landed = local_batch_payload(&mut storage, 0); - storage.land(200, &[(SENDER_A, landed)]); // append fresh safe input — batch 0 becomes gold - // current=500: Tip's age = 490, below threshold 1000. - storage.advance_safe_head(500); // advance safe head, but not past danger threshold - - let invalidated = storage - .recover_post_flush(1000) - .expect("recover_post_flush with fresh Tip"); - assert!( - invalidated.is_empty(), - "Tip below danger_threshold must not be cascaded, got: {invalidated:?}" - ); - - // Tip preserved. - let head = storage.open_state().expect("load").expect("Tip"); - assert_eq!(head.batch_index, 1); - } -} - -mod tip_staleness { - use super::*; - - // ── Tip staleness — `recover_aging_tip` and post-flush combined cases ── - // - // `recover_aging_tip` backs the RecoverTip startup action: all closed - // batches are outside the danger zone, but the open Tip has aged past the - // configured threshold. The Tip has no L1 footprint (no wallet nonce, no - // `safe_input`), so it can be invalidated directly without a flush. - // - // The first four tests cover that path: - // - positive: Tip IS stale → invalidated - // - negative: Tip is fresh → NOT invalidated (no false positives) - // - boundary at threshold: invalidated - // - boundary just below threshold: not invalidated - // - // The remaining tests cover the guarded post-flush Cascade phase's combined - // closed+open behavior (`recover_post_flush`'s `batch_index >= N` - // cascade rule catches the Tip too). - - #[test] - fn open_batch_stale_by_current_safe_block_is_invalidated() { - // Scenario: sequencer opened batch 0 at safe_block=10, never closed it, - // then stayed down until safe advanced to 1500 (>= the 1200-block threshold - // past safe_block). - // Recovery must invalidate the open batch. - let (_db, mut storage) = open_temp_storage("open-batch-stale"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize open state at safe_block=10"); - - // Advance the safe head so the open batch's first frame (safe_block=10) - // is now stale: 1500 - 10 >= 1200. - storage.advance_safe_head(1500); // advance safe head past the 1200-block danger threshold - - let invalidated = storage - .recover_aging_tip(1200) - .expect("recover from stale open batch"); - assert_eq!( - invalidated, - vec![0], - "open batch 0 should be invalidated by current staleness" - ); - assert_eq!( - storage - .history_state() - .expect("history after Tip recovery") - .version - .recovery_generation - .get(), - 1, - "the direct RecoverTip path advances the shared generation boundary" - ); - - // A fresh recovery batch must be opened at batch_index=1. - let head = storage.open_state().expect("load").expect("head"); - assert_eq!(head.batch_index, 1, "recovery batch is the next index"); - } - - #[test] - fn open_batch_not_yet_stale_is_not_invalidated() { - // Negative: open batch's first frame safe_block=10 with current safe=1100. - // 1100 - 10 = 1090 < 1200. Must NOT cascade. - // Catches false-positive regressions in `recover_aging_tip`. - let (_db, mut storage) = open_temp_storage("open-batch-fresh"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize open state at safe_block=10"); - - storage.advance_safe_head(1100); // advance safe head below threshold - - let invalidated = storage - .recover_aging_tip(1200) - .expect("recover with non-stale open batch"); - assert!( - invalidated.is_empty(), - "fresh open batch must not be cascade-invalidated, got: {invalidated:?}" - ); - - // The open batch must still be the live one (no recovery batch opened). - let head = storage.open_state().expect("load").expect("head"); - assert_eq!( - head.batch_index, 0, - "original open batch 0 must still be the head" - ); - } - - #[test] - fn open_batch_exactly_at_threshold_is_invalidated() { - // Boundary: 1210 - 10 = 1200, which is >= the danger_threshold argument (1200). - // The staleness comparison is `>=`, so this must invalidate. - let (_db, mut storage) = open_temp_storage("open-batch-boundary"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - - storage.advance_safe_head(1210); // advance safe head to exact threshold - - let invalidated = storage.recover_aging_tip(1200).expect("recover"); - assert_eq!(invalidated, vec![0], "boundary (>= threshold) invalidates"); - } - - #[test] - fn open_batch_one_block_below_threshold_is_not_invalidated() { - // Boundary: 1209 - 10 = 1199 < 1200. One-block margin must NOT invalidate. - let (_db, mut storage) = open_temp_storage("open-batch-below-boundary"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - - storage.advance_safe_head(1209); // advance safe head to one block below threshold - - let invalidated = storage.recover_aging_tip(1200).expect("recover"); - assert!( - invalidated.is_empty(), - "one-block-below-threshold must not invalidate, got: {invalidated:?}" - ); - } - - #[test] - fn closed_unsubmitted_stale_and_open_stale_both_cascade() { - // Scenario: batch 0 is closed and nonced but never submitted to L1 - // (safe_accepted_batches is empty). Batch 1 is open and also stale. - // `find_first_batch_in_danger` should return closed batch 0 at the - // frontier (nonce 0, no acceptance yet) and cascade through batch 1. - let (_db, mut storage) = open_temp_storage("closed-unsubmitted-and-open-stale"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize at safe_block=10"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - // Advance safe head so batch 0's first frame (safe_block=10) is stale. - storage.advance_safe_head(1500); // advance safe head past staleness - - let invalidated = storage.recover_post_flush(1200).expect("recover"); - assert_eq!( - invalidated, - vec![0, 1], - "closed unsubmitted batch 0 and subsequent open batch 1 cascade together" - ); - } - - #[test] - fn opens_batch_after_torn_invalidation() { - let (_db, mut storage) = open_temp_storage("detect-torn"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - storage.insert_invalid_batch(0).expect("invalidate 0"); - storage.insert_invalid_batch(1).expect("invalidate 1"); - storage.advance_safe_head(10); // record observed safe head - - let invalidated = storage - .recover_post_flush(1200) - .expect("recover from torn state"); - assert!(invalidated.is_empty(), "no new invalidations"); - assert_eq!( - storage - .history_state() - .expect("history after structural repair") - .version - .recovery_generation - .get(), - 0, - "opening a missing Tip without invalidating history is not a recovery generation" - ); - assert!(generation_cuts(&storage).is_empty()); - - let head = storage.open_state().expect("load open state"); - assert!(head.is_some(), "recovery should have opened a fresh batch"); - assert_eq!(head.unwrap().batch_index, 2); - } - - #[test] - fn rolls_back_when_cascade_update_aborts() { - let (db, mut storage) = open_temp_storage("detect-cascade-abort"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - // Advance safe head so batch 0's first frame (safe_block=10) is stale. - storage.advance_safe_head(1500); // advance safe head past staleness - - storage - .conn - .execute_batch( - "CREATE TRIGGER fail_cascade_invalidation - AFTER UPDATE OF invalidated_at_ms ON batches - WHEN NEW.invalidated_at_ms IS NOT NULL - AND OLD.invalidated_at_ms IS NULL - BEGIN - SELECT RAISE(ABORT, 'injected cascade failure'); - END;", - ) - .expect("install failure trigger"); - - let err = storage - .recover_post_flush(1200) - .expect_err("trigger should abort recovery transaction"); - assert!( - err.to_string().contains("injected cascade failure"), - "unexpected error: {err:?}" - ); - drop(storage); - - let conn = Storage::open_connection(db.path.as_str()).expect("open read conn"); - let invalidated_count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", - [], - |row| row.get(0), - ) - .expect("count invalidated"); - assert_eq!( - invalidated_count, 0, - "failed cascade must not persist torn invalidation state" - ); - - let batch_count: i64 = conn - .query_row("SELECT COUNT(*) FROM batches", [], |row| row.get(0)) - .expect("count batches"); - assert_eq!( - batch_count, 2, - "failed recovery must not open an extra batch" - ); - - let open_batch_index: i64 = conn - .query_row("SELECT batch_index FROM valid_open_batch", [], |row| { - row.get(0) - }) - .expect("query valid open batch"); - assert_eq!( - open_batch_index, 1, - "failed recovery must leave the original Tip in place" - ); - } - - #[test] - fn rolls_back_generation_and_invalidation_when_tip_reopen_aborts() { - let (_db, mut storage) = open_temp_storage("detect-tip-reopen-abort"); - - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.land(10, &[(Address::ZERO, vec![0xd1])]); // append mapped direct - storage - .close_frame_only_with_executions( - &mut head, - 10, - SafeInputRange::new(0, 1), - &[DirectInputExecution { - safe_input_index: 0, - executed_input_offset: ExecutedInputCount::ZERO, - }], - ) - .expect("attribute mapped direct"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - assert_eq!( - storage - .next_executed_input_count() - .expect("pre-recovery history head"), - ExecutedInputCount::new(1) - ); - storage.advance_safe_head(1500); // advance safe head past staleness - - storage - .conn - .execute_batch( - "CREATE TRIGGER fail_recovery_tip_insert - BEFORE INSERT ON batches - BEGIN - SELECT RAISE(ABORT, 'injected Tip reopen failure'); - END;", - ) - .expect("install failure trigger"); - - let err = storage - .recover_post_flush(1200) - .expect_err("Tip reopen failure must abort the whole recovery transaction"); - assert!( - err.to_string().contains("injected Tip reopen failure"), - "unexpected error: {err:?}" - ); - assert_eq!( - storage - .history_state() - .expect("history after rollback") - .version - .recovery_generation - .get(), - 0, - "the generation bump must roll back with the failed Tip reopen" - ); - assert!( - generation_cuts(&storage).is_empty(), - "the pre-reopen cut must roll back too" - ); - let invalidated_count: i64 = storage - .conn - .query_row( - "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", - [], - |row| row.get(0), - ) - .expect("count invalidated"); - assert_eq!( - invalidated_count, 0, - "the cascade must roll back with the generation bump" - ); - let mappings: Vec = storage - .conn - .prepare("SELECT offset FROM application_inputs ORDER BY offset") - .expect("prepare mapping query") - .query_map([], |row| row.get(0)) - .expect("query mappings") - .collect::>() - .expect("collect mappings"); - assert_eq!( - mappings, - vec![0], - "trigger-deleted mappings must roll back with the failed recovery" - ); - assert_eq!( - storage - .next_executed_input_count() - .expect("history head after rollback"), - ExecutedInputCount::new(1), - "the canonical history head must roll back with its mapping" - ); - let open_batch_index: i64 = storage - .conn - .query_row("SELECT batch_index FROM valid_open_batch", [], |row| { - row.get(0) - }) - .expect("query original Tip"); - assert_eq!(open_batch_index, 1); - } - - #[test] - fn recovery_redrains_direct_inputs_and_replay_sees_them_once() { - let (_db, mut storage) = open_temp_storage("recovery-redrain-e2e"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.land( - 10, - &[(Address::ZERO, vec![0xd1]), (Address::ZERO, vec![0xd2])], - ); // two deposits - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 2)) - .expect("close frame with deposits"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - let before = all_ordered_l2_txs(&mut storage); - assert_eq!(before.len(), 2, "both deposits should be visible"); - - let batch_submitter = Address::repeat_byte(0xAA); - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let invalidated = storage - .recover_post_flush(1200) - .expect("detect and recover"); - assert!(!invalidated.is_empty(), "should have invalidated batches"); - assert_eq!( - generation_cuts(&storage), - [(1, 0)], - "replacement directs must not enlarge the preserved prefix" - ); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::new(2) - ); - - let after = all_ordered_l2_txs(&mut storage); - let direct_payloads: Vec<&[u8]> = after - .iter() - .filter_map(|tx| match tx { - SequencedL2Tx::Direct(d) if d.sender != batch_submitter => { - Some(d.payload.as_slice()) - } - _ => None, - }) - .collect(); - assert_eq!( - direct_payloads, - vec![&[0xd1][..], &[0xd2][..]], - "deposits must appear exactly once in replay after recovery" - ); - - let recovery_batch = storage.open_state().expect("load").unwrap(); - let recovery_txs = storage - .ordered_l2_txs_for_batch(recovery_batch.batch_index) - .expect("load recovery batch txs"); - let recovery_direct_count = recovery_txs - .iter() - .filter(|tx| matches!(tx, SequencedL2Tx::Direct(d) if d.sender != batch_submitter)) - .count(); - assert_eq!( - recovery_direct_count, 2, - "both deposits should be in the recovery batch" - ); - } - - #[test] - fn undrained_safe_input_appears_in_recovery_batch_first_frame() { - // a deposit ingested into safe_inputs but not yet drained - // into any frame must be sequenced into the recovery batch's first - // frame after cascade. Complements - // `recovery_redrains_direct_inputs_and_replay_sees_them_once` (re-drain from - // invalidated) with the never-drained case. - let (_db, mut storage) = open_temp_storage("recovery-includes-undrained"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0 with no deposits"); - - let non_submitter = Address::repeat_byte(0xCC); - storage.land(20, &[(non_submitter, vec![0xde, 0xad])]); // append undrained deposit - let before = all_ordered_l2_txs(&mut storage); - assert!( - before.iter().all(|tx| !matches!( - tx, - SequencedL2Tx::Direct(d) if d.sender == non_submitter - )), - "undrained deposit must not be sequenced before drain", - ); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let invalidated = storage.recover_post_flush(1200).expect("recover"); - assert!(!invalidated.is_empty(), "stale batch must cascade"); - - let recovery = storage.open_state().expect("load").unwrap(); - let recovery_txs = storage - .ordered_l2_txs_for_batch(recovery.batch_index) - .expect("load recovery batch txs"); - let deposit_payloads: Vec<&[u8]> = recovery_txs - .iter() - .filter_map(|tx| match tx { - SequencedL2Tx::Direct(d) if d.sender == non_submitter => Some(d.payload.as_slice()), - _ => None, - }) - .collect(); - assert_eq!( - deposit_payloads, - vec![&[0xde, 0xad][..]], - "undrained deposit must land in the recovery batch's first frame", - ); - } - - #[test] - fn recovery_batch_opens_empty_when_no_direct_inputs_pending() { - // no drained-into-invalidated inputs AND no undrained safe - // inputs → recovery batch opens with an empty first frame (aside - // from the batch-submitter's own self-submission, which is drained - // but carries no user-visible payload). - let (_db, mut storage) = open_temp_storage("recovery-empty-first-frame"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - let batch_submitter = SENDER_A; - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let invalidated = storage.recover_post_flush(1200).expect("recover"); - assert_eq!(invalidated, vec![0, 1]); - - let recovery = storage.open_state().expect("load").unwrap(); - let recovery_txs = storage - .ordered_l2_txs_for_batch(recovery.batch_index) - .expect("load recovery batch txs"); - let user_visible: Vec<_> = recovery_txs - .iter() - .filter(|tx| match tx { - SequencedL2Tx::Direct(d) => d.sender != batch_submitter, - SequencedL2Tx::UserOp(_) => true, - }) - .collect(); - assert!( - user_visible.is_empty(), - "recovery batch must have no deposits or user-ops when none were pending: {user_visible:?}", - ); - } - - #[test] - fn first_batch_stale_recovery_reuses_nonce_zero() { - // first-ever batch (nonce 0) goes stale before reaching - // Gold. Cascade invalidates it; recovery opens a fresh batch that - // reuses nonce 0 (no valid ancestor exists to advance the nonce). - let (db, mut storage) = open_temp_storage("first-batch-stale-nonce-zero"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0 (nonce 0)"); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let invalidated = storage.recover_post_flush(1200).expect("recover"); - assert_eq!( - invalidated, - vec![0, 1], - "closed batch 0 and open batch 1 must both invalidate", - ); - - let recovery = storage.open_state().expect("load").unwrap(); - assert_eq!(recovery.batch_index, 2, "batch_index is monotonic (PK)"); - drop(storage); - - // Read the new Tip's nonce and parent pointer via raw SQL — no - // public accessor surfaces them. - let conn = Storage::open_connection(db.path.as_str()).expect("open read conn"); - let recovery_i64 = recovery.batch_index as i64; - let nonce: i64 = conn - .query_row( - "SELECT nonce FROM batches WHERE batch_index = ?1", - [recovery_i64], - |row| row.get(0), - ) - .expect("query nonce"); - assert_eq!( - nonce, 0, - "recovery batch must reuse nonce 0 after torn cascade", - ); - let parent: Option = conn - .query_row( - "SELECT parent_batch_index FROM batches WHERE batch_index = ?1", - [recovery_i64], - |row| row.get(0), - ) - .expect("query parent"); - assert_eq!( - parent, None, - "torn recovery has no valid ancestor; parent_batch_index is NULL", - ); - } - - #[test] - fn after_post_recovery_crash_is_no_op() { - // simulate a crash AFTER open_recovery_batch has run. On - // restart, the state contains a valid open recovery batch (no stale - // tail remains). A fresh `recover_post_flush` call must be a no-op: - // no new invalidations, and the same recovery batch remains the Tip. - // - // Distinct from `is_idempotent` (idempotent back-to-back call on the - // same Storage handle): this test drops and reopens Storage to model - // a full restart over the persisted DB. - let (db, mut storage) = open_temp_storage("post-recovery-crash-idempotent"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - // First call: full recovery runs to completion and opens a new Tip. - let invalidated = storage.recover_post_flush(1200).expect("recover"); - assert_eq!(invalidated, vec![0, 1]); - let recovery_index = storage - .open_state() - .expect("load open") - .expect("recovery batch exists") - .batch_index; - - // Simulate "crash immediately after open_recovery_batch" by - // dropping Storage (mimics process exit) and reopening against the - // same on-disk DB. - drop(storage); - let mut storage = Storage::open(db.path.as_str()).expect("reopen storage"); - - let second = storage.recover_post_flush(1200).expect("second detect"); - assert!( - second.is_empty(), - "post-recovery restart must be a no-op, got invalidations: {second:?}", - ); - let after = storage - .open_state() - .expect("load after restart") - .expect("recovery batch still Tip after restart"); - assert_eq!( - after.batch_index, recovery_index, - "the same recovery batch must remain the Tip after restart", - ); - } -} - -mod check_danger_zone { - use super::*; - - // ── check_danger_zone ────────────────────────────────────────────── - - #[test] - fn check_danger_zone_ignores_old_gold_batches() { - // Batch 0 is Gold (accepted, first_frame_safe_block=10). Batch 1 is - // the open tip at first_frame_safe_block=100. Advance safe head to - // 1200 so batch 0 is age=1190 > 1125 (past threshold, but it's Gold - // and therefore excluded) while batch 1 is age=1100 < 1125 (fresh). - // - // `check_danger_zone` must return None: no unresolved batch is in - // danger. Gold batches (accepted past the frontier) never participate, - // and the open tip isn't old enough to trip the threshold. - let (_db, mut storage) = open_temp_storage("danger-zone-gold"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 0"); - - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); - // Advance to a current safe block where batch 0 (safe_block=10) is - // past threshold (1200-10=1190>=1125) but batch 1 (safe_block=100) - // is still fresh (1200-100=1100<1125). - storage.advance_safe_head(1200); // advance safe block - - let result = - find_closed_frontier_batch_in_danger(&storage.conn, 1125).expect("check danger zone"); - assert!( - result.is_none(), - "old Gold batches should not trigger danger zone; got batch_index={result:?}" - ); - } - - #[test] - fn check_danger_zone_does_not_flag_open_batch_zombie() { - // `check_danger_zone` is for zombie detection: it must NOT flag the - // open batch (which has no L1 tx to become a zombie). Flagging open - // batches here would put the live submitter into a shutdown/restart - // loop when an open batch ages into the danger zone without any - // pending wallet-nonce slots to flush. - // - // Scenario: only an open batch exists, aged past the danger - // threshold. `check_danger_zone` returns None. - let (_db, mut storage) = open_temp_storage("danger-zone-open-no-zombie"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize open batch at safe_block=10"); - - storage.advance_safe_head(1200); // advance safe head past danger threshold - - let result = - find_closed_frontier_batch_in_danger(&storage.conn, 1125).expect("check danger zone"); - assert!( - result.is_none(), - "open batch (no zombie) must not trigger check_danger_zone; got batch_index={result:?}" - ); - } -} - -mod check_any_unresolved { - use super::*; - - // ── check_any_unresolved_batch_in_danger ─────────────────────────────── - - #[test] - fn check_any_unresolved_flags_stale_open_batch() { - // Wall-clock fallback regression: `check_any_unresolved_batch_in_danger` - // MUST flag a stale open batch. This is the semantic the wall-clock - // fallback relies on — if L1 is unreachable and an open batch may be - // past the threshold, refuse to boot rather than accept user ops - // into a batch that can't land. - let (_db, mut storage) = open_temp_storage("any-unresolved-open-stale"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize open batch at safe_block=10"); - - storage.advance_safe_head(1200); // advance safe head past threshold - - let result = find_first_batch_in_danger(&storage.conn, 1125) - .expect("check any unresolved in danger"); - assert_eq!( - result, - Some(0), - "stale open batch (batch 0) must be flagged by the unified check" - ); - } - - #[test] - fn check_any_unresolved_does_not_flag_fresh_open_batch() { - // Negative counterpart. Fresh open batch below threshold must not - // trigger false positives in the unified check. - let (_db, mut storage) = open_temp_storage("any-unresolved-open-fresh"); - - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize open batch at safe_block=10"); - - storage.advance_safe_head(1100); // advance safe head below threshold - - let result = find_first_batch_in_danger(&storage.conn, 1125) - .expect("check any unresolved in danger"); - assert!( - result.is_none(), - "fresh open batch must not trigger the unified check; got batch_index={result:?}" - ); - } - - #[test] - fn check_danger_zone_triggers_on_frontier_batch() { - let (_db, mut storage) = open_temp_storage("danger-zone-frontier"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 1"); - - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); - storage.advance_safe_head(1200); // advance safe block - - let result = - find_closed_frontier_batch_in_danger(&storage.conn, 1125).expect("check danger zone"); - assert_eq!(result, Some(1), "frontier batch should trigger danger zone"); - } - - #[test] - fn check_danger_zone_does_not_trigger_below_threshold() { - let (_db, mut storage) = open_temp_storage("danger-zone-below"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 10) - .expect("close batch 0"); - storage - .close_batch_at(&mut head, 100) - .expect("close batch 1"); - - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); - storage.advance_safe_head(1134); // advance safe block - - let result = - find_closed_frontier_batch_in_danger(&storage.conn, 1125).expect("check danger zone"); - assert!( - result.is_none(), - "should not trigger below threshold; got batch_index={result:?}" - ); - } - - #[test] - fn find_first_batch_in_danger_prefers_closed_frontier_over_aged_tip() { - // When BOTH the closed frontier batch and the open Tip are aged past the - // threshold, find_first_batch_in_danger must return the CLOSED frontier: - // cascading from it covers the Tip (batch_index >= pivot), and the scoped - // pending-snapshot clear keys on the pivot's nonce. Both existing - // find_*_in_danger tests use open-batch-only scenarios; the closed-over-Tip - // preference (the helper's whole point) was unasserted. - let (_db, mut storage) = open_temp_storage("danger-prefers-closed-frontier"); - - // batch 0 (frame sb 10) → made gold below; batch 1 (frame sb 50) → the - // closed frontier; batch 2 (Tip, frame sb 50) → open at the same safe - // block (the lane rotated without a safe-block advance). - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_batch_at(&mut head, 50) - .expect("seal batch 0, open batch 1 @ sb 50"); - storage - .close_batch_at(&mut head, 50) - .expect("seal batch 1, open the Tip @ sb 50"); - - // Make batch 0 gold: land its genuine bytes (accepted) → frontier → nonce 1. - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); // land batch 0 - assert_eq!( - storage - .submitter_frontier() - .expect("frontier") - .accepted_next_nonce, - 1, - "batch 0 is gold; the closed frontier is now batch 1 (nonce 1)" - ); - - // Advance the safe head so both batch 1 and the Tip (both sb 50) are aged: - // 1200 - 50 = 1150 >= threshold 1125. - storage.advance_safe_head(1200); - - let result = find_first_batch_in_danger(&storage.conn, 1125).expect("find first in danger"); - assert_eq!( - result, - Some(1), - "must return the closed frontier batch (1), not the aged Tip (2)" - ); - } -} - -mod boundary { - use super::*; - - // ── boundary tests ───────────────────────────────────────────────── - - #[test] - fn boundary_exactly_max_wait_is_stale() { - let (_db, mut storage) = open_temp_storage("detect-boundary-exact"); - - let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.close_batch_at(&mut head, 100).expect("close batch"); - - storage.land(1300, &[(SENDER_A, make_stale_batch_payload(0, 100))]); - let invalidated = storage.recover_post_flush(1200).expect("detect"); - assert_eq!(invalidated, vec![0, 1], "exactly at max_wait must be stale"); - assert_eq!(storage.open_state().expect("load").unwrap().batch_index, 2); - } - - #[test] - fn boundary_one_below_max_wait_is_not_stale() { - let (_db, mut storage) = open_temp_storage("detect-boundary-one-below"); - - let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.close_batch_at(&mut head, 100).expect("close batch"); - - let landed = local_batch_payload(&mut storage, 0); - storage.land(1299, &[(SENDER_A, landed)]); - let invalidated = storage.recover_post_flush(1200).expect("detect"); - assert!( - invalidated.is_empty(), - "one below max_wait must not be stale" - ); - } - - #[test] - fn all_batches_invalidated_frontier_zero() { - let (_db, mut storage) = open_temp_storage("detect-frontier-zero"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - for _ in 0..3 { - storage.close_batch_at(&mut head, 10).expect("close"); - } - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let inv = storage.recover_post_flush(1200).expect("detect"); - assert_eq!(inv, vec![0, 1, 2, 3]); - assert!(storage.open_state().expect("open").is_some()); - } - - #[test] - fn recovery_batch_itself_becomes_stale() { - let (_db, mut storage) = open_temp_storage("detect-recovery-stale"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage.close_batch_at(&mut head, 10).expect("close"); - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); // append gen1 - let inv1 = storage.recover_post_flush(1200).expect("recover gen1"); - assert_eq!(inv1, vec![0, 1]); - - let mut head2 = storage.open_state().expect("load").unwrap(); - storage - .close_batch_at(&mut head2, 1210) - .expect("close gen2"); - - storage.land(2410, &[(SENDER_A, make_stale_batch_payload(0, 1210))]); // append gen2 - let inv2 = storage.recover_post_flush(1200).expect("recover gen2"); - assert_eq!(inv2, vec![2, 3]); - assert!(storage.open_state().expect("open").is_some()); - } - - #[test] - fn multi_round_gen3_recovery() { - let (_db, mut storage) = open_temp_storage("detect-gen3"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("init"); - storage.close_batch_at(&mut head, 10).expect("close"); - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - storage.recover_post_flush(1200).expect("recover gen1"); - - let mut head2 = storage.open_state().expect("load").unwrap(); - storage - .close_batch_at(&mut head2, 1210) - .expect("close gen2"); - storage.land(2410, &[(SENDER_A, make_stale_batch_payload(0, 1210))]); // append gen2 - storage.recover_post_flush(1200).expect("recover gen2"); - - let mut head3 = storage.open_state().expect("load").unwrap(); - storage - .close_batch_at(&mut head3, 2410) - .expect("close gen3"); - let landed = local_batch_payload(&mut storage, 0); - storage.land(2420, &[(SENDER_A, landed)]); // append gen3 - let inv3 = storage.recover_post_flush(1200).expect("recover gen3"); - assert!(inv3.is_empty(), "gen3 should be healthy"); - } - - #[test] - fn large_cascade_50_batches() { - let (_db, mut storage) = open_temp_storage("detect-large-cascade"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - for _ in 0..50 { - storage.close_batch_at(&mut head, 10).expect("close"); - } - - storage.land(1210, &[(SENDER_A, make_stale_batch_payload(0, 10))]); - let inv = storage.recover_post_flush(1200).expect("detect"); - assert_eq!(inv.len(), 51); - } -} - -mod tree_invariants { - use super::*; - - // ── Parent-pointer tree invariants ────────────────────────────── - use crate::storage::convert::{i64_to_u64, u64_to_i64}; - use rusqlite::params; - - /// The schema triggers check parent links, the root and the single Tip on - /// every write; invalidation is an UPDATE no trigger checks, so assert the - /// valid spine here. The e2e harness asserts the same on the binary's DB. - fn assert_valid_spine_from_anchor(storage: &mut Storage) { - let anchor = u64_to_i64(storage.batch_tree_anchor().expect("anchor")); - let nonces: Vec = storage - .conn - .prepare("SELECT nonce FROM valid_batches ORDER BY nonce ASC") - .expect("prepare") - .query_map([], |row| row.get(0)) - .expect("query") - .collect::>() - .expect("collect"); - let expected: Vec = (anchor..).take(nonces.len()).collect(); - assert_eq!( - nonces, expected, - "valid spine must be contiguous from anchor {anchor}" - ); - } - - #[test] - fn tree_invariants_hold_across_mixed_workload() { - // Exercises every mutating code path: genesis, rotations, partial - // cascades (ancestor survives), cascades across accepted frontier, - // torn cascades (no valid ancestor), and back-to-back generations. - // Every write is checked by the schema triggers (an ABORT fails its - // expect); the valid spine is asserted after each step. - let (_db, mut storage) = open_temp_storage("tree-invariants-workload"); - - // Phase 1: genesis + 4 rotations. Simple chain. - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - assert_valid_spine_from_anchor(&mut storage); - for _ in 0..4 { - storage.close_batch_at(&mut head, 100).expect("close"); - assert_valid_spine_from_anchor(&mut storage); - } - // Tree: 0(Gold sentinel in concept)→1→2→3→4 (Tip) - - // Phase 2: cascade with a valid ancestor. Batch 0 is accepted first. - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); // append accepted - storage.advance_safe_head(1400); // advance past threshold - let inv = storage.recover_post_flush(1200).expect("recover"); - assert!(!inv.is_empty(), "partial cascade should invalidate"); - assert_valid_spine_from_anchor(&mut storage); - - // Phase 3: more rotations after partial cascade. - let mut head = storage.open_state().expect("load").unwrap(); - for _ in 0..3 { - storage.close_batch_at(&mut head, 1500).expect("close gen2"); - assert_valid_spine_from_anchor(&mut storage); - } - - // Phase 4: torn cascade — invalidate everything including batch 0. - let latest = storage.latest_batch_index().expect("latest").unwrap(); - for bi in 0..=latest { - storage.insert_invalid_batch(bi).expect("invalidate"); - } - storage.recover_post_flush(1200).expect("recover from torn"); - assert_valid_spine_from_anchor(&mut storage); - - // Phase 5: rotations after torn cascade — new Tip has parent=NULL, nonce=0. - let mut head = storage.open_state().expect("load").unwrap(); - for _ in 0..5 { - storage.close_batch_at(&mut head, 2000).expect("close gen3"); - assert_valid_spine_from_anchor(&mut storage); - } - } - - #[test] - fn subtree_by_batch_index_equals_subtree_by_parent_walk() { - // cascade queries use `batch_index >= N` as a shortcut for - // "subtree rooted at N". This test asserts the equivalence on a - // realistic scenario with multiple cascade generations. - let (_db, mut storage) = open_temp_storage("subtree-equivalence"); - - // Build: 5 batches, cascade from 2 (partial), 3 more, cascade from 1 (torn-ish). - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - for _ in 0..4 { - storage.close_batch_at(&mut head, 100).expect("close"); - } - let landed = local_batch_payload(&mut storage, 0); - storage.land(20, &[(SENDER_A, landed)]); // append accepted - storage.advance_safe_head(1400); // advance - let _ = storage.recover_post_flush(1200).expect("cascade 1"); - - let mut head = storage.open_state().expect("load").unwrap(); - for _ in 0..2 { - storage.close_batch_at(&mut head, 1500).expect("close"); - } - - // Assert equivalence among VALID batches for every valid N. - // Restricting both sides to `valid_batches` is the invariant cascade - // relies on: its WHERE filters invalidated rows, so the two sets need - // only agree on the valid subset. - let valid_bi: Vec = { - let mut stmt = storage - .conn - .prepare("SELECT batch_index FROM valid_batches ORDER BY batch_index") - .expect("prepare"); - stmt.query_map([], |row| row.get::<_, i64>(0).map(i64_to_u64)) - .expect("query") - .collect::>() - .expect("collect") - }; - for &n in &valid_bi { - let by_index: Vec = { - let mut stmt = storage - .conn - .prepare( - "SELECT batch_index FROM valid_batches \ - WHERE batch_index >= ?1 ORDER BY batch_index", - ) - .expect("prepare"); - stmt.query_map(params![u64_to_i64(n)], |row| { - row.get::<_, i64>(0).map(i64_to_u64) - }) - .expect("query") - .collect::>() - .expect("collect") - }; - let by_subtree: Vec = { - let mut stmt = storage - .conn - .prepare( - "WITH RECURSIVE subtree(batch_index) AS ( \ - SELECT batch_index FROM valid_batches WHERE batch_index = ?1 \ - UNION ALL \ - SELECT b.batch_index FROM valid_batches b \ - JOIN subtree s ON b.parent_batch_index = s.batch_index \ - ) \ - SELECT batch_index FROM subtree ORDER BY batch_index", - ) - .expect("prepare"); - stmt.query_map(params![u64_to_i64(n)], |row| { - row.get::<_, i64>(0).map(i64_to_u64) - }) - .expect("query") - .collect::>() - .expect("collect") - }; - assert_eq!( - by_index, by_subtree, - "cascade root {n}: valid batch_index >= N diverged from valid parent-walk subtree" - ); - } - } -} - -mod recovery_snapshot_selection { - use super::*; - use std::path::Path; - - fn baseline(storage: &mut Storage) -> i64 { - storage - .insert_baseline_snapshot( - Path::new("/tmp/recovery-baseline"), - ExecutedInputCount::ZERO, - ) - .expect("baseline") - } - - fn batch_snapshot(storage: &mut Storage, batch_index: u64) -> i64 { - storage - .insert_batch_snapshot( - Path::new(&format!("/tmp/recovery-batch-{batch_index}")), - batch_index, - ) - .expect("batch snapshot") - } - - #[test] - fn cascade_hides_every_doomed_snapshot_and_retains_baseline() { - let (_db, mut storage) = open_temp_storage("recovery-baseline-fallback"); - let baseline_id = baseline(&mut storage); - seed_closed_batches(&mut storage, 3); - for index in 0..3 { - batch_snapshot(&mut storage, index); - } - assert_ne!( - storage.latest_snapshot().unwrap().unwrap().dump.id, - baseline_id - ); - storage.advance_safe_head(1500); // safe head - assert_eq!( - storage.recover_post_flush(1200).expect("cascade"), - vec![0, 1, 2, 3] - ); - assert_eq!( - storage.latest_snapshot().unwrap().unwrap().dump.id, - baseline_id - ); - assert!( - storage - .has_rollback_safe_snapshot() - .expect("rollback checkpoint") - ); - let garbage = storage - .gc_unreferenced_dumps() - .expect("gc doomed snapshots"); - assert_eq!(garbage.len(), 3); - assert!(garbage.iter().all(|dump| dump.id != baseline_id)); - } - - #[test] - fn accepted_snapshot_survives_cascade_after_baseline_gc_and_restart() { - let (db, mut storage) = open_temp_storage("recovery-accepted-fallback"); - let baseline_id = baseline(&mut storage); - seed_closed_batches(&mut storage, 2); - let accepted_id = batch_snapshot(&mut storage, 0); - let doomed_id = batch_snapshot(&mut storage, 1); - seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0]); - let garbage = storage.gc_unreferenced_dumps().expect("retire baseline"); - assert_eq!( - garbage.iter().map(|d| d.id).collect::>(), - vec![baseline_id] - ); - assert_eq!( - storage.latest_snapshot().unwrap().unwrap().dump.id, - doomed_id - ); - drop(storage); - let mut storage = Storage::open(&db.path).expect("restart"); - assert!( - storage - .has_rollback_safe_snapshot() - .expect("accepted checkpoint") - ); - assert_eq!( - storage.recover_post_flush(1200).expect("cascade suffix"), - vec![1, 2] - ); - assert_eq!( - storage.latest_snapshot().unwrap().unwrap().dump.id, - accepted_id - ); - assert_eq!( - storage.finalized_dump().unwrap().unwrap().dump.id, - accepted_id - ); - assert_eq!( - storage - .gc_unreferenced_dumps() - .unwrap() - .iter() - .map(|d| d.id) - .collect::>(), - vec![doomed_id] - ); - assert!( - storage - .recover_post_flush(1200) - .expect("idempotent recovery") - .is_empty() - ); - } - - #[test] - fn an_optimistic_snapshot_alone_does_not_admit_recovery() { - let (_db, mut storage) = open_temp_storage("recovery-optimistic-is-not-fallback"); - seed_closed_batches(&mut storage, 1); - batch_snapshot(&mut storage, 0); - let protocol = default_protocol_timing(); - let now = crate::clock::unix_now_ms(); - storage - .append_safe_inputs_with_timestamp( - 1500, - now / 1000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("fresh observation"); - assert!( - storage - .latest_snapshot() - .expect("optimistic snapshot") - .is_some() - ); - assert!( - !storage - .has_rollback_safe_snapshot() - .expect("no stable fallback") - ); - let err = storage - .recover_post_flush_for_recovery(1500, &protocol, now) - .expect_err("refuse before losing only snapshot"); - assert!(matches!( - err, - crate::storage::RecoveryMutationError::MissingRecoveryCheckpoint - )); - assert_eq!( - storage - .latest_snapshot() - .unwrap() - .unwrap() - .executed_input_count, - ExecutedInputCount::ZERO - ); - assert_eq!( - storage - .conn - .query_row( - "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", - [], - |r| r.get::<_, i64>(0) - ) - .unwrap(), - 0 - ); - } -} - -mod check_danger { - use crate::clock::unix_now_ms; - use crate::storage::test_helpers::{SENDER_A, default_protocol_timing, local_batch_payload}; - use crate::storage::{SafeInputRange, Storage, StoredSafeInput}; - use crate::test_support::temp_db; - - #[test] - fn check_danger_reports_observed_closed_batch_danger() { - let db = temp_db("check-danger-observed-closed"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 1"); - - let protocol = default_protocol_timing(); - let landed = local_batch_payload(&mut storage, 0); - storage - .append_safe_inputs( - 1135, - &[StoredSafeInput { - sender: SENDER_A, - payload: landed, - block_number: 20, - }], - SENDER_A, - &protocol, - ) - .expect("append accepted batch 0"); - - let status = storage - .check_danger(&protocol, unix_now_ms()) - .expect("check_danger"); - assert_eq!(status, crate::storage::DangerStatus::ClosedBatchInDanger(1)); - } - - #[test] - fn check_danger_reports_estimated_batch_danger_on_wall_clock_drift() { - // Observed block-based checks wouldn't fire (batch 1 has - // first_frame_safe_block = 100 and safe_block = 1200, age = 1100 < - // 1125). But wall-clock says the safe head hasn't advanced in ~25 - // blocks: the effective threshold drops to 1100, so batch 1 reaches - // danger via the wall-clock correction. - let db = temp_db("check-danger-estimated"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch 0"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch 1"); - - let protocol = default_protocol_timing(); - let landed = local_batch_payload(&mut storage, 0); - storage - .append_safe_inputs( - 1200, - &[StoredSafeInput { - sender: SENDER_A, - payload: landed, - block_number: 200, - }], - SENDER_A, - &protocol, - ) - .expect("append accepted batch 0"); - - // Pretend safe-progress was recorded 25 blocks' worth of wall-clock ago. - let now_ms = unix_now_ms(); - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms.saturating_sub(25 * 12 * 1000)).unwrap_or(i64::MAX)], - ) - .expect("rewind safe-progress timestamp"); - - let status = storage - .check_danger(&protocol, now_ms) - .expect("check_danger"); - assert_eq!( - status, - crate::storage::DangerStatus::EstimatedBatchInDanger(1) - ); - } - - #[test] - fn check_danger_prefers_observed_closed_over_estimated_when_both_fire() { - // If the observed safe head already puts the closed frontier past - // `danger_threshold`, route to observed closed-batch recovery. The - // wall-clock arm is a fallback, not a mask over observed danger. - let db = temp_db("check-danger-observed-closed-wins"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - - let protocol = default_protocol_timing(); - // Advance safe head so batch 0 is past `danger_threshold` (1125): - // current=1200 - first_frame=10 = 1190 >= 1125. - // No safe_input ingested for batch 0, so it stays non-gold. - storage - .append_safe_inputs(1200, &[], SENDER_A, &protocol) - .expect("advance safe head past observed danger threshold"); - - // Pretend safe-progress was recorded 25 blocks' worth of wall-clock - // ago. With the wall-clock correction, the adjusted threshold is - // 1100 — batch 0's age (1190) also clears that, so wall-clock fires. - let now_ms = unix_now_ms(); - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms.saturating_sub(25 * 12 * 1000)).unwrap_or(i64::MAX)], - ) - .expect("rewind safe-progress timestamp"); - - let status = storage - .check_danger(&protocol, now_ms) - .expect("check_danger"); - assert_eq!( - status, - crate::storage::DangerStatus::ClosedBatchInDanger(0), - "observed closed-batch danger must win once safe-state has crossed danger", - ); - } - - #[test] - fn check_danger_prefers_observed_tip_over_estimated_when_both_fire() { - // Same fallback rule for the open Tip: if observed safe-state already - // puts the Tip in danger, recover the Tip directly instead of refusing - // under the wall-clock classification. - let db = temp_db("check-danger-observed-tip-wins"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0; batch 1 is Tip"); - - let protocol = default_protocol_timing(); - let landed = local_batch_payload(&mut storage, 0); - storage - .append_safe_inputs( - 1200, - &[StoredSafeInput { - sender: SENDER_A, - payload: landed, - block_number: 20, - }], - SENDER_A, - &protocol, - ) - .expect("append accepted batch 0"); - - // The Tip's observed age is 1190 >= 1125, and the wall-clock-adjusted - // arm would also fire. Observed Tip danger must win. - let now_ms = unix_now_ms(); - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms.saturating_sub(25 * 12 * 1000)).unwrap_or(i64::MAX)], - ) - .expect("rewind safe-progress timestamp"); - - let status = storage - .check_danger(&protocol, now_ms) - .expect("check_danger"); - assert_eq!( - status, - crate::storage::DangerStatus::TipInDanger(1), - "Tip must win when observed safe-state has already crossed danger", - ); - } - - #[test] - fn check_danger_refuses_when_l1_view_is_stale() { - let db = temp_db("check-danger-l1-view-stale"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - - let protocol = default_protocol_timing(); - let old_safe_timestamp = 1_000_u64; - storage - .append_safe_inputs_with_timestamp( - 1200, - old_safe_timestamp, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("advance safe head with stale L1 timestamp"); - - let now_ms = - (old_safe_timestamp + protocol.l1_read_stale_after_secs()).saturating_mul(1000); - let status = storage - .check_danger(&protocol, now_ms) - .expect("check_danger"); - assert_eq!( - status, - crate::storage::DangerStatus::L1ViewStale, - "global L1 view staleness must refuse before observed batch recovery", - ); - } - - #[test] - fn check_danger_refuses_when_safe_block_timestamp_is_in_the_future() { - let db = temp_db("check-danger-future-safe-timestamp"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); - let now_ms = 1_000_000_u64; - - // A full block-time (12 s) ahead of the local clock: unusable. Sub-block - // ahead-ness is NTP-scale skew and tolerated (see the companion assert). - storage - .append_safe_inputs_with_timestamp( - 10, - 1_012, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("record future L1 timestamp"); - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms).expect("test time fits")], - ) - .expect("make local progress baseline usable"); - - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::L1ViewStale - ); - - // Sub-block skew (1 s ahead) is the freshest possible view, not a fault. - storage - .conn - .execute( - "UPDATE l1_safe_head SET block_timestamp = 1001 WHERE singleton_id = 0", - [], - ) - .expect("set sub-block-ahead timestamp"); - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::Safe - ); - } - - #[test] - fn check_danger_refuses_when_local_progress_timestamp_is_in_the_future() { - let db = temp_db("check-danger-future-progress"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let protocol = default_protocol_timing(); - let now_ms = 1_000_000_u64; - - storage - .append_safe_inputs_with_timestamp( - 10, - 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("record safe head"); - // One full block-time (12 s) of regression: the wall-clock - // extrapolation is genuinely invalid — refuse. A sub-block step is - // tolerated (companion assert below). - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms + 12_000).expect("test time fits")], - ) - .expect("move local progress baseline into future"); - - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::L1ViewStale - ); - - // A 1 ms step is quantization noise for a block-granular estimate. - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms + 1).expect("test time fits")], - ) - .expect("move baseline a sub-block step into the future"); - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::Safe - ); - } - - #[test] - fn observed_batch_danger_outranks_regressed_progress_clock() { - let db = temp_db("check-danger-future-progress-priority"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - let protocol = default_protocol_timing(); - let now_ms = 1_000_000_u64; - - storage - .append_safe_inputs_with_timestamp( - 1200, - 1_000, - &[], - SENDER_A, - &protocol, - crate::storage::FrontierMode::Populate, - ) - .expect("advance observed safe block past danger"); - // Even a full-block clock regression (a real fault, not noise) must - // not suppress the observed arm: batch age here is pure block - // arithmetic on persisted L1 observations, valid regardless of the - // local clock. The refusal only wins when no observed danger stands - // (2026-07-31 review of the containment commit). - storage - .conn - .execute( - "UPDATE l1_safe_head SET synced_at_ms = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms + 12_000).expect("test time fits")], - ) - .expect("move local progress baseline into future"); - - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::ClosedBatchInDanger(0), - "observed danger stands on L1 observation alone; a wall-clock \ - fault must not delay recovery of a batch aging toward staleness" - ); - - // Same verdict with BOTH baselines faulted: the safe-block timestamp - // a full block ahead of the clock AND the progress baseline regressed. - storage - .conn - .execute( - "UPDATE l1_safe_head SET block_timestamp = ?1 WHERE singleton_id = 0", - [i64::try_from(now_ms / 1000 + 12).expect("test time fits")], - ) - .expect("move safe-block timestamp a block into the future"); - assert_eq!( - storage - .check_danger(&protocol, now_ms) - .expect("check danger"), - crate::storage::DangerStatus::ClosedBatchInDanger(0), - "both clock-fault baselines together still yield to observed danger" - ); - } - - #[test] - fn check_danger_safe_when_never_synced() { - // Fresh DB, no prior safe block timestamp. The L1 view is unusable - // until the input reader records a real safe-head observation. - let db = temp_db("check-danger-never-synced"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let status = storage - .check_danger(&default_protocol_timing(), unix_now_ms()) - .expect("check_danger"); - assert_eq!(status, crate::storage::DangerStatus::L1ViewStale); - } - - #[test] - fn check_danger_errors_when_valid_tip_has_no_first_frame() { - let db = temp_db("check-danger-missing-first-frame"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - let protocol = default_protocol_timing(); - storage - .append_safe_inputs(10, &[], SENDER_A, &protocol) - .expect("record fresh safe head"); - storage - .conn - .execute("DELETE FROM frames WHERE batch_index = 0", []) - .expect("inject missing-frame corruption"); - - let err = storage - .check_danger(&protocol, unix_now_ms()) - .expect_err("a valid batch without a first frame must fail loud"); - assert!(matches!(err, rusqlite::Error::QueryReturnedNoRows)); - } -} diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs deleted file mode 100644 index ecf8b33c..00000000 --- a/sequencer/src/storage/snapshot_dumps.rs +++ /dev/null @@ -1,757 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -//! Immutable batch-close and baseline snapshots, derived acceptance, and leases. -//! Files are durable before registration; GC deletes rows before their files. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use rusqlite::{Connection, OptionalExtension, Result, Transaction, params}; -use sequencer_core::history::{ExecutedInputCount, HistoryVersion}; - -use super::convert::{i64_to_u64, u64_to_i64}; -use super::history::{next_executed_input_count_in, query_history_state}; -use super::{Storage, is_persistent_storage_error, is_persistent_storage_open_error}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DumpRow { - pub id: i64, - pub prefix: PathBuf, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Snapshot { - pub dump: DumpRow, - pub executed_input_count: ExecutedInputCount, -} - -/// A batch-close snapshot whose complete predicted prefix was accepted on L1. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FinalizedDump { - pub dump: DumpRow, - pub inclusion_block: u64, - pub next_batch_nonce: u64, - pub executed_input_count: ExecutedInputCount, -} - -#[derive(Debug, thiserror::Error)] -pub enum FinalizedSelectionError { - #[error("canonical divergence prevents accepted checkpoint selection")] - CanonicalDivergence, - #[error("selecting finalized checkpoint: {0}")] - Storage(#[from] rusqlite::Error), -} - -pub type ReleaseScheduler = Arc) + Send + Sync + 'static>; -pub type PersistentReleaseFailureReporter = Arc; - -/// An armed lease release, inseparable from the lease it holds. Handed out -/// bundled inside a [`LeasedDump`]: while it lives, `lease_count > 0` keeps GC -/// off the dump; its `Drop` releases the lease. The release re-opens a brief -/// writer connection from the storage path (the egress handlers open per-op, -/// so the guard owns the path rather than borrowing a `Storage`) and runs via -/// the injected [`ReleaseScheduler`]. Dropping it without a runtime still -/// releases — just inline. `reset_dump_leases` at startup is the crash backstop. -pub struct LeaseGuard { - path: String, - dump_id: i64, - schedule: ReleaseScheduler, - report_persistent_failure: PersistentReleaseFailureReporter, -} - -impl Drop for LeaseGuard { - fn drop(&mut self) { - let path = std::mem::take(&mut self.path); - let dump_id = self.dump_id; - let report_persistent_failure = self.report_persistent_failure.clone(); - (self.schedule)(Box::new(move || match Storage::open_writer(&path) { - Ok(mut storage) => { - if let Err(err) = storage.release_dump_lease(dump_id) { - if is_persistent_storage_error(&err) { - report_persistent_failure(&format!( - "snapshot lease release for dump {dump_id} failed persistently: {err}" - )); - } else { - tracing::warn!( - error = %err, dump_id, - "snapshot lease release failed; will be reset at next startup", - ); - } - } - } - Err(err) => { - if is_persistent_storage_open_error(&err) { - report_persistent_failure(&format!( - "snapshot lease release for dump {dump_id}: writer open failed persistently: {err}" - )); - } else { - tracing::warn!( - error = %err, dump_id, - "snapshot lease release: open failed; will be reset at next startup", - ); - } - } - })); - } -} - -/// Artifact and history metadata selected atomically with the lease increment. -pub struct LeasedDump { - pub prefix: PathBuf, - pub executed_input_count: ExecutedInputCount, - pub history_version: HistoryVersion, - pub next_batch_nonce: u64, - pub guard: LeaseGuard, -} - -pub struct FinalizedLease { - pub inclusion_block: u64, - pub dump: LeasedDump, -} - -impl Storage { - pub fn release_dump_lease(&mut self, dump_id: i64) -> Result<()> { - self.write(|tx| release_dump_lease_in(tx, dump_id)) - } - - pub fn reset_dump_leases(&mut self) -> Result { - self.write(|tx| tx.execute("UPDATE dumps SET lease_count = 0", [])) - } - - /// GC keeps the accepted rollback checkpoint and every valid snapshot beyond - /// it. Intermediate optimistic snapshots may become the next accepted head. - /// Row selection/deletion serializes with lease acquisition; files follow. - pub fn gc_unreferenced_dumps(&mut self) -> Result> { - self.write(|tx| { - let anchor = rollback_snapshot_in(tx)?; - let anchor_id = anchor.as_ref().map(|s| s.dump.id); - let accepted_nonce = - latest_accepted_boundary_in(tx)?.map(|(_, nonce, _)| u64_to_i64(nonce)); - let candidates = { - let mut stmt = tx.prepare( - "SELECT d.id, d.prefix FROM dumps d \ - LEFT JOIN snapshots s ON s.dump_id = d.id \ - LEFT JOIN valid_closed_batches b ON b.batch_index = s.batch_index \ - WHERE d.lease_count = 0 AND (?1 IS NULL OR d.id != ?1) \ - AND (b.batch_index IS NULL OR (?2 IS NOT NULL AND b.nonce <= ?2)) \ - ORDER BY d.id", - )?; - stmt.query_map(params![anchor_id, accepted_nonce], row_to_dump_row)? - .collect::>>()? - }; - for dump in &candidates { - tx.execute("DELETE FROM snapshots WHERE dump_id = ?1", [dump.id])?; - tx.execute("DELETE FROM dumps WHERE id = ?1", [dump.id])?; - } - Ok(candidates) - }) - } - - /// The latest accepted batch must have its own snapshot. A missing artifact - /// is corruption, never a request to use an older accepted snapshot. - pub fn finalized_dump( - &mut self, - ) -> std::result::Result, FinalizedSelectionError> { - self.read(|tx| Ok(finalized_dump_in(tx)))? - } - - pub fn latest_snapshot(&mut self) -> Result> { - self.read(|tx| latest_snapshot_in(tx)) - } - - pub fn rollback_snapshot(&mut self) -> Result> { - self.read(|tx| rollback_snapshot_in(tx)) - } - - pub fn acquire_finalized_lease( - &mut self, - schedule: ReleaseScheduler, - report_persistent_failure: PersistentReleaseFailureReporter, - ) -> std::result::Result, FinalizedSelectionError> { - let acquired = self.write(|tx| { - let snapshot = match finalized_dump_in(tx) { - Ok(Some(snapshot)) => snapshot, - Ok(None) => return Ok(Ok(None)), - Err(error) => return Ok(Err(error)), - }; - let history_version = query_history_state(tx)?.version; - acquire_dump_lease_in(tx, snapshot.dump.id)?; - Ok(Ok(Some((snapshot, history_version)))) - })??; - Ok(acquired.map(|(snapshot, history_version)| FinalizedLease { - inclusion_block: snapshot.inclusion_block, - dump: LeasedDump { - prefix: snapshot.dump.prefix, - executed_input_count: snapshot.executed_input_count, - next_batch_nonce: snapshot.next_batch_nonce, - history_version, - // Only a committed increment arms a release. A failed COMMIT - // rolls back the lease and must never schedule its decrement. - guard: LeaseGuard { - path: self.path.clone(), - dump_id: snapshot.dump.id, - schedule, - report_persistent_failure, - }, - }, - })) - } - - pub fn acquire_latest_snapshot_lease( - &mut self, - schedule: ReleaseScheduler, - report_persistent_failure: PersistentReleaseFailureReporter, - ) -> Result> { - let acquired = self.write(|tx| { - let Some(snapshot) = latest_snapshot_in(tx)? else { - return Ok(None); - }; - let history_version = query_history_state(tx)?.version; - let next_batch_nonce = snapshot_next_nonce_in(tx, snapshot.dump.id)?; - acquire_dump_lease_in(tx, snapshot.dump.id)?; - Ok(Some((snapshot, history_version, next_batch_nonce))) - })?; - Ok( - acquired.map(|(snapshot, history_version, next_batch_nonce)| LeasedDump { - prefix: snapshot.dump.prefix, - executed_input_count: snapshot.executed_input_count, - history_version, - next_batch_nonce, - guard: LeaseGuard { - path: self.path.clone(), - dump_id: snapshot.dump.id, - schedule, - report_persistent_failure, - }, - }), - ) - } - - pub fn list_dump_rows(&mut self) -> Result> { - self.read(|tx| { - let mut stmt = tx.prepare("SELECT id, prefix FROM dumps ORDER BY id")?; - stmt.query_map([], row_to_dump_row)?.collect() - }) - } - - pub fn batch_nonce(&mut self, batch_index: u64) -> Result { - self.read(|tx| batch_nonce_in(tx, batch_index)) - } -} - -pub(super) fn insert_baseline_snapshot_in( - tx: &Transaction<'_>, - prefix: &Path, - count: ExecutedInputCount, -) -> Result { - insert_snapshot_in(tx, prefix, None, count) -} - -pub(super) fn insert_batch_snapshot_in( - tx: &Transaction<'_>, - prefix: &Path, - batch_index: u64, - count: ExecutedInputCount, -) -> Result { - // Registration is composed with sealing, so the batch must already be closed. - tx.query_row( - "SELECT batch_index FROM valid_closed_batches WHERE batch_index = ?1", - [u64_to_i64(batch_index)], - |_| Ok(()), - )?; - insert_snapshot_in(tx, prefix, Some(batch_index), count) -} - -fn insert_snapshot_in( - tx: &Transaction<'_>, - prefix: &Path, - batch_index: Option, - count: ExecutedInputCount, -) -> Result { - assert_eq!( - count, - next_executed_input_count_in(tx)?, - "snapshot count differs from canonical storage history" - ); - tx.execute( - "INSERT INTO dumps (prefix) VALUES (?1)", - [prefix.to_string_lossy().as_ref()], - )?; - let id = tx.last_insert_rowid(); - tx.execute( - "INSERT INTO snapshots (dump_id, batch_index, executed_input_count) VALUES (?1, ?2, ?3)", - params![id, batch_index.map(u64_to_i64), u64_to_i64(count.get())], - )?; - Ok(id) -} - -/// Required by recovery/admission inside their existing transaction. -pub(super) fn has_rollback_safe_snapshot_in(conn: &Connection) -> Result { - Ok(rollback_snapshot_in(conn)?.is_some()) -} - -fn rollback_snapshot_in(conn: &Connection) -> Result> { - match latest_accepted_boundary_in(conn)? { - Some((batch_index, _, _)) => snapshot_for_batch_in(conn, batch_index).map(Some), - None => baseline_snapshot_in(conn), - } -} - -fn latest_accepted_boundary_in(conn: &Connection) -> Result> { - conn.query_row( - "SELECT b.batch_index, a.nonce, a.inclusion_block FROM safe_accepted_batches a \ - LEFT JOIN valid_closed_batches b ON b.nonce = a.nonce \ - ORDER BY a.safe_input_index DESC LIMIT 1", - [], - |row| { - Ok(( - i64_to_u64(row.get(0)?), - i64_to_u64(row.get(1)?), - i64_to_u64(row.get(2)?), - )) - }, - ) - .optional() -} - -fn snapshot_for_batch_in(conn: &Connection, batch_index: u64) -> Result { - conn.query_row( - "SELECT s.dump_id, d.prefix, s.executed_input_count FROM snapshots s \ - LEFT JOIN dumps d ON d.id = s.dump_id WHERE s.batch_index = ?1", - [u64_to_i64(batch_index)], - row_to_snapshot, - ) -} - -fn baseline_snapshot_in(conn: &Connection) -> Result> { - conn.query_row( - "SELECT s.dump_id, d.prefix, s.executed_input_count FROM snapshots s \ - LEFT JOIN dumps d ON d.id = s.dump_id WHERE s.batch_index IS NULL", - [], - row_to_snapshot, - ) - .optional() -} - -pub(super) fn finalized_dump_in( - conn: &Connection, -) -> std::result::Result, FinalizedSelectionError> { - // A matched batch can precede a divergent accepted batch in the same L1 - // block. Its snapshot cannot represent that complete block boundary. - if super::safe_accepted_batches::canonical_divergence_in(conn)?.is_some() { - return Err(FinalizedSelectionError::CanonicalDivergence); - } - if let Some((batch_index, nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { - let snapshot = snapshot_for_batch_in(conn, batch_index)?; - return Ok(Some(FinalizedDump { - dump: snapshot.dump, - executed_input_count: snapshot.executed_input_count, - inclusion_block, - next_batch_nonce: nonce.checked_add(1).expect("accepted nonce overflow"), - })); - } - // Genesis is independently known canonical. A rebuilt baseline may contain - // the recovery fold's speculative final drain and is never comparable at C. - let Some(snapshot) = baseline_snapshot_in(conn)? else { - return Ok(None); - }; - let history = query_history_state(conn)?; - if history.base_safe_block == 0 && history.base_executed_input_count == 0 { - Ok(Some(FinalizedDump { - dump: snapshot.dump, - executed_input_count: snapshot.executed_input_count, - inclusion_block: 0, - next_batch_nonce: super::mutations::batch_tree_anchor_in(conn)?, - })) - } else { - Ok(None) - } -} - -fn latest_snapshot_in(conn: &Connection) -> Result> { - let batch_index: Option = conn - .query_row( - "SELECT batch_index FROM valid_closed_batches ORDER BY batch_index DESC LIMIT 1", - [], - |r| r.get(0), - ) - .optional()?; - match batch_index { - Some(index) => snapshot_for_batch_in(conn, i64_to_u64(index)).map(Some), - None => baseline_snapshot_in(conn), - } -} - -fn snapshot_next_nonce_in(conn: &Connection, dump_id: i64) -> Result { - let batch_index: Option = conn.query_row( - "SELECT batch_index FROM snapshots WHERE dump_id = ?1", - [dump_id], - |r| r.get(0), - )?; - match batch_index { - Some(index) => Ok(batch_nonce_in(conn, i64_to_u64(index))? - .checked_add(1) - .expect("batch nonce overflow")), - None => super::mutations::batch_tree_anchor_in(conn), - } -} - -fn acquire_dump_lease_in(tx: &Transaction<'_>, id: i64) -> Result<()> { - let changed = tx.execute( - "UPDATE dumps SET lease_count = lease_count + 1 WHERE id = ?1", - [id], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(()) -} - -fn release_dump_lease_in(tx: &Transaction<'_>, id: i64) -> Result<()> { - let changed = tx.execute( - "UPDATE dumps SET lease_count = lease_count - 1 WHERE id = ?1", - [id], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(()) -} - -pub(super) fn batch_nonce_in(conn: &Connection, batch_index: u64) -> Result { - conn.query_row( - "SELECT nonce FROM batches WHERE batch_index = ?1", - [u64_to_i64(batch_index)], - |row| Ok(i64_to_u64(row.get(0)?)), - ) -} - -fn row_to_dump_row(row: &rusqlite::Row<'_>) -> Result { - Ok(DumpRow { - id: row.get(0)?, - prefix: PathBuf::from(row.get::<_, String>(1)?), - }) -} - -fn row_to_snapshot(row: &rusqlite::Row<'_>) -> Result { - Ok(Snapshot { - dump: row_to_dump_row(row)?, - executed_input_count: ExecutedInputCount::new(i64_to_u64(row.get(2)?)), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; - use crate::storage::history::{advance_recovery_generation_in, initialize_history_in}; - use crate::storage::test_helpers::{SENDER_A, seed_safe_inputs_with_batch_nonces}; - use crate::storage::{LifecycleCommand, SafeInputRange, WriteHead}; - use crate::test_support::{pin_test_deployment_identity, temp_db}; - use alloy_primitives::{Address, Signature}; - use sequencer_core::user_op::{SignedUserOp, UserOp}; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - use std::time::SystemTime; - - fn inline(release: Box) { - release(); - } - fn reporter() -> PersistentReleaseFailureReporter { - Arc::new(|_| {}) - } - fn prefix(n: u64) -> PathBuf { - PathBuf::from(format!("/snapshot/{n}")) - } - - fn close(storage: &mut Storage, head: &mut WriteHead) -> i64 { - let index = head.batch_index; - let safe_block = head.safe_block; - let count = storage.next_executed_input_count().unwrap(); - storage - .close_frame_and_batch_with_snapshot(head, safe_block, &prefix(index + 1), count) - .unwrap(); - storage.latest_snapshot().unwrap().unwrap().dump.id - } - - #[test] - fn acceptance_derives_checkpoint_and_gc_preserves_every_future_candidate() { - let db = temp_db("derived-snapshot-gc"); - let mut storage = Storage::open(&db.path).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); - let baseline = storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - let first = close(&mut storage, &mut head); - let second = close(&mut storage, &mut head); - let third = close(&mut storage, &mut head); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, baseline); - seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); - assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, first); - assert_eq!( - storage - .gc_unreferenced_dumps() - .unwrap() - .iter() - .map(|r| r.id) - .collect::>(), - vec![baseline] - ); - seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 2, &[1]); - assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, second); - assert_eq!(storage.latest_snapshot().unwrap().unwrap().dump.id, third); - assert_eq!( - storage - .gc_unreferenced_dumps() - .unwrap() - .iter() - .map(|r| r.id) - .collect::>(), - vec![first] - ); - } - - #[test] - fn missing_exact_accepted_snapshot_never_falls_back_or_collects() { - let db = temp_db("missing-accepted-snapshot"); - let mut storage = Storage::open(&db.path).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); - storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - close(&mut storage, &mut head); - close(&mut storage, &mut head); - seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0, 1]); - storage - .conn - .execute("DELETE FROM snapshots WHERE batch_index=1", []) - .unwrap(); - assert!(matches!( - storage.finalized_dump(), - Err(FinalizedSelectionError::Storage( - rusqlite::Error::QueryReturnedNoRows - )) - )); - assert!(matches!( - storage.latest_snapshot(), - Err(rusqlite::Error::QueryReturnedNoRows) - )); - assert!(storage.has_rollback_safe_snapshot().is_err()); - assert!(storage.gc_unreferenced_dumps().is_err()); - assert_eq!(storage.list_dump_rows().unwrap().len(), 3); - } - - #[test] - fn rebuilt_baseline_is_restorable_but_not_an_accepted_comparison_checkpoint() { - let db = temp_db("rebuilt-snapshot"); - let mut storage = - Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - storage - .write(|tx| { - initialize_history_in(tx, ExecutedInputCount::new(41), 100)?; - insert_baseline_snapshot_in(tx, &prefix(0), ExecutedInputCount::new(41))?; - Ok(()) - }) - .unwrap(); - assert_eq!( - storage - .latest_snapshot() - .unwrap() - .unwrap() - .executed_input_count - .get(), - 41 - ); - assert!(storage.finalized_dump().unwrap().is_none()); - assert!(storage.has_rollback_safe_snapshot().unwrap()); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - } - - #[test] - fn leases_keep_artifact_count_and_version_together_after_acceptance_and_generation_change() { - let db = temp_db("snapshot-version"); - let mut storage = Storage::open(&db.path).unwrap(); - pin_test_deployment_identity(&mut storage, SENDER_A); - let baseline = storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - let old = storage - .acquire_latest_snapshot_lease(Arc::new(inline), reporter()) - .unwrap() - .unwrap(); - let old_version = old.history_version; - let (respond_to, _) = tokio::sync::oneshot::channel(); - storage - .append_executed_user_ops_chunk( - &mut head, - &[IncludedUserOp { - pending: PendingUserOp { - signed: SignedUserOp { - sender: Address::ZERO, - signature: Signature::test_signature(), - user_op: UserOp { - nonce: 0, - max_fee: u16::MAX, - data: vec![].into(), - }, - }, - respond_to, - received_at: SystemTime::now(), - }, - executed_input_offset: ExecutedInputCount::ZERO, - }], - ) - .unwrap(); - close(&mut storage, &mut head); - seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); - storage.write(advance_recovery_generation_in).unwrap(); - let new = storage - .acquire_finalized_lease(Arc::new(inline), reporter()) - .unwrap() - .unwrap(); - assert_eq!(old.prefix, prefix(0)); - assert_eq!(old.executed_input_count, ExecutedInputCount::ZERO); - assert_eq!(old.history_version, old_version); - assert_eq!(new.dump.executed_input_count.get(), 1); - assert_eq!(new.dump.next_batch_nonce, 1); - assert_ne!(new.dump.history_version, old_version); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - drop(old); - assert_eq!(storage.gc_unreferenced_dumps().unwrap()[0].id, baseline); - } - - #[test] - fn invalidated_snapshot_is_not_selected_and_its_lease_blocks_collection() { - let db = temp_db("invalidated-snapshot"); - let mut storage = Storage::open(&db.path).unwrap(); - storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .unwrap(); - let id = close(&mut storage, &mut head); - let lease = storage - .acquire_latest_snapshot_lease(Arc::new(inline), reporter()) - .unwrap() - .unwrap(); - storage - .conn - .execute( - "UPDATE batches SET invalidated_at_ms=1 WHERE batch_index=0", - [], - ) - .unwrap(); - assert_eq!( - storage.latest_snapshot().unwrap().unwrap().dump.prefix, - prefix(0) - ); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - drop(lease); - assert_eq!(storage.gc_unreferenced_dumps().unwrap()[0].id, id); - } - - #[test] - fn failed_lease_commit_never_arms_a_release() { - let db = temp_db("failed-lease-commit"); - let mut storage = Storage::open(&db.path).unwrap(); - let id = storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - storage.conn.execute_batch( - "CREATE TABLE lease_parent(id INTEGER PRIMARY KEY); - CREATE TABLE lease_child(parent_id INTEGER REFERENCES lease_parent(id) DEFERRABLE INITIALLY DEFERRED); - CREATE TRIGGER fail_lease AFTER UPDATE OF lease_count ON dumps WHEN NEW.lease_count > OLD.lease_count - BEGIN INSERT INTO lease_child VALUES(1); END;" - ).unwrap(); - let count = Arc::new(AtomicUsize::new(0)); - let counter = count.clone(); - let schedule: ReleaseScheduler = Arc::new(move |_| { - counter.fetch_add(1, Ordering::SeqCst); - }); - assert!( - storage - .acquire_latest_snapshot_lease(schedule.clone(), reporter()) - .is_err() - ); - assert!( - storage - .acquire_finalized_lease(schedule, reporter()) - .is_err() - ); - assert_eq!(count.load(Ordering::SeqCst), 0); - assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); - } - - #[test] - fn failed_history_query_never_leases_or_arms_a_release() { - let db = temp_db("failed-lease-query"); - let mut storage = Storage::open(&db.path).unwrap(); - let id = storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - storage - .conn - .execute_batch("DROP TABLE history_state") - .unwrap(); - let count = Arc::new(AtomicUsize::new(0)); - let counter = count.clone(); - let schedule: ReleaseScheduler = Arc::new(move |_| { - counter.fetch_add(1, Ordering::SeqCst); - }); - assert!( - storage - .acquire_latest_snapshot_lease(schedule.clone(), reporter()) - .is_err() - ); - assert!( - storage - .acquire_finalized_lease(schedule, reporter()) - .is_err() - ); - assert_eq!(count.load(Ordering::SeqCst), 0); - assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); - } - - #[test] - fn lease_release_cannot_underflow() { - let db = temp_db("lease-underflow"); - let mut storage = Storage::open(&db.path).unwrap(); - let id = storage - .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) - .unwrap(); - let error = storage.release_dump_lease(id).unwrap_err(); - assert_eq!( - error.sqlite_error_code(), - Some(rusqlite::ErrorCode::ConstraintViolation) - ); - assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); - } - - #[test] - fn persistent_release_failure_reaches_reporter() { - let db = temp_db("persistent-lease"); - let _storage = Storage::open(&db.path).unwrap(); - let reported = Arc::new(AtomicBool::new(false)); - let flag = reported.clone(); - drop(LeaseGuard { - path: db.path, - dump_id: i64::MAX, - schedule: Arc::new(inline), - report_persistent_failure: Arc::new(move |_| { - flag.store(true, Ordering::SeqCst); - }), - }); - assert!(reported.load(Ordering::SeqCst)); - } -} diff --git a/sequencer/src/storage/snapshots.rs b/sequencer/src/storage/snapshots.rs new file mode 100644 index 00000000..c349b190 --- /dev/null +++ b/sequencer/src/storage/snapshots.rs @@ -0,0 +1,427 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Immutable batch-close and baseline snapshots, derived acceptance, and the +//! in-process download leases. Files are durable before registration; GC deletes +//! rows before their files, which callers delete: no filesystem work here (I13). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; + +use rusqlite::{Connection, OptionalExtension, Result, Transaction, TransactionBehavior, params}; +use sequencer_core::history::{EraId, L2TxCount}; + +use super::Storage; +use super::convert::{i64_to_u64, u64_to_i64}; +use super::history::{era_in, next_l2_tx_count_in}; + +/// One `snapshots` row: the artifact directory and the count it holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Snapshot { + pub id: i64, + pub prefix: PathBuf, + pub l2_tx_count: L2TxCount, +} + +/// A batch-close snapshot whose complete predicted prefix was accepted on L1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalizedDump { + pub snapshot: Snapshot, + pub inclusion_block: u64, + pub next_batch_nonce: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum FinalizedSelectionError { + #[error("canonical divergence prevents accepted checkpoint selection")] + CanonicalDivergence, + #[error("selecting finalized checkpoint: {0}")] + Storage(#[from] rusqlite::Error), +} + +type LeaseCounters = HashMap>; + +/// Download leases, shared by the egress handlers and the lane's GC. A leased +/// snapshot's row and files are never collected. Process-local: only `run` +/// serves and collects, under the data directory's process lock. +/// +/// Acquire locks the registry, runs its selection in one read transaction, +/// increments the snapshot's counter and unlocks, so its read postdates any GC +/// commit. GC begins its IMMEDIATE transaction, locks the registry, deletes +/// only unleased candidates and unlocks after its transaction ends, so no +/// acquire reads a row GC is deleting. Release is the guard's decrement: no +/// lock, no I/O. GC holds SQLite's writer lock, then the registry; acquire +/// holds the registry and only reads, which WAL never makes wait on the writer +/// lock, so the two cannot deadlock (docs/snapshots/lifecycle.md). +#[derive(Debug, Clone, Default)] +pub struct SnapshotLeases(Arc>); + +/// One download lease. Dropping it releases the lease; a zero entry stays in +/// the registry until the next GC removes it. +#[derive(Debug)] +pub struct LeaseGuard(Arc); + +impl Drop for LeaseGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +impl SnapshotLeases { + fn lock(&self) -> MutexGuard<'_, LeaseCounters> { + self.0.lock().expect("snapshot lease registry poisoned") + } + + /// The acquire half of the protocol: `select` returns the snapshot id to + /// lease and the data read with it, in one read transaction under the lock. + fn lease( + &self, + storage: &mut Storage, + select: impl FnOnce(&Transaction<'_>) -> std::result::Result, E>, + ) -> std::result::Result, E> + where + E: From, + { + // A fresh connection parses the schema and maps the WAL index on its + // first statement: do that before the lock, so only the selection + // runs under it. + storage.read(|tx| tx.prepare("SELECT 1 FROM snapshots").map(drop))?; + let mut registry = self.lock(); + let Some((id, selected)) = storage.read(|tx| Ok(select(tx)))?? else { + return Ok(None); + }; + let counter = registry.entry(id).or_default().clone(); + counter.fetch_add(1, Ordering::SeqCst); + drop(registry); + Ok(Some((selected, LeaseGuard(counter)))) + } + + #[cfg(test)] + pub(crate) fn count(&self, snapshot_id: i64) -> Option { + self.lock() + .get(&snapshot_id) + .map(|counter| counter.load(Ordering::SeqCst)) + } + + #[cfg(test)] + fn entries(&self) -> usize { + self.lock().len() + } +} + +/// Artifact and history metadata selected in the read that took the lease. +pub struct LeasedDump { + pub prefix: PathBuf, + pub l2_tx_count: L2TxCount, + pub era_id: EraId, + pub next_batch_nonce: u64, + pub guard: LeaseGuard, +} + +pub struct FinalizedLease { + pub inclusion_block: u64, + pub dump: LeasedDump, +} + +impl Storage { + /// GC keeps the accepted checkpoint, every snapshot beyond + /// it (an intermediate optimistic snapshot may become the next accepted + /// head) and every leased snapshot. Returns the collected prefixes, whose + /// directories the caller deletes after this commit (I13). + pub fn collect_snapshots(&mut self, leases: &SnapshotLeases) -> Result> { + // SQLite's writer lock first, then the registry (`SnapshotLeases`). + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let mut registry = leases.lock(); + let collected = match collect_unleased_in(&tx, &mut registry) { + Ok(collected) => collected, + Err(error) => { + drop(tx); + drop(registry); + return Err(error); + } + }; + let committed = tx.commit(); + // Only now may an acquire read: before the commit returns, a reader + // still sees the rows this transaction deletes. + drop(registry); + committed?; + Ok(collected) + } + + /// The latest accepted batch must have its own snapshot. A missing artifact + /// is corruption, never a request to use an older accepted snapshot. + pub fn finalized_dump( + &mut self, + ) -> std::result::Result, FinalizedSelectionError> { + self.read(|tx| Ok(finalized_dump_in(tx)))? + } + + pub fn latest_snapshot(&mut self) -> Result> { + self.read(|tx| latest_snapshot_in(tx)) + } + + pub fn accepted_snapshot(&mut self) -> Result> { + self.read(|tx| accepted_snapshot_in(tx)) + } + + /// Lease the accepted checkpoint; egress calls it on a read-only handle. + pub fn acquire_finalized_lease( + &mut self, + leases: &SnapshotLeases, + ) -> std::result::Result, FinalizedSelectionError> { + let leased = leases.lease::<_, FinalizedSelectionError>(self, |tx| { + let Some(finalized) = finalized_dump_in(tx)? else { + return Ok(None); + }; + let era_id = era_in(tx)?.id; + Ok(Some((finalized.snapshot.id, (finalized, era_id)))) + })?; + Ok(leased.map(|((finalized, era_id), guard)| FinalizedLease { + inclusion_block: finalized.inclusion_block, + dump: LeasedDump { + prefix: finalized.snapshot.prefix, + l2_tx_count: finalized.snapshot.l2_tx_count, + next_batch_nonce: finalized.next_batch_nonce, + era_id, + guard, + }, + })) + } + + /// Lease the newest snapshot; egress calls it on a read-only handle. + pub fn acquire_latest_snapshot_lease( + &mut self, + leases: &SnapshotLeases, + ) -> Result> { + let leased = leases.lease::<_, rusqlite::Error>(self, |tx| { + let Some(snapshot) = latest_snapshot_in(tx)? else { + return Ok(None); + }; + let era_id = era_in(tx)?.id; + let next_batch_nonce = snapshot_next_nonce_in(tx, snapshot.id)?; + Ok(Some((snapshot.id, (snapshot, era_id, next_batch_nonce)))) + })?; + Ok( + leased.map(|((snapshot, era_id, next_batch_nonce), guard)| LeasedDump { + prefix: snapshot.prefix, + l2_tx_count: snapshot.l2_tx_count, + era_id, + next_batch_nonce, + guard, + }), + ) + } + + pub fn list_snapshots(&mut self) -> Result> { + self.read(|tx| { + let mut stmt = tx.prepare( + "SELECT snapshot_id, prefix, l2_tx_count FROM snapshots ORDER BY snapshot_id", + )?; + stmt.query_map([], row_to_snapshot)?.collect() + }) + } +} + +/// Select the collectible snapshots, keep the leased ones, drop the registry's +/// zero entries, and delete the remaining rows. +fn collect_unleased_in(tx: &Transaction<'_>, registry: &mut LeaseCounters) -> Result> { + let anchor_id = accepted_snapshot_in(tx)?.map(|snapshot| snapshot.id); + // Collectible: not the accepted snapshot, and either the baseline or an + // accepted batch's. + let candidates = { + let mut stmt = tx.prepare( + "SELECT s.snapshot_id, s.prefix FROM snapshots s \ + LEFT JOIN batches b ON b.nonce = s.batch_nonce \ + WHERE (?1 IS NULL OR s.snapshot_id != ?1) \ + AND (s.batch_nonce IS NULL OR b.accepted_input IS NOT NULL) \ + ORDER BY s.snapshot_id", + )?; + stmt.query_map(params![anchor_id], |row| { + Ok(( + row.get::<_, i64>(0)?, + PathBuf::from(row.get::<_, String>(1)?), + )) + })? + .collect::>>()? + }; + let leased = |id: &i64| { + registry + .get(id) + .is_some_and(|counter| counter.load(Ordering::SeqCst) > 0) + }; + let collected: Vec<(i64, PathBuf)> = candidates + .into_iter() + .filter(|(id, _)| !leased(id)) + .collect(); + registry.retain(|_, counter| counter.load(Ordering::SeqCst) > 0); + let mut prefixes = Vec::with_capacity(collected.len()); + for (id, prefix) in collected { + tx.execute("DELETE FROM snapshots WHERE snapshot_id = ?1", [id])?; + prefixes.push(prefix); + } + Ok(prefixes) +} + +pub(super) fn insert_baseline_snapshot_in( + tx: &Transaction<'_>, + prefix: &Path, + count: L2TxCount, +) -> Result { + insert_snapshot_in(tx, prefix, None, count) +} + +pub(super) fn insert_batch_snapshot_in( + tx: &Transaction<'_>, + prefix: &Path, + batch_nonce: u64, + count: L2TxCount, +) -> Result { + // Registration is composed with sealing, so the batch must already be closed. + tx.query_row( + "SELECT nonce FROM closed_batches WHERE nonce = ?1", + [u64_to_i64(batch_nonce)], + |_| Ok(()), + )?; + insert_snapshot_in(tx, prefix, Some(batch_nonce), count) +} + +fn insert_snapshot_in( + tx: &Transaction<'_>, + prefix: &Path, + batch_nonce: Option, + count: L2TxCount, +) -> Result { + assert_eq!( + count, + next_l2_tx_count_in(tx)?, + "snapshot count differs from canonical storage history" + ); + tx.execute( + "INSERT INTO snapshots (batch_nonce, l2_tx_count, prefix) VALUES (?1, ?2, ?3)", + params![ + batch_nonce.map(u64_to_i64), + u64_to_i64(count.get()), + prefix.to_string_lossy().as_ref() + ], + )?; + Ok(tx.last_insert_rowid()) +} + +/// Whether a accepted snapshot exists, read inside the caller's +/// transaction (the history metadata reads it). +pub(super) fn has_accepted_snapshot_in(conn: &Connection) -> Result { + Ok(accepted_snapshot_in(conn)?.is_some()) +} + +fn accepted_snapshot_in(conn: &Connection) -> Result> { + match latest_accepted_boundary_in(conn)? { + Some((nonce, _)) => snapshot_for_batch_in(conn, nonce).map(Some), + None => baseline_snapshot_in(conn), + } +} + +/// The greatest accepted batch: `(nonce, inclusion block)`, the block of the +/// safe input that carried its accepted landing. +fn latest_accepted_boundary_in(conn: &Connection) -> Result> { + conn.query_row( + "SELECT b.nonce, s.block_number FROM batches b \ + JOIN safe_inputs s ON s.safe_input_index = b.accepted_input \ + WHERE b.accepted_input = (SELECT MAX(accepted_input) FROM batches)", + [], + |row| Ok((i64_to_u64(row.get(0)?), i64_to_u64(row.get(1)?))), + ) + .optional() +} + +fn snapshot_for_batch_in(conn: &Connection, batch_nonce: u64) -> Result { + conn.query_row( + "SELECT snapshot_id, prefix, l2_tx_count FROM snapshots WHERE batch_nonce = ?1", + [u64_to_i64(batch_nonce)], + row_to_snapshot, + ) +} + +fn baseline_snapshot_in(conn: &Connection) -> Result> { + conn.query_row( + "SELECT snapshot_id, prefix, l2_tx_count FROM snapshots WHERE batch_nonce IS NULL", + [], + row_to_snapshot, + ) + .optional() +} + +pub(super) fn finalized_dump_in( + conn: &Connection, +) -> std::result::Result, FinalizedSelectionError> { + // A matched batch can precede a divergent accepted batch in the same L1 + // block. Its snapshot cannot represent that complete block boundary. + if super::acceptance::canonical_divergence_in(conn)?.is_some() { + return Err(FinalizedSelectionError::CanonicalDivergence); + } + if let Some((nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { + return Ok(Some(FinalizedDump { + snapshot: snapshot_for_batch_in(conn, nonce)?, + inclusion_block, + next_batch_nonce: nonce.checked_add(1).expect("accepted nonce overflow"), + })); + } + // The baseline is the application's state at the end of the stop block, + // which the canonical machine reaches too, so it is the era's first + // comparison checkpoint, genesis base or not. + let Some(snapshot) = baseline_snapshot_in(conn)? else { + return Ok(None); + }; + let era = era_in(conn)?; + assert_eq!( + snapshot.l2_tx_count, era.rebuilt_end, + "the era baseline sits at the rebuilt history's end" + ); + Ok(Some(FinalizedDump { + snapshot, + inclusion_block: era.stop_block, + next_batch_nonce: era.anchor_batch_nonce, + })) +} + +fn latest_snapshot_in(conn: &Connection) -> Result> { + let nonce: Option = conn + .query_row( + "SELECT nonce FROM closed_batches ORDER BY nonce DESC LIMIT 1", + [], + |r| r.get(0), + ) + .optional()?; + match nonce { + Some(nonce) => snapshot_for_batch_in(conn, i64_to_u64(nonce)).map(Some), + None => baseline_snapshot_in(conn), + } +} + +fn snapshot_next_nonce_in(conn: &Connection, snapshot_id: i64) -> Result { + let batch_nonce: Option = conn.query_row( + "SELECT batch_nonce FROM snapshots WHERE snapshot_id = ?1", + [snapshot_id], + |r| r.get(0), + )?; + match batch_nonce { + Some(nonce) => Ok(i64_to_u64(nonce) + .checked_add(1) + .expect("batch nonce overflow")), + None => Ok(era_in(conn)?.anchor_batch_nonce), + } +} + +fn row_to_snapshot(row: &rusqlite::Row<'_>) -> Result { + Ok(Snapshot { + id: row.get(0)?, + prefix: PathBuf::from(row.get::<_, String>(1)?), + l2_tx_count: L2TxCount::new(i64_to_u64(row.get(2)?)), + }) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/snapshots/tests.rs b/sequencer/src/storage/snapshots/tests.rs new file mode 100644 index 00000000..ef7d248c --- /dev/null +++ b/sequencer/src/storage/snapshots/tests.rs @@ -0,0 +1,389 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::*; +use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; +use crate::storage::test_helpers::{SENDER_A, era, seed_safe_inputs_with_batch_nonces}; +use crate::storage::{LifecycleCommand, SafeInputRange, WriteHead}; +use crate::test_support::{pin_test_deployment_identity, temp_db}; +use alloy_primitives::{Address, Signature}; +use sequencer_core::user_op::{SignedUserOp, UserOp}; +use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::time::SystemTime; + +fn prefix(n: u64) -> PathBuf { + PathBuf::from(format!("/snapshot/{n}")) +} + +/// Close the open batch with a snapshot at `prefix(batch_nonce + 1)`. +fn close(storage: &mut Storage, head: &mut WriteHead) -> i64 { + let index = head.batch_nonce(); + let count = storage.next_l2_tx_count().unwrap(); + storage + .close_frame_and_batch_with_snapshot(head, &prefix(index + 1), count) + .unwrap(); + storage.latest_snapshot().unwrap().unwrap().id +} + +/// A baseline at `prefix(0)` and an open genesis Tip. +fn baseline_and_tip(name: &str) -> (crate::test_support::TestDb, Storage, i64, WriteHead) { + let db = temp_db(name); + // No baseline yet: the fixture registers its own at `prefix(0)`. + let mut storage = Storage::open_with_era(&db.path, &crate::storage::NewEra::EMPTY_GENESIS); + let baseline = storage + .insert_baseline_snapshot(&prefix(0), L2TxCount::ZERO) + .unwrap(); + let head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + (db, storage, baseline, head) +} + +#[test] +fn acceptance_derives_checkpoint_and_gc_preserves_every_future_candidate() { + let (_db, mut storage, baseline, mut head) = baseline_and_tip("derived-snapshot-gc"); + let leases = SnapshotLeases::default(); + let first = close(&mut storage, &mut head); + let second = close(&mut storage, &mut head); + let third = close(&mut storage, &mut head); + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + assert_eq!( + storage.finalized_dump().unwrap().unwrap().snapshot.id, + baseline + ); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + assert_eq!( + storage.finalized_dump().unwrap().unwrap().snapshot.id, + first + ); + assert_eq!(storage.collect_snapshots(&leases).unwrap(), vec![prefix(0)]); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 2, &[1]); + assert_eq!( + storage.finalized_dump().unwrap().unwrap().snapshot.id, + second + ); + assert_eq!(storage.latest_snapshot().unwrap().unwrap().id, third); + assert_eq!(storage.collect_snapshots(&leases).unwrap(), vec![prefix(1)]); +} + +#[test] +fn missing_exact_accepted_snapshot_never_falls_back_or_collects() { + let (_db, mut storage, _, mut head) = baseline_and_tip("missing-accepted-snapshot"); + close(&mut storage, &mut head); + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0, 1]); + storage + .conn + .execute("DELETE FROM snapshots WHERE batch_nonce=1", []) + .unwrap(); + assert!(matches!( + storage.finalized_dump(), + Err(FinalizedSelectionError::Storage( + rusqlite::Error::QueryReturnedNoRows + )) + )); + assert!(matches!( + storage.latest_snapshot(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(storage.has_accepted_snapshot().is_err()); + assert!( + storage + .collect_snapshots(&SnapshotLeases::default()) + .is_err() + ); + assert_eq!(storage.list_snapshots().unwrap().len(), 2); +} + +#[test] +fn the_era_baseline_is_the_comparison_checkpoint_at_the_stop_block() { + let db = temp_db("rebuilt-snapshot"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + storage + .publish_era( + &prefix(0), + &crate::storage::NewEra { + anchor_batch_nonce: 7, + ..era(41, 41, 100) + }, + ) + .unwrap(); + assert_eq!( + storage + .latest_snapshot() + .unwrap() + .unwrap() + .l2_tx_count + .get(), + 41 + ); + let checkpoint = storage.finalized_dump().unwrap().expect("baseline"); + assert_eq!(checkpoint.snapshot.prefix, prefix(0)); + assert_eq!( + ( + checkpoint.inclusion_block, + checkpoint.snapshot.l2_tx_count.get(), + checkpoint.next_batch_nonce + ), + (100, 41, 7) + ); + assert!(storage.has_accepted_snapshot().unwrap()); + assert!( + storage + .collect_snapshots(&SnapshotLeases::default()) + .unwrap() + .is_empty() + ); +} + +#[test] +fn snapshot_prefix_is_immutable() { + let (_db, storage, baseline, _) = baseline_and_tip("snapshot-prefix-immutable"); + let error = storage + .conn + .execute( + "UPDATE snapshots SET prefix = '/elsewhere' WHERE snapshot_id = ?1", + [baseline], + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("snapshot boundaries are immutable"), + "{error}" + ); +} + +#[test] +fn leases_keep_artifact_count_and_era_together_after_acceptance() { + let (_db, mut storage, baseline, mut head) = baseline_and_tip("snapshot-version"); + let leases = SnapshotLeases::default(); + let old = storage + .acquire_latest_snapshot_lease(&leases) + .unwrap() + .unwrap(); + let old_era = old.era_id; + let (respond_to, _) = tokio::sync::oneshot::channel(); + storage + .append_included_user_ops( + &mut head, + &[IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: Address::ZERO, + signature: Signature::test_signature(), + user_op: UserOp { + nonce: 0, + max_fee: u16::MAX, + data: vec![].into(), + }, + }, + respond_to, + received_at: SystemTime::now(), + }, + l2_tx_index: L2TxCount::ZERO, + }], + ) + .unwrap(); + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + let new = storage.acquire_finalized_lease(&leases).unwrap().unwrap(); + assert_eq!(old.prefix, prefix(0)); + assert_eq!(old.l2_tx_count, L2TxCount::ZERO); + assert_eq!(old.era_id, old_era); + assert_eq!(new.dump.l2_tx_count.get(), 1); + assert_eq!(new.dump.next_batch_nonce, 1); + assert_eq!(new.dump.era_id, old_era); + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + assert_eq!(leases.count(baseline), Some(1)); + drop(old); + assert_eq!(storage.collect_snapshots(&leases).unwrap(), vec![prefix(0)]); +} + +#[test] +fn a_lease_taken_before_gc_keeps_its_row_until_the_guard_drops() { + let (_db, mut storage, baseline, mut head) = baseline_and_tip("lease-before-gc"); + let leases = SnapshotLeases::default(); + let lease = storage.acquire_finalized_lease(&leases).unwrap().unwrap(); + assert_eq!(lease.dump.prefix, prefix(0)); + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + assert!( + storage + .list_snapshots() + .unwrap() + .iter() + .any(|snapshot| snapshot.id == baseline), + "a leased row is never collected" + ); + drop(lease); + assert_eq!(leases.count(baseline), Some(0)); + assert_eq!(storage.collect_snapshots(&leases).unwrap(), vec![prefix(0)]); + assert!( + storage + .list_snapshots() + .unwrap() + .iter() + .all(|snapshot| snapshot.id != baseline) + ); +} + +#[test] +fn gc_removes_zero_entries_and_keeps_live_ones() { + let (_db, mut storage, baseline, mut head) = baseline_and_tip("lease-zero-entries"); + let leases = SnapshotLeases::default(); + drop(storage.acquire_finalized_lease(&leases).unwrap().unwrap()); + let first = close(&mut storage, &mut head); + let live = storage + .acquire_latest_snapshot_lease(&leases) + .unwrap() + .unwrap(); + assert_eq!(leases.entries(), 2); + assert_eq!(leases.count(baseline), Some(0)); + + // Nothing is collectible yet: removing zero entries does not need a candidate. + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + assert_eq!(leases.entries(), 1); + assert_eq!(leases.count(baseline), None); + assert_eq!(leases.count(first), Some(1)); + drop(live); + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + assert_eq!(leases.entries(), 0); +} + +#[test] +fn two_leases_on_one_snapshot_both_keep_it() { + let (_db, mut storage, baseline, mut head) = baseline_and_tip("two-leases"); + let leases = SnapshotLeases::default(); + let first = storage.acquire_finalized_lease(&leases).unwrap().unwrap(); + let second = storage + .acquire_latest_snapshot_lease(&leases) + .unwrap() + .unwrap(); + assert_eq!( + (first.dump.prefix.clone(), second.prefix.clone()), + (prefix(0), prefix(0)) + ); + assert_eq!(leases.count(baseline), Some(2)); + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + + drop(first); + assert_eq!(leases.count(baseline), Some(1)); + assert!(storage.collect_snapshots(&leases).unwrap().is_empty()); + drop(second); + assert_eq!(storage.collect_snapshots(&leases).unwrap(), vec![prefix(0)]); +} + +#[test] +fn a_failed_selection_takes_no_lease() { + let (db, storage, _, _) = baseline_and_tip("failed-selection-no-lease"); + storage + .conn + .execute_batch("DROP TRIGGER trg_era_not_deletable; DELETE FROM era") + .unwrap(); + let leases = SnapshotLeases::default(); + let mut reader = Storage::open_read_only(&db.path).unwrap(); + assert!(reader.acquire_finalized_lease(&leases).is_err()); + assert!(reader.acquire_latest_snapshot_lease(&leases).is_err()); + assert_eq!(leases.entries(), 0); +} + +fn is_busy(error: &rusqlite::Error) -> bool { + matches!( + error.sqlite_error_code(), + Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) + ) +} + +/// One thread leases the oldest snapshot in a loop while another supersedes and +/// collects it in a loop. After every GC that may have overlapped a lease has +/// returned, the leased row must still exist. +#[test] +fn concurrent_leases_never_hold_a_collected_row() { + const ROUNDS: u64 = 200; + let (db, mut writer, _, mut head) = baseline_and_tip("lease-gc-race"); + let leases = SnapshotLeases::default(); + let gc_started = AtomicU64::new(0); + let gc_finished = AtomicU64::new(0); + let stop = AtomicBool::new(false); + + /// Releases the leaser when the collector finishes or panics: the scope + /// joins the leaser, which otherwise waits on the collector forever. + struct ReleaseLeaser<'a> { + stop: &'a AtomicBool, + gc_finished: &'a AtomicU64, + } + impl Drop for ReleaseLeaser<'_> { + fn drop(&mut self) { + self.gc_finished.store(u64::MAX, Ordering::SeqCst); + self.stop.store(true, Ordering::SeqCst); + } + } + + std::thread::scope(|scope| { + let leaser = scope.spawn(|| { + let mut reader = Storage::open_read_only(&db.path).unwrap(); + let mut leased = 0_u64; + while !stop.load(Ordering::SeqCst) { + let oldest = |tx: &Transaction<'_>| { + tx.query_row( + "SELECT snapshot_id FROM snapshots ORDER BY snapshot_id LIMIT 1", + [], + |row| row.get::<_, i64>(0), + ) + .optional() + .map(|id| id.map(|id| (id, id))) + }; + let (id, guard) = match leases.lease(&mut reader, oldest) { + Ok(Some(leased)) => leased, + Ok(None) => continue, + Err(error) if is_busy(&error) => continue, + Err(error) => panic!("lease: {error}"), + }; + // Every GC that could have overlapped the lease has committed. + let started = gc_started.load(Ordering::SeqCst); + while gc_finished.load(Ordering::SeqCst) < started { + std::thread::yield_now(); + } + let exists = loop { + match reader.read(|tx| { + tx.query_row( + "SELECT COUNT(*) FROM snapshots WHERE snapshot_id = ?1", + [id], + |row| row.get::<_, i64>(0), + ) + }) { + Ok(count) => break count == 1, + Err(error) if is_busy(&error) => continue, + Err(error) => panic!("check leased row: {error}"), + } + }; + assert!(exists, "GC collected leased snapshot {id}"); + drop(guard); + leased += 1; + } + leased + }); + + let release = ReleaseLeaser { + stop: &stop, + gc_finished: &gc_finished, + }; + let mut collected = 0; + for round in 0..ROUNDS { + close(&mut writer, &mut head); + seed_safe_inputs_with_batch_nonces(&mut writer, SENDER_A, round + 1, &[round]); + gc_started.fetch_add(1, Ordering::SeqCst); + collected += writer.collect_snapshots(&leases).unwrap().len(); + gc_finished.fetch_add(1, Ordering::SeqCst); + } + drop(release); + let leased = leaser.join().expect("leaser thread"); + assert!(leased > 0, "the leaser never took a lease"); + assert!(collected > 0, "GC never collected a superseded snapshot"); + }); +} diff --git a/sequencer/src/storage/test_helpers.rs b/sequencer/src/storage/test_helpers.rs index b28f5219..4f8f349e 100644 --- a/sequencer/src/storage/test_helpers.rs +++ b/sequencer/src/storage/test_helpers.rs @@ -1,44 +1,182 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Storage-domain test fixtures: seeding, the recovery fixture trait and +//! Storage-domain test fixtures: seeding, the landing fixture trait and //! divergence markers. Crate-wide fixtures live in `crate::test_support`. use std::path::Path; use alloy_primitives::{Address, B256}; +use app_core::application::WALLET_SCHEDULER_LIMITS; use rusqlite::params; -use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; -use sequencer_core::protocol::ProtocolTiming; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; -use super::convert::{from_unix_ms, i64_to_u16, i64_to_u64, now_unix_ms, u64_to_i64}; -use super::history::next_executed_input_count_in; +use super::convert::{i64_to_u16, i64_to_u32, i64_to_u64, now_unix_ms, u64_to_i64}; +use super::history::next_l2_tx_count_in; use super::ingress::{ insert_tip_rows, open_fresh_tip_in_tx, open_successor_frame_in, seal_and_open_next_batch, }; use super::mutations::persist_frame_direct_sequence_derived; use super::queries::{load_current_write_head, query_batch_policy}; -use super::recovery::{recover_aging_tip_inner, recover_post_flush_inner}; -use super::snapshot_dumps::{insert_baseline_snapshot_in, insert_batch_snapshot_in}; +use super::snapshots::{insert_baseline_snapshot_in, insert_batch_snapshot_in}; use super::{ - BatchPolicy, ExecutedInputCount, FrontierMode, IngestedSafeInput, SafeInputRange, Storage, - StorageOpenError, StoredSafeInput, WriteHead, + BatchPolicy, IngestedSafeInput, L2TxCount, LifecycleCommand, NewEra, SafeInputRange, Storage, + StorageOpenError, WriteHead, }; use crate::test_support::{TestDb, pin_test_deployment_identity, temp_db}; +/// One safe input as stored on the L1 InputBox: sender, opaque payload, and +/// the L1 block where it was included. Test fixtures seed inputs with it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct StoredSafeInput { + pub sender: Address, + pub payload: Vec, + /// Chain block number where this input was included (e.g. InputAdded event block). + pub block_number: u64, +} + +impl NewEra { + /// An era at genesis that rebuilt nothing, with stop block 0 and anchor 0: + /// what [`set_up_storage`], the crate-test open path and seeded harnesses + /// publish. + pub(crate) const EMPTY_GENESIS: Self = Self { + base: L2TxCount::ZERO, + rebuilt_end: L2TxCount::ZERO, + stop_block: 0, + first_queued_input: 0, + anchor_batch_nonce: 0, + }; +} + +impl SafeInputRange { + pub(crate) fn empty_at(index: u64) -> Self { + Self::new(index, index) + } +} + +impl WriteHead { + pub(crate) fn frame_in_batch(&self) -> u32 { + self.frame_in_batch + } + + pub(crate) fn open_frame_user_op_count(&self) -> u32 { + self.open_frame_user_op_count + } +} + +/// A database in the state `setup` leaves it, built through setup's own +/// calls: the schema, the test deployment identity with `batch_submitter`, +/// and the genesis era published with its baseline snapshot (a `baseline` +/// prefix beside the database file). A test that needs no identity or no +/// baseline yet opens through `Storage::open_with_era` instead; one that +/// needs no era, through `Storage::initialize_for_command`. +pub(crate) fn set_up_storage(path: &str, batch_submitter: Address) -> Storage { + set_up_storage_with_window( + path, + batch_submitter, + WALLET_SCHEDULER_LIMITS.max_wait_blocks, + ) +} + +/// [`set_up_storage`] with the identity pinning `max_wait_blocks` instead of +/// the wallet's window. +pub(crate) fn set_up_storage_with_window( + path: &str, + batch_submitter: Address, + max_wait_blocks: u64, +) -> Storage { + let mut storage = + Storage::initialize_for_command(path, LifecycleCommand::Setup).expect("create schema"); + crate::test_support::pin_test_deployment_identity_with_window( + &mut storage, + batch_submitter, + max_wait_blocks, + ); + storage + .publish_era( + &Path::new(path).with_file_name("baseline"), + &NewEra::EMPTY_GENESIS, + ) + .expect("publish the genesis era"); + storage +} + pub(crate) const SENDER_A: Address = Address::repeat_byte(0xAA); pub(crate) const SENDER_B: Address = Address::repeat_byte(0xBB); -/// Default protocol timing for tests that don't care about specific tuning. -/// Sender-independent (timing carries no address); pair with a submitter -/// address as needed at the call site. -pub(crate) fn default_protocol_timing() -> ProtocolTiming { - ProtocolTiming::try_new(sequencer_core::MAX_WAIT_BLOCKS, 75, 900, 12) - .expect("valid test timing") +/// The direct payload cap every test identity pins: the wallet's, which +/// `StubApp` shares, so fixtures and either test lane drain the same directs. +pub(crate) const TEST_MAX_DIRECT_PAYLOAD_BYTES: u64 = + WALLET_SCHEDULER_LIMITS.max_direct_payload_bytes; + +/// An era with nothing queued and anchor 0; fixtures override fields. +pub(crate) fn era(base: u64, rebuilt_end: u64, stop_block: u64) -> super::NewEra { + super::NewEra { + base: L2TxCount::new(base), + rebuilt_end: L2TxCount::new(rebuilt_end), + stop_block, + first_queued_input: 0, + anchor_batch_nonce: 0, + } +} + +/// Every field of a [`WriteHead`], for the few tests that build one by hand: +/// the lane's pure dequeue tests, the counter-overflow tests and the stale-head +/// tests. Production heads come only from storage's loader. +#[derive(Debug, Clone, Copy)] +pub(crate) struct WriteHeadParts { + pub batch_nonce: u64, + pub batch_created_at: std::time::SystemTime, + pub frame_fee: u16, + pub safe_block: u64, + pub batch_user_op_count: u64, + pub open_frame_user_op_count: u32, + pub frame_in_batch: u32, + pub max_batch_user_op_bytes: u64, +} + +impl WriteHeadParts { + /// Nonce 0, frame 0, everything zero, a one-byte size target. + pub(crate) const ZERO: Self = Self { + batch_nonce: 0, + batch_created_at: std::time::UNIX_EPOCH, + frame_fee: 0, + safe_block: 0, + batch_user_op_count: 0, + open_frame_user_op_count: 0, + frame_in_batch: 0, + max_batch_user_op_bytes: 1, + }; + + pub(crate) fn of(head: &WriteHead) -> Self { + Self { + batch_nonce: head.batch_nonce, + batch_created_at: head.batch_created_at, + frame_fee: head.frame_fee, + safe_block: head.safe_block, + batch_user_op_count: head.batch_user_op_count, + open_frame_user_op_count: head.open_frame_user_op_count, + frame_in_batch: head.frame_in_batch, + max_batch_user_op_bytes: head.max_batch_user_op_bytes, + } + } + + pub(crate) fn build(self) -> WriteHead { + WriteHead { + batch_nonce: self.batch_nonce, + batch_created_at: self.batch_created_at, + frame_fee: self.frame_fee, + safe_block: self.safe_block, + batch_user_op_count: self.batch_user_op_count, + open_frame_user_op_count: self.open_frame_user_op_count, + frame_in_batch: self.frame_in_batch, + max_batch_user_op_bytes: self.max_batch_user_op_bytes, + } + } } /// Lane fixtures. Production opens the Tip through -/// `ensure_open_tip_for_recovery` and closes frames and batches with +/// `open_batch_if_none` and closes frames and batches with /// attributed executions and a snapshot (I7). impl Storage { /// Open the genesis Tip with an explicit leading range; asserts none exists. @@ -52,14 +190,14 @@ impl Storage { load_current_write_head(tx)?.is_none(), "open state already exists" ); - let batch_index = insert_tip_rows(tx, Some(0), None, safe_block)?; - persist_frame_direct_sequence_derived(tx, batch_index, 0, leading_direct_range)?; + let batch_nonce = insert_tip_rows(tx, safe_block)?; + persist_frame_direct_sequence_derived(tx, batch_nonce, 0, leading_direct_range)?; Ok(load_current_write_head(tx)?.expect("genesis tip just inserted")) }) } /// Open the genesis Tip at the observed safe head unless one exists; the - /// unguarded twin of `ensure_open_tip_for_recovery`. + /// unguarded twin of `open_batch_if_none`. pub(crate) fn ensure_open_tip(&mut self) -> rusqlite::Result<()> { self.write(|tx| { if load_current_write_head(tx)?.is_some() { @@ -83,19 +221,9 @@ impl Storage { } /// Close the batch without registering a snapshot. - pub(crate) fn close_frame_and_batch( - &mut self, - head: &mut WriteHead, - next_safe_block: u64, - ) -> rusqlite::Result<()> { - let (next_batch_index, now_ms, policy) = - self.write(|tx| seal_and_open_next_batch(tx, head.batch_index, next_safe_block))?; - head.move_to_next_batch( - next_batch_index, - from_unix_ms(now_ms), - policy, - next_safe_block, - ); + pub(crate) fn close_frame_and_batch(&mut self, head: &mut WriteHead) -> rusqlite::Result<()> { + let next = self.write(|tx| seal_and_open_next_batch(tx, head))?; + next.apply_to(head); Ok(()) } @@ -103,106 +231,131 @@ impl Storage { query_batch_policy(&self.conn) } - /// The batch-tree anchor nonce (default 0). - pub(crate) fn batch_tree_anchor(&mut self) -> rusqlite::Result { - self.read(|tx| super::mutations::batch_tree_anchor_in(tx)) + /// The published era; errors before setup publishes one. + pub(crate) fn era(&mut self) -> rusqlite::Result { + self.read(|tx| super::history::era_in(tx)) + } +} + +/// Reads only tests make. +impl Storage { + /// Unix-ms of the last successful `log_gas_price` write. `0` means never + /// written (migration default); completed setup guarantees a nonzero value. + pub(crate) fn log_gas_price_updated_at_ms(&self) -> rusqlite::Result { + self.conn.query_row( + "SELECT log_gas_price_updated_at_ms FROM batch_policy WHERE singleton_id = 0", + [], + |row| Ok(i64_to_u64(row.get::<_, i64>(0)?)), + ) } } /// Seeding and inspection levers. Each bypasses the guard, provenance or /// wall clock its production counterpart enforces. impl Storage { - /// Set the batch-tree anchor nonce directly; production sets it once, in - /// `complete_baseline_setup`. Aborts if `setup_complete` already exists - /// (`trg_batch_tree_anchor_write_once`). - pub(crate) fn set_batch_tree_anchor(&mut self, nonce: u64) -> rusqlite::Result<()> { - self.write(|tx| super::mutations::set_batch_tree_anchor_in(tx, nonce)) + /// A fresh database with `era`: setup's schema, then the era row through + /// `publish_era`'s insert, with no identity and no baseline snapshot + /// (callers that need either add it). For fixtures whose era is not the + /// genesis one, or that need no identity or no baseline yet. + pub(crate) fn open_with_era(path: &str, era: &super::NewEra) -> Storage { + let mut storage = + Storage::initialize_for_command(path, super::LifecycleCommand::Setup).expect("schema"); + storage + .write(|tx| super::history::insert_era_in(tx, era)) + .expect("insert era"); + storage } - pub(crate) fn next_executed_input_count(&mut self) -> rusqlite::Result { - self.read(|tx| next_executed_input_count_in(tx)) + pub(crate) fn next_l2_tx_count(&mut self) -> rusqlite::Result { + self.read(|tx| next_l2_tx_count_in(tx)) } - pub(crate) fn has_rollback_safe_snapshot(&mut self) -> rusqlite::Result { - self.read(|tx| super::snapshot_dumps::has_rollback_safe_snapshot_in(tx)) + pub(crate) fn has_accepted_snapshot(&mut self) -> rusqlite::Result { + self.read(|tx| super::snapshots::has_accepted_snapshot_in(tx)) } - pub(crate) fn dump_lease_count(&mut self, dump_id: i64) -> rusqlite::Result> { - use rusqlite::OptionalExtension; + /// [`Storage::read`]'s deferred transaction, for tests outside storage + /// that read raw SQL. + pub(crate) fn raw_read( + &mut self, + f: impl FnOnce(&rusqlite::Transaction<'_>) -> rusqlite::Result, + ) -> rusqlite::Result { + self.read(f) + } - self.read(|tx| { - tx.query_row( - "SELECT lease_count FROM dumps WHERE id = ?1", - [dump_id], - |r| r.get(0), - ) - .optional() - }) + /// [`Storage::write`]'s immediate transaction, for tests outside storage + /// that write raw SQL. + pub(crate) fn raw_write( + &mut self, + f: impl FnOnce(&rusqlite::Transaction<'_>) -> rusqlite::Result, + ) -> rusqlite::Result { + self.write(f) } /// A raw connection with [`Storage::open`]'s pragmas, for tests that reach - /// past the typed API (rewinding `synced_at_ms`, installing failure - /// triggers). + /// past the typed API (raw reads, installing failure triggers). pub(crate) fn open_connection(path: &str) -> Result { super::open::open_writer_connection(path) } - /// Append safe inputs at the current wall clock, with zero provenance. + /// Append safe inputs with the wall clock as the safe block's L1 + /// timestamp, with zero provenance. See + /// [`Storage::append_safe_inputs_with_timestamp`] for `batch_submitter`. pub(crate) fn append_safe_inputs( &mut self, safe_block: u64, inputs: &[StoredSafeInput], batch_submitter: Address, - timing: &ProtocolTiming, ) -> rusqlite::Result<()> { self.append_safe_inputs_with_timestamp( safe_block, i64_to_u64(now_unix_ms()) / 1000, inputs, batch_submitter, - timing, - FrontierMode::Populate, ) } /// [`Storage::append_safe_inputs`] at an explicit L1 timestamp of - /// `safe_block`. `frontier` gates the `safe_accepted_batches` update. + /// `safe_block`. Stands in for setup's identity pin: on an unpinned + /// database it pins a test identity whose submitter is `batch_submitter`; + /// on a pinned one it asserts the pinned submitter is `batch_submitter`. + /// Storage reads the submitter from the pin, never from this argument. pub(crate) fn append_safe_inputs_with_timestamp( &mut self, safe_block: u64, safe_block_timestamp: u64, inputs: &[StoredSafeInput], batch_submitter: Address, - timing: &ProtocolTiming, - frontier: FrontierMode, ) -> rusqlite::Result<()> { + match self.deployment_identity()? { + Some(identity) => assert_eq!( + identity.batch_submitter_address, batch_submitter, + "the fixture's submitter differs from the pinned identity's" + ), + None => pin_test_deployment_identity(self, batch_submitter), + } let rows: Vec = inputs.iter().map(synthetic_row).collect(); self.append_ingested_safe_inputs_with_timestamp( safe_block, safe_block_timestamp, rows.as_slice(), - batch_submitter, - timing, - frontier, ) } - /// Highest valid `batch_index` (the open batch included), or `None`. - pub(crate) fn latest_batch_index(&mut self) -> rusqlite::Result> { + /// Highest batch nonce (the open batch included), or `None`. + pub(crate) fn latest_batch_nonce(&mut self) -> rusqlite::Result> { let value: Option = self.conn - .query_row("SELECT MAX(batch_index) FROM valid_batches", [], |row| { - row.get(0) - })?; + .query_row("SELECT MAX(nonce) FROM batches", [], |row| row.get(0))?; Ok(value.map(i64_to_u64)) } - /// The sequenced L2 txs of one batch; `[]` for an invalidated batch. - /// Production replay pages the valid stream by offset, never per batch. + /// The sequenced L2 txs of one batch. + /// Production replay pages the stream by offset, never per batch. pub(crate) fn ordered_l2_txs_for_batch( &mut self, - batch_index: u64, - ) -> rusqlite::Result> { + batch_nonce: u64, + ) -> rusqlite::Result> { const SQL: &str = " SELECT CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END AS kind, @@ -211,78 +364,63 @@ impl Storage { WHEN s.safe_input_index IS NOT NULL THEN d.sender ELSE NULL END AS sender, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END AS nonce, + f.safe_block, + s.safe_input_index, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END AS block_timestamp FROM application_inputs s LEFT JOIN user_ops u - ON u.batch_index = s.batch_index + ON u.batch_nonce = s.batch_nonce AND u.frame_in_batch = s.frame_in_batch AND u.pos_in_frame = s.user_op_pos_in_frame LEFT JOIN frames f - ON f.batch_index = s.batch_index + ON f.batch_nonce = s.batch_nonce AND f.frame_in_batch = s.frame_in_batch LEFT JOIN safe_inputs d ON d.safe_input_index = s.safe_input_index - WHERE s.batch_index = ?1 - ORDER BY s.offset ASC + WHERE s.batch_nonce = ?1 + ORDER BY s.l2_tx_count ASC "; let mut stmt = self.conn.prepare_cached(SQL)?; - let rows = stmt.query_map(params![u64_to_i64(batch_index)], |row| { - Ok(decode_l2_tx_row( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - )) - })?; + let rows = stmt.query_map(params![u64_to_i64(batch_nonce)], decode_l2_tx_row)?; rows.collect::>>() } - /// Mark one batch invalid; leaves an already-invalid row alone. - pub(crate) fn insert_invalid_batch(&mut self, batch_index: u64) -> rusqlite::Result<()> { - let now_ms = now_unix_ms(); - self.conn.execute( - "UPDATE batches SET invalidated_at_ms = ?1 \ - WHERE batch_index = ?2 AND invalidated_at_ms IS NULL", - params![now_ms, u64_to_i64(batch_index)], - )?; - Ok(()) - } - - /// The unguarded twin of `recover_post_flush_for_recovery`. - pub(crate) fn recover_post_flush( - &mut self, - danger_threshold: u64, - ) -> rusqlite::Result> { - self.write(|tx| recover_post_flush_inner(tx, danger_threshold)) - } - - /// The unguarded twin of `recover_aging_tip_for_recovery`. - pub(crate) fn recover_aging_tip( + pub(crate) fn insert_batch_snapshot( &mut self, - danger_threshold: u64, - ) -> rusqlite::Result> { - self.write(|tx| recover_aging_tip_inner(tx, danger_threshold)) + prefix: &Path, + batch_nonce: u64, + ) -> rusqlite::Result { + self.write(|tx| insert_batch_snapshot_in(tx, prefix, batch_nonce, next_l2_tx_count_in(tx)?)) } - pub(crate) fn insert_batch_snapshot( + /// Close the open batch (the genesis Tip when none is open) and register + /// `prefix` as its snapshot: the fixture for a non-baseline row, since a + /// second baseline is refused. Returns the snapshot id and batch nonce. + pub(crate) fn insert_closed_batch_snapshot( &mut self, prefix: &Path, - batch_index: u64, - ) -> rusqlite::Result { - self.write(|tx| { - insert_batch_snapshot_in(tx, prefix, batch_index, next_executed_input_count_in(tx)?) - }) + ) -> rusqlite::Result<(i64, u64)> { + let mut head = match self.open_state()? { + Some(head) => head, + None => self.initialize_open_state(0, SafeInputRange::empty_at(0))?, + }; + let batch_nonce = head.batch_nonce; + self.close_frame_and_batch(&mut head)?; + Ok(( + self.insert_batch_snapshot(prefix, batch_nonce)?, + batch_nonce, + )) } pub(crate) fn insert_baseline_snapshot( &mut self, prefix: &Path, - count: ExecutedInputCount, + count: L2TxCount, ) -> rusqlite::Result { self.write(|tx| insert_baseline_snapshot_in(tx, prefix, count)) } @@ -315,35 +453,45 @@ fn synthetic_row(input: &StoredSafeInput) -> IngestedSafeInput { } } -fn decode_l2_tx_row( - kind: i64, - sender: Option>, - data: Option>, - fee: Option, - payload: Option>, - block_number: Option, -) -> SequencedL2Tx { - let sender_bytes = sender.expect("ordered replay row: missing sender"); +/// One row of [`Storage::ordered_l2_txs_for_batch`], built the way the +/// canonical history page builds its rows: a user op carries its frame's fee +/// and safe block, a direct its L1 index, block and timestamp. +fn decode_l2_tx_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let sender_bytes: Vec = row + .get::<_, Option>>(1)? + .expect("ordered replay row: missing sender"); assert_eq!( sender_bytes.len(), 20, "ordered replay row: sender must be 20 bytes" ); - if kind == 0 { - SequencedL2Tx::UserOp(ValidUserOp { - sender: Address::from_slice(sender_bytes.as_slice()), - // Replay uses the persisted frame fee (log-space exponent) to mirror canonical execution. - fee: i64_to_u16(fee.expect("ordered replay row: missing fee")), - data: data.expect("ordered replay row: missing data"), - }) + let sender = Address::from_slice(sender_bytes.as_slice()); + let required = |index: usize, column: &str| -> rusqlite::Result { + Ok(row + .get::<_, Option>(index)? + .unwrap_or_else(|| panic!("ordered replay row: missing {column}"))) + }; + if row.get::<_, i64>(0)? == 0 { + Ok(L2Tx::UserOp(UserOpTx { + sender, + nonce: i64_to_u32(required(4, "nonce")?), + // The persisted frame fee (log-space exponent), as canonical execution sees it. + fee: i64_to_u16(required(3, "fee")?), + block: i64_to_u64(required(5, "safe_block")?), + data: row + .get::<_, Option>>(2)? + .expect("ordered replay row: missing data"), + })) } else { - SequencedL2Tx::Direct(DirectInput { - sender: Address::from_slice(sender_bytes.as_slice()), - block_number: i64_to_u64( - block_number.expect("ordered replay row: missing block_number"), - ), - payload: payload.expect("ordered replay row: missing payload"), - }) + Ok(L2Tx::Direct(Direct { + input_index: i64_to_u64(required(6, "safe_input_index")?), + sender, + block: i64_to_u64(required(8, "block_number")?), + timestamp: i64_to_u64(required(9, "block_timestamp")?), + payload: row + .get::<_, Option>>(7)? + .expect("ordered replay row: missing payload"), + })) } } @@ -371,27 +519,27 @@ fn close_frame_derived_in( let (policy, next_frame_in_batch) = open_successor_frame_in(tx, head, next_safe_block)?; persist_frame_direct_sequence_derived( tx, - head.batch_index, + head.batch_nonce, next_frame_in_batch, leading_direct_range, )?; Ok(policy) } -/// Recovery-storage fixture with the same prerequisite ordering as setup: -/// baseline schema first, then the persisted deployment identity, then any L1 -/// facts or batch-tree mutations driven by the individual test. +/// A fresh database with the genesis era and the test identity +/// ([`SENDER_A`]) but no baseline snapshot, for recovery tests that register +/// their own snapshots or test their absence. pub(crate) fn temp_db_with_default_deployment_identity(name: &str) -> TestDb { let db = temp_db(name); - let mut storage = Storage::open(db.path.as_str()).expect("open test storage"); + let mut storage = Storage::open_with_era(&db.path, &NewEra::EMPTY_GENESIS); pin_test_deployment_identity(&mut storage, SENDER_A); db } -/// Wire bytes of the local valid closed batch at `nonce` — the +/// Wire bytes of the local closed batch at `nonce` — the /// production-faithful "our batch landed on L1" payload. Hash-matches the /// seal-time stamp, so the content-identity check accepts it. -/// Panics if no valid closed local batch carries `nonce` (test bug: an +/// Panics if no closed local batch carries `nonce` (test bug: an /// accepted landing without a matching local batch is, by design, a /// canonical divergence). pub(crate) fn local_batch_payload(storage: &mut Storage, nonce: u64) -> Vec { @@ -400,16 +548,16 @@ pub(crate) fn local_batch_payload(storage: &mut Storage, nonce: u64) -> Vec .expect("load local closed batches") .into_iter() .find(|b| b.nonce == nonce) - .unwrap_or_else(|| panic!("no valid closed local batch at nonce {nonce}")) + .unwrap_or_else(|| panic!("no closed local batch at nonce {nonce}")) .encoded } /// Insert safe inputs whose payloads are SSZ-encoded batches with the given /// nonces, all attributed to `sender`. `sender` doubles as the -/// batch-submitter address passed to `append_safe_inputs`, so the populated -/// `safe_accepted_batches` view matches this sender. +/// batch-submitter address passed to `append_safe_inputs`, so the recorded +/// acceptances follow this sender. /// -/// Nonces with a valid closed local batch land with that batch's real wire +/// Nonces with a closed local batch land with that batch's real wire /// bytes (the landing will be *accepted* — the content-identity check /// requires byte equality with the seal-time hash). Nonces without one get /// a synthetic empty-batch payload — sound **only** for landings the @@ -445,36 +593,30 @@ pub(crate) fn seed_safe_inputs_with_batch_nonces( }) .collect(); storage - .append_safe_inputs( - safe_block, - inputs.as_slice(), - sender, - &default_protocol_timing(), - ) + .append_safe_inputs(safe_block, inputs.as_slice(), sender) .expect("append safe inputs"); } -/// Create N closed batches (batch indices `0..count-1`) plus one open batch (index `count`). +/// Create N closed batches plus one open batch, from the era's anchor nonce. pub(crate) fn seed_closed_batches(storage: &mut Storage, count: u64) { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); for _ in 0..count { - let safe_block = head.safe_block; storage - .close_frame_and_batch(&mut head, safe_block) + .close_frame_and_batch(&mut head) .expect("close batch"); } } -/// Pull every valid sequenced L2 tx out of storage, dropping the offset. +/// Pull every sequenced L2 tx out of storage, dropping the offset. /// Test-only convenience around `canonical_history_page`. -pub(crate) fn all_ordered_l2_txs(storage: &mut Storage) -> Vec { +pub(crate) fn all_ordered_l2_txs(storage: &mut Storage) -> Vec { let bounds = storage.history_bounds().expect("history bounds"); storage .canonical_history_page( sequencer_core::history::HistoryClaim { - version: bounds.version, + era_id: bounds.era_id, next_input: bounds.available_from, }, 1_000_000, @@ -483,8 +625,8 @@ pub(crate) fn all_ordered_l2_txs(storage: &mut Storage) -> Vec { .rows .into_iter() .map(|row| match row.context { - super::L2TxContext::UserOp { tx, .. } => SequencedL2Tx::UserOp(tx), - super::L2TxContext::DirectInput { tx, .. } => SequencedL2Tx::Direct(tx), + super::L2TxContext::UserOp { tx, .. } => L2Tx::UserOp(tx), + super::L2TxContext::Direct { tx, .. } => L2Tx::Direct(tx), }) .collect() } @@ -523,10 +665,10 @@ pub(crate) fn record_canonical_divergence( .expect("record canonical divergence"); } -/// Recovery-test plumbing over [`Storage`]. `land` and `advance_safe_head` -/// append through the production path with [`SENDER_A`] as the submitter and -/// default timing; every input lands at `safe_block`. -pub(crate) trait RecoveryFixture { +/// Landing-test plumbing over [`Storage`]. `land` and `advance_safe_head` +/// append through the production path with [`SENDER_A`] as the submitter; +/// every input lands at `safe_block`. +pub(crate) trait LandingFixture { /// Exercise the same frame-advance-before-close sequence as the live lane. fn close_batch_at( &mut self, @@ -536,7 +678,7 @@ pub(crate) trait RecoveryFixture { fn land(&mut self, safe_block: u64, inputs: &[(Address, Vec)]); fn advance_safe_head(&mut self, safe_block: u64); } -impl RecoveryFixture for Storage { +impl LandingFixture for Storage { fn close_batch_at( &mut self, head: &mut crate::storage::WriteHead, @@ -557,7 +699,7 @@ impl RecoveryFixture for Storage { SafeInputRange::new(start, u64::try_from(end).unwrap()), )?; } - self.close_frame_and_batch(head, safe_block) + self.close_frame_and_batch(head) } #[track_caller] @@ -570,7 +712,7 @@ impl RecoveryFixture for Storage { block_number: safe_block, }) .collect(); - self.append_safe_inputs(safe_block, &inputs, SENDER_A, &default_protocol_timing()) + self.append_safe_inputs(safe_block, &inputs, SENDER_A) .expect("append safe inputs"); } @@ -580,10 +722,11 @@ impl RecoveryFixture for Storage { } } -/// A storage over a fresh pinned database. Bind the [`TestDb`] as `_db`, never -/// `_`: dropping it deletes the directory under the open connection. +/// A storage over a fresh set-up database ([`set_up_storage`], submitter +/// [`SENDER_A`]). Bind the [`TestDb`] as `_db`, never `_`: dropping it +/// deletes the directory under the open connection. pub(crate) fn open_temp_storage(name: &str) -> (TestDb, Storage) { - let db = temp_db_with_default_deployment_identity(name); - let storage = Storage::open(db.path.as_str()).expect("open storage"); + let db = temp_db(name); + let storage = set_up_storage(&db.path, SENDER_A); (db, storage) } diff --git a/sequencer/src/test_support/mod.rs b/sequencer/src/test_support/mod.rs index 55aeea9e..95644b79 100644 --- a/sequencer/src/test_support/mod.rs +++ b/sequencer/src/test_support/mod.rs @@ -40,14 +40,30 @@ pub(crate) fn temp_db(name: &str) -> TestDb { pub(crate) fn pin_test_deployment_identity( storage: &mut Storage, batch_submitter_address: Address, +) { + pin_test_deployment_identity_with_window( + storage, + batch_submitter_address, + app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks, + ); +} + +/// [`pin_test_deployment_identity`] with another staleness window, for tests +/// that pin where storage reads the window from. +pub(crate) fn pin_test_deployment_identity_with_window( + storage: &mut Storage, + batch_submitter_address: Address, + max_wait_blocks: u64, ) { storage - .load_or_insert_deployment_identity(DeploymentIdentity { + .pin_deployment_identity(DeploymentIdentity { chain_id: 1, app_address: Address::repeat_byte(0x11), input_box_address: Address::repeat_byte(0x22), app_deployment_block: 0, batch_submitter_address, + max_direct_payload_bytes: crate::storage::test_helpers::TEST_MAX_DIRECT_PAYLOAD_BYTES, + max_wait_blocks, fee_oracle: FeeOracleIdentity::Fixed { log_gas_price: 0 }, }) .expect("pin test deployment identity"); diff --git a/sequencer/src/test_support/server.rs b/sequencer/src/test_support/server.rs index f964ac06..82a6efcc 100644 --- a/sequencer/src/test_support/server.rs +++ b/sequencer/src/test_support/server.rs @@ -11,7 +11,7 @@ use alloy_primitives::{Address, Signature}; use alloy_sol_types::Eip712Domain; use app_core::application::{MAX_METHOD_PAYLOAD_BYTES, WalletApp, WalletConfig}; use sequencer_core::application::Application; -use sequencer_core::history::ExecutedInputCount; +use sequencer_core::history::L2TxCount; use sequencer_core::user_op::{SignedUserOp, UserOp}; use tokio::sync::mpsc; @@ -21,7 +21,7 @@ use crate::ingress::inclusion_lane::{ InclusionLane, InclusionLaneConfig, InclusionLaneError, PendingUserOp, dump_info, }; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::Storage; +use crate::storage::{SnapshotLeases, Storage}; /// Two ephemeral loopback listeners; returns (listeners, public, internal). pub(crate) async fn bind_api_listeners() -> ( @@ -90,6 +90,7 @@ impl TestServerBuilder { pub(crate) async fn start(self) -> TestServer { let (listeners, public_addr, internal_addr) = bind_api_listeners().await; let shutdown = RuntimeScope::default(); + let snapshot_leases = SnapshotLeases::default(); let db_path = self.db_path.as_str(); let (tx, lane_handle, queue_rx) = match self.queue { QueueMode::Idle => { @@ -97,7 +98,7 @@ impl TestServerBuilder { (tx, None, Some(rx)) } QueueMode::Lane => { - let (tx, handle) = start_wallet_lane(db_path, &shutdown); + let (tx, handle) = start_wallet_lane(db_path, &shutdown, &snapshot_leases); (tx, Some(handle), None) } QueueMode::Full => { @@ -144,11 +145,14 @@ impl TestServerBuilder { state_file_in_dump: |dump_dir| { WalletApp::state_file_in_dump(&dump_info::app_prefix(dump_dir)) }, + leases: snapshot_leases.clone(), }, + crate::landing::test_feed(1200), ); TestServer { public_addr, internal_addr, + snapshot_leases, lane_handle, shutdown, server_task: Some(server_task), @@ -162,6 +166,7 @@ impl TestServerBuilder { fn start_wallet_lane( db_path: &str, shutdown: &RuntimeScope, + snapshot_leases: &SnapshotLeases, ) -> ( mpsc::Sender, tokio::task::JoinHandle>, @@ -185,21 +190,21 @@ fn start_wallet_lane( ) .expect("genesis dump"); storage - .insert_baseline_snapshot(&genesis_dir, ExecutedInputCount::ZERO) + .insert_baseline_snapshot(&genesis_dir, L2TxCount::ZERO) .expect("register genesis"); } // Tip-existence invariant: open the genesis Tip only after the genesis // snapshot exists. Opening the Tip may attribute already-ingested direct // inputs, so doing this first would give the fresh application dump the - // wrong executed-input count. + // wrong L2 tx count. storage.ensure_open_tip().expect("establish open tip"); let head = storage .open_state() .expect("load open state") .expect("open state exists"); // Default log_gas_price=0 -> 0+296+20+419+621 = 1356. - assert_eq!(head.frame_fee, 1356); + assert_eq!(head.frame_fee(), 1356); InclusionLane::::start( 128, @@ -207,6 +212,7 @@ fn start_wallet_lane( storage, InclusionLaneConfig { dumps_dir, + snapshot_leases: snapshot_leases.clone(), max_user_ops_per_chunk: 32, safe_input_buffer_capacity: 32, max_batch_open: Duration::from_secs(60 * 60), @@ -222,6 +228,9 @@ fn start_wallet_lane( pub(crate) struct TestServer { pub(crate) public_addr: SocketAddr, pub(crate) internal_addr: SocketAddr, + /// The registry the egress handlers lease from; a test's GC passes it to + /// share the protocol with the server. + pub(crate) snapshot_leases: SnapshotLeases, pub(crate) lane_handle: Option>>, shutdown: RuntimeScope, server_task: Option, diff --git a/sequencer/src/test_support/stub_app.rs b/sequencer/src/test_support/stub_app.rs index ef40f09d..b6b0ca13 100644 --- a/sequencer/src/test_support/stub_app.rs +++ b/sequencer/src/test_support/stub_app.rs @@ -1,17 +1,18 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +use std::collections::BTreeMap; use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::time::Duration; use alloy_primitives::Address; -use app_core::application::MAX_METHOD_PAYLOAD_BYTES; +use app_core::application::{MAX_METHOD_PAYLOAD_BYTES, WALLET_SCHEDULER_LIMITS}; use sequencer_core::application::{ - AppError, AppOutputs, Application, ApplicationProgress, InvalidReason, ValidationOutcome, + Admission, AppError, AppOutputs, Application, PolicyRefusal, SchedulerLimits, }; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::history::L2TxCount; +use sequencer_core::l2_tx::{Direct, UserOpTx}; use sequencer_core::user_op::UserOp; use crate::ingress::inclusion_lane::dump_info::{self, create_dump_dir_with_info}; @@ -24,23 +25,32 @@ pub(crate) enum ReplayEvent { data: Vec, }, DirectInput { + input_index: u64, sender: Address, block_number: u64, + timestamp: u64, payload: Vec, }, } -/// Accepts and executes every input. Dumps persist only progress (two LE -/// `u64`s at `prefix/state`); `from_dump` resets every other field, so a test -/// observes what the lane executed through the dumps it writes. +/// Executes every transaction it is handed. Dumps persist the count and the +/// per-sender nonces at `prefix/state`; `from_dump` resets every other field, +/// so a test observes what the lane executed through the dumps it writes. #[derive(Default)] pub(crate) struct StubApp { - pub(crate) progress: ApplicationProgress, - /// Inputs this instance executed; not persisted. + pub(crate) count: L2TxCount, + /// Each sender's next nonce; an absent sender is at zero. + pub(crate) nonces: BTreeMap, + /// Transactions this instance executed; not persisted. pub(crate) replayed: Vec, - /// Test-only scheduling seam used to keep a rejected queue saturated long - /// enough to distinguish one bounded turn from an unbounded drain. + /// Test-only scheduling seam that keeps a refused queue saturated long + /// enough to distinguish one bounded turn from an unbounded drain: each + /// nonce read sleeps, then reports one past the sender's next nonce, so + /// every live op refuses with a wrong nonce (and a stored op would skip). pub(crate) reject_user_ops_after: Option, + /// `admit_user_op` refuses every op with this policy text. + pub(crate) refuse_user_ops: Option, + /// `admit_user_op` fails with an internal error. pub(crate) fail_validation: bool, pub(crate) fail_apply: bool, pub(crate) fail_dump: bool, @@ -53,16 +63,16 @@ pub(crate) struct StubApp { } impl StubApp { - pub(crate) fn with_progress(executed_input_count: u64, last_executed_safe_block: u64) -> Self { + pub(crate) fn with_progress(l2_tx_count: u64) -> Self { Self { - progress: ApplicationProgress::try_new( - ExecutedInputCount::new(executed_input_count), - last_executed_safe_block, - ) - .expect("coherent progress"), + count: L2TxCount::new(l2_tx_count), ..Self::default() } } + + fn next_nonce(&self, sender: Address) -> u32 { + self.nonces.get(&sender).copied().unwrap_or(0) + } } fn internal_failure() -> AppError { @@ -72,68 +82,79 @@ fn internal_failure() -> AppError { } impl Application for StubApp { - fn max_method_payload_bytes() -> usize { - MAX_METHOD_PAYLOAD_BYTES - } - - fn validate_user_op( - &self, - _sender: Address, - user_op: &UserOp, - _current_fee: u16, - ) -> Result { - if self.fail_validation { - return Err(internal_failure()); - } + fn user_nonce(&self, sender: Address) -> Result { + let next = self.next_nonce(sender); if let Some(delay) = self.reject_user_ops_after { std::thread::sleep(delay); - return Ok(ValidationOutcome::Reject(InvalidReason::InvalidNonce { - expected: user_op.nonce.wrapping_add(1), - got: user_op.nonce, - })); + return Ok(next.wrapping_add(1)); } - Ok(ValidationOutcome::Accept) + Ok(next) } - fn apply_valid_user_op( - &mut self, - user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { + fn apply_user_op(&mut self, op: &UserOpTx) -> Result { if self.fail_apply { return Err(internal_failure()); } self.replayed.push(ReplayEvent::UserOp { - sender: user_op.sender, - data: user_op.data.clone(), + sender: op.sender, + data: op.data.clone(), }); - self.progress.advance(safe_block); + let next = self + .next_nonce(op.sender) + .checked_add(1) + .expect("the executor never applies an op at nonce u32::MAX"); + self.nonces.insert(op.sender, next); + self.count.advance(); Ok(Vec::new()) } - fn apply_direct_input(&mut self, input: &DirectInput) -> Result { + fn apply_direct(&mut self, direct: &Direct) -> Result { if let Some(shutdown) = &self.shutdown_on_direct { shutdown.request_shutdown(); } self.replayed.push(ReplayEvent::DirectInput { - sender: input.sender, - block_number: input.block_number, - payload: input.payload.clone(), + input_index: direct.input_index, + sender: direct.sender, + block_number: direct.block, + timestamp: direct.timestamp, + payload: direct.payload.clone(), }); - self.progress.advance(input.block_number); + self.count.advance(); Ok(Vec::new()) } - fn progress(&self) -> ApplicationProgress { - self.progress + fn l2_tx_count(&self) -> L2TxCount { + self.count + } + + fn max_method_payload_bytes() -> usize { + MAX_METHOD_PAYLOAD_BYTES + } + + /// The wallet's limits, so storage fixtures that open frames with the + /// stub's cap select the same directs a `WalletApp` lane would. + fn scheduler_limits() -> SchedulerLimits { + WALLET_SCHEDULER_LIMITS + } + + fn admit_user_op( + &self, + _sender: Address, + _op: &UserOp, + _fee: u16, + ) -> Result { + if self.fail_validation { + return Err(internal_failure()); + } + if let Some(reason) = &self.refuse_user_ops { + return Ok(Admission::Refuse(PolicyRefusal::Other(reason.clone()))); + } + Ok(Admission::Admit) } fn from_dump(prefix: &Path) -> Result { let bytes = std::fs::read(Self::state_file_in_dump(prefix))?; - Ok(Self { - progress: decode_progress(&bytes)?, - ..Self::default() - }) + decode_state(&bytes) } fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { @@ -141,37 +162,65 @@ impl Application for StubApp { return Err(std::io::Error::other("injected application dump failure").into()); } std::fs::create_dir(prefix)?; - std::fs::write( - Self::state_file_in_dump(prefix), - encode_progress(self.progress), - )?; + std::fs::write(Self::state_file_in_dump(prefix), encode_state(self))?; Ok(()) } fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } + + fn from_canonical_state(state_file: &Path, l2_tx_count: L2TxCount) -> Result { + let app = decode_state(&std::fs::read(state_file)?)?; + if app.count != l2_tx_count { + return Err(AppError::Internal { + reason: format!( + "StubApp state holds count {}, but the machine reports {}", + app.count.get(), + l2_tx_count.get() + ), + }); + } + Ok(app) + } } -fn encode_progress(progress: ApplicationProgress) -> [u8; 16] { - let mut bytes = [0_u8; 16]; - bytes[..8].copy_from_slice(&progress.executed_input_count().get().to_le_bytes()); - bytes[8..].copy_from_slice(&progress.last_executed_safe_block().to_le_bytes()); +const NONCE_ENTRY_BYTES: usize = 20 + 4; + +/// The count as a LE `u64`, then each sender with a nonce as its 20 address +/// bytes and a LE `u32`, in address order. +fn encode_state(app: &StubApp) -> Vec { + let mut bytes = Vec::with_capacity(8 + app.nonces.len() * NONCE_ENTRY_BYTES); + bytes.extend_from_slice(&app.count.get().to_le_bytes()); + for (sender, nonce) in &app.nonces { + bytes.extend_from_slice(sender.as_slice()); + bytes.extend_from_slice(&nonce.to_le_bytes()); + } bytes } -fn decode_progress(bytes: &[u8]) -> Result { - if bytes.len() != 16 { - return Err(AppError::Internal { - reason: "StubApp dump must be exactly 16 bytes".to_owned(), - }); +fn decode_state(bytes: &[u8]) -> Result { + let malformed = || AppError::Internal { + reason: "StubApp dump must be a count and whole nonce entries".to_owned(), + }; + let (count, entries) = bytes.split_first_chunk::<8>().ok_or_else(malformed)?; + let (entries, rest) = entries.as_chunks::(); + if !rest.is_empty() { + return Err(malformed()); } - let count = u64::from_le_bytes(bytes[..8].try_into().expect("checked slice length")); - let safe_block = u64::from_le_bytes(bytes[8..].try_into().expect("checked slice length")); - Ok( - ApplicationProgress::try_new(ExecutedInputCount::new(count), safe_block) - .expect("coherent progress"), - ) + let nonces = entries + .iter() + .map(|entry| { + let (sender, nonce) = entry.split_at(20); + let nonce: [u8; 4] = nonce.try_into().expect("checked entry length"); + (Address::from_slice(sender), u32::from_le_bytes(nonce)) + }) + .collect(); + Ok(StubApp { + count: L2TxCount::new(u64::from_le_bytes(*count)), + nonces, + ..StubApp::default() + }) } /// Mirror of the production dump layout: a structured directory with diff --git a/tests/benchmarks/BENCHMARK_SPEC.md b/tests/benchmarks/BENCHMARK_SPEC.md index 773a233f..7e26cd9c 100644 --- a/tests/benchmarks/BENCHMARK_SPEC.md +++ b/tests/benchmarks/BENCHMARK_SPEC.md @@ -42,8 +42,8 @@ This spec makes that requirement measurable and repeatable. 1. `logical_db_bytes`: `(page_count - freelist_count) * page_size`, read over a read-only connection at run end. It counts pages still in the WAL and excludes free pages. 2. `db_bytes` and `wal_bytes`: on-disk footprint of the SQLite file and its WAL at run end. 3. `growth_bytes_per_1k_accepted`: growth of `logical_db_bytes` over the measured steps, normalized by accepted tx volume. -4. `batches_pending`: valid sealed batches whose nonce is not yet in `safe_accepted_batches` when the drain starts. The drain starts when the last measured step returns: every load worker is past its deadline, and in round-trip mode the WS matcher has finished, bounded by `--max-ws-wait-ms`. -5. `drain_secs`: time from the drain start until `safe_accepted_batches` holds the highest valid sealed batch nonce read at the start. The open batch is excluded, so this is the submission backlog, not the batch-open window. +4. `batches_pending`: sealed batches not yet accepted (`batches.accepted_input` is NULL) when the drain starts. The drain starts when the last measured step returns: every load worker is past its deadline, and in round-trip mode the WS matcher has finished, bounded by `--max-ws-wait-ms`. +5. `drain_secs`: time from the drain start until the highest sealed batch nonce read at the start is accepted. The open batch is excluded, so this is the submission backlog, not the batch-open window. ## 3. Measurement Method @@ -87,7 +87,6 @@ This spec makes that requirement measurable and repeatable. 3. During the drain the harness mines one L1 block per second at Anvil's own clock and stops after 120 s; `drain_secs` is then reported as not reached. 4. `drain_secs` is shaped by the environment: the submitter's idle poll (5 s by default), Anvil's safe-head lag (`--slots-in-an-epoch 1`) and the 1 s poll. It does not measure submitter throughput. 5. Neither metric has a threshold in this phase. -6. A recovery cascade during the run reuses batch nonces, which makes `drain_secs` meaningless. ## 4. Benchmark Scenarios diff --git a/tests/benchmarks/src/runtime.rs b/tests/benchmarks/src/runtime.rs index 6d4385ea..e5792a07 100644 --- a/tests/benchmarks/src/runtime.rs +++ b/tests/benchmarks/src/runtime.rs @@ -282,8 +282,8 @@ pub struct DrainReport { pub drain_secs: Option, } -/// Time from now, with the load stopped, until `safe_accepted_batches` holds -/// the highest valid sealed batch nonce read now. The open batch is excluded, +/// Time from now, with the load stopped, until the accepted batches reach +/// the highest sealed batch nonce read now. The open batch is excluded, /// so this measures the submission backlog, not the batch-open window. Mines /// one L1 block per second at Anvil's own clock. pub async fn measure_drain(runtime: &ManagedSequencer) -> BenchResult { diff --git a/tests/e2e/src/cold_replica.rs b/tests/e2e/src/cold_replica.rs index 0edfd1bd..15313808 100644 --- a/tests/e2e/src/cold_replica.rs +++ b/tests/e2e/src/cold_replica.rs @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! A cold consumer restores an HTTP archive and follows its claim through replay, -//! live writes, and a process-boundary recovery. Its reference starts at genesis. +//! live writes, and a clean restart of the sequencer. Its reference starts at +//! genesis. use std::time::Duration; @@ -12,10 +13,7 @@ use rollups_harness::replay::apply_ws_message; use rollups_harness::{ManagedSequencer, ReplayWalletApp, TestSigner, WsClient}; use sequencer_core::api::WsTxMessage; use sequencer_core::application::Application; -use sequencer_core::fee::fee_to_linear; -use sequencer_rust_client::{ - HistoryClaim, HistoryPolicyError, SequencerClient, SnapshotResponse, SubscribeError, -}; +use sequencer_rust_client::{HistoryClaim, SequencerClient, SnapshotResponse}; use crate::ScenarioResult; use crate::test_cases::advance_live_frame_until_covers; @@ -54,7 +52,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari record(&mut reference_ws, &mut reference, &mut history).await?; // Size closure supplies a nonempty immutable snapshot while leaving a suffix - // in the next batch. Only the first batch is submitted before the outage. + // in the next batch. for _ in 0..150 { alice_l2.transfer(alice_address, U256::from(1)).await?; record(&mut reference_ws, &mut reference, &mut history).await?; @@ -67,7 +65,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari snapshot_count > 0, "the consumer must restore nonempty state" ); - assert!(snapshot_count <= reference.executed_input_count()); + assert!(snapshot_count <= reference.l2_tx_count()); // Hold the response without consuming its body. This write is acknowledged // after snapshot selection and before archive restoration or subscription. @@ -77,7 +75,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari assert_eq!(downloaded_claim, original_claim); let snapshot_reference = replay_prefix(&history, snapshot_count)?; assert_same_state(&mut replica, &snapshot_reference)?; - assert!(replica.executed_input_count().get() < reference.executed_input_count()); + assert!(replica.l2_tx_count().get() < reference.l2_tx_count()); let backlog_deposit = alice_l1 .mint_and_deposit_supported_token(U256::from(70_000)) @@ -88,7 +86,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari record(&mut reference_ws, &mut reference, &mut history).await?; let mut replica_ws = WsClient::connect(&client, downloaded_claim).await?; - let backlog_head = reference.executed_input_count(); + let backlog_head = reference.l2_tx_count(); let catch_up_target = backlog_head + 2; let (first_replayed, replay_started) = tokio::sync::oneshot::channel(); let (writes_committed, resume_replay) = tokio::sync::oneshot::channel(); @@ -105,7 +103,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari }; let consumer = async { apply_ws_message(&mut replica, replica_ws.next_message().await?)?; - assert!(replica.executed_input_count().get() < backlog_head); + assert!(replica.l2_tx_count().get() < backlog_head); first_replayed .send(()) .map_err(|_| "producer dropped the replay barrier")?; @@ -119,7 +117,11 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari .await?; // Already at the tip: these inputs must arrive through live continuation. - let clock_before_live = replica.last_executed_safe_block(); + // The replica matches the reference, so the reference's last block is + // also the replica's. + let block_before_live = reference + .last_applied_block() + .ok_or("the reference applied messages before going live")?; let live_deposit = alice_l1 .mint_and_deposit_supported_token(U256::from(80_000)) .await?; @@ -127,14 +129,10 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari record(&mut reference_ws, &mut reference, &mut history).await?; alice_l2.transfer(bob_address, U256::from(5_000)).await?; record(&mut reference_ws, &mut reference, &mut history).await?; - consume_until( - &mut replica_ws, - &mut replica, - reference.executed_input_count(), - ) - .await?; + let live_block = consume_until(&mut replica_ws, &mut replica, reference.l2_tx_count()).await?; assert_same_state(&mut replica, &reference)?; - assert!(replica.last_executed_safe_block() > clock_before_live); + assert_eq!(live_block, reference.last_applied_block()); + assert!(live_block.is_some_and(|block| block > block_before_live)); assert_eq!( reference.current_user_balance(bob_address), U256::from(15_000) @@ -143,7 +141,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari // Preserve the replica's claim across a clean restart, then make the // restarted host execute against its restored nonempty snapshot and suffix. let restart_claim = HistoryClaim { - next_input: replica.executed_input_count(), + next_input: replica.l2_tx_count(), ..downloaded_claim }; drop(replica_ws); @@ -164,107 +162,14 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari alice_l2.set_next_nonce(reference.current_user_nonce(alice_address)); alice_l2.transfer(bob_address, U256::from(6_000)).await?; record(&mut reference_ws, &mut reference, &mut history).await?; - consume_until( - &mut replica_ws, - &mut replica, - reference.executed_input_count(), - ) - .await?; + consume_until(&mut replica_ws, &mut replica, reference.l2_tx_count()).await?; assert_same_state(&mut replica, &reference)?; assert_eq!( reference.current_user_balance(bob_address), U256::from(21_000) ); - let stale_claim = HistoryClaim { - next_input: replica.executed_input_count(), - ..downloaded_claim - }; - drop(replica_ws); - drop(reference_ws); - runtime.stop().await?; - runtime - .advance_wall_and_mine(Duration::from_secs( - (sequencer_core::MAX_WAIT_BLOCKS + 50) * 12, - )) - .await?; - runtime.respawn().await?; - - // Respawn chooses a fresh listener. The claim comes exclusively from the - // downloaded snapshot and consumed inputs, never the harness's DB helpers. - let client = SequencerClient::new(runtime.internal_endpoint())?; - assert!(matches!( - client.subscribe(stale_claim).await, - Err(SubscribeError::History( - HistoryPolicyError::StaleGeneration { .. } - )) - )); - let (mut recovered, fresh_claim) = restore::(client.latest_snapshot().await?).await?; - assert_eq!(fresh_claim.version.era_id, original_claim.version.era_id); - assert_eq!( - fresh_claim.version.recovery_generation.get(), - original_claim.version.recovery_generation.get() + 1, - ); - assert_eq!(fresh_claim.next_input.get(), snapshot_count); - - // Reconstruct the expected replacement branch independently: the accepted - // prefix survives, optimistic user ops disappear, and L1 directs replay. - let mut recovered_reference = replay_prefix(&history, snapshot_count)?; - assert_same_state(&mut recovered, &recovered_reference)?; - for message in &history[snapshot_count as usize..] { - if let WsTxMessage::DirectInput { .. } = message { - let mut direct = message.clone(); - if let WsTxMessage::DirectInput { offset, .. } = &mut direct { - *offset = recovered_reference.executed_input_count(); - } - recovered_reference.apply(direct)?; - } - } - let mut recovered_ws = WsClient::connect(&client, fresh_claim).await?; - consume_until( - &mut recovered_ws, - &mut recovered, - recovered_reference.executed_input_count(), - ) - .await?; - assert_same_state(&mut recovered, &recovered_reference)?; - assert_eq!( - recovered_reference.current_user_balance(bob_address), - U256::ZERO - ); - assert!(recovered.executed_input_count().get() < replica.executed_input_count().get()); - - let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; - let expected_nonce = recovered_reference.current_user_nonce(alice_address); - let balance_before = recovered_reference.current_user_balance(alice_address); - // The fixed oracle keeps this quote valid across frame rotations. Derive the - // expected debit before receiving the event so wrong feed fees cannot agree by replay. - let quote = SequencerClient::new(runtime.endpoint())?.get_fee().await?; - assert_eq!(quote.fee, quote.recommended_fee); - let amount = U256::from(6_000); - let expected_balance = balance_before - amount - fee_to_linear(quote.fee); - assert_eq!(alice_l2.served_next_nonce().await?, expected_nonce); - alice_l2.set_next_nonce(expected_nonce); - alice_l2.transfer(bob_address, amount).await?; - let resumed = recovered_ws.expect_user_op_from(alice_address).await?; - assert!(matches!(resumed, WsTxMessage::UserOp { fee, nonce, .. } - if fee == quote.fee && nonce == expected_nonce)); - apply_ws_message(&mut recovered, resumed.clone())?; - recovered_reference.apply(resumed)?; - assert_same_state(&mut recovered, &recovered_reference)?; - assert_eq!( - recovered_reference.current_user_nonce(alice_address), - expected_nonce + 1 - ); - assert_eq!( - recovered_reference.current_user_balance(alice_address), - expected_balance - ); - assert_eq!( - recovered_reference.current_user_balance(bob_address), - U256::from(6_000) - ); - recovered_ws + replica_ws .expect_no_message_for(Duration::from_millis(100)) .await?; Ok(()) @@ -283,7 +188,7 @@ pub(crate) async fn restore( tar::Archive::new(archive.as_ref()).unpack(directory.path())?; assert!(directory.path().join("info.toml").is_file()); let app = A::from_dump(&directory.path().join("state"))?; - assert_eq!(app.executed_input_count(), claim.next_input); + assert_eq!(app.l2_tx_count(), claim.next_input); // Subsequent replay also checks that restoring does not retain a dependency // on the downloaded source directory. directory.close()?; @@ -301,16 +206,19 @@ async fn record( Ok(()) } +/// Applies feed messages until `app` reaches `target`. Returns the L1 block +/// of the last message applied, which the application no longer keeps. async fn consume_until( ws: &mut WsClient, app: &mut A, target: u64, -) -> ScenarioResult<()> { - while app.executed_input_count().get() < target { - apply_ws_message(app, ws.next_message().await?)?; +) -> ScenarioResult> { + let mut last_block = None; + while app.l2_tx_count().get() < target { + last_block = Some(apply_ws_message(app, ws.next_message().await?)?); } - assert_eq!(app.executed_input_count().get(), target); - Ok(()) + assert_eq!(app.l2_tx_count().get(), target); + Ok(last_block) } fn replay_prefix(history: &[WsTxMessage], count: u64) -> ScenarioResult { @@ -325,27 +233,20 @@ pub(crate) fn assert_same_state( actual: &mut A, expected: &ReplayWalletApp, ) -> ScenarioResult<()> { - assert_eq!( - actual.executed_input_count().get(), - expected.executed_input_count() - ); - assert_eq!( - actual.last_executed_safe_block(), - expected.last_executed_safe_block() - ); - let progress = actual.progress(); + let count = actual.l2_tx_count(); + assert_eq!(count.get(), expected.l2_tx_count()); let directory = tempfile::tempdir()?; let checkpoint = directory.path().join("checkpoint"); actual.create_dump(&checkpoint)?; assert_eq!( - actual.progress(), - progress, + actual.l2_tx_count(), + count, "checkpoint creation preserves state" ); assert_eq!( std::fs::read(A::state_file_in_dump(&checkpoint))?, expected.canonical_snapshot_bytes()?, - "all wallet state, including balances, nonces, config, count, and clock" + "all wallet state, including balances, nonces, config and count" ); Ok(()) } diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index c42bd065..0b4df5e8 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) mod cold_replica; +mod recovery_round_trip; pub mod test_cases; mod watchdog_compare; diff --git a/tests/e2e/src/recovery_round_trip.rs b/tests/e2e/src/recovery_round_trip.rs new file mode 100644 index 00000000..e7ca6677 --- /dev/null +++ b/tests/e2e/src/recovery_round_trip.rs @@ -0,0 +1,666 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Setup as an operator runs it: in an empty data directory only, from a +//! base (genesis or a stored canonical machine) through a stop block, then +//! `run`, then a wallet client of the old era adopting the new one at its own +//! count, and a watchdog fresh from the template agreeing with the new era's +//! first accepted batch. The procedure is docs/recovery/cockroach.md ("What +//! setup does"). + +use std::path::Path; +use std::time::Duration; + +use alloy_primitives::{Address, U256}; +use rollups_harness::{ + EXIT_TERMINAL, ManagedSequencer, RebuiltHistoryRows, ReplayWalletApp, TestSigner, + WalletL2Client, WsClient, +}; +use sequencer_core::api::WsTxMessage; +use sequencer_core::scheduler::{SCHEDULER_INSPECT_QUERY, SchedulerExport}; +use sequencer_rust_client::{ + AdoptEraError, AdoptError, HistoryClaim, HistoryPolicyError, L2TxCount, SequencerClient, + SubscribeError, +}; + +use crate::ScenarioResult; +use crate::test_cases::{ + NO_WS_MESSAGE_WAIT, TRANSFERS_TO_FORCE_BATCH_CLOSE, apply_reconciled_supported_deposit, + mine_until_finalized_advances, +}; +use crate::watchdog_compare::{ + DEVNET_MACHINE_IMAGE, StoredMachine, Watchdog, cm_inspect, head_block, machine_image, + require_cartesi_machine, +}; + +/// Bound on single-block mines while waiting for Anvil's safe tag to cover a +/// block; its safe head trails the chain head by a few blocks. +const SAFE_HEAD_MINE_ATTEMPTS: usize = 64; + +/// A genesis setup on a fresh chain publishes an empty era; the dry run +/// reports it without touching the data directory; and a second setup on the +/// set-up directory is refused with exit 30, leaving it as it was for `run`. +pub(crate) async fn run_setup_refuses_a_set_up_data_directory_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { + let era = runtime.published_era()?; + assert_eq!(era.base_count, 0, "a genesis base starts at count zero"); + assert_eq!( + era.rebuilt_history_end, 0, + "a fresh chain has nothing to rebuild" + ); + assert_eq!(era.first_queued_input, 0, "nothing is queued"); + assert_eq!( + era.next_batch_nonce, 0, + "the batch list starts at nonce zero" + ); + assert_eq!( + runtime.rebuilt_history_rows()?, + RebuiltHistoryRows { + total: 0, + executed: 0 + } + ); + runtime.stop().await?; + let contents = runtime.data_dir_contents()?; + + let dry_run = runtime.setup_dry_run().await?; + assert!(dry_run.success(), "the dry run must exit 0: {dry_run:?}"); + assert!( + dry_run.stdout.contains("base: 0 L1 inputs, L2 tx count 0"), + "the dry run reports its base: {}", + dry_run.stdout + ); + assert_eq!( + runtime.data_dir_contents()?, + contents, + "the dry run never touches the data directory" + ); + + let refused = runtime.run_setup().await?; + assert_eq!( + refused.code(), + Some(EXIT_TERMINAL), + "setup refuses a set-up directory with the terminal class: {refused:?}" + ); + assert_eq!( + runtime.data_dir_contents()?, + contents, + "a refused setup leaves the directory as it was" + ); + assert_eq!( + runtime.published_era()?, + era, + "a refused setup changes no era field" + ); + + // `run` still starts on the untouched directory. + runtime.respawn().await?; + let _ws = runtime.ws(0).await?; + Ok(()) +} + +/// The round trip from genesis: any base works, genesis included, though the +/// rebuild re-judges every batch the old instance landed. +pub(crate) async fn run_recovery_from_genesis_round_trip_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { + run_round_trip(runtime, RecoveryBase::Genesis).await +} + +/// The round trip from a stored canonical machine, taken by the watchdog's +/// `replay` from the template image at a block where a direct of the old era +/// is still queued, so setup resumes the machine's non-empty fridge. +pub(crate) async fn run_recovery_from_stored_machine_round_trip_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { + run_round_trip(runtime, RecoveryBase::StoredMachine).await +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecoveryBase { + Genesis, + StoredMachine, +} + +/// A toy-wallet client of the feed: its state, and every row it applied, +/// kept by count as a backup would keep them. +struct WalletClient { + app: ReplayWalletApp, + rows: Vec, +} + +impl WalletClient { + fn new() -> Self { + Self { + app: ReplayWalletApp::devnet(), + rows: Vec::new(), + } + } + + fn apply(&mut self, message: WsTxMessage) -> ScenarioResult<()> { + self.app.apply(message.clone())?; + self.rows.push(message); + Ok(()) + } + + fn count(&self) -> u64 { + self.app.l2_tx_count() + } + + /// The client restored from its backup at `count`. + fn restored_at(&self, count: u64) -> ScenarioResult { + let mut restored = Self::new(); + for row in self + .rows + .iter() + .take(usize::try_from(count).map_err(|_| "count overflows usize")?) + { + restored.apply(row.clone())?; + } + if restored.count() != count { + return Err(format!( + "the client kept {} rows, cannot restore count {count}", + restored.count() + ) + .into()); + } + Ok(restored) + } +} + +/// What the old instance left on L1 that the scenario checks the new era +/// against. +struct OldEra { + /// The accepted checkpoint after the first batch landed. + first_accepted: (u64, u64), + /// A direct deposited after the first batch landed, as its input index + /// and block: the second batch drains it, so it is queued between. + queued_between: (u64, u64), + /// The accepted checkpoint after the second batch landed: the client's + /// trusted count is its count. + last_accepted_count: u64, + /// The claim the client held in the old era. + claim: HistoryClaim, + /// The old era's head; the rows past the last accepted count were served + /// but never submitted. + head: u64, +} + +async fn run_round_trip(runtime: &mut ManagedSequencer, base: RecoveryBase) -> ScenarioResult<()> { + require_cartesi_machine(); + let template = machine_image(DEVNET_MACHINE_IMAGE)?; + let alice = TestSigner::from_default(1)?; + let bob = TestSigner::from_default(2)?; + let alice_address = alice.address(); + let bob_address = bob.address(); + let alice_l1 = runtime.wallet_l1(alice.clone()).await?; + let mut client = WalletClient::new(); + + // ── The old era: two batches land, and the old instance serves more + // rows in a batch it never submits. + let old = { + let mut ws = runtime.ws(0).await?; + let mut alice_l2 = runtime.wallet_l2(alice.clone())?; + let mut bob_l2 = runtime.wallet_l2(bob.clone())?; + let deposit = apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut client.app, + &alice_l1, + // Fees for the three batch-sealing pumps of self-transfers: two in + // the old era, one in the new. + U256::from(45_000_000_u64), + ) + .await?; + client.rows.push(deposit); + alice_l2 + .transfer(bob_address, U256::from(400_000_u64)) + .await?; + client.apply(ws.expect_user_op_from(alice_address).await?)?; + let first_accepted = + land_a_batch(runtime, &mut ws, &mut client, &mut alice_l2, alice_address).await?; + + let between = apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut client.app, + &alice_l1, + U256::from(200_000_u64), + ) + .await?; + let queued_between = match &between { + WsTxMessage::DirectInput { + input_index, + block_number, + .. + } => (*input_index, *block_number), + other => return Err(format!("expected the deposit, got {other:?}").into()), + }; + client.rows.push(between); + + bob_l2 + .transfer(alice_address, U256::from(1_000_u64)) + .await?; + client.apply(ws.expect_user_op_from(bob_address).await?)?; + let (_, last_accepted_count) = + land_a_batch(runtime, &mut ws, &mut client, &mut alice_l2, alice_address).await?; + OldEra { + first_accepted, + queued_between, + last_accepted_count, + claim: runtime.history_claim(last_accepted_count)?, + head: client.count(), + } + }; + + // ── The old instance stops for good; two directs arrive, in two blocks, + // while it is down and become safe, so they are queued at the stop block. + runtime.stop().await?; + let bob_l1 = runtime.wallet_l1(bob.clone()).await?; + let alice_queued = U256::from(500_000_u64); + let bob_queued = U256::from(300_000_u64); + alice_l1.mint_supported_token(alice_queued).await?; + let first_queued_block = alice_l1.deposit_supported_token(alice_queued).await?; + bob_l1.mint_supported_token(bob_queued).await?; + let queued_block = bob_l1.deposit_supported_token(bob_queued).await?; + assert!( + queued_block > first_queued_block, + "the queued directs land in two blocks" + ); + mine_until_safe(runtime, queued_block).await?; + let input_blocks = runtime.input_blocks().await?; + let queued_input = u64::try_from(input_blocks.len())? + .checked_sub(2) + .ok_or("the two queued deposits are inputs")?; + + // ── The base, and the client's trusted state at the last accepted count. + let (base_machine, base_count, base_inputs) = match base { + RecoveryBase::Genesis => (None, 0, 0), + RecoveryBase::StoredMachine => { + let (queued_index, block) = old.queued_between; + let machine = stored_machine_at(runtime, &template, block).await?; + let inputs = input_blocks.iter().filter(|b| **b <= block).count(); + let report = cm_inspect(&machine.path, SCHEDULER_INSPECT_QUERY).await?; + let export = SchedulerExport::decode(&report)?; + assert_eq!( + ( + export.l1_input_count, + export.l2_tx_count, + export.first_queued_input, + export.fridge_len + ), + ( + u64::try_from(inputs)?, + old.first_accepted.1, + queued_index, + 1 + ), + "the base holds the first batch's rows and queues the deposit after it" + ); + (Some(machine), old.first_accepted.1, inputs) + } + }; + let trusted = old.last_accepted_count; + let mut client = client.restored_at(trusted)?; + + // ── The operator's playbook: a new data directory, the dry run, setup, + // run. + runtime.discard_data_dir(); + runtime.set_setup_base(base_machine.as_ref().map(|machine| machine.path.clone())); + let dry_run = runtime.setup_dry_run().await?; + assert!(dry_run.success(), "the dry run must exit 0: {dry_run:?}"); + for expected in [ + format!("base: {base_inputs} L1 inputs, L2 tx count {base_count}"), + "the rebuild re-judges the old instance's batches past the base".to_string(), + ] { + assert!( + dry_run.stdout.contains(&expected), + "the dry run must report `{expected}`: {}", + dry_run.stdout + ); + } + assert!( + runtime.data_dir_entries()?.is_empty(), + "the dry run never creates the data directory" + ); + runtime.respawn().await?; + + // ── The published era. + let era = runtime.published_era()?; + assert_eq!(era.base_count, base_count, "the era starts at the base"); + assert_eq!( + era.rebuilt_history_end, trusted, + "the rebuilt history ends where the landed batches end: the old \ + instance's unsubmitted rows never landed, and the queued directs wait \ + for the root frame" + ); + assert_eq!( + era.first_queued_input, queued_input, + "the directs that arrived while the old instance was down are queued" + ); + assert!( + era.stop_block >= queued_block, + "the queued directs are safe at the stop block" + ); + assert_eq!( + era.next_batch_nonce, 2, + "two batches landed before the stop block" + ); + let rebuilt = runtime.rebuilt_history_rows()?; + assert_eq!( + rebuilt, + RebuiltHistoryRows { + total: trusted - base_count, + executed: trusted - base_count, + }, + "one row per transaction executed past the base" + ); + assert_eq!(runtime.canonical_divergence()?, None); + + let sequencer = runtime.sequencer_client()?; + let info = sequencer.history(None).await?; + assert_eq!(info.history.available_from.get(), base_count); + assert_eq!(info.baseline.rebuilt_history_end.get(), trusted); + assert_eq!(info.baseline.l1_stop_block, era.stop_block); + assert_eq!(info.baseline.next_batch_nonce, 2); + let checkpoint = info + .accepted_checkpoint + .ok_or("the era's baseline is its first comparison checkpoint")?; + assert_eq!( + ( + checkpoint.inclusion_block, + checkpoint.executed_input_count.get(), + checkpoint.next_batch_nonce + ), + (era.stop_block, trusted, 2) + ); + assert_ne!( + info.history.era_id, old.claim.era_id, + "setup starts a new era" + ); + + // ── The client's playbook. + assert!(matches!( + sequencer.subscribe(old.claim).await, + Err(SubscribeError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + // The lane seals a batch at a fixed user-op count, and each transfer waits + // for its inclusion before the next is sent, so where the pumps split + // into batches is fixed: the second pump leaves rows in the open batch + // that the old instance served but never submitted. A client at its head + // cannot adopt the new era there. + assert!( + old.head > trusted, + "the old instance served rows past its last landed batch: head {}, last accepted {trusted}", + old.head + ); + assert!(matches!( + sequencer.adopt_current_era(L2TxCount::new(old.head)).await, + Err(AdoptEraError::Adopt(AdoptError::PastRebuiltEnd { .. })) + )); + if base_count > 0 { + assert!(matches!( + sequencer + .adopt_current_era(L2TxCount::new(base_count - 1)) + .await, + Err(AdoptEraError::Adopt(AdoptError::BelowBase { .. })) + )); + } + check_rebuilt_rows(runtime, &client, base_count, trusted).await?; + + let alice_before = client.app.current_user_balance(alice_address); + let bob_before = client.app.current_user_balance(bob_address); + let mut ws = runtime.ws_adopting(trusted).await?; + for (i, queued_block) in (0_u64..).zip([first_queued_block, queued_block]) { + let drained = ws + .expect_direct_input_from(runtime.erc20_portal_address()) + .await?; + let WsTxMessage::DirectInput { + offset, + input_index, + block_number, + batch_nonce, + .. + } = &drained + else { + return Err(format!("expected a queued direct, got {drained:?}").into()); + }; + assert_eq!( + (*offset, *input_index, *block_number, *batch_nonce), + (trusted + i, queued_input + i, queued_block, 2), + "the root frame drains the queued directs first, in order, in the era's first batch" + ); + client.apply(drained)?; + } + assert_eq!( + client.app.current_user_balance(alice_address), + alice_before + alice_queued + ); + assert_eq!( + client.app.current_user_balance(bob_address), + bob_before + bob_queued + ); + + // Live rows follow; the client signs on from its own nonce. + let mut alice_l2 = runtime.wallet_l2(alice)?; + alice_l2.set_next_nonce(client.app.current_user_nonce(alice_address)); + let bob_before = client.app.current_user_balance(bob_address); + alice_l2 + .transfer(bob_address, U256::from(7_000_u64)) + .await?; + let live = ws.expect_user_op_from(alice_address).await?; + let WsTxMessage::UserOp { + offset, + batch_nonce, + .. + } = &live + else { + return Err(format!("expected the live op, got {live:?}").into()); + }; + assert_eq!( + (*offset, *batch_nonce), + (trusted + 2, 2), + "the live op follows the drain in the era's first batch" + ); + client.apply(live)?; + assert_eq!( + client.app.current_user_balance(bob_address), + bob_before + U256::from(7_000_u64) + ); + ws.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; + + // ── The canonical machine's verdict. First on the era's baseline: the + // rebuilt history through the stop block. + expect_watchdog_agrees( + runtime, + &template, + Agreement::At(era.stop_block), + "the era's baseline", + ) + .await?; + + // Then on the new era's first batch: the root frame's drain and the live + // rows, accepted on L1. A watchdog fresh from the template replays every + // input from genesis, so agreement past the stop block proves the rebuilt + // history plus the root drain is what the canonical machine executes. + let (landed_block, landed_count) = + land_a_batch(runtime, &mut ws, &mut client, &mut alice_l2, alice_address).await?; + assert!( + landed_block > era.stop_block, + "the new era's first batch lands past the stop block: {landed_block} vs {}", + era.stop_block + ); + assert!( + landed_count > trusted + 2, + "the new era's first batch holds the drained directs and the live rows: \ + count {landed_count}, rebuilt end {trusted}" + ); + let info = sequencer.history(None).await?; + let landed = info + .accepted_checkpoint + .ok_or("the landed batch is an accepted checkpoint")?; + assert_eq!( + landed.next_batch_nonce, 3, + "the new era's first batch takes the nonce after the old era's two" + ); + expect_watchdog_agrees( + runtime, + &template, + Agreement::AtOrAfter(landed_block), + "the new era's first accepted batch", + ) + .await?; + Ok(()) +} + +/// The block a watchdog's agreement must be at. +#[derive(Debug, Clone, Copy)] +enum Agreement { + /// Exactly this checkpoint, so nothing later stands in for it. + At(u64), + /// This checkpoint or a later one. + AtOrAfter(u64), +} + +/// A watchdog started fresh from the template image replays L1 from genesis +/// and must agree with the sequencer's accepted checkpoint where `expected` +/// says. +async fn expect_watchdog_agrees( + runtime: &ManagedSequencer, + template: &Path, + expected: Agreement, + what: &str, +) -> ScenarioResult<()> { + let watchdog = Watchdog::new(template.to_path_buf(), 0)?; + watchdog.expect(runtime, &["init"], 0).await?; + watchdog.expect(runtime, &["tick"], 0).await?; + let status = watchdog.status(runtime).await?; + let agreed = head_block(&status).ok_or("status without a head")?; + let placed = match expected { + Agreement::At(block) => agreed == block, + Agreement::AtOrAfter(block) => agreed >= block, + }; + if !placed || status["last_tick"]["outcome"] != "agreed" { + return Err( + format!("expected the watchdog to agree with {what} ({expected:?}): {status}").into(), + ); + } + Ok(()) +} + +/// Seal the open batch by size, land it, and wait for its acceptance. +/// Returns the accepted checkpoint's inclusion block and count. +async fn land_a_batch( + runtime: &ManagedSequencer, + ws: &mut WsClient, + client: &mut WalletClient, + alice_l2: &mut WalletL2Client, + alice_address: Address, +) -> ScenarioResult<(u64, u64)> { + let floor = runtime.finalized_inclusion_block().await?.unwrap_or(0); + let sealed_before = runtime.count_batches()?.sealed; + for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { + alice_l2.transfer(alice_address, U256::from(1_u64)).await?; + client.apply(ws.expect_user_op_from(alice_address).await?)?; + } + if runtime.count_batches()?.sealed <= sealed_before { + return Err("the transfers did not seal a batch".into()); + } + let block = mine_until_finalized_advances(runtime, floor).await?; + let checkpoint = runtime + .sequencer_client()? + .history(None) + .await? + .accepted_checkpoint + .ok_or("a landed batch is an accepted checkpoint")?; + if checkpoint.inclusion_block != block { + return Err(format!( + "/history reports checkpoint block {}, /finalized_state {block}", + checkpoint.inclusion_block + ) + .into()); + } + Ok((block, checkpoint.executed_input_count.get())) +} + +/// Mine single blocks until Anvil's safe head covers `block`. +async fn mine_until_safe(runtime: &ManagedSequencer, block: u64) -> ScenarioResult<()> { + for _ in 0..SAFE_HEAD_MINE_ATTEMPTS { + if runtime.l1_safe_block_number().await? >= block { + return Ok(()); + } + runtime.mine_live_l1_blocks(1).await?; + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(format!("Anvil's safe head never reached block {block}").into()) +} + +/// A canonical machine stopped after every input through `block`, as an +/// operator sources one: replayed from the template image by the watchdog. +async fn stored_machine_at( + runtime: &ManagedSequencer, + template: &Path, + block: u64, +) -> ScenarioResult { + let base = StoredMachine::new("base")?; + let out = &base.path; + // `replay` reads the chain and network from an initialized state directory. + let watchdog = Watchdog::new(template.to_path_buf(), 0)?; + watchdog.expect(runtime, &["init"], 0).await?; + let stdout = watchdog + .expect( + runtime, + &[ + "replay", + "--from", + &template.to_string_lossy(), + "--from-block", + "0", + "--to-block", + &block.to_string(), + "--out", + &out.to_string_lossy(), + ], + 0, + ) + .await?; + let replay: serde_json::Value = + serde_json::from_str(&stdout).map_err(|err| format!("replay JSON: {err}: {stdout}"))?; + if !replay["stop"].is_null() { + return Err(format!("the canonical machine stopped while replaying: {replay}").into()); + } + Ok(base) +} + +/// The client's optional row check: the new era serves, from the base to the +/// client's count, exactly the rows the client kept from the old era. +async fn check_rebuilt_rows( + runtime: &ManagedSequencer, + client: &WalletClient, + base_count: u64, + trusted: u64, +) -> ScenarioResult<()> { + let sequencer: SequencerClient = runtime.sequencer_client()?; + let claim = sequencer + .adopt_current_era(L2TxCount::new(base_count)) + .await?; + let mut ws = WsClient::connect(&sequencer, claim).await?; + for count in base_count..trusted { + let served = serde_json::to_value(ws.next_message().await?)?; + let kept = serde_json::to_value( + client + .rows + .get(usize::try_from(count)?) + .ok_or("the client kept no row at this count")?, + )?; + if served != kept { + return Err( + format!("row {count} differs between eras: served {served}, kept {kept}").into(), + ); + } + } + Ok(()) +} diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index f7052896..39b566dd 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -6,18 +6,18 @@ use std::time::Duration; use crate::{Host, Scenario, ScenarioFn, ScenarioResult}; use alloy_primitives::{Address, U256}; use rollups_harness::{ - DEVNET_PREEMPTIVE_MARGIN_BLOCKS, DEVNET_SECONDS_PER_BLOCK, ManagedSequencer, - ManagedSequencerConfig, RecoveryCheckpoint, RecoverySetupParams, ReplayWalletApp, - RespawnAttemptOutcome, RespawnPolicy, TcpProxy, TestSigner, WalletL1Client, WalletL2Client, - WsClient, sign_user_op_hex, + DEVNET_SECONDS_PER_BLOCK, EXIT_TERMINAL, LandingHealth, ManagedSequencer, + ManagedSequencerConfig, ReplayWalletApp, RunStart, TcpProxy, TestSigner, UnacceptedBatch, + WalletL1Client, WalletL2Client, WsClient, batch_submitter_signer, encode_batch, + is_terminal_abort, sign_user_op_hex, }; use sequencer_core::api::{TxRequest, WsTxMessage}; -use sequencer_core::application::Application; +use sequencer_core::batch::Batch; use sequencer_core::fee::fee_to_linear; use sequencer_core::user_op::UserOp; -use sequencer_rust_client::{HistoryPolicyError, SequencerClient, SubscribeError}; +use sequencer_rust_client::SequencerClient; -const NO_WS_MESSAGE_WAIT: Duration = Duration::from_secs(1); +pub(crate) const NO_WS_MESSAGE_WAIT: Duration = Duration::from_secs(1); /// Default log_recommended_fee exponent (0 + 296 + 20 + 419 + 621 = 1356). const DEFAULT_FRAME_FEE: u16 = 1356; @@ -28,7 +28,7 @@ const DEFAULT_MAX_FEE: u16 = 2500; /// Self-transfers that force the inclusion lane to seal a batch by size, so a /// finalized "gold" snapshot promotes for a watchdog compare — independent of /// the batch-open timer. -const TRANSFERS_TO_FORCE_BATCH_CLOSE: usize = 150; +pub(crate) const TRANSFERS_TO_FORCE_BATCH_CLOSE: usize = 150; /// Poll budget for [`mine_until_finalized_advances`]: mine + recheck the /// finalized snapshot up to this many times before timing out. With @@ -41,7 +41,7 @@ const PROMOTION_POLL_ATTEMPTS: usize = 40; const PROMOTION_POLL_INTERVAL: Duration = Duration::from_secs(1); /// Poll budget for waiting until a landed batch reaches the reader's -/// `safe_accepted_batches` frontier. +/// accepted frontier (`batches.accepted_input`). const ACCEPTANCE_POLL_ATTEMPTS: usize = 40; /// Per-attempt pause while waiting for batch submission and safe-head ingestion. @@ -50,93 +50,56 @@ const ACCEPTANCE_POLL_INTERVAL: Duration = Duration::from_secs(1); /// Product-visible live frame-clock interval. E2E advancement must derive from /// the protocol constant rather than incidental Anvil transaction counts. const FRAME_CLOCK_INTERVAL_SAFE_BLOCKS: u64 = - sequencer_core::protocol::ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS; - + sequencer_core::protocol::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS; const FRAME_CLOCK_POLL_ATTEMPTS: usize = 80; const FRAME_CLOCK_POLL_INTERVAL: Duration = Duration::from_millis(250); -// ── Zone-math constants for the outage-matrix and recovery tests ───────── -// -// These derive from `MAX_WAIT_BLOCKS` (sequencer-core) and the timing the -// harness passes to the sequencer (`DEVNET_PREEMPTIVE_MARGIN_BLOCKS`, -// `DEVNET_SECONDS_PER_BLOCK`), so a change to either flows through here. The -// compile-time asserts below catch drift that would break the zone framing -// (e.g., a per-retry advance that no longer crosses MAX_WAIT in the -// orchestrator loop). +// ── The window ─────────────────────────────────────────────────────────── // -// The picks (PRE / DANGER / PAST_STALE) are deliberately well inside their -// zones to give tests slack against scheduling jitter and timing drift. - -/// Source of truth: shared between sequencer + scheduler via -/// `sequencer_core::MAX_WAIT_BLOCKS`. -const MAX_WAIT_BLOCKS: u64 = sequencer_core::MAX_WAIT_BLOCKS; - -/// Derived: the preemptive-recovery danger threshold. Below this we're safe; -/// above it (but below `MAX_WAIT_BLOCKS`) is the danger zone where the -/// sequencer triggers flush + shutdown but no cascade. -const DANGER_THRESHOLD_BLOCKS: u64 = MAX_WAIT_BLOCKS - DEVNET_PREEMPTIVE_MARGIN_BLOCKS; - -/// Pre-danger pick — well below `DANGER_THRESHOLD_BLOCKS` so background drift -/// can't accidentally tip a test into the danger zone. -const PRE_DANGER_BLOCKS: u64 = 500; - -/// Danger-zone pick — comfortably past `DANGER_THRESHOLD_BLOCKS`, comfortably -/// below `MAX_WAIT_BLOCKS`. Used by tests that want "danger detected, no -/// cascade" framing. -const DANGER_ZONE_BLOCKS: u64 = 1150; - -/// Past-stale pick — comfortably past `MAX_WAIT_BLOCKS`. Startup recovery -/// must cascade at this point. -const PAST_STALE_BLOCKS: u64 = 1250; - -/// Per-retry L1 + wall-clock advance for `respawn_until_stable` loops that -/// start in the danger zone. The closed in-danger batch only cascades once -/// it ages past `MAX_WAIT_BLOCKS`, so each retry has to push the system -/// across that boundary within `RespawnPolicy::max_attempts`. The -/// compile-time check below pins the load-bearing relationship. -const RESPAWN_RETRY_ADVANCE_BLOCKS: u64 = 100; +// Every outage drill counts blocks from the wallet's staleness window, which +// its guest image and its sequencer binary both read, so a change to the +// window flows through here. + +const MAX_WAIT_BLOCKS: u64 = app_core::application::WALLET_SCHEDULER_LIMITS.max_wait_blocks; + +/// Blocks a drill leaves between the safe head and the void boundary when its +/// oldest batch must still land: the lane's seal, the poster's tick, the +/// inclusion and Anvil's safe-head lag fit in a few. +const LANDING_ROOM_BLOCKS: u64 = 50; + +/// The lane's default batch-open deadline (two hours), in harness blocks. +const BATCH_OPEN_DEADLINE_BLOCKS: u64 = 2 * 60 * 60 / DEVNET_SECONDS_PER_BLOCK; + +/// A short outage: well inside the window, and under the lane's batch-open +/// deadline, so the open batch stays open. +const SHORT_OUTAGE_BLOCKS: u64 = if MAX_WAIT_BLOCKS / 3 < BATCH_OPEN_DEADLINE_BLOCKS - 1 { + MAX_WAIT_BLOCKS / 3 +} else { + BATCH_OPEN_DEADLINE_BLOCKS - 1 +}; + +const _: () = assert!(LANDING_ROOM_BLOCKS < MAX_WAIT_BLOCKS); + +/// Poll budget for a `/healthz` reading; the landing watch publishes every 2 s. +const LANDING_POLL_ATTEMPTS: usize = 40; +const LANDING_POLL_INTERVAL: Duration = Duration::from_millis(250); + +/// How long a running sequencer may take to stop on a terminal fact: the +/// reader's poll plus the landing watch's, with slack. +const TERMINAL_STOP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Longer than the reader's and the landing watch's 2 s polls: a process alive +/// after it has seen what they read. +const STABILITY_WINDOW: Duration = Duration::from_secs(5); + +/// `run`'s retry class: restart with backoff. +const EXIT_RESTART_TRANSIENT: i32 = 20; /// Convert a block count to wall-clock duration at the harness block time. const fn blocks_as_duration(blocks: u64) -> Duration { Duration::from_secs(blocks * DEVNET_SECONDS_PER_BLOCK) } -// Compile-time guards: drift in the constants above that breaks the test -// framing fails the build instead of failing tests at runtime. -const _: () = { - assert!( - DANGER_THRESHOLD_BLOCKS < MAX_WAIT_BLOCKS, - "danger threshold must precede the staleness boundary", - ); - assert!( - PRE_DANGER_BLOCKS < DANGER_THRESHOLD_BLOCKS, - "PRE_DANGER_BLOCKS must stay below DANGER_THRESHOLD_BLOCKS", - ); - assert!( - DANGER_ZONE_BLOCKS > DANGER_THRESHOLD_BLOCKS, - "DANGER_ZONE_BLOCKS must clear DANGER_THRESHOLD_BLOCKS", - ); - assert!( - DANGER_ZONE_BLOCKS < MAX_WAIT_BLOCKS, - "DANGER_ZONE_BLOCKS must stay below MAX_WAIT_BLOCKS (no premature cascade)", - ); - assert!( - PAST_STALE_BLOCKS > MAX_WAIT_BLOCKS, - "PAST_STALE_BLOCKS must exceed MAX_WAIT_BLOCKS (cascade must fire)", - ); - // Load-bearing for the `respawn_until_stable` tests: starting from a closed - // in-danger batch, one retry advance must push it past MAX_WAIT_BLOCKS - // so cascade fires before max_attempts is exhausted. If - // `RESPAWN_RETRY_ADVANCE_BLOCKS` shrinks or `MAX_WAIT_BLOCKS` grows - // such that this no longer holds, tests would silently start failing - // by exhausting their retries — the compile-time check makes the - // breakage visible immediately. - assert!( - DANGER_ZONE_BLOCKS + RESPAWN_RETRY_ADVANCE_BLOCKS > MAX_WAIT_BLOCKS, - "RESPAWN_RETRY_ADVANCE_BLOCKS must cross MAX_WAIT from DANGER_ZONE in one retry", - ); -}; - struct ExpectedWalletState { address: Address, balance: U256, @@ -179,24 +142,18 @@ const FIXED_LOG_GAS_PRICE: i16 = 100; pub fn test_cases() -> Vec { vec![ c_host( - "c_host_cold_replica_snapshot_backlog_live_recovery_test", + "c_host_cold_replica_snapshot_backlog_live_restart_test", |runtime| Box::pin(crate::cold_replica::run_c(runtime)), ), - c_host("c_host_setup_recovery_round_trip_test", |runtime| { - Box::pin(run_setup_recovery_round_trip_test::(runtime)) - }), c_host("c_host_deposit_transfer_withdrawal_test", |runtime| { Box::pin(run_deposit_transfer_withdrawal_test(runtime)) }) .with(real_clock), - c_host("c_host_recovery_after_stale_batches_test", |runtime| { - Box::pin(run_recovery_after_stale_batches_test(runtime)) - }), c_host("c_host_restart_and_replay_test", |runtime| { Box::pin(run_restart_and_replay_test(runtime)) }), rust( - "cold_replica_snapshot_backlog_live_recovery_test", + "cold_replica_snapshot_backlog_live_restart_test", |runtime| Box::pin(crate::cold_replica::run(runtime)), ), rust("deposit_transfer_withdrawal_test", |runtime| { @@ -238,134 +195,93 @@ pub fn test_cases() -> Vec { "restart_after_committed_tx_replays_cleanly_test", |runtime| Box::pin(run_restart_after_committed_tx_replays_cleanly_test(runtime)), ), - rust("recovery_after_stale_batches_test", |runtime| { - Box::pin(run_recovery_after_stale_batches_test(runtime)) - }), - rust("setup_recovery_round_trip_test", |runtime| { - Box::pin(run_setup_recovery_round_trip_test::< - app_core::application::WalletApp, - >(runtime)) - }), - rust("sequencer_outage_pre_danger_no_recovery_test", |runtime| { - Box::pin(run_sequencer_outage_pre_danger_no_recovery_test(runtime)) - }), - rust("sequencer_outage_danger_zone_tip_cascade_test", |runtime| { - Box::pin(run_sequencer_outage_danger_zone_tip_cascade_test(runtime)) - }), - rust("provider_outage_past_stale_cascades_test", |runtime| { - Box::pin(run_provider_outage_past_stale_cascades_test(runtime)) - }), - rust("provider_outage_wall_clock_refuses_boot_test", |runtime| { - Box::pin(run_provider_outage_wall_clock_refuses_boot_test(runtime)) - }), - rust( - "warm_restart_from_fresh_persisted_facts_with_l1_down_test", - |runtime| { - Box::pin(run_warm_restart_from_fresh_persisted_facts_with_l1_down_test(runtime)) - }, - ), + // Batch landing and outages (docs/recovery/README.md, "Scenarios"). rust( - "wall_clock_backward_jump_retries_then_recovers_test", - |runtime| { - Box::pin(run_wall_clock_backward_jump_retries_then_recovers_test( - runtime, - )) - }, + "restart_after_a_short_outage_keeps_history_test", + |runtime| Box::pin(run_restart_after_a_short_outage_keeps_history_test(runtime)), ), - rust("stalled_safe_head_startup_refuses_boot_test", |runtime| { - Box::pin(run_stalled_safe_head_startup_refuses_boot_test(runtime)) - }), rust( - "provider_outage_pre_danger_sequencer_continues_test", + "restart_inside_the_window_keeps_history_then_lands_test", |runtime| { - Box::pin(run_provider_outage_pre_danger_sequencer_continues_test( + Box::pin(run_restart_inside_the_window_keeps_history_then_lands_test( runtime, )) }, ), rust( - "provider_outage_danger_zone_sequencer_self_exits_test", + "dropped_submission_is_posted_again_after_restart_test", |runtime| { - Box::pin(run_provider_outage_danger_zone_sequencer_self_exits_test( + Box::pin(run_dropped_submission_is_posted_again_after_restart_test( runtime, )) }, ), - rust("provider_outage_short_hiccup_no_recovery_test", |runtime| { - Box::pin(run_provider_outage_short_hiccup_no_recovery_test(runtime)) - }), rust( - "both_down_danger_zone_sequencer_first_refuses_boot_test", + "batches_stop_landing_inside_the_window_then_land_test", |runtime| { - Box::pin(run_both_down_danger_zone_sequencer_first_refuses_boot_test( + Box::pin(run_batches_stop_landing_inside_the_window_then_land_test( runtime, )) }, ), rust( - "both_down_danger_zone_proxy_first_restart_cycle_recovers_test", - |runtime| { - Box::pin(run_both_down_danger_zone_proxy_first_restart_cycle_recovers_test(runtime)) - }, + "void_history_while_running_stops_for_good_test", + |runtime| Box::pin(run_void_history_while_running_stops_for_good_test(runtime)), ), rust( - "sequencer_outage_danger_zone_coupled_restart_cycle_recovers_test", + "blind_past_the_window_stops_at_the_first_sync_that_shows_it_test", |runtime| { Box::pin( - run_sequencer_outage_danger_zone_coupled_restart_cycle_recovers_test(runtime), + run_blind_past_the_window_stops_at_the_first_sync_that_shows_it_test(runtime), ) }, ), rust( - "provider_outage_danger_zone_mid_run_exit_then_restart_cycle_recovers_test", + "blind_start_boots_on_the_stored_view_then_stops_at_the_first_sync_test", |runtime| { Box::pin( - run_provider_outage_danger_zone_mid_run_exit_then_restart_cycle_recovers_test( + run_blind_start_boots_on_the_stored_view_then_stops_at_the_first_sync_test( runtime, ), ) }, ), rust( - "first_boot_l1_unreachable_never_synced_refuses_boot_test", - |runtime| { - Box::pin(run_first_boot_l1_unreachable_never_synced_refuses_boot_test(runtime)) - }, - ), - rust("delayed_inclusion_cascades_on_restart_test", |runtime| { - Box::pin(run_delayed_inclusion_cascades_on_restart_test(runtime)) - }), - rust("aging_open_tip_runtime_danger_zone_exit_test", |runtime| { - Box::pin(run_aging_open_tip_runtime_danger_zone_exit_test(runtime)) - }), - rust("stalled_safe_head_live_exit_test", |runtime| { - Box::pin(run_stalled_safe_head_live_exit_test(runtime)) - }), - rust( - "ws_reconnect_with_invalidated_claim_is_refused_test", + "stopped_past_the_window_refuses_every_start_test", |runtime| { - Box::pin(run_ws_reconnect_with_invalidated_claim_is_refused_test( + Box::pin(run_stopped_past_the_window_refuses_every_start_test( runtime, )) }, ), - rust("ws_subscribe_ahead_of_head_is_refused_test", |runtime| { - Box::pin(run_ws_subscribe_ahead_of_head_is_refused_test(runtime)) - }), rust( - "recovery_drains_safe_but_undrained_direct_input_test", + "stopped_past_the_window_with_nothing_confirmed_refuses_test", |runtime| { - Box::pin(run_recovery_drains_safe_but_undrained_direct_input_test( - runtime, - )) + Box::pin(run_stopped_past_the_window_with_nothing_confirmed_refuses_test(runtime)) }, ), rust( - "recovery_batch_opens_empty_when_no_direct_inputs_pending_test", + "foreign_batch_at_our_nonce_stops_for_good_test", + |runtime| Box::pin(run_foreign_batch_at_our_nonce_stops_for_good_test(runtime)), + ), + rust("first_run_of_an_era_with_l1_down_retries_test", |runtime| { + Box::pin(run_first_run_of_an_era_with_l1_down_retries_test(runtime)) + }), + rust("warm_restart_with_l1_down_test", |runtime| { + Box::pin(run_warm_restart_with_l1_down_test(runtime)) + }), + rust( + "provider_outage_inside_the_window_sequencer_continues_test", |runtime| { - Box::pin(run_recovery_batch_opens_empty_when_no_direct_inputs_pending_test(runtime)) + Box::pin(run_provider_outage_inside_the_window_sequencer_continues_test(runtime)) }, ), + rust("provider_outage_short_hiccup_test", |runtime| { + Box::pin(run_provider_outage_short_hiccup_test(runtime)) + }), + rust("ws_subscribe_ahead_of_head_is_refused_test", |runtime| { + Box::pin(run_ws_subscribe_ahead_of_head_is_refused_test(runtime)) + }), rust("replay_matches_live_for_mixed_workload_test", |runtime| { Box::pin(run_replay_matches_live_for_mixed_workload_test(runtime)) }), @@ -391,14 +307,6 @@ pub fn test_cases() -> Vec { )) }, ), - rust( - "nonce_zero_recovery_invalidates_then_accepts_at_nonce_zero_test", - |runtime| { - Box::pin( - run_nonce_zero_recovery_invalidates_then_accepts_at_nonce_zero_test(runtime), - ) - }, - ), rust("watchdog_genesis_compare_test", |runtime| { Box::pin(crate::watchdog_compare::run_watchdog_genesis_compare_test( runtime, @@ -409,6 +317,51 @@ pub fn test_cases() -> Vec { Box::pin(run_watchdog_divergence_drill_test(runtime)) }) .with(real_clock), + rust("setup_refuses_a_set_up_data_directory_test", |runtime| { + Box::pin( + crate::recovery_round_trip::run_setup_refuses_a_set_up_data_directory_test(runtime), + ) + }), + c_host( + "c_host_setup_refuses_a_set_up_data_directory_test", + |runtime| { + Box::pin( + crate::recovery_round_trip::run_setup_refuses_a_set_up_data_directory_test( + runtime, + ), + ) + }, + ), + rust("recovery_from_genesis_round_trip_test", |runtime| { + Box::pin(crate::recovery_round_trip::run_recovery_from_genesis_round_trip_test(runtime)) + }) + .with(real_clock), + // The rebuild's execute pass, the baseline dump and the feed of a + // rebuilt era, through the C engine. + c_host("c_host_recovery_from_genesis_round_trip_test", |runtime| { + Box::pin(crate::recovery_round_trip::run_recovery_from_genesis_round_trip_test(runtime)) + }) + .with(real_clock), + rust("recovery_from_stored_machine_round_trip_test", |runtime| { + Box::pin( + crate::recovery_round_trip::run_recovery_from_stored_machine_round_trip_test( + runtime, + ), + ) + }) + .with(real_clock), + // The base state taken out of a stored machine, opened by the C engine. + c_host( + "c_host_recovery_from_stored_machine_round_trip_test", + |runtime| { + Box::pin( + crate::recovery_round_trip::run_recovery_from_stored_machine_round_trip_test( + runtime, + ), + ) + }, + ) + .with(real_clock), ] } @@ -487,7 +440,7 @@ async fn prepare_non_genesis_watchdog_state(runtime: &mut ManagedSequencer) -> S /// time, so the submitter and promoter get to run without synthesizing a future /// L1 clock; returns the new inclusion block, or times out loudly. Robust /// against CI jitter — it waits exactly as long as promotion takes, no more. -async fn mine_until_finalized_advances( +pub(crate) async fn mine_until_finalized_advances( runtime: &ManagedSequencer, floor: u64, ) -> ScenarioResult { @@ -506,20 +459,6 @@ async fn mine_until_finalized_advances( ) } -async fn mine_until_batch_is_safe_accepted( - runtime: &ManagedSequencer, -) -> ScenarioResult<(u64, Option)> { - for _ in 0..ACCEPTANCE_POLL_ATTEMPTS { - runtime.mine_l1_block_at_wall_clock().await?; - tokio::time::sleep(ACCEPTANCE_POLL_INTERVAL).await; - let accepted = runtime.count_safe_accepted_batches()?; - if accepted.0 > 0 { - return Ok(accepted); - } - } - Err("timed out waiting for a batch to reach safe_accepted_batches".into()) -} - /// Advance Anvil only as far as the next semantic live-frame tick needs, then /// wait for the input reader and inclusion lane to commit a frame covering /// `required_safe_block`. The live safe-head query prevents stale SQLite @@ -924,9 +863,14 @@ async fn run_restart_and_replay_test(runtime: &mut ManagedSequencer) -> Scenario 3, ); assert_eq!( - replay_after_restart.last_executed_safe_block(), - replay_before_restart.last_executed_safe_block(), - "mirror safe-block clock must match the pre-restart live replay clock", + replay_after_restart.canonical_snapshot_bytes()?, + replay_before_restart.canonical_snapshot_bytes()?, + "the replayed mirror's whole state must match the pre-restart live mirror", + ); + assert_eq!( + replay_after_restart.last_applied_block(), + replay_before_restart.last_applied_block(), + "the replayed feed must carry the same L1 blocks as the live feed", ); // Reading the persisted feed alone does not prove the restarted engine @@ -992,11 +936,23 @@ async fn run_unsupported_token_deposit_noop_test( assert_eq!(replay.current_user_balance(alice_address), U256::ZERO); assert_eq!(replay.current_user_nonce(alice_address), 0); - assert_eq!(replay.executed_input_count(), 1); + assert_eq!(replay.l2_tx_count(), 1); + Ok(()) +} + +/// Setup reads L1 before its first write, so a refusal there leaves the data +/// directory empty but for the lock file, and a retry may run in it. +pub(crate) fn assert_data_dir_holds_at_most_the_lock( + runtime: &ManagedSequencer, +) -> ScenarioResult<()> { + let entries = runtime.data_dir_entries()?; + if entries.iter().any(|entry| entry != "sequencer.lock") { + return Err(format!("a setup refused before writing left {entries:?}").into()); + } Ok(()) } -async fn apply_reconciled_supported_deposit( +pub(crate) async fn apply_reconciled_supported_deposit( runtime: &ManagedSequencer, ws: &mut WsClient, replay: &mut ReplayWalletApp, @@ -1245,7 +1201,7 @@ async fn run_multi_deposit_reconciliation_test( assert_eq!(replay.current_user_balance(alice_address), alice_deposit); assert_eq!(replay.current_user_balance(bob_address), bob_deposit); - assert_eq!(replay.executed_input_count(), 2); + assert_eq!(replay.l2_tx_count(), 2); Ok(()) } @@ -1303,7 +1259,233 @@ async fn run_restart_after_committed_tx_replays_cleanly_test( Ok(()) } -async fn run_recovery_after_stale_batches_test( +// ── Batch landing and outages ───────────────────────────────────── +// +// The drills of docs/recovery/README.md ("Scenarios"). Each derives its block +// counts from the oldest unaccepted batch's first frame `s` and the window: +// the history is void once the safe head `H` reaches `s + MAX_WAIT_BLOCKS - 1` +// (`H + 1 - s >= max_wait_blocks`). Outages advance L1 and the sequencer's +// wall clock together, so the safe head's age follows the outage. + +/// The oldest batch L1 has not accepted. +fn oldest_unaccepted(runtime: &ManagedSequencer) -> ScenarioResult { + Ok(runtime + .oldest_unaccepted_batch()? + .ok_or("expected a batch L1 has not accepted")?) +} + +/// The safe head at which a batch opened at `first_frame_safe_block` is void. +const fn void_safe_head(first_frame_safe_block: u64) -> u64 { + first_frame_safe_block + MAX_WAIT_BLOCKS - 1 +} + +/// The safe head that leaves a batch opened at `first_frame_safe_block` +/// [`LANDING_ROOM_BLOCKS`] to land in. +const fn inside_window_safe_head(first_frame_safe_block: u64) -> u64 { + void_safe_head(first_frame_safe_block) - LANDING_ROOM_BLOCKS +} + +/// Advance L1 and the sequencer's wall clock together until Anvil's safe +/// head is `target`, in one `anvil_mine`. +async fn advance_safe_head_to(runtime: &mut ManagedSequencer, target: u64) -> ScenarioResult<()> { + let live = runtime.l1_safe_block_number().await?; + let blocks = target + .checked_sub(live) + .ok_or_else(|| format!("Anvil's safe head {live} is already past {target}"))?; + runtime + .advance_wall_and_mine(blocks_as_duration(blocks)) + .await?; + let reached = runtime.l1_safe_block_number().await?; + if reached != target { + return Err(format!("mined to safe head {reached}, expected {target}").into()); + } + Ok(()) +} + +/// Mine single blocks until Anvil's safe head covers `block`. +async fn mine_until_safe(runtime: &ManagedSequencer, block: u64) -> ScenarioResult<()> { + for _ in 0..ACCEPTANCE_POLL_ATTEMPTS { + if runtime.l1_safe_block_number().await? >= block { + return Ok(()); + } + runtime.mine_live_l1_blocks(1).await?; + } + Err(format!("Anvil's safe head never reached block {block}").into()) +} + +/// Poll `/healthz` until the running sequencer reports `landing_lag_blocks` +/// `expected` (`H + 1 - s`) beside the pinned window. +async fn wait_for_landing_lag( + runtime: &ManagedSequencer, + expected: u64, +) -> ScenarioResult { + let mut health = runtime.landing_health().await?; + for _ in 0..LANDING_POLL_ATTEMPTS { + if health.landing_lag_blocks == Some(expected) { + assert_eq!(health.max_wait_blocks, MAX_WAIT_BLOCKS); + return Ok(health); + } + tokio::time::sleep(LANDING_POLL_INTERVAL).await; + health = runtime.landing_health().await?; + } + Err(format!("expected landing_lag_blocks {expected}, /healthz reports {health:?}").into()) +} + +/// A running sequencer that sees a terminal fact aborts, and its log names +/// the fact. Every terminal runtime fault aborts, so the log tells them apart. +async fn expect_terminal_abort( + runtime: &mut ManagedSequencer, + facts: &[String], +) -> ScenarioResult<()> { + let status = runtime.wait_for_exit(TERMINAL_STOP_TIMEOUT).await?; + if !is_terminal_abort(&status) { + return Err(format!("expected the terminal abort (SIGABRT), got {status:?}").into()); + } + let log = runtime.read_log_contents()?; + if !facts.iter().all(|fact| log.contains(fact.as_str())) { + return Err(format!( + "the aborted run must name {facts:?}; see {}", + runtime.log_path().display() + ) + .into()); + } + Ok(()) +} + +/// The facts a void-history refusal names. `safe_head` is the safe block it +/// names, where the drill knows it exactly: a poster that lands a void batch +/// as the provider returns can move it by a block. +fn void_history_facts(first_frame_safe_block: u64, safe_head: Option) -> Vec { + let mut facts = vec![ + "the history is void".to_owned(), + format!("opened at safe block {first_frame_safe_block}"), + format!("max_wait_blocks {MAX_WAIT_BLOCKS}"), + ]; + if let Some(safe_head) = safe_head { + facts.push(format!("not accepted by safe block {safe_head}")); + } + facts +} + +/// Start `run` and expect the terminal refusal: exit 30 before readiness and +/// its exit line naming `facts`. The terminal-fault black box stays as it +/// was: the database already holds a terminal fact, so a refusal copies none. +async fn expect_start_refused( + runtime: &mut ManagedSequencer, + facts: &[String], +) -> ScenarioResult<()> { + let recorded = runtime.terminal_fault_causes()?; + match runtime.try_respawn().await? { + RunStart::Exited(status) if status.code() == Some(EXIT_TERMINAL) => {} + other => { + return Err(format!( + "expected a start refused with exit {EXIT_TERMINAL}, got {other:?}" + ) + .into()); + } + } + let log = runtime.read_log_contents()?; + if !facts.iter().all(|fact| log.contains(fact.as_str())) { + return Err(format!( + "the refusal must name {facts:?}; see {}", + runtime.log_path().display() + ) + .into()); + } + let causes = runtime.terminal_fault_causes()?; + if causes != recorded { + return Err(format!( + "a start refused on {facts:?} must leave the black box as it was ({recorded:?}); \ + it holds {causes:?}" + ) + .into()); + } + Ok(()) +} + +/// Mine one block per second until L1 has accepted every batch through +/// `nonce`. +async fn mine_until_accepted_through(runtime: &ManagedSequencer, nonce: u64) -> ScenarioResult<()> { + for _ in 0..ACCEPTANCE_POLL_ATTEMPTS { + runtime.mine_l1_block_at_wall_clock().await?; + tokio::time::sleep(ACCEPTANCE_POLL_INTERVAL).await; + if runtime + .batch_nonce_frontier()? + .safe_accepted + .is_some_and(|accepted| accepted >= nonce) + { + return Ok(()); + } + } + Err(format!("timed out waiting for L1 to accept batch nonce {nonce}").into()) +} + +/// The accepted batches are the nonces from zero, accepted in nonce order: a +/// batch lands under the nonce it was sealed with. +fn assert_accepted_in_order(runtime: &ManagedSequencer) -> ScenarioResult<()> { + let accepted = runtime.accepted_batches()?; + for (position, (nonce, _)) in (0_u64..).zip(&accepted) { + assert_eq!( + *nonce, position, + "accepted nonces run from zero: {accepted:?}" + ); + } + for pair in accepted.windows(2) { + assert!( + pair[0].1 < pair[1].1, + "batches are accepted in nonce order: {accepted:?}" + ); + } + Ok(()) +} + +/// Self-transfers until the lane seals at least one more batch by size. +async fn seal_a_batch_by_size( + runtime: &ManagedSequencer, + ws: &mut WsClient, + replay: &mut ReplayWalletApp, + wallet: &mut WalletL2Client, + address: Address, +) -> ScenarioResult<()> { + let sealed_before = runtime.count_batches()?.sealed; + for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { + wallet.transfer(address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(address).await?)?; + } + let sealed = runtime.count_batches()?.sealed; + if sealed <= sealed_before { + return Err( + format!("the transfers did not seal a batch: {sealed_before} -> {sealed}").into(), + ); + } + Ok(()) +} + +/// The feed from offset zero carries exactly what `expected` applied. +async fn assert_feed_matches( + runtime: &ManagedSequencer, + expected: &ReplayWalletApp, +) -> ScenarioResult<()> { + let mut ws = runtime.ws(0).await?; + let mut replay = ReplayWalletApp::devnet(); + for _ in 0..expected.l2_tx_count() { + replay.apply(ws.next_message().await?)?; + } + ws.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; + assert_eq!( + replay.canonical_snapshot_bytes()?, + expected.canonical_snapshot_bytes()?, + "the feed serves the history as it was", + ); + Ok(()) +} + +// (e) A short stop, well inside the window: the restart is nothing special. +// History, the served nonce and the open batch are unchanged; a direct that +// arrived while the sequencer was down is drained after it. Also the suite's +// process-level pin of the exit-code contract's clean class: SIGTERM on a +// healthy process exits 0. +async fn run_restart_after_a_short_outage_keeps_history_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { let alice = TestSigner::from_default(1)?; @@ -1314,1222 +1496,358 @@ async fn run_recovery_after_stale_batches_test( let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); + let mut replay = ReplayWalletApp::devnet(); - // Large deposit leaves headroom for the post-recovery batch-close pump - // that drives a finalized snapshot for watchdog compare. - let deposit_amount = U256::from(10_000_000_u64); + let deposit_amount = U256::from(600_000_u64); let transfer_amount = U256::from(100_000_u64); - let post_recovery_transfer = U256::from(200_000_u64); + let late_amount = U256::from(50_000_u64); let gas = fee_to_linear(DEFAULT_FRAME_FEE); - // Step 1: Fund Alice via L1 deposit. - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - deposit_amount, - ) - .await?; - - // Step 2: Alice transfers to Bob (this will be lost after recovery). + apply_reconciled_supported_deposit(runtime, &mut ws, &mut replay, &alice_l1, deposit_amount) + .await?; alice_l2.transfer(bob_address, transfer_amount).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Verify pre-recovery state. - assert_eq!( - replay_before.current_user_balance(alice_address), - deposit_amount - transfer_amount - gas, - ); - assert_eq!( - replay_before.current_user_balance(bob_address), - transfer_amount, - ); + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + let open_batch = oldest_unaccepted(runtime)?; - // Step 3: Stop the healthy sequencer (Anvil stays up). This is the - // suite's process-level pin of the exit-code contract's clean class: - // SIGTERM on a healthy process must exit 0. drop(ws); runtime.stop_expecting_clean_exit().await?; - // Step 4: Simulate ~4h of outage: advance both L1 and wall clock by - // MAX_WAIT_BLOCKS * SECONDS_PER_BLOCK = 1200 * 12 = 14400s. On respawn, - // l1_safe_head will be >1200 blocks past the frames' safe_block. + // A deposit while the sequencer is down, then a short outage. + alice_l1.mint_supported_token(late_amount).await?; + let late_block = alice_l1.deposit_supported_token(late_amount).await?; runtime - .advance_wall_and_mine(blocks_as_duration(MAX_WAIT_BLOCKS)) + .advance_wall_and_mine(blocks_as_duration(SHORT_OUTAGE_BLOCKS)) .await?; - - // Step 5: Respawn the sequencer. Startup recovery should detect staleness. runtime.respawn().await?; - // Step 6: Replay from offset 0 after recovery. - // The deposit should be re-drained into the recovery batch. - // The transfer should be GONE (it was in an invalidated batch). + assert_eq!( + oldest_unaccepted(runtime)?, + open_batch, + "the restart keeps the open batch: same nonce, same first frame", + ); + let safe_head = runtime.l1_safe_block_number().await?; + wait_for_landing_lag(runtime, safe_head + 1 - open_batch.first_frame_safe_block).await?; + let mut ws_after = runtime.ws(0).await?; let mut replay_after = ReplayWalletApp::devnet(); - - // Expect the re-drained deposit. replay_after.apply( ws_after .expect_direct_input_from(runtime.erc20_portal_address()) .await?, )?; - - // No more messages — the transfer was invalidated. + replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; + let late = ws_after + .expect_direct_input_from(runtime.erc20_portal_address()) + .await?; + assert!( + matches!(late, WsTxMessage::DirectInput { block_number, .. } if block_number == late_block), + "the deposit made while the sequencer was down is drained after the restart: {late:?}", + ); + replay_after.apply(late)?; ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - // Alice should have her full deposit back (no transfer deducted). - assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount, - "after recovery, Alice should have full deposit (transfer was invalidated)" + assert_wallet_state( + &replay_after, + ExpectedWalletState { + address: alice_address, + balance: deposit_amount - transfer_amount - gas + late_amount, + nonce: 1, + }, + ExpectedWalletState { + address: bob_address, + balance: transfer_amount, + nonce: 0, + }, + 3, ); assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "after recovery, Bob should have zero (transfer was invalidated)" + runtime.wallet_l2(alice)?.served_next_nonce().await?, + 1, + "GET /nonce keeps the confirmed transfer", ); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - - // Step 8: Verify new work succeeds after recovery. - let mut alice_l2_fresh = runtime.wallet_l2(alice)?; - assert_eq!( - alice_l2_fresh.served_next_nonce().await?, - 0, - "GET /nonce must fall back with the invalidated transfer" - ); - alice_l2_fresh - .transfer(bob_address, post_recovery_transfer) - .await?; - assert_eq!(alice_l2_fresh.served_next_nonce().await?, 1); - replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount - post_recovery_transfer - gas, - ); - assert_eq!( - replay_after.current_user_balance(bob_address), - post_recovery_transfer, - ); - assert_eq!(replay_after.current_user_nonce(alice_address), 1); - - drive_finalized_gold_batch_for_watchdog( - runtime, - &mut ws_after, - &mut replay_after, - &mut alice_l2_fresh, - alice_address, - ) - .await?; - crate::watchdog_compare::run_watchdog_non_genesis_compare_test(runtime).await?; - Ok(()) } -// ── Cockroach recovery: setup --recovery round-trip ──────── -// -// The affirmative end-to-end test of cockroach recovery: build state, promote a -// real finalized snapshot (the checkpoint), wipe the DB, rebuild it from the -// checkpoint via `setup --recovery` (flush → fold → fill), boot `run`, and -// prove the recovered state is correct — the resumed sequencer accepts a new -// user-op at the *continuing* nonce (so the fold preserved the sender's nonce + -// balance), and the rebuilt tree is anchored at the resume nonce N' (validated -// structurally by the post-scenario `assert_schema_invariants`). -// -// `max_batch_open` is set to 5s (not the 2h default) so a batch seals + gets -// promoted via plain L1 mining — no 2h clock jump, so the L1 view stays fresh -// and no staleness/cascade fires. - -/// Mine + wait until a finalized snapshot is promoted past genesis (B > 0), -/// then capture it as a checkpoint. Times out loudly if no promotion lands. -async fn drive_promotion_and_capture( - runtime: &ManagedSequencer, -) -> ScenarioResult { - mine_until_finalized_advances(runtime, 0).await?; - runtime.capture_finalized_checkpoint().await -} - -async fn run_setup_recovery_round_trip_test( +// (e) then (a): a stop that leaves the open batch just inside the window. The +// restart keeps history and the served nonce; the stop outlasted the lane's +// batch-open deadline, so the lane seals the batch at once, and it lands +// inside the window under its own nonce. +async fn run_restart_inside_the_window_keeps_history_then_lands_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - runtime.set_max_batch_open_seconds(Some(5)); - runtime.restart().await?; - let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); - let gas = fee_to_linear(DEFAULT_FRAME_FEE); let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; let mut replay = ReplayWalletApp::devnet(); - // Build state: a deposit (direct) + a transfer (user-op). These land in the - // batch that will seal + promote into the checkpoint. Size the deposit for - // the post-recovery gold-batch pump (~150 self-transfers × ~38k fee). - let deposit = U256::from(10_000_000_u64); - let transfer1 = U256::from(100_000_u64); - apply_reconciled_supported_deposit(runtime, &mut ws, &mut replay, &alice_l1, deposit).await?; - alice_l2.transfer(bob_address, transfer1).await?; + apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut replay, + &alice_l1, + U256::from(600_000_u64), + ) + .await?; + alice_l2 + .transfer(bob_address, U256::from(100_000_u64)) + .await?; replay.apply(ws.expect_user_op_from(alice_address).await?)?; - let expected_alice = deposit - transfer1 - gas; - assert_eq!(replay.current_user_balance(alice_address), expected_alice); - assert_eq!(replay.current_user_balance(bob_address), transfer1); - assert_eq!(replay.current_user_nonce(alice_address), 1); - - // Drive the batch to seal + be accepted + promoted, and capture the - // resulting finalized snapshot as the recovery checkpoint. - let checkpoint = drive_promotion_and_capture(runtime).await?; - let old_claim = SequencerClient::new(runtime.internal_endpoint())? - .latest_snapshot() - .await? - .claim; - eprintln!( - "recovery checkpoint: B={} N={}", - checkpoint.checkpoint_block, checkpoint.resume_nonce - ); + let batch = oldest_unaccepted(runtime)?; - // Wipe the DB and rebuild it from the checkpoint via `setup --recovery`. drop(ws); runtime.stop().await?; - runtime.set_recovery_setup(Some(RecoverySetupParams { - checkpoint_block: checkpoint.checkpoint_block, - checkpoint_dump_dir: checkpoint.dir, - })); - runtime.reset_database()?; - // The recovery setup's flush + the post-recovery run boot both need L1 to - // keep advancing. - runtime.set_mine_l1_during_boot(true); + advance_safe_head_to( + runtime, + inside_window_safe_head(batch.first_frame_safe_block), + ) + .await?; runtime.respawn().await?; - // A rebuilt database cannot authorize the old replica, even when its - // numerical generation/count match. Restore and subscribe in the new era. - let client = SequencerClient::new(runtime.internal_endpoint())?; - assert!(matches!( - client.subscribe(old_claim).await, - Err(SubscribeError::History( - HistoryPolicyError::EraChanged { .. } - )) - )); - let (mut restored, fresh_claim) = - crate::cold_replica::restore::(client.latest_snapshot().await?).await?; - assert_ne!(fresh_claim.version.era_id, old_claim.version.era_id); - assert_eq!(fresh_claim.version.recovery_generation.get(), 0); - assert!(fresh_claim.next_input.get() > 0); - crate::cold_replica::assert_same_state(&mut restored, &replay)?; - let mut resumed_ws = WsClient::connect(&client, fresh_claim).await?; - - // A continuing nonce exercises the recovered host's state as well as the - // independently restored replica and the retained reference history. - let mut alice_l2_after = runtime.wallet_l2(alice)?; - // Known gap: the rebuilt database has no ops for the folded prefix, so a - // sender idle since the rebuild reads 0 (application contract, "User - // nonces"). Flip this when setup seeds baseline nonces. - assert_eq!(alice_l2_after.served_next_nonce().await?, 0); - alice_l2_after.set_next_nonce(1); - let post_transfer = U256::from(70_000_u64); - alice_l2_after.transfer(bob_address, post_transfer).await?; - assert_eq!( - alice_l2_after.served_next_nonce().await?, - 2, - "stored nonces are absolute, so one op in the new era makes it exact" - ); - let message = resumed_ws.expect_user_op_from(alice_address).await?; - rollups_harness::replay::apply_ws_message(&mut restored, message.clone())?; - replay.apply(message)?; - crate::cold_replica::assert_same_state(&mut restored, &replay)?; - - // Explicit recovery-correctness assertions, beyond the structural - // `assert_schema_invariants` (which checks `0..`-from-anchor contiguity): - // 1. the rebuilt tree is anchored at N' (I16): the checkpoint's N plus the - // batches accepted in (B, C], counted from L1; - // 2. recovery's resync did NOT spuriously freeze the frontier — the I15 - // content-identity false-positive against below-anchor collapsed history - // is otherwise a silent failure, invisible at the e2e level (the same - // silence the (C, H1] bug shipped behind). - // The 5 s timer keeps closing batches until the stop, so whether one lands - // after B varies run to run. Counting consecutive nonces from N mirrors the - // scheduler's acceptance; batches this fresh cannot be stale. - let stop_block = runtime.baseline_safe_block()?; - let mut resume_nonce = checkpoint.resume_nonce; - for nonce in runtime - .l1_batch_nonces(checkpoint.checkpoint_block, stop_block) - .await? - { - if nonce == resume_nonce { - resume_nonce += 1; - } - } - eprintln!("recovery fold: C={stop_block} N'={resume_nonce}"); - assert_eq!( - runtime.batch_tree_anchor()?, - resume_nonce, - "the rebuilt tree must be anchored at N', the checkpoint nonce plus the \ - batches accepted after the checkpoint block" - ); - assert_eq!( - runtime.canonical_divergence()?, - None, - "recovery must not falsely freeze the frontier (no canonical divergence)" - ); - - // ── Watchdog agreement on the rebuilt + resumed state ──────────────── - // - // The strongest end-to-end proof that `setup --recovery` reconstructed - // canonical state: drive a finalized gold batch on the *resumed* sequencer, - // then assert the watchdog's from-genesis CM replay is byte-identical to the - // finalized snapshot the sequencer now serves. The CM replays every L1 input - // from genesis — the pre-wipe batches (nonce < N') plus the post-recovery - // batches at the resume nonce N' — so agreement proves the fold-rebuilt state - // S' and the resumed submission both land exactly on the canonical chain - // state. This checks native execution against the independent CM target. - let floor_inclusion_block = runtime.finalized_inclusion_block().await?.unwrap_or(0); - let batches_before = runtime.count_batches()?; - for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { - alice_l2_after - .transfer(alice_address, U256::from(1_u64)) - .await?; - } - let batches_after = runtime.count_batches()?; - assert!( - batches_after.sealed > batches_before.sealed, - "expected a sealed batch before the recovery watchdog compare: \ - before={batches_before:?} after={batches_after:?}" - ); - mine_until_finalized_advances(runtime, floor_inclusion_block).await?; - crate::watchdog_compare::run_watchdog_non_genesis_compare_test(runtime).await?; + assert_feed_matches(runtime, &replay).await?; + assert_eq!(runtime.wallet_l2(alice)?.served_next_nonce().await?, 1); + mine_until_accepted_through(runtime, batch.nonce).await?; + assert_accepted_in_order(runtime)?; + assert_feed_matches(runtime, &replay).await?; Ok(()) } -// ── Sequencer outage, pre-danger zone ──────────────────────────── -// -// Sequencer stops with an open batch (deposit + transfer); L1 advances 500 -// blocks (well below the danger threshold of 900). On restart: -// - Startup recovery runs but finds no danger zone → no flush. -// - No batches are stale → no cascade invalidation. -// - The deposit and transfer persist across the restart. -// - New txs succeed against the unchanged state. -// -// This is the positive control for the recovery procedure: it must NOT fire -// (or over-fire) when L1 hasn't drifted enough to cause trouble. - -async fn run_sequencer_outage_pre_danger_no_recovery_test( +// (a) at restart: the submission of a sealed batch is lost from the mempool +// while the sequencer is down. Restarted inside the window, the poster posts +// the same batch nonce again and L1 accepts it. +async fn run_dropped_submission_is_posted_again_after_restart_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Pick an advance safely below the danger threshold - // (MAX_WAIT_BLOCKS 1200 - default margin 300 = 900). - const PRE_DANGER: Duration = blocks_as_duration(PRE_DANGER_BLOCKS); - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); - let bob_address = bob.address(); - - let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - - let deposit_amount = U256::from(600_000_u64); - let transfer_amount = U256::from(100_000_u64); - let gas = fee_to_linear(DEFAULT_FRAME_FEE); + let mut replay = ReplayWalletApp::devnet(); - // Step 1: Fund Alice and record a transfer. + let mut ws = runtime.ws(0).await?; apply_reconciled_supported_deposit( runtime, &mut ws, - &mut replay_before, + &mut replay, &alice_l1, - deposit_amount, + U256::from(10_000_000_u64), ) .await?; - alice_l2.transfer(bob_address, transfer_amount).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - let expected_alice_balance = deposit_amount - transfer_amount - gas; - let expected_bob_balance = transfer_amount; + // Hold the submission in the mempool, then lose it. + runtime.set_automine(false).await?; + seal_a_batch_by_size(runtime, &mut ws, &mut replay, &mut alice_l2, alice_address).await?; + runtime.wait_for_pending_batch_submission().await?; + let batch = oldest_unaccepted(runtime)?; + let served_nonce = alice_l2.served_next_nonce().await?; - // Step 2: Stop the sequencer. Leave Anvil running. drop(ws); runtime.stop().await?; - - // Step 3: Advance L1 + wall-clock a pre-danger amount (500 blocks ≈ 100min - // < 900 block danger threshold). - runtime.advance_wall_and_mine(PRE_DANGER).await?; - - // Step 4: Restart. No recovery should fire. + runtime.drop_all_pending_txs().await?; + runtime.set_automine(true).await?; + advance_safe_head_to( + runtime, + inside_window_safe_head(batch.first_frame_safe_block), + ) + .await?; runtime.respawn().await?; - // Step 5: Replay via WS from offset 0. Both the deposit and transfer must - // still be present (no invalidation). - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - expected_alice_balance, - "pre-danger restart must preserve Alice's balance", - ); + mine_until_accepted_through(runtime, batch.nonce).await?; + assert_accepted_in_order(runtime)?; + assert_feed_matches(runtime, &replay).await?; assert_eq!( - replay_after.current_user_balance(bob_address), - expected_bob_balance, - "pre-danger restart must preserve Bob's balance", - ); - assert_eq!( - replay_after.current_user_nonce(alice_address), - 1, - "Alice's nonce must NOT be reset", + runtime.wallet_l2(alice)?.served_next_nonce().await?, + served_nonce ); - - // Step 6: No further messages queued. Confirm nothing else comes through. - // (A follow-up "new work succeeds" step is omitted here because the - // harness's `wallet_l2` initializes its local nonce counter at 0, and - // this scenario explicitly does NOT reset the on-chain nonce — the - // post-restart nonce is 1. Adding a "submit at nonce 1" check would - // require harness plumbing beyond the scope of this regression test.) - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - Ok(()) } -// ── Sequencer outage, danger zone (Tip cascade) ─────────────────── -// -// Sequencer stops; L1 advances into the danger zone (past `danger_threshold`) -// but strictly below `MAX_WAIT_BLOCKS`. On restart: -// - `check_danger` returns `TipInDanger(idx)` — the closed-frontier check finds -// nothing past gold, but the open Tip's first frame has aged past -// `danger_threshold`. -// - the startup reducer selects `RecoverTip` (no flush — the Tip has -// no L1 footprint). -// - `recover_aging_tip` cascades the Tip; pre-outage soft-confirmed user -// ops are rolled back (this is the documented "soft confirmations may -// be invalidated under recovery" semantics). -// -// This exercises the Tip-recovery-without-flush path. The Tip is invalidated at -// `danger_threshold`, not `MAX_WAIT_BLOCKS`, so the system stops issuing soft -// confirmations on a Tip that is already operationally suspect. - -async fn run_sequencer_outage_danger_zone_tip_cascade_test( +// (a) live: batches stop landing while the sequencer runs (the submitter's +// transactions are refused), and the cause is fixed with the oldest batch +// just inside the window. The sequencer keeps serving and sealing, reports +// the lag, then posts the backlog in order once it can; a claim taken before +// the outage subscribes with no gap. +async fn run_batches_stop_landing_inside_the_window_then_land_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Pick an advance in the danger zone: > danger_threshold (900) but < - // MAX_WAIT (1200). L1 block time and process time advance together so the - // observed Tip age selects `RecoverTip` without leaving the repaired L1 - // view future-dated during the mandatory post-recovery inspection. - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); - let bob_address = bob.address(); - - let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - - let deposit_amount = U256::from(600_000_u64); - let transfer_amount = U256::from(100_000_u64); - let gas = fee_to_linear(DEFAULT_FRAME_FEE); + let mut replay = ReplayWalletApp::devnet(); + let mut ws = runtime.ws(0).await?; apply_reconciled_supported_deposit( runtime, &mut ws, - &mut replay_before, + &mut replay, &alice_l1, - deposit_amount, + U256::from(20_000_000_u64), ) .await?; - alice_l2.transfer(bob_address, transfer_amount).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - - let expected_alice_balance = deposit_amount - transfer_amount - gas; - let expected_bob_balance = transfer_amount; - - drop(ws); - runtime.stop().await?; - - // L1 advances into the danger zone but strictly below the staleness - // threshold. The Tip's first frame has aged past `danger_threshold`, - // so the startup `RecoverTip` path cascades it (no flush — the Tip - // has no L1 footprint). Alice's pre-outage transfer was a soft - // confirmation against the Tip; it's rolled back. - runtime - .advance_wall_and_mine(blocks_as_duration(DANGER_ZONE_BLOCKS)) - .await?; - let _ = expected_alice_balance; - let _ = expected_bob_balance; - - runtime.respawn().await?; - - let counts = runtime.count_batches()?; - assert!( - counts.invalidated >= 1, - "RecoverTip must invalidate the aged Tip before admission: {counts:?}", - ); + let claim = runtime.history_claim(replay.l2_tx_count())?; - // After Tip cascade: balances roll back to the post-deposit / pre-transfer - // state, nonces reset, and the WS feed should not replay the invalidated - // user op. - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; + runtime.hold_batch_submissions().await?; + seal_a_batch_by_size(runtime, &mut ws, &mut replay, &mut alice_l2, alice_address).await?; + let batch = oldest_unaccepted(runtime)?; + advance_safe_head_to( + runtime, + inside_window_safe_head(batch.first_frame_safe_block), + ) + .await?; + // Inside the window: up, serving, sealing, reporting the lag. + wait_for_landing_lag(runtime, MAX_WAIT_BLOCKS - LANDING_ROOM_BLOCKS).await?; + seal_a_batch_by_size(runtime, &mut ws, &mut replay, &mut alice_l2, alice_address).await?; + let frontier = runtime.batch_nonce_frontier()?; assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount, - "Tip cascade must roll Alice back to her deposit (transfer was \ - soft-confirmed against the cascaded Tip)", - ); - assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "Bob never received the rolled-back transfer", - ); - assert_eq!( - replay_after.current_user_nonce(alice_address), - 0, - "nonce must reset when Tip cascade rolls back the user op", + frontier.safe_accepted, None, + "nothing lands while submissions are held" ); + let sealed = frontier.sealed.ok_or("batches were sealed")?; + let served_nonce = alice_l2.served_next_nonce().await?; + assert_eq!(served_nonce, replay.current_user_nonce(alice_address)); + + runtime.release_batch_submissions().await?; + mine_until_accepted_through(runtime, sealed).await?; + assert_accepted_in_order(runtime)?; assert_eq!( - runtime.wallet_l2(alice)?.served_next_nonce().await?, - 0, - "GET /nonce must reset with the cascaded Tip", + alice_l2.served_next_nonce().await?, + served_nonce, + "GET /nonce is unchanged" ); - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - + let mut resumed = WsClient::connect(&runtime.sequencer_client()?, claim).await?; + for offset in claim.next_input.get()..replay.l2_tx_count() { + let message = resumed.next_message().await?; + assert_eq!( + message.offset(), + offset, + "the pre-outage claim resumes with no gap" + ); + } + drop(ws); Ok(()) } -// ── Provider outage, past-stale (recovery through proxy) ────────── -// -// Scenario: the sequencer is routed through a `TcpProxy`, simulating a -// gateway in front of the real L1 node. While the sequencer is stopped, -// a temporary outage happens (proxy disconnected), L1 advances past the -// staleness threshold, and the outage ends (proxy reconnected). The next -// sequencer restart connects via the proxy, sees the advanced safe head, -// and cascade-invalidates the stale open batch. -// -// What this locks down that the sequencer-outage tests don't: -// - The proxy is actually wired into the RPC path. Subsequent RPC calls -// from the sequencer (safe-head sync, batch submission) route through -// it. If `set_l1_endpoint_override` ever regressed (e.g., respawn -// ignored the override), this test would fail. -// - Recovery over a non-direct connection works end-to-end. -// -// Note on wall-clock fallback: in principle this scenario would also test -// the fallback refusing to boot when L1 is unreachable AND real time has -// elapsed past the danger threshold. In practice, `anvil_mine(N)` takes -// milliseconds of real wall-clock time, so the fallback correctly reports -// "not yet in danger by wall-clock" and lets the sequencer boot with stale -// data. Exercising the wall-clock-refuses-to-boot path requires either -// direct `synced_at_ms` DB manipulation or a time-skew tool — deferred. - -async fn run_provider_outage_past_stale_cascades_test( +// (b) Not fixed: submissions stay refused until the oldest batch is void +// while the sequencer runs. One block short of the boundary it is up and +// reports a lag of one block under the window; at the boundary it stops for +// good, and every start refuses. The operator's playbook (a new data +// directory, setup, run) starts a new era, which a client of the old one +// sees as ERA_CHANGED. A transaction of the old instance still in the +// mempool is replaced by setup's flush, and the new era's first batch lands +// at the era's anchor nonce. +async fn run_void_history_while_running_stops_for_good_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Advance comfortably past staleness so the test is robust to small - // scheduling drifts. - const PAST_STALE: Duration = blocks_as_duration(PAST_STALE_BLOCKS); - let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); + // Fees for the new era's batch-sealing transfers. + let deposit_amount = U256::from(10_000_000_u64); - // Step 1: Normal setup — deposit + transfer (the transfer will be lost). - let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - - let deposit_amount = U256::from(600_000_u64); - let transfer_amount = U256::from(100_000_u64); - - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - deposit_amount, - ) - .await?; - alice_l2.transfer(bob_address, transfer_amount).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Step 2: Stop the sequencer and insert a proxy into the L1 path. - drop(ws); - runtime.stop().await?; - - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - - // Step 3: Simulate a gateway outage that spans the staleness window. - // - Disconnect the proxy (gateway is down). - // - Mine 1250 blocks directly on Anvil (bypasses the proxy). - // - Reconnect the proxy (gateway is back). - // During the outage the sequencer is stopped; when it comes back up, - // it will see the advanced safe head through the proxy. - proxy.disconnect(); - runtime.advance_wall_and_mine(PAST_STALE).await?; - proxy.reconnect(); + let mut replay = ReplayWalletApp::devnet(); + let claim = { + let mut ws = runtime.ws(0).await?; + apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut replay, + &alice_l1, + deposit_amount, + ) + .await?; + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + runtime.history_claim(replay.l2_tx_count())? + }; - // Step 4: Respawn. The sequencer dials the proxy, the proxy forwards - // to Anvil, `sync_to_current_safe_head` returns 1250+ blocks past the - // open Tip's first frame. `check_danger` fires `TipInDanger(idx)`, and - // the reducer's guarded `RecoverTip` phase cascades the Tip and opens a - // fresh one. + runtime.hold_batch_submissions().await?; + let batch = oldest_unaccepted(runtime)?; + let void_at = void_safe_head(batch.first_frame_safe_block); + advance_safe_head_to(runtime, void_at - 1).await?; + wait_for_landing_lag(runtime, MAX_WAIT_BLOCKS - 1).await?; + assert_eq!(runtime.observe_for(STABILITY_WINDOW).await?, None); + + advance_safe_head_to(runtime, void_at).await?; + let facts = void_history_facts(batch.first_frame_safe_block, Some(void_at)); + expect_terminal_abort(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + + // The cause is fixed, but a transaction signed with the submitter's key + // is still in the mempool when the operator follows the playbook. + runtime.release_batch_submissions().await?; + runtime.set_automine(false).await?; + let stranded = runtime + .broadcast_input_as( + &batch_submitter_signer()?, + encode_batch(&Batch { + nonce: batch.nonce, + frames: Vec::new(), + }), + ) + .await?; + runtime.discard_data_dir(); + runtime.set_mine_l1_during_setup(true); runtime.respawn().await?; + runtime.set_mine_l1_during_setup(false); + runtime.set_automine(true).await?; + assert!( + !runtime.transaction_mined(stranded).await?, + "setup's flush replaces the stranded transaction with a no-op", + ); - // Step 5: Verify via WS replay. - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) + assert!(matches!( + runtime.sequencer_client()?.subscribe(claim).await, + Err(sequencer_rust_client::SubscribeError::History( + sequencer_rust_client::HistoryPolicyError::EraChanged { .. } + )) + )); + + // The new era serves the rebuilt deposit, and its first batch lands at + // the anchor nonce. + let anchor = runtime.published_era()?.next_batch_nonce; + let mut ws = runtime.ws(0).await?; + let mut replay = ReplayWalletApp::devnet(); + replay.apply( + ws.expect_direct_input_from(runtime.erc20_portal_address()) .await?, )?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount, - "transfer must be invalidated after past-stale outage routed through proxy", - ); + assert_eq!(replay.current_user_balance(alice_address), deposit_amount); + let mut alice_l2 = runtime.wallet_l2(alice)?; + seal_a_batch_by_size(runtime, &mut ws, &mut replay, &mut alice_l2, alice_address).await?; + mine_until_accepted_through(runtime, anchor).await?; assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "Bob's receiving balance must be rolled back", + runtime.accepted_batches()?.first().map(|(nonce, _)| *nonce), + Some(anchor), + "the new era's first accepted batch takes the anchor nonce", ); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - - // Step 6: Tear down the proxy cleanly. - proxy.shutdown().await?; - Ok(()) } -// ── Wall-clock fallback refuses to boot past danger threshold ───── -// -// Scenario: L1 is unreachable AND wall-clock time has elapsed past the -// danger threshold since the last successful L1 sync. The sequencer must -// refuse to boot — proceeding would mean issuing soft confirmations against -// stale L1 state, potentially missing that batches are already doomed. -// -// This test only became possible after the `find_first_batch_in_danger` -// unification. Prior to that fix, an open batch was invisible to -// `check_danger_zone`, so the wall-clock fallback could "miss" an open -// batch aging into danger while L1 was unreachable and boot anyway. -// -// The wall-clock illusion is created without OS tooling: `rewind_synced_at_ms` -// rewrites `l1_safe_head.synced_at_ms` to an older timestamp, equivalent -// to advancing the wall clock from the sequencer's perspective. We mine -// an equivalent number of blocks on Anvil to keep the block-time coupling -// documented in `docs/threat-model/README.md`. - -async fn run_provider_outage_wall_clock_refuses_boot_test( +// (c) live: the provider goes down while the sequencer runs, and L1 passes +// the window behind it. Blind, the sequencer keeps running: its landing lag +// freezes below the window while the safe head's age grows. It stops at the +// first sync that shows the void history, and a respawn refuses. +async fn run_blind_past_the_window_stops_at_the_first_sync_that_shows_it_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Pick an elapsed time comfortably past the danger threshold. Defaults: - // seconds_per_block=12, danger_threshold=MAX_WAIT_BLOCKS(1200)-margin(300)=900. - // We need elapsed_secs / 12 > 900 → elapsed_secs > 10800. Use 5h. - const OUTAGE: Duration = Duration::from_secs(5 * 60 * 60); - let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); - // Step 1: Normal setup — deposit + transfer (transfer will be lost). - let mut ws = runtime.ws(0).await?; let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2 - .transfer(bob_address, U256::from(100_000_u64)) - .await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Step 2: Stop the sequencer, insert proxy, disconnect it, advance both - // the wall clock and L1 by the outage duration — block-time coupled so - // the sequencer sees a consistent view (5h ≈ 1500 blocks at 12s/block). - drop(ws); - runtime.stop().await?; - - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - proxy.disconnect(); - runtime.advance_wall_and_mine(OUTAGE).await?; - - // Step 3: Attempt respawn with proxy disconnected. The sequencer: - // - dials the proxy → sync_to_current_safe_head fails (L1 unreachable). - // - sees the persisted safe block timestamp is older than the L1 - // read-staleness threshold. - // - the startup reducer returns Retry(L1ViewStale), so the process exits - // without runtime admission. - let respawn_result = runtime.respawn().await; - assert!( - respawn_result.is_err(), - "respawn must fail: wall-clock says past-danger AND open batch is in danger", - ); - - // Step 4: Reconnect the proxy and respawn normally. Sync now succeeds, - // the stale open batch is cascade-invalidated, recovery batch opens. - proxy.reconnect(); - runtime.respawn().await?; - - // Step 5: Verify the invalidation: only the re-drained deposit appears. - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(600_000_u64), - "transfer must be invalidated after wall-clock-triggered recovery", - ); - assert_eq!(replay_after.current_user_balance(bob_address), U256::ZERO,); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - - proxy.shutdown().await?; - Ok(()) -} - -// Warm restart from a fresh persisted L1 view while the RPC is unreachable. -// -// This is the production-shaped discriminator for startup admission: an -// initial refresh attempt may fail, but a recent persisted view that still -// reduces to `Admit` is sufficient authority to prepare and launch the -// runtime. The test keeps the proxy disconnected through readiness, one full -// background-worker retry cadence, POST /tx, and WS broadcast. A boot path -// that makes provider reachability an unconditional admission gate fails at -// `respawn`; a runtime that launches without a usable inclusion lane fails the -// transaction or WS assertion. -async fn run_warm_restart_from_fresh_persisted_facts_with_l1_down_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - const STABILITY_WINDOW: Duration = Duration::from_secs(3); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - let deposit_amount = U256::from(600_000_u64); - let transfer_amount = U256::from(100_000_u64); - let fee = fee_to_linear(DEFAULT_FRAME_FEE); - - // Persist a recent safe-head observation plus replayable wallet state, - // then stop cleanly so the next run boots over fresh facts without a - // recovery phase. - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut ws = runtime.ws(0).await?; - let mut replay = ReplayWalletApp::devnet(); - apply_reconciled_supported_deposit(runtime, &mut ws, &mut replay, &alice_l1, deposit_amount) - .await?; - drop(ws); - runtime.stop().await?; - - // Route the next process through an already-disconnected gateway. The - // proxy stays down until after the admitted runtime serves the write and - // its corresponding ordered-tx feed event. - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - proxy.disconnect(); - runtime.respawn().await?; - assert!( - !proxy.is_connected(), - "warm restart must reach readiness without reconnecting the L1 proxy", - ); - - // Cross the 2s input-reader/danger-detector cadence before submitting, so - // this proves more than racing a request ahead of the first failed RPC - // retry. Fresh local facts must keep the process admitted and live. - let exit = runtime.observe_for(STABILITY_WINDOW).await?; - assert!( - exit.is_none(), - "warm runtime must remain live while the fresh persisted L1 view is valid; got {exit:?}", - ); - - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - let mut alice_l2 = runtime.wallet_l2(alice)?; - alice_l2.transfer(bob_address, transfer_amount).await?; - replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - - assert_wallet_state( - &replay_after, - ExpectedWalletState { - address: alice_address, - balance: deposit_amount - transfer_amount - fee, - nonce: 1, - }, - ExpectedWalletState { - address: bob_address, - balance: transfer_amount, - nonce: 0, - }, - 2, - ); - assert!( - !proxy.is_connected(), - "POST /tx and WS must succeed before L1 connectivity is restored", - ); - assert_eq!( - runtime.count_batches()?.invalidated, - 0, - "a fresh-view warm restart must not invalidate the batch tree", - ); - - proxy.reconnect(); - runtime.set_l1_endpoint_override(None); - proxy.shutdown().await?; - Ok(()) -} - -// A wall-clock regression of at least one configured block interval makes a -// persisted L1-progress timestamp unusable. With L1 unreachable, startup must -// return the retryable `L1ViewStale` verdict (exit 20) without mutating the -// batch tree. Restoring both time and connectivity must make the next boot -// healthy. Sub-block regressions remain tolerated and are pinned by protocol -// unit tests. -async fn run_wall_clock_backward_jump_retries_then_recovers_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - let alice = TestSigner::from_default(1)?; - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut ws = runtime.ws(0).await?; - let mut replay_before = ReplayWalletApp::devnet(); - - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(100_000_u64), - ) - .await?; - drop(ws); - - let counts_before = runtime.count_batches()?; - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - proxy.disconnect(); - runtime.set_faketime_offset(Some("-1h".to_string()))?; - - let respawn_error = runtime - .respawn() - .await - .expect_err("a one-hour clock regression with L1 unreachable must refuse admission"); - let respawn_error = respawn_error.to_string(); - assert!( - respawn_error.contains("status=exit status: 20"), - "backward-clock refusal must use the retryable exit class, not panic: {respawn_error}", - ); - - let counts_after_retry = runtime.count_batches()?; - assert_eq!( - counts_after_retry, counts_before, - "retryable clock refusal must not mutate the batch tree", - ); - - // Restore the usable clock and L1 connection. The same persisted facts - // are now ageable again, so startup admits and the runtime remains healthy - // past the danger detector's two-second cadence. - runtime.set_faketime_offset(None)?; - proxy.reconnect(); - runtime.respawn().await?; - - let exit = runtime.observe_for(Duration::from_secs(3)).await?; - assert!( - exit.is_none(), - "restoring time and L1 connectivity must produce a stable runtime; got {exit:?}", - ); - assert_eq!( - runtime.count_batches()?, - counts_before, - "recovering from a retryable clock refusal must not invalidate batches", - ); - - proxy.shutdown().await?; - Ok(()) -} - -// Provider reachable, safe head frozen, startup refuses to boot. -// -// Tests the stale-L1-view startup arm in isolation: when L1 is -// reachable but the safe head hasn't advanced since the last observation, -// the safe block timestamp crosses the read-staleness threshold and startup -// refuses. -// -// Scenario: -// 1. Set up baseline state (deposit + transfer in the open Tip). The Tip -// stays well below `danger_threshold` — we don't pre-age it, because -// the runtime detector now exits on `DangerStatus::TipInDanger` and we want -// a clean stop. -// 2. Stop the sequencer. -// 3. Advance only the sequencer's wall clock (no mining → safe head -// frozen). The offset must exceed `danger_threshold * SECONDS_PER_BLOCK` -// so the wall-clock-adjusted threshold saturates to 0 and catches even -// a fresh Tip. -// 4. Respawn → `check_danger` fires `L1ViewStale` → `Refuse(L1ViewStale)`. -// 5. Mine one L1 block and clear the faketime offset; safe-head progress -// resumes, the timestamp refreshes on sync, and the sequencer stays up. -async fn run_stalled_safe_head_startup_refuses_boot_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // 4 hours: comfortably past `danger_threshold * SECONDS_PER_BLOCK` - // (900 blocks * 12 s = 10800 s = 3 h), so missed_blocks > danger_threshold, - // adjusted threshold saturates to 0, and the wall-clock arm catches even - // a fresh Tip. - const STALLED_SAFE_HEAD_OFFSET: &str = "+14400s"; - const SAFE_HEAD_SYNC_WINDOW: Duration = Duration::from_secs(8); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; - let mut replay = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - runtime.stop().await?; - runtime.set_faketime_offset(Some(STALLED_SAFE_HEAD_OFFSET.to_string()))?; - - let respawn_result = runtime.respawn().await; - assert!( - respawn_result.is_err(), - "startup must refuse when L1 is reachable but the safe head stayed frozen \ - long enough that the L1 view is stale", - ); - - let counts = runtime.count_batches()?; - assert_eq!( - counts.invalidated, 0, - "startup refusal on a reachable-but-stalled safe head must not cascade batches: {counts:?}", - ); - - // Resume safe-head progress: mine a block and reset the faketime offset. - // On respawn, sync refreshes `last_safe_progress_ms` to faketime-now, - // wall-clock arm sees `missed = 0`, `Safe` → `Proceed`, sequencer stays up. - runtime.mine_l1_blocks(1).await?; - runtime.set_faketime_offset(None)?; - runtime.respawn().await?; - - let stable_after_progress = runtime.observe_for(SAFE_HEAD_SYNC_WINDOW).await?; - assert!( - stable_after_progress.is_none(), - "once safe-head progress resumes, the sequencer should boot and remain stable; got {stable_after_progress:?}", - ); - - Ok(()) -} - -// provider outage in the pre-danger zone while the sequencer stays -// running. -// -// Load-under-outage check: the sequencer must continue to accept user ops, -// persist them, broadcast on WS, and CLOSE BATCHES BY SIZE while its L1 -// connection is down. Proves the inclusion lane is independent of L1 -// reachability — as long as the wall-clock fallback keeps the pre-danger -// verdict, the sequencer keeps doing useful work. -// -// Scenario: -// 1. Spawn + apply a large deposit so Alice can fund many transfers. -// 2. Route the sequencer through a proxy (stop → set override → respawn). -// 3. Disconnect the proxy, advance L1 by a pre-danger amount (500 blocks). -// 4. Submit enough transfers (~150 × ~100 B each ≈ 15 KB) to exceed the -// default ~12 KB batch-size target, guaranteeing at least one size- -// triggered batch close during the outage. -// 5. Assert `count_batches().sealed` strictly increased during the outage. -// 6. Reconnect the proxy; confirm one more transfer goes through and the -// schema invariants hold post-test. -async fn run_provider_outage_pre_danger_sequencer_continues_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Pre-danger budget: see module-level `PRE_DANGER_BLOCKS` (500 blocks = - // 100min at 12s/block, well below the danger threshold). - // Enough transfers to exceed the default ~12 KB batch size target. Each - // transfer user_op is ≈ 100 B (SSZ-encoded Transfer + signature + nonce), - // so 150 ops ≈ 15 KB — one batch close is guaranteed; two or more is - // typical. - const TRANSFERS_DURING_OUTAGE: usize = 150; - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - // Step 1: Deposit big — Alice needs to cover 150+ transfers and their fees. - // Default fee per user-op ≈ 38276 units (log-fee 1356); reserve margin. - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let deposit_amount = U256::from(10_000_000_u64); - let mut replay = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay, - &alice_l1, - deposit_amount, - ) - .await?; - } - - // Step 2: Insert the proxy and route the sequencer through it via - // stop → set override → respawn. The initial spawn (direct to Anvil) is - // treated as setup only; from here on, all sequencer → L1 traffic flows - // through the proxy. - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - runtime.respawn().await?; - - // Step 3: Connect a fresh WS (catches up the deposit from offset 0) and - // a fresh L2 wallet. Consume the deposit replay so subsequent - // `expect_user_op_from` calls line up. - let mut ws = runtime.ws(0).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - ws.expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - - // Baseline: one transfer while the proxy is still connected, confirming - // end-to-end plumbing works through the proxy. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - let batches_before = runtime.count_batches()?; - - // Step 4: Cut the L1 connection, advance Anvil by 500 blocks (pre-danger). - // The sequencer is still running; its wall-clock fallback sees real time - // not yet past the threshold, so it keeps retrying rather than shutting - // down. - proxy.disconnect(); - runtime.mine_l1_blocks(PRE_DANGER_BLOCKS).await?; - - // Step 5: Submit many transfers during the outage. Each should be - // accepted (POST /tx succeeds), broadcast on WS, and eventually packed - // into a new batch. Size-triggered close fires when the cumulative user-op - // bytes exceed the default target. - for _ in 0..TRANSFERS_DURING_OUTAGE { - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Step 6: Batch closure during outage — the whole point of this test. - let batches_mid = runtime.count_batches()?; - assert!( - batches_mid.sealed > batches_before.sealed, - "sequencer must continue closing batches during L1 outage: \ - before={before:?}, after={after:?}", - before = batches_before, - after = batches_mid, - ); - - // Step 7: Restore L1 connectivity. The batch submitter's next tick - // reaches L1 again and starts draining the pending batches. - proxy.reconnect(); - - // Final check: one more transfer goes through after reconnect, proving - // the sequencer didn't just survive — it's fully operational. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Sanity: total sealed batches grew from baseline to final, and nothing - // got invalidated (pre-danger → no recovery triggered). - let batches_final = runtime.count_batches()?; - assert!( - batches_final.sealed > batches_before.sealed, - "final sealed count {final:?} must exceed baseline {before:?}", - final = batches_final, - before = batches_before, - ); - assert_eq!( - batches_final.invalidated, 0, - "pre-danger outage must not invalidate any batches, got {:?}", - batches_final, - ); - - proxy.shutdown().await?; - Ok(()) -} - -// provider outage aging into the danger zone while the sequencer is -// running — sequencer detects via its live wall-clock fallback and self-exits -// with `DangerDetected`. Also verifies the startup freshness checks refuse -// subsequent boots while L1 is still unreachable. -// -// The full "reconnect → recover → no cascade" cycle needs the harness to -// handle an orchestrator-style restart loop (the first post-reconnect boot -// may still trip the danger check and exit, requiring another boot after -// enough blocks age out). That's tracked separately and -// deliberately out of scope here. -// -// Uses dynamic faketime (FAKETIME_TIMESTAMP_FILE re-read on every time call) -// to jump the sequencer's clock past the danger threshold mid-run without -// respawning — the scenario we'd otherwise need 3h45min of real wall-clock -// time to reproduce. -async fn run_provider_outage_danger_zone_sequencer_self_exits_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Defaults: MAX_WAIT_BLOCKS=1200, margin=300, danger_threshold=900 - // blocks at 12s/block = 10800s = 3h. Use 3h55min: comfortably past - // danger, under MAX_WAIT (so no cascade fires later). - const INTO_DANGER: Duration = Duration::from_secs(3 * 60 * 60 + 55 * 60); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - // Step 1: Baseline — deposit + transfer so there's observable state. - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut replay = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay, - &alice_l1, - U256::from(500_000_u64), - ) - .await?; - } - - // Step 2: Switch routing to the proxy (stop → set override → respawn). - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - runtime.respawn().await?; - - let mut ws = runtime.ws(0).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - ws.expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Step 3: Disconnect the proxy and advance both clocks into the danger - // zone. The running `DangerDetector` polls `Storage::check_danger` - // every cadence; with the proxy down the input reader can't refresh - // `last_safe_progress_ms` or the safe block timestamp, so the freshness - // checks eventually emit a non-Safe status and the process exits. - proxy.disconnect(); - runtime.advance_wall_and_mine(INTO_DANGER).await?; - - // Step 4: Wait for the sequencer to detect and self-exit. Dynamic - // faketime means the shift hits the submitter's next tick immediately — - // no real-time wait needed. - let exit_status = runtime.wait_for_exit(Duration::from_secs(30)).await?; - assert!( - !exit_status.success(), - "sequencer must self-exit with non-zero status on danger detection, got {exit_status:?}", - ); - - // Step 5: Try to respawn while proxy is still disconnected. The startup - // reducer runs the same wall-clock fallback and must return - // `Retry(L1ViewStale)` without runtime admission. - let respawn_result = runtime.respawn().await; - assert!( - respawn_result.is_err(), - "respawn must fail while proxy disconnected and wall-clock past danger", - ); - - // No cascade happened yet — batches under MAX_WAIT are not invalidated - // by startup recovery, only preemptively shut-down-and-flushed. - let counts = runtime.count_batches()?; - assert_eq!( - counts.invalidated, 0, - "danger-zone (not past-stale) must not invalidate batches: {counts:?}", - ); - - proxy.shutdown().await?; - Ok(()) -} - -// Short-duration provider hiccup, heals within pre-danger. -// -// The most-common production fault: an RPC gateway flakes briefly, retries -// succeed. No recovery should fire. -// -// What this tests that the longer-disconnect provider-outage test doesn't: the longer test disconnects for a -// 500-block L1 advance + 150 transfers worth of real time, exercising the -// inclusion lane under load. This test instead exercises the "pure retry -// loop" path: **no** L1 advance, **no** faketime advance, just a few seconds -// of real-time wall-clock downtime across at least one -// `idle_poll_interval_ms` (default 5 s) so the submitter definitely attempts -// and fails a tick, then the reconnect path lets the next tick succeed. -// -// Scenario: -// 1. Route through proxy; establish a baseline transfer. -// 2. Disconnect, submit one more transfer (inclusion lane must still -// accept), sleep >5 s so the submitter's tick hits the disconnect. -// 3. Reconnect, submit another transfer. -// 4. Assert no batches were invalidated and POST /tx still works. -async fn run_provider_outage_short_hiccup_no_recovery_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Long enough to straddle the default 5 s submitter idle_poll_interval so - // at least one retry actually fails against the disconnected proxy. - const HICCUP_DURATION: Duration = Duration::from_secs(6); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let deposit_amount = U256::from(2_000_000_u64); let mut replay = ReplayWalletApp::devnet(); { let mut ws = runtime.ws(0).await?; @@ -2538,652 +1856,310 @@ async fn run_provider_outage_short_hiccup_no_recovery_test( &mut ws, &mut replay, &alice_l1, - deposit_amount, + U256::from(500_000_u64), ) .await?; } - // Route through the proxy (stop → override → respawn). runtime.stop().await?; let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; runtime.set_l1_endpoint_override(Some(proxy.endpoint())); runtime.respawn().await?; - - let mut ws = runtime.ws(0).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; - ws.expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - - // Baseline transfer via the proxy, proving the proxy path works. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - let batches_before = runtime.count_batches()?; - - // Disconnect: the submitter's next tick (within 5 s) fails against the - // disconnected proxy, runs wall_clock_danger_estimate with ~zero - // elapsed — far below danger threshold — and just retries. - proxy.disconnect(); - - // Inclusion lane is independent of L1; POST /tx still accepts. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - // Wait at least one full submitter idle_poll_interval (default 5 s) so the - // failed-retry path is definitely exercised under the disconnect. - tokio::time::sleep(HICCUP_DURATION).await; - - proxy.reconnect(); - - // Reconnect: another transfer goes through normally — proves the - // sequencer didn't just sit there, its next tick genuinely recovered. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - - let batches_after = runtime.count_batches()?; - assert_eq!( - batches_after.invalidated, 0, - "a short pre-danger hiccup must not invalidate any batch: {batches_after:?}", - ); - assert!( - batches_after.sealed >= batches_before.sealed, - "sealed-batch count must be monotonic across a hiccup: \ - before={batches_before:?}, after={batches_after:?}", - ); - - proxy.shutdown().await?; - Ok(()) -} - -// Both down, sequencer returns first into the danger zone, refuses -// to boot. -// -// Companion to `run_provider_outage_past_stale_cascades_test` (past-stale -// cascades through proxy): this is the *danger-zone* window of the same -// setup. Sequencer is stopped AND the proxy is disconnected; wall-clock and -// L1 advance into the danger zone but stay below `MAX_WAIT_BLOCKS`; the -// sequencer comes back first while L1 is still unreachable. Startup's -// wall-clock fallback must see "past danger" and refuse the boot — advancing -// the safe head off stale data would risk issuing soft confirmations against -// a state that may already be doomed. -// -// No cascade is expected yet (we haven't crossed MAX_WAIT_BLOCKS). The test -// stops at the refuse-to-boot assertion — the full reconnect+recovery cycle -// is covered by the both-down-proxy-first restart-cycle test below. -async fn run_both_down_danger_zone_sequencer_first_refuses_boot_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Safely inside the danger zone: past 900-block threshold, below 1200. - // 3h55min at 12 s/block = 1175 blocks. - const INTO_DANGER: Duration = Duration::from_secs(3 * 60 * 60 + 55 * 60); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - // Baseline: deposit + transfer (both will survive — no cascade expected - // in the danger-zone window). - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2 - .transfer(bob_address, U256::from(100_000_u64)) - .await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Both down: stop sequencer, insert proxy, disconnect proxy. - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - proxy.disconnect(); - - // Coupled advance into the danger zone. Anvil mines behind the proxy - // (direct connection via `mine_l1_blocks`), and faketime shifts the - // sequencer's wall clock cumulatively. - runtime.advance_wall_and_mine(INTO_DANGER).await?; - - // Respawn while proxy is still disconnected: sync fails → wall-clock - // fallback computes past-danger → refuses to boot. - let respawn_result = runtime.respawn().await; - assert!( - respawn_result.is_err(), - "sequencer must refuse to boot while L1 unreachable and wall-clock past danger", - ); - - // No cascade should have run yet — we haven't crossed MAX_WAIT_BLOCKS. - let counts = runtime.count_batches()?; - assert_eq!( - counts.invalidated, 0, - "refuse-to-boot must not invalidate any batch: {counts:?}", - ); - - proxy.shutdown().await?; - Ok(()) -} - -// Both down, proxy returns first, then sequencer — restart cycle -// converges. -// -// Complement to `run_both_down_danger_zone_sequencer_first_refuses_boot_test` -// (sequencer first): here L1 comes back before the sequencer does. Once the -// sequencer restarts, startup recovery sees L1 reachable and the Tip aged -// past `danger_threshold`, so `check_danger` returns `TipInDanger(idx)` and -// the guarded `RecoverTip` phase cascades the Tip and opens a fresh one. -// Convergence typically happens on the first respawn. -// -// Other paths can fire under different timings — e.g., the lane might -// close the Tip into a nonced closed batch during boot, the submitter -// might land it on L1 fresh (Gold) or stale (Silver), and a subsequent -// respawn picks up the post-flush cascade or the existing safe_input as -// canonical. The test asserts only convergence to `Stable`, not which -// specific path fires. -// -// The key invariant this tests that the other outage tests don't: the full -// *restart-loop* works, driven to convergence by `respawn_until_stable`. -async fn run_both_down_danger_zone_proxy_first_restart_cycle_recovers_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - const INTO_DANGER: Duration = Duration::from_secs(3 * 60 * 60 + 55 * 60); - // Each restart attempt advances ~20 min (100 blocks) of additional L1 - // time, simulating the real orchestrator-restart cadence. One extra - // tick past the first failed attempt is enough to push an aged Tip's - // closed-batch form past MAX_WAIT_BLOCKS (1175 + 100 > 1200). - const ADVANCE_PER_RETRY: Duration = blocks_as_duration(RESPAWN_RETRY_ADVANCE_BLOCKS); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - // Baseline: deposit + transfer — the transfer will be invalidated when - // cascade finally fires. - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); { let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2 - .transfer(bob_address, U256::from(100_000_u64)) + let mut alice_l2 = runtime.wallet_l2(alice)?; + ws.expect_direct_input_from(runtime.erc20_portal_address()) .await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; } + let batch = oldest_unaccepted(runtime)?; - // Both down: stop sequencer, insert proxy, disconnect. - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); proxy.disconnect(); - - // Coupled advance into the danger zone. - runtime.advance_wall_and_mine(INTO_DANGER).await?; - - // L1 recovers first — proxy back online while sequencer is still stopped. - proxy.reconnect(); - - // Simulated orchestrator loop. Each failed attempt advances L1 (and - // wall-clock) by ~20 min; the aged Tip eventually ages past - // `MAX_WAIT_BLOCKS` and cascade fires on a subsequent respawn. - let outcomes = runtime - .respawn_until_stable(RespawnPolicy { - max_attempts: 5, - stabilization: Duration::from_secs(8), - advance_per_retry: Some(ADVANCE_PER_RETRY), - }) - .await?; - assert!( - matches!(outcomes.last(), Some(RespawnAttemptOutcome::Stable)), - "restart cycle must converge to Stable, got: {outcomes:?}", - ); - - // The cascade fired somewhere in the loop — the transfer was invalidated. - let counts = runtime.count_batches()?; - assert!( - counts.invalidated >= 1, - "expected at least one invalidation after restart-cycle cascade: {counts:?}", - ); - - // Verify via WS replay: only the re-drained deposit appears, the transfer - // is gone. - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(600_000_u64), - "Alice must get her full deposit back after cascade", - ); + advance_safe_head_to(runtime, void_safe_head(batch.first_frame_safe_block)).await?; assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "Bob's balance must roll back", - ); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - - proxy.shutdown().await?; - Ok(()) -} - -// Sequencer outage, coupled wall+L1 advance into the danger zone, -// orchestrator restart cycle converges. -// -// Realistic counterpart to the decoupled -// (`sequencer_outage_danger_zone_tip_cascade_test`). In a real outage both -// L1 and wall clock advance together. -// -// The most likely outcome on respawn is `RecoverTip`: the open Tip's first -// frame has aged past `danger_threshold` and `check_danger` fires `TipInDanger(idx)` -// (the closed-frontier check finds nothing past the gold frontier). Recovery -// cascades the Tip directly without a flush; on a healthy L1, the first -// respawn typically converges to Stable. -// -// Other paths can fire depending on timing — e.g., the lane might close the -// Tip into a nonced batch before the detector trips, the submitter might -// get the batch onto L1 fresh, and convergence happens by the next respawn -// seeing it in `safe_inputs`. Or the closed batch lands stale, routes through -// the guarded Flush → Sync → Cascade phases, and converges after a flush cycle. -// -// The test's load-bearing assertion is restart-loop convergence under a -// realistic coupled outage, not which specific recovery path fires nor how -// many attempts that path requires. -async fn run_sequencer_outage_danger_zone_coupled_restart_cycle_recovers_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - const INTO_DANGER: Duration = Duration::from_secs(3 * 60 * 60 + 55 * 60); - const ADVANCE_PER_RETRY: Duration = blocks_as_duration(RESPAWN_RETRY_ADVANCE_BLOCKS); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - let mut replay_before = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2 - .transfer(bob_address, U256::from(100_000_u64)) - .await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Sequencer outage: stop, do NOT insert a proxy. Coupled L1+wall advance - // into the danger zone. - runtime.stop().await?; - runtime.advance_wall_and_mine(INTO_DANGER).await?; - - let outcomes = runtime - .respawn_until_stable(RespawnPolicy { - max_attempts: 5, - stabilization: Duration::from_secs(8), - advance_per_retry: Some(ADVANCE_PER_RETRY), - }) - .await?; - assert!( - matches!(outcomes.last(), Some(RespawnAttemptOutcome::Stable)), - "restart cycle must converge to Stable, got: {outcomes:?}", - ); - - // Convergence is the load-bearing claim. The number of attempts depends - // on which recovery path fires (`RecoverTip` typically converges on the - // first respawn; Flush → Sync → Cascade may take more), so we don't pin a - // minimum here. - assert!( - !outcomes.is_empty(), - "respawn_until_stable must record at least one attempt" - ); - - let counts = runtime.count_batches()?; - - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - if counts.invalidated >= 1 { - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(600_000_u64), - "cascade must roll Alice back to the full deposit", - ); - assert_eq!(replay_after.current_user_balance(bob_address), U256::ZERO); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - } else { - replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(500_000_u64), - "if no cascade fired, the pre-outage transfer must have remained canonical", - ); - assert_eq!( - replay_after.current_user_balance(bob_address), - U256::from(100_000_u64), - ); - assert_eq!(replay_after.current_user_nonce(alice_address), 1); - } + runtime.observe_for(STABILITY_WINDOW).await?, + None, + "no exit while blind", + ); + assert_blind(runtime).await?; + proxy.reconnect(); + let facts = void_history_facts(batch.first_frame_safe_block, None); + expect_terminal_abort(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + proxy.shutdown().await?; Ok(()) } -// Follow-up to `run_provider_outage_danger_zone_sequencer_self_exits_test`: -// provider outage into the danger zone while the sequencer is running, mid-run -// danger-detected exit, then reconnect + restart cycle converges. -// -// The provider-outage danger-zone test stops at "refuse to reboot while proxy still -// disconnected". This completes that story: after the sequencer self-exits -// mid-run via its live wall-clock fallback and the proxy reconnects, the -// orchestrator restart cycle eventually converges — same -// `respawn_until_stable` pattern as -// `run_sequencer_outage_danger_zone_coupled_restart_cycle_recovers_test` and -// `run_both_down_danger_zone_proxy_first_restart_cycle_recovers_test`. -// -// Ordering detail: the wall-clock advance only advances the sequencer's -// clock; the proxy has been disconnecting Anvil traffic, so Anvil's block -// count advanced via `mine_l1_blocks` (which bypasses the proxy). When the -// proxy reconnects, the sequencer sees both the shifted wall clock and the -// fresh safe head via the same RPC connection. -async fn run_provider_outage_danger_zone_mid_run_exit_then_restart_cycle_recovers_test( +// (c) at start: the sequencer is stopped and the provider is down while L1 +// passes the window. The start boots on its stored view, where the history is +// not void yet, and serves; the first sync after the provider returns shows +// the void history and stops it; a respawn refuses. +async fn run_blind_start_boots_on_the_stored_view_then_stops_at_the_first_sync_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - const INTO_DANGER: Duration = Duration::from_secs(3 * 60 * 60 + 55 * 60); - const ADVANCE_PER_RETRY: Duration = blocks_as_duration(RESPAWN_RETRY_ADVANCE_BLOCKS); - let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); - // Baseline — deposit + transfer while running directly against Anvil, - // then route through the proxy for the outage. - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut replay_before = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - } - - runtime.stop().await?; - let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; - runtime.set_l1_endpoint_override(Some(proxy.endpoint())); - runtime.respawn().await?; - let mut ws = runtime.ws(0).await?; + let alice_l1 = runtime.wallet_l1(alice.clone()).await?; let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - ws.expect_direct_input_from(runtime.erc20_portal_address()) - .await?; + let mut replay = ReplayWalletApp::devnet(); + apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut replay, + &alice_l1, + U256::from(600_000_u64), + ) + .await?; alice_l2 .transfer(bob_address, U256::from(100_000_u64)) .await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - drop(ws); + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + let batch = oldest_unaccepted(runtime)?; - // Mid-run outage: proxy goes down, coupled wall+L1 advance into danger. - // The running sequencer's submitter tick hits the disconnect, runs - // danger check, sees a non-Safe status, and exits. + drop(ws); + runtime.stop().await?; + let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; + runtime.set_l1_endpoint_override(Some(proxy.endpoint())); proxy.disconnect(); - runtime.advance_wall_and_mine(INTO_DANGER).await?; + advance_safe_head_to(runtime, void_safe_head(batch.first_frame_safe_block)).await?; - let exit_status = runtime.wait_for_exit(Duration::from_secs(30)).await?; - assert!( - !exit_status.success(), - "sequencer must self-exit on mid-run danger detection, got {exit_status:?}", + runtime.respawn().await?; + assert_eq!( + runtime.observe_for(STABILITY_WINDOW).await?, + None, + "a blind start runs on its stored view", ); + assert_blind(runtime).await?; + assert_feed_matches(runtime, &replay).await?; - // L1 comes back. Run the orchestrator cycle: the aged closed batch - // eventually ages past `MAX_WAIT_BLOCKS` and startup recovery cascades. proxy.reconnect(); + let facts = void_history_facts(batch.first_frame_safe_block, None); + expect_terminal_abort(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + proxy.shutdown().await?; + Ok(()) +} - let outcomes = runtime - .respawn_until_stable(RespawnPolicy { - max_attempts: 5, - stabilization: Duration::from_secs(8), - advance_per_retry: Some(ADVANCE_PER_RETRY), - }) - .await?; +/// What a blind sequencer reports: a landing lag frozen below the window, and +/// a safe head as old as the outage. +async fn assert_blind(runtime: &ManagedSequencer) -> ScenarioResult<()> { + let health = runtime.landing_health().await?; + let lag = health + .landing_lag_blocks + .ok_or("a blind sequencer still reports its lag")?; assert!( - matches!(outcomes.last(), Some(RespawnAttemptOutcome::Stable)), - "restart cycle must converge to Stable, got: {outcomes:?}", - ); - - let counts = runtime.count_batches()?; + lag < MAX_WAIT_BLOCKS, + "the lag freezes below the window: {health:?}" + ); + let age = health + .safe_head_age_seconds + .ok_or("a blind sequencer reports its safe head's age")?; + // The outage spans most of the window; live mining before it can stamp + // the stored safe block a little ahead of the wall clock. assert!( - counts.invalidated >= 1, - "expected cascade after mid-run exit + restart cycle: {counts:?}", + age >= blocks_as_duration(MAX_WAIT_BLOCKS / 2).as_secs(), + "the safe head's age covers the outage the lag cannot see: {health:?}", ); + assert_eq!(health.max_wait_blocks, MAX_WAIT_BLOCKS); + Ok(()) +} - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; +// (d) Stopped past the window: the open batch aged while the sequencer was +// down. One block short of the boundary a start boots; at the boundary the +// first start finds the void history at its sync and refuses, the next +// refuses on the database alone, and neither changes the batches. +async fn run_stopped_past_the_window_refuses_every_start_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { + let alice = TestSigner::from_default(1)?; + let bob = TestSigner::from_default(2)?; + let alice_address = alice.address(); + let bob_address = bob.address(); - assert_eq!( - replay_after.current_user_balance(alice_address), + let mut ws = runtime.ws(0).await?; + let alice_l1 = runtime.wallet_l1(alice.clone()).await?; + let mut alice_l2 = runtime.wallet_l2(alice)?; + let mut replay = ReplayWalletApp::devnet(); + apply_reconciled_supported_deposit( + runtime, + &mut ws, + &mut replay, + &alice_l1, U256::from(600_000_u64), + ) + .await?; + alice_l2 + .transfer(bob_address, U256::from(100_000_u64)) + .await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + let batch = oldest_unaccepted(runtime)?; + let void_at = void_safe_head(batch.first_frame_safe_block); + + // One block short of the boundary a start boots. Submissions are held, + // so the batch its batch-open deadline seals cannot land. + drop(ws); + runtime.stop().await?; + runtime.hold_batch_submissions().await?; + advance_safe_head_to(runtime, void_at - 1).await?; + runtime.respawn().await?; + wait_for_landing_lag(runtime, MAX_WAIT_BLOCKS - 1).await?; + runtime.stop().await?; + + // At the boundary every start refuses. + advance_safe_head_to(runtime, void_at).await?; + let facts = void_history_facts(batch.first_frame_safe_block, Some(void_at)); + let batches = runtime.count_batches()?; + let frontier = runtime.batch_nonce_frontier()?; + expect_start_refused(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + assert_eq!( + runtime.count_batches()?, + batches, + "a refused start changes no batch" ); - assert_eq!(replay_after.current_user_balance(bob_address), U256::ZERO); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); + assert_eq!(runtime.batch_nonce_frontier()?, frontier); + assert_eq!(oldest_unaccepted(runtime)?, batch); + Ok(()) +} + +// (d) with nothing confirmed: the first run's batch holds no user op, and it +// ages all the same. The start refuses, and so does one with L1 unreachable: +// the void history is a fact of the database. +async fn run_stopped_past_the_window_with_nothing_confirmed_refuses_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { + let batch = oldest_unaccepted(runtime)?; + runtime.stop().await?; + let void_at = void_safe_head(batch.first_frame_safe_block); + advance_safe_head_to(runtime, void_at).await?; + + let facts = void_history_facts(batch.first_frame_safe_block, Some(void_at)); + expect_start_refused(runtime, &facts).await?; + let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; + runtime.set_l1_endpoint_override(Some(proxy.endpoint())); + proxy.disconnect(); + expect_start_refused(runtime, &facts).await?; proxy.shutdown().await?; Ok(()) } -// First-boot-with-L1-down refuses to boot (wall-clock fallback -// treats "never synced" as danger). -// -// `wall_clock_danger_estimate` has a distinguished branch for `last_sync_ms -// == 0`: it refuses to proceed because the sequencer has no baseline to -// measure drift against, so issuing soft confirmations against whatever -// stale safe head we last saw is unsafe. There's a unit test covering that -// branch in isolation; this e2e confirms the full `run()` boot path -// respects it end-to-end. -// -// How we reach the condition: the harness's `spawn()` does a successful -// first boot (needs L1 reachable to deploy contracts and pin the deployment -// identity). We stop, rewrite the recorded L1 safe-head observation to -// unknown, then respawn with the proxy disconnected. The deployment identity -// is still populated — so the sequencer gets past the identity gate — but -// `check_danger` sees the missing safe-head row and the reducer returns -// `Retry(L1ViewStale)`. -// -// Scope note: a "truly" first-ever boot would fail even earlier (no -// deployment identity, can't discover contracts). That's a separate test; this -// one targets only the L1-view freshness branch. -async fn run_first_boot_l1_unreachable_never_synced_refuses_boot_test( +// (f) A foreign batch at the sequencer's nonce: another process signs a batch +// with the submitter's key and lands it at the open batch's nonce. The running +// sequencer records the divergence and stops for good; every start refuses. +async fn run_foreign_batch_at_our_nonce_stops_for_good_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Baseline boot so the deployment identity lands on disk. - { - let _ws = runtime.ws(0).await?; - } + let batch = oldest_unaccepted(runtime)?; + // An empty batch is never stale, so the scheduler accepts it at the nonce. + let foreign = encode_batch(&Batch { + nonce: batch.nonce, + frames: Vec::new(), + }); + let block = runtime + .add_input_as(&batch_submitter_signer()?, foreign) + .await?; + mine_until_safe(runtime, block).await?; + + let facts = [format!( + "canonical divergence at batch nonce {}", + batch.nonce + )]; + expect_terminal_abort(runtime, &facts).await?; + assert_eq!(runtime.canonical_divergence()?.as_deref(), Some("foreign")); + expect_start_refused(runtime, &facts).await?; + expect_start_refused(runtime, &facts).await?; + Ok(()) +} +// The first `run` of an era with L1 unreachable: it has no batch to start on, +// so it exits with the retry class and opens none; once L1 is back, the next +// start opens the era's first batch. +async fn run_first_run_of_an_era_with_l1_down_retries_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { runtime.stop().await?; + runtime.discard_data_dir(); + let setup = runtime.run_setup().await?; + assert!(setup.success(), "setup with L1 up must succeed: {setup:?}"); - // Simulate "never synced L1" by clearing the safe-head observation. - runtime.clear_l1_safe_head_observation()?; - - // Route the sequencer through a disconnected proxy so L1 is unreachable - // from the sequencer's perspective. let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; runtime.set_l1_endpoint_override(Some(proxy.endpoint())); proxy.disconnect(); - - let respawn_result = runtime.respawn().await; - assert!( - respawn_result.is_err(), - "never-synced + L1-unreachable must refuse to boot, got {respawn_result:?}", + match runtime.try_respawn().await? { + RunStart::Exited(status) if status.code() == Some(EXIT_RESTART_TRANSIENT) => {} + other => { + return Err(format!("expected exit {EXIT_RESTART_TRANSIENT}, got {other:?}").into()); + } + } + assert_eq!( + runtime.count_batches()?.total, + 0, + "the refused start opens no batch" ); - // Confirm the refusal is reversible: reconnect the proxy and the - // sequencer boots normally (the wall-clock fallback path is gated on - // L1 unreachability, not on any persistent flag). proxy.reconnect(); runtime.respawn().await?; - + assert_eq!( + oldest_unaccepted(runtime)?.nonce, + 0, + "the era's first batch" + ); proxy.shutdown().await?; Ok(()) } -// Past-stale closed+submitted batch (delayed-inclusion cascade). -// -// Scenario: a batch closes and the submitter's L1 tx is never mined (the -// gateway dropped it, mempool evicted it, whatever). Blocks accumulate. On -// the sequencer's next startup recovery, the batch's first frame is > -// `MAX_WAIT_BLOCKS` behind current_safe_block, so the scheduler skips it -// in `populate_safe_accepted_batches` and `find_first_batch_in_danger` -// flags it — cascade fires. -// -// This is the closed-and-submitted sibling of `run_recovery_after_stale_batches_test` -// (open-batch variant): `find_first_batch_in_danger` must cascade both "open batch -// got old" and "closed batch submission got lost" (this one). -// -// Setup shape: we use Anvil's `setAutomine(false)` + `dropAllPendingTxs` -// (harness primitives) to hold the sequencer's batch-submission tx -// out of the chain, then drop it entirely — cleaner than the mempool-hold -// approach, because `anvil_mine(N)` with a pending tx would include it in -// the first mined block, not the Nth. Dropping simulates gateway packet -// loss directly and advances 1250 genuinely empty blocks. -async fn run_delayed_inclusion_cascades_on_restart_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Past-stale: 1250 blocks > MAX_WAIT_BLOCKS (1200). - const PAST_STALE: Duration = blocks_as_duration(PAST_STALE_BLOCKS); - // Enough transfers to trigger at least one size-based batch close. - // Same sizing as `run_provider_outage_pre_danger_sequencer_continues_test` - // (≈100 B/op × 150 ops ≈ 15 KB > 12 KB target). - const TRANSFERS_TO_FORCE_BATCH_CLOSE: usize = 150; - +// Warm restart while the RPC is unreachable: the startup sync fails on the +// provider, and `run` starts on its stored L1 view and serves. The test keeps +// the proxy disconnected through readiness, one full background-worker retry +// cadence, POST /tx, and WS broadcast. A boot path that makes provider +// reachability a start condition fails at `respawn`; a runtime that launches +// without a usable inclusion lane fails the transaction or WS assertion. +async fn run_warm_restart_with_l1_down_test(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); + let deposit_amount = U256::from(600_000_u64); + let transfer_amount = U256::from(100_000_u64); + let fee = fee_to_linear(DEFAULT_FRAME_FEE); - // Fund Alice generously — 151 transfers + fees is well under 10 M. + // Persist replayable wallet state, then stop cleanly. let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let deposit_amount = U256::from(10_000_000_u64); - let mut replay_before = ReplayWalletApp::devnet(); - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - deposit_amount, - ) - .await?; - - // Capture the sealed-batch baseline BEFORE we disable auto-mining so - // we can assert that at least one new batch sealed during the - // mempool-held phase. - let batches_before_close = runtime.count_batches()?; - - // Hold the mempool. From here, txs go to Anvil but don't mine until - // we either re-enable auto-mining or call `anvil_mine`. - runtime.set_automine(false).await?; - - // Submit enough transfers to trigger at least one size-triggered - // batch close. Each POST /tx is processed by the sequencer - // synchronously; the inclusion lane seals a batch when cumulative - // user-op bytes exceed the target. - let mut alice_l2 = runtime.wallet_l2(alice)?; - for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Wait for the submitter's tick to put the batch-submission tx into the - // (held) mempool. - runtime.wait_for_pending_batch_submission().await?; - - let batches_after_close = runtime.count_batches()?; - assert!( - batches_after_close.sealed > batches_before_close.sealed, - "expected at least one new sealed batch: before={batches_before_close:?} after={batches_after_close:?}", - ); - - // Shut the sequencer down, then drop the mempool so the submitted - // batch tx never lands. The sequencer's DB still shows a sealed - // batch; L1 has no corresponding event. + let mut replay = ReplayWalletApp::devnet(); + apply_reconciled_supported_deposit(runtime, &mut ws, &mut replay, &alice_l1, deposit_amount) + .await?; drop(ws); runtime.stop().await?; - runtime.drop_all_pending_txs().await?; - - // Advance past MAX_WAIT_BLOCKS. With auto-mining still off but the - // mempool empty, these are genuinely empty blocks — nothing to - // include. `advance_wall_and_mine` also shifts the sequencer's - // faketime offset so the wall-clock fallback stays in sync with L1. - runtime.advance_wall_and_mine(PAST_STALE).await?; - - // Re-enable auto-mining AND mine L1 throughout the recovery boot. Startup - // recovery's mempool flush submits a no-op at the stranded wallet-nonce slot - // and blocks until that slot is *safe* (`safe_wallet_nonce >= watermark + 1`). - // Auto-mining alone lands the no-op but mints no further blocks, so Anvil's - // `safe` tag never advances past it and the boot would hang; the boot miner - // supplies the steady block production a live chain would (see - // `ManagedSequencer::set_mine_l1_during_boot`). - runtime.set_automine(true).await?; - runtime.set_mine_l1_during_boot(true); + // Route the next process through an already-disconnected gateway. The + // proxy stays down until after the runtime serves the write and + // its corresponding ordered-tx feed event. + let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; + runtime.set_l1_endpoint_override(Some(proxy.endpoint())); + proxy.disconnect(); runtime.respawn().await?; + assert!( + !proxy.is_connected(), + "warm restart must reach readiness without reconnecting the L1 proxy", + ); - // Verify cascade fired. - let counts = runtime.count_batches()?; + // Cross the 2s input-reader/landing-watch cadence before submitting, so + // this proves more than racing a request ahead of the first failed RPC + // retry. + let exit = runtime.observe_for(STABILITY_WINDOW).await?; assert!( - counts.invalidated >= 1, - "expected cascade-invalidation of the delayed-inclusion batch: {counts:?}", + exit.is_none(), + "a warm runtime must remain live with L1 down; got {exit:?}", ); - // Replay from offset 0: the deposit must be re-drained (it's still a - // safe L1 input), and the sealed batch's transfers must be gone. let mut ws_after = runtime.ws(0).await?; let mut replay_after = ReplayWalletApp::devnet(); replay_after.apply( @@ -3191,62 +2167,70 @@ async fn run_delayed_inclusion_cascades_on_restart_test( .expect_direct_input_from(runtime.erc20_portal_address()) .await?, )?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; + let mut alice_l2 = runtime.wallet_l2(alice)?; + alice_l2.transfer(bob_address, transfer_amount).await?; + replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount, - "Alice must have her full deposit back (all transfers invalidated)", + assert_wallet_state( + &replay_after, + ExpectedWalletState { + address: alice_address, + balance: deposit_amount - transfer_amount - fee, + nonce: 1, + }, + ExpectedWalletState { + address: bob_address, + balance: transfer_amount, + nonce: 0, + }, + 2, ); - assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "Bob's receiving balance must roll back", + assert!( + !proxy.is_connected(), + "POST /tx and WS must succeed before L1 connectivity is restored", ); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); + proxy.reconnect(); + runtime.set_l1_endpoint_override(None); + proxy.shutdown().await?; Ok(()) } -// Aging open Tip trips the runtime detector and the next startup -// runs RecoverTip. +// Provider outage inside the window while the sequencer stays running. // -// The runtime danger detector intentionally fires on `DangerStatus::TipInDanger(_)`: -// once the open Tip's first frame has aged past `danger_threshold`, the -// inclusion lane has failed to rotate it within `max_batch_open` (2 h ≈ -// 600 blocks, comfortably below `danger_threshold` ≈ 900 blocks), which -// means the lane is stuck. The detector exits the process; the orchestrator -// respawns; startup dispatches `RecoverTip` and `recover_aging_tip` -// cascades the Tip without a flush (the Tip has no L1 footprint). -// -// Earlier behavior tolerated an aging Tip at runtime under the assumption -// that only closed batches with pending L1 transactions could zombie on -// confirm. That left "lane stuck" as a silent failure mode. The new policy -// makes it loud. +// Load-under-outage check: the sequencer must continue to accept user ops, +// persist them, broadcast on WS, and CLOSE BATCHES BY SIZE while its L1 +// connection is down. Proves the inclusion lane is independent of L1 +// reachability: nothing stops intake on a stale L1 view. // -// Staging: -// 1. Baseline: deposit + transfer → Tip at first_frame_safe_block X. -// 2. Advance only L1 by `DANGER_ZONE_BLOCKS` — current_safe_block jumps -// ~1150 past X before the wall-clock batch deadline can close the Tip. -// Once the reader persists that head, observed Tip age outranks the -// deliberately future-dated L1 clock and clears `danger_threshold`. -// 3. `wait_for_exit` — input reader catches up; detector ticks; sees -// `DangerStatus::TipInDanger(_)`; process exits non-zero. -// 4. Align wall time with the already-mined L1 time, then respawn. Startup -// can now age the persisted head normally and sees Tip in danger → -// `RecoverTip` → `recover_aging_tip` cascades the Tip + opens a fresh -// one. Alice's pre-outage transfer was a soft confirmation against the -// cascaded Tip; it's rolled back. -async fn run_aging_open_tip_runtime_danger_zone_exit_test( +// Scenario: +// 1. Spawn + apply a large deposit so Alice can fund many transfers. +// 2. Route the sequencer through a proxy (stop → set override → respawn). +// 3. Disconnect the proxy, advance L1 well inside the window. +// 4. Submit enough transfers (~150 × ~100 B each ≈ 15 KB) to exceed the +// default ~12 KB batch-size target, guaranteeing at least one size- +// triggered batch close during the outage. +// 5. Assert `count_batches().sealed` strictly increased during the outage. +// 6. Reconnect the proxy; confirm one more transfer goes through and the +// schema invariants hold post-test. +async fn run_provider_outage_inside_the_window_sequencer_continues_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { + // Enough transfers to exceed the default ~12 KB batch size target. Each + // transfer user_op is ≈ 100 B (SSZ-encoded Transfer + signature + nonce), + // so 150 ops ≈ 15 KB — one batch close is guaranteed; two or more is + // typical. + const TRANSFERS_DURING_OUTAGE: usize = 150; + let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; let alice_address = alice.address(); let bob_address = bob.address(); + // Step 1: Deposit big — Alice needs to cover 150+ transfers and their fees. + // Default fee per user-op ≈ 38276 units (log-fee 1356); reserve margin. let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; + let deposit_amount = U256::from(10_000_000_u64); let mut replay = ReplayWalletApp::devnet(); { let mut ws = runtime.ws(0).await?; @@ -3255,106 +2239,106 @@ async fn run_aging_open_tip_runtime_danger_zone_exit_test( &mut ws, &mut replay, &alice_l1, - U256::from(600_000_u64), + deposit_amount, ) .await?; + } + + // Step 2: Insert the proxy and route the sequencer through it via + // stop → set override → respawn. The initial spawn (direct to Anvil) is + // treated as setup only; from here on, all sequencer → L1 traffic flows + // through the proxy. + runtime.stop().await?; + let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; + runtime.set_l1_endpoint_override(Some(proxy.endpoint())); + runtime.respawn().await?; + + // Step 3: Connect a fresh WS (catches up the deposit from offset 0) and + // a fresh L2 wallet. Consume the deposit replay so subsequent + // `expect_user_op_from` calls line up. + let mut ws = runtime.ws(0).await?; + let mut alice_l2 = runtime.wallet_l2(alice.clone())?; + ws.expect_direct_input_from(runtime.erc20_portal_address()) + .await?; + + // Baseline: one transfer while the proxy is still connected, confirming + // end-to-end plumbing works through the proxy. + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + + let batches_before = runtime.count_batches()?; + + // Step 4: Cut the L1 connection and advance Anvil well inside the + // window. The sequencer keeps running and its workers keep retrying. + proxy.disconnect(); + runtime.mine_l1_blocks(SHORT_OUTAGE_BLOCKS).await?; + + // Step 5: Submit many transfers during the outage. Each should be + // accepted (POST /tx succeeds), broadcast on WS, and eventually packed + // into a new batch. Size-triggered close fires when the cumulative user-op + // bytes exceed the default target. + for _ in 0..TRANSFERS_DURING_OUTAGE { alice_l2.transfer(bob_address, U256::from(1_u64)).await?; replay.apply(ws.expect_user_op_from(alice_address).await?)?; } - // Advance L1 atomically while leaving wall time still: this injects the - // "lane failed to close its Tip" condition without letting the normal - // wall-clock batch deadline repair it first. - runtime.mine_l1_blocks(DANGER_ZONE_BLOCKS).await?; - - // The detector must trip on `DangerStatus::TipInDanger` once the input reader - // catches up. Allow a window for input-reader poll (~2 s) plus - // detector poll (2 s) plus margin. - let exit = runtime.wait_for_exit(Duration::from_secs(15)).await?; - // Exit 10 (`EXIT_RESTART_EXPECT_RECOVERY`) is the observed-danger class - // (`TipInDanger` / `ClosedBatchInDanger`); a clock fallback (`L1ViewStale`, - // `EstimatedBatchInDanger`) would exit 20. The scenario has no closed - // batch, so 10 here means the detector saw the aging Tip, not the - // future-dated L1 clock. The exit code is the contract; the rendered log - // line is not. - assert_eq!( - exit.code(), - Some(10), - "runtime detector must exit with the observed-danger class on \ - `DangerStatus::TipInDanger`, not a clock fallback; got {exit:?}", - ); - - // No cascade fires on detector exit alone. The recovery happens at - // the next startup: `RecoverTip` → `recover_aging_tip` cascades the - // Tip and opens a fresh one. Alice's transfer was inside the Tip; - // it's rolled back. - let counts_before = runtime.count_batches()?; - assert_eq!( - counts_before.invalidated, 0, - "detector exit alone must not invalidate batches; that happens at startup: {counts_before:?}", + // Step 6: Batch closure during outage — the whole point of this test. + let batches_mid = runtime.count_batches()?; + assert!( + batches_mid.sealed > batches_before.sealed, + "sequencer must continue closing batches during L1 outage: \ + before={before:?}, after={after:?}", + before = batches_before, + after = batches_mid, ); - // The reader's persisted L1 head is future-dated because Anvil advanced - // atomically. Add ordinary live progress, then align process time before - // restart: initial sync observes a strictly newer safe head and refreshes - // the wall-progress witness, so the mandatory post-repair inspection can - // admit the fresh Tip rather than correctly retaining a stale-view retry. - runtime.mine_live_l1_blocks(1).await?; - let aligned_offset_secs = blocks_as_duration(DANGER_ZONE_BLOCKS).as_secs() + 1; - runtime.set_faketime_offset(Some(format!("+{aligned_offset_secs}s")))?; - runtime.respawn().await?; + // Step 7: Restore L1 connectivity. The batch submitter's next tick + // reaches L1 again and starts draining the pending batches. + proxy.reconnect(); - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(600_000_u64), - "Tip cascade must roll Alice back to her deposit", - ); - assert_eq!( - replay_after.current_user_balance(bob_address), - U256::ZERO, - "Bob never received the rolled-back transfer", - ); - assert_eq!( - replay_after.current_user_nonce(alice_address), - 0, - "nonce must reset when the Tip carrying the user op is cascaded", - ); + // Final check: one more transfer goes through after reconnect, proving + // the sequencer didn't just survive — it's fully operational. + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; - let counts_after = runtime.count_batches()?; + // Sanity: total sealed batches grew from baseline to final. + let batches_final = runtime.count_batches()?; assert!( - counts_after.invalidated >= 1, - "RecoverTip must invalidate at least the cascaded Tip; got {counts_after:?}", + batches_final.sealed > batches_before.sealed, + "final sealed count {final:?} must exceed baseline {before:?}", + final = batches_final, + before = batches_before, ); - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - + proxy.shutdown().await?; Ok(()) } -// Provider reachable, safe head frozen, live freshness check fires and the -// running detector self-exits. +// Short-duration provider hiccup. +// +// The most-common production fault: an RPC gateway flakes briefly, retries +// succeed. +// +// What this tests that the longer-disconnect provider-outage test doesn't: the longer test disconnects for a +// 500-block L1 advance + 150 transfers worth of real time, exercising the +// inclusion lane under load. This test instead exercises the "pure retry +// loop" path: **no** L1 advance, **no** faketime advance, just a few seconds +// of real-time wall-clock downtime across at least one +// `idle_poll_interval_ms` (default 5 s) so the submitter definitely attempts +// and fails a tick, then the reconnect path lets the next tick succeed. // -// Runtime twin of `run_stalled_safe_head_startup_refuses_boot_test`. Tests that -// the wall-clock-adjusted danger arm fires at runtime when L1 stays reachable -// but the safe head stops advancing. We don't pre-age the Tip — the detector -// now exits on `DangerStatus::TipInDanger` independently, which would conflate -// test signals. Instead, we let the Tip stay fresh and jump the wall clock past -// `danger_threshold * SECONDS_PER_BLOCK` so the wall-clock arm saturates to -// threshold 0 and catches the fresh Tip via `find_first_batch_in_danger`. -async fn run_stalled_safe_head_live_exit_test( +// Scenario: +// 1. Route through proxy; establish a baseline transfer. +// 2. Disconnect, submit one more transfer (inclusion lane must still +// accept), sleep >5 s so the submitter's tick hits the disconnect. +// 3. Reconnect, submit another transfer. +// 4. Assert POST /tx still works and the sealed count never falls. +async fn run_provider_outage_short_hiccup_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // 4 hours: same rationale as `run_stalled_safe_head_startup_refuses_boot_test`. - // missed_blocks > danger_threshold → adjusted threshold saturates to 0 - // → wall-clock arm catches even a fresh Tip. - const STALLED_SAFE_HEAD_OFFSET: &str = "+14400s"; + // Long enough to straddle the default 5 s submitter idle_poll_interval so + // at least one retry actually fails against the disconnected proxy. + const HICCUP_DURATION: Duration = Duration::from_secs(6); let alice = TestSigner::from_default(1)?; let bob = TestSigner::from_default(2)?; @@ -3362,7 +2346,7 @@ async fn run_stalled_safe_head_live_exit_test( let bob_address = bob.address(); let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; + let deposit_amount = U256::from(2_000_000_u64); let mut replay = ReplayWalletApp::devnet(); { let mut ws = runtime.ws(0).await?; @@ -3371,95 +2355,55 @@ async fn run_stalled_safe_head_live_exit_test( &mut ws, &mut replay, &alice_l1, - U256::from(600_000_u64), + deposit_amount, ) .await?; - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; } - // Don't mine any L1 blocks — keep the safe head where it is. Just - // jump the wall clock far enough that the wall-clock arm fires. - runtime.set_faketime_offset(Some(STALLED_SAFE_HEAD_OFFSET.to_string()))?; - - let exit = runtime.wait_for_exit(Duration::from_secs(15)).await?; - assert!( - !exit.success(), - "reachable-but-stalled safe head must force a non-zero self-exit via \ - the wall-clock-adjusted danger arm, got {exit:?}", - ); + // Route through the proxy (stop → override → respawn). + runtime.stop().await?; + let proxy = TcpProxy::spawn(runtime.l1_endpoint()).await?; + runtime.set_l1_endpoint_override(Some(proxy.endpoint())); + runtime.respawn().await?; - let counts = runtime.count_batches()?; - assert_eq!( - counts.invalidated, 0, - "live stalled-safe-head shutdown must not cascade batches on its own: {counts:?}", - ); + let mut ws = runtime.ws(0).await?; + let mut alice_l2 = runtime.wallet_l2(alice)?; + ws.expect_direct_input_from(runtime.erc20_portal_address()) + .await?; - Ok(()) -} + // Baseline transfer via the proxy, proving the proxy path works. + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; -// Reconnect at a previously-observed offset that got invalidated -// after the WS connection dropped. -// -// A WS connection cannot span invalidation: the sequencer necessarily exits -// (danger detection or stop) before any cascade runs. Recovery invalidates -// the identity attached to a previously consumed prefix. -async fn run_ws_reconnect_with_invalidated_claim_is_refused_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Past-stale: matches `recovery_after_stale_batches_test` sizing. - const PAST_STALE: Duration = blocks_as_duration(MAX_WAIT_BLOCKS); + let batches_before = runtime.count_batches()?; - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); + // Disconnect: the submitter's next tick (within 5 s) fails against the + // disconnected proxy and retries. + proxy.disconnect(); - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; - let mut replay_before = ReplayWalletApp::devnet(); + // Inclusion lane is independent of L1; POST /tx still accepts. + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; - // Build up offsets 0 (deposit) and 1 (transfer) and capture the - // transfer's offset so we can later reconnect at it. - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(600_000_u64), - ) - .await?; - alice_l2 - .transfer(bob_address, U256::from(100_000_u64)) - .await?; - let transfer_msg = ws.expect_user_op_from(alice_address).await?; - let resume_claim = runtime.history_claim(transfer_msg.offset().checked_add(1).unwrap())?; - replay_before.apply(transfer_msg)?; + // Wait at least one full submitter idle_poll_interval (default 5 s) so the + // failed-retry path is definitely exercised under the disconnect. + tokio::time::sleep(HICCUP_DURATION).await; - // Kill the WS socket and the sequencer (same way a real reconnect arc - // works — process dies, client dials back in). - drop(ws); - runtime.stop().await?; + proxy.reconnect(); - runtime.advance_wall_and_mine(PAST_STALE).await?; - runtime.respawn().await?; + // Reconnect: another transfer goes through normally — proves the + // sequencer didn't just sit there, its next tick genuinely recovered. + alice_l2.transfer(bob_address, U256::from(1_u64)).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; - assert!(matches!( - SequencerClient::new(runtime.internal_endpoint())? - .subscribe(resume_claim) - .await, - Err(sequencer_rust_client::SubscribeError::History( - sequencer_rust_client::HistoryPolicyError::StaleGeneration { .. } - )) - )); - let mut replay = runtime.ws(0).await?; - let redrained = replay - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - assert_eq!(redrained.offset(), 0); - replay.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; + let batches_after = runtime.count_batches()?; + assert!( + batches_after.sealed >= batches_before.sealed, + "sealed-batch count must be monotonic across a hiccup: \ + before={batches_before:?}, after={batches_after:?}", + ); + proxy.shutdown().await?; Ok(()) } @@ -3478,116 +2422,6 @@ async fn run_ws_subscribe_ahead_of_head_is_refused_test( Ok(()) } -// Safe direct input that was NOT yet drained before the cascade -// must be drained into the recovery batch's first frame. -// -// Distinct from `recovery_after_stale_batches_test`, where the -// direct input was drained into an invalidated batch and gets *re*-drained -// on recovery. Here we exercise the simpler case: the input hit the -// sequencer's view post-stop, so it was never referenced by any frame; -// recovery must include it in the fresh batch's leading range. -// -// Setup: -// 1. Spawn + stop immediately. Initial Tip is empty and anchored at an -// early safe_block. -// 2. Deposit on L1 directly (sequencer is stopped, so the event isn't -// consumed yet). -// 3. Advance L1 past MAX_WAIT_BLOCKS to age the empty initial Tip past -// stale. -// 4. Respawn. Startup recovery syncs the new safe head, sees the -// deposit in `safe_inputs`, cascades the aged initial Tip, and opens -// a recovery batch with `leading_range = [next_undrained, end)` — -// including the undrained deposit. -// 5. WS replay at offset 0 must deliver the deposit event (drained -// exactly once, into the recovery batch's first frame). -async fn run_recovery_drains_safe_but_undrained_direct_input_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - const PAST_STALE: Duration = blocks_as_duration(PAST_STALE_BLOCKS); - - let alice = TestSigner::from_default(1)?; - let alice_address = alice.address(); - - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - - // Stop the sequencer before any user-level activity. The initial Tip - // is empty and anchored at whatever safe_block the lane saw on first - // boot. - runtime.stop().await?; - - // Deposit happens entirely on L1 while the sequencer is offline — - // WalletL1Client dials Anvil directly, not through the sequencer. - let deposit_amount = U256::from(600_000_u64); - alice_l1.mint_supported_token(deposit_amount).await?; - alice_l1.deposit_supported_token(deposit_amount).await?; - - // Advance L1 past MAX_WAIT_BLOCKS + safe-depth so the aged empty - // initial Tip gets cascaded and the deposit event is safe. - runtime.advance_wall_and_mine(PAST_STALE).await?; - - runtime.respawn().await?; - - // WS from offset 0. Recovery batch's first frame must contain the - // deposit (never drained before), and nothing else. - let mut ws_after = runtime.ws(0).await?; - let mut replay_after = ReplayWalletApp::devnet(); - let deposit_msg = ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - replay_after.apply(deposit_msg)?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - assert_eq!( - replay_after.current_user_balance(alice_address), - deposit_amount, - "the deposit, never previously drained, must land in the recovery \ - batch's first frame", - ); - - // Cascade fired on the empty initial Tip. - let counts = runtime.count_batches()?; - assert!( - counts.invalidated >= 1, - "expected the empty initial Tip to be cascaded: {counts:?}", - ); - - Ok(()) -} - -// Recovery batch opens empty when no direct inputs are pending. -// -// Negative control for -// `run_recovery_drains_safe_but_undrained_direct_input_test`: same overall -// shape but with no L1 deposit before respawn. The recovery batch's -// `leading_range` is `[0, 0)` and the batch's first frame is empty. WS replay -// delivers nothing. -async fn run_recovery_batch_opens_empty_when_no_direct_inputs_pending_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - const PAST_STALE: Duration = blocks_as_duration(PAST_STALE_BLOCKS); - - runtime.stop().await?; - - // No deposits, no user ops. Just age the initial Tip past stale. - runtime.advance_wall_and_mine(PAST_STALE).await?; - - runtime.respawn().await?; - - // WS from offset 0 must deliver nothing — the recovery batch is empty. - let mut ws_after = runtime.ws(0).await?; - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - // Cascade still fired (empty initial Tip past MAX_WAIT). - let counts = runtime.count_batches()?; - assert!( - counts.invalidated >= 1, - "expected the empty initial Tip to be cascaded even without direct \ - inputs: {counts:?}", - ); - - Ok(()) -} - // Replay determinism: for any workload accepted live, catch-up // replay must produce an identical per-user state. // @@ -3662,7 +2496,7 @@ async fn run_replay_matches_live_for_mixed_workload_test( bob_l2.withdraw(U256::from(50_000_u64)).await?; replay_live.apply(ws.expect_user_op_from(bob_address).await?)?; - let expected_input_count = replay_live.executed_input_count(); + let expected_input_count = replay_live.l2_tx_count(); // Restart + catch-up replay. Each WS catch-up event feeds the fresh // replay identically to how the live stream fed the original; if the @@ -3691,14 +2525,16 @@ async fn run_replay_matches_live_for_mixed_workload_test( "nonce divergence for {addr:?}: live vs. replay must match", ); } + assert_eq!(replay_post.l2_tx_count(), replay_live.l2_tx_count(),); assert_eq!( - replay_post.executed_input_count(), - replay_live.executed_input_count(), + replay_post.canonical_snapshot_bytes()?, + replay_live.canonical_snapshot_bytes()?, + "the replayed mirror's whole state must match the live mirror", ); assert_eq!( - replay_post.last_executed_safe_block(), - replay_live.last_executed_safe_block(), - "mirror safe-block clock must match the live replay clock", + replay_post.last_applied_block(), + replay_live.last_applied_block(), + "the replayed feed must carry the same L1 blocks as the live feed", ); Ok(()) @@ -3708,7 +2544,7 @@ async fn run_replay_matches_live_for_mixed_workload_test( // retry on provider errors (connection refused, timeout) without // crashing, and pick up the backlog on reconnect. // -// Distinct from `provider_outage_short_hiccup_no_recovery_test`, which tests +// Distinct from `provider_outage_short_hiccup_test`, which tests // the batch submitter's retry path via POST activity. Here the interesting // component is the **input reader**: its only job is polling L1 for new // events, so the only observable signal that its retry loop works is @@ -3798,31 +2634,26 @@ async fn run_provider_outage_input_reader_retries_after_reconnect_test( Ok(()) } -// First-ever boot with no deployment identity + L1 unreachable refuses before -// recovery logic runs. +// First-ever boot with no deployment identity + L1 unreachable: setup refuses. // -// Distinct from `run_first_boot_l1_unreachable_never_synced_refuses_boot_test` -// (already covered): that test exercises the run reducer's wall-clock fallback, -// which only runs after setup has succeeded once (so deployment identity is -// pinned). This test targets the earlier failure: setup's L1 discovery, which -// first checks the chain id and then asks L1 for the InputBox address. With no -// deployment identity, those reads have no fallback and the boot fails before -// recovery logic runs. +// Distinct from `run_first_run_of_an_era_with_l1_down_retries_test`, where +// setup succeeded and `run` meets the outage. This test targets the earlier +// failure: setup's L1 discovery, which first checks the chain id and then asks +// L1 for the InputBox address. With no deployment identity, those reads have +// no fallback and setup fails before it writes anything. // -// The harness simulates "no deployment identity" by `reset_database()` after -// a normal boot has populated it (truly first-ever boot would also lack -// a pinned identity, but the failure mode is identical: the bootstrap -// step has nothing to fall back to). +// The harness simulates "no deployment identity" by moving to a new data +// directory after a normal boot (a truly first-ever boot also lacks a pinned +// identity, and the failure mode is identical: the bootstrap step has nothing +// to fall back to). async fn run_first_boot_no_identity_l1_unreachable_refuses_boot_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Baseline boot to ensure the schema is fully migrated. We then - // reset the DB to mimic a first-ever boot. { let _ws = runtime.ws(0).await?; } runtime.stop().await?; - runtime.reset_database()?; + runtime.discard_data_dir(); // Route through a disconnected proxy so setup's first L1 read (the chain // id) fails with a provider error. @@ -3835,9 +2666,12 @@ async fn run_first_boot_no_identity_l1_unreachable_refuses_boot_test( respawn_result.is_err(), "first boot with no deployment identity + L1 unreachable must refuse boot, got {respawn_result:?}", ); + // Setup reads L1 before its first write, so the refusal leaves the + // directory empty (the lock file aside) and a retry may set it up. + assert_data_dir_holds_at_most_the_lock(runtime)?; - // Verify reversibility: reconnect proxy, respawn, this time the - // bootstrap step succeeds and pins the deployment identity. + // Verify reversibility: reconnect proxy, respawn in the same directory, + // this time the bootstrap step succeeds and pins the deployment identity. proxy.reconnect(); runtime.respawn().await?; @@ -3874,9 +2708,9 @@ async fn run_chain_id_mismatch_via_live_rpc_refuses_boot_test( } runtime.stop().await?; - // Reset the DB so the live RPC path runs (otherwise the identity-fallback - // path would catch the mismatch first). - runtime.reset_database()?; + // A new data directory, so setup's live RPC path runs (otherwise the + // identity-fallback path would catch the mismatch first). + runtime.discard_data_dir(); // Configure a mismatched chain id and respawn. The bootstrap-time // RPC check returns the actual chain id (31337), compares it with @@ -3887,165 +2721,16 @@ async fn run_chain_id_mismatch_via_live_rpc_refuses_boot_test( respawn_result.is_err(), "chain-id mismatch via live RPC must refuse boot, got {respawn_result:?}", ); + assert_data_dir_holds_at_most_the_lock(runtime)?; - // Reset to the correct chain id. Respawn must succeed — proves the - // failed attempt didn't poison the cache or other DB state. + // Reset to the correct chain id. Respawn in the same directory must + // succeed — proves the failed attempt wrote nothing. runtime.set_chain_id_override(None); runtime.respawn().await?; Ok(()) } -// Nonce-0 first batch recovery edge. -// -// Two coupled invariants: -// - Cascade from no valid ancestor: if the first-ever batch (nonce 0) -// goes stale before any -// batch reaches `Gold` (i.e., before any batch is L1-accepted), -// recovery cascades it and opens a fresh recovery batch that itself -// has nonce 0 (parent NULL — there's no valid ancestor to point -// at). No genesis sentinel exists in the implementation; the -// parent-pointer schema must handle "all batches invalidated" -// natively. -// - Reused nonce acceptance: after recovery, the recovery batch (with -// nonce 0 reused) -// submits to L1, gets accepted by `populate_safe_accepted_batches`, -// and lands in `safe_accepted_batches` — proving the scheduler- -// simulation cursor handles a reused nonce after cascade correctly. -// -// The structural invariants are validated by -// `assert_schema_invariants` (post-test hook in `tests/e2e/src/main.rs`): -// it checks that NULL-parent batches have nonce 0 and that valid-path -// nonces form a contiguous `0..N`. So this test asserts those -// observable consequences plus the explicit `safe_accepted_batches` -// post-condition for reused-nonce acceptance. -// -// Setup turns auto-mining off and drops the mempool so the first batch's L1 -// submission is dropped before reaching the chain — guaranteeing it -// never reaches `Gold` before being cascaded. -async fn run_nonce_zero_recovery_invalidates_then_accepts_at_nonce_zero_test( - runtime: &mut ManagedSequencer, -) -> ScenarioResult<()> { - // Past stale to ensure the cascade fires. - const PAST_STALE: Duration = blocks_as_duration(PAST_STALE_BLOCKS); - // Force a size-triggered batch close. Same sizing as - // `run_delayed_inclusion_cascades_on_restart_test`. - const TRANSFERS_TO_FORCE_CLOSE: usize = 150; - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - - // Fund Alice and queue many transfers into the open batch (which is - // the FIRST EVER batch — nonce 0). Using auto-mining-off across the - // submitter's tick so the batch's L1 tx hits the mempool but never - // mines, then dropping it. - let mut replay_before = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - let mut alice_l2 = runtime.wallet_l2(alice.clone())?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay_before, - &alice_l1, - U256::from(10_000_000_u64), - ) - .await?; - - runtime.set_automine(false).await?; - - for _ in 0..TRANSFERS_TO_FORCE_CLOSE { - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay_before.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Wait for the submitter's tick to put the (nonce-0) batch's L1 tx - // into the held mempool. - runtime.wait_for_pending_batch_submission().await?; - } - - runtime.stop().await?; - runtime.drop_all_pending_txs().await?; - - runtime.advance_wall_and_mine(PAST_STALE).await?; - // Re-enable auto-mining AND mine L1 throughout the recovery boot: the mempool - // flush submits a no-op at the stranded nonce-0 slot and waits for it to - // become safe. Auto-mining lands the no-op but mints no further blocks, so - // the boot miner supplies the steady block production needed to advance - // Anvil's `safe` tag past it (see `set_mine_l1_during_boot`). - runtime.set_automine(true).await?; - runtime.set_mine_l1_during_boot(true); - - runtime.respawn().await?; - - // Nonce-0 recovery assertions: the only existing batch (the original nonce-0 - // one) was cascaded, and a recovery batch was opened. The recovery - // batch's invariants (NULL parent → nonce 0) are checked structurally - // by the post-test `assert_schema_invariants` hook. - let counts = runtime.count_batches()?; - assert!( - counts.invalidated >= 1, - "expected the original nonce-0 batch to be invalidated: {counts:?}", - ); - assert!( - counts.total > counts.invalidated, - "recovery batch must exist alongside the invalidated original: {counts:?}", - ); - - // Replay shows the deposit re-drained, transfers gone (rolled back). - // Recreate WS + wallet against the post-respawn HTTP endpoint - // (`runtime.endpoint()` rebinds to a fresh port on every respawn). - let mut ws_after = runtime.ws(0).await?; - let mut alice_l2_fresh = runtime.wallet_l2(alice)?; - let mut replay_after = ReplayWalletApp::devnet(); - replay_after.apply( - ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?, - )?; - assert_eq!( - replay_after.current_user_balance(alice_address), - U256::from(10_000_000_u64), - "Alice must have her full deposit back after nonce-0 cascade", - ); - assert_eq!(replay_after.current_user_balance(bob_address), U256::ZERO,); - assert_eq!(replay_after.current_user_nonce(alice_address), 0); - - // drive enough work into the recovery batch that the - // submitter closes it by size and submits to L1. With auto-mining - // back on, the submission lands and the input reader picks it up - // into `safe_inputs`; `populate_safe_accepted_batches` accepts it - // at the expected nonce (0, reused). - for _ in 0..TRANSFERS_TO_FORCE_CLOSE { - alice_l2_fresh - .transfer(bob_address, U256::from(1_u64)) - .await?; - replay_after.apply(ws_after.expect_user_op_from(alice_address).await?)?; - } - - // Mine and poll until submission, confirmation, and the input-reader sync - // have all completed. Fixed sleeps made this assertion race under CI load. - let (accepted_count, min_accepted_nonce) = mine_until_batch_is_safe_accepted(runtime).await?; - assert!( - accepted_count >= 1, - "expected at least one batch to land in safe_accepted_batches \ - post-recovery (proves reused-nonce-0 was accepted): \ - count={accepted_count}", - ); - assert_eq!( - min_accepted_nonce, - Some(0), - "the first L1-accepted batch must have nonce 0 (reused after \ - cascade) — got {min_accepted_nonce:?}", - ); - - Ok(()) -} - fn eip712_domain(runtime: &ManagedSequencer) -> alloy_sol_types::Eip712Domain { sequencer_core::build_input_domain(runtime.domain_chain_id(), runtime.verifying_contract()) } @@ -4059,11 +2744,11 @@ fn assert_wallet_state( replay: &ReplayWalletApp, first: ExpectedWalletState, second: ExpectedWalletState, - executed_input_count: u64, + l2_tx_count: u64, ) { assert_eq!(replay.current_user_balance(first.address), first.balance); assert_eq!(replay.current_user_nonce(first.address), first.nonce); assert_eq!(replay.current_user_balance(second.address), second.balance); assert_eq!(replay.current_user_nonce(second.address), second.nonce); - assert_eq!(replay.executed_input_count(), executed_input_count); + assert_eq!(replay.l2_tx_count(), l2_tx_count); } diff --git a/tests/e2e/src/watchdog_compare.rs b/tests/e2e/src/watchdog_compare.rs index 058d67fd..41c93908 100644 --- a/tests/e2e/src/watchdog_compare.rs +++ b/tests/e2e/src/watchdog_compare.rs @@ -19,10 +19,15 @@ use tokio::process::Command; use crate::ScenarioResult; -const DEVNET_MACHINE_IMAGE: &str = "examples/canonical-app/out/canonical-machine-image"; +pub(crate) const DEVNET_MACHINE_IMAGE: &str = "examples/canonical-app/out/canonical-machine-image"; const SEPOLIA_MACHINE_IMAGE: &str = "examples/canonical-app/out/canonical-machine-image-sepolia"; -fn require_cartesi_machine() { +/// Bound on one watchdog command. On the devnet a tick or replay from genesis +/// covers a few dozen inputs and ends in seconds; a hung L1 or sequencer call +/// must fail the scenario instead of stalling the suite. +const WATCHDOG_COMMAND_TIMEOUT: Duration = Duration::from_secs(120); + +pub(crate) fn require_cartesi_machine() { assert!( std::process::Command::new("cartesi-machine") .arg("--version") @@ -34,7 +39,7 @@ fn require_cartesi_machine() { ); } -fn machine_image(relative: &str) -> ScenarioResult { +pub(crate) fn machine_image(relative: &str) -> ScenarioResult { let image = paths::workspace_root().join(relative); if !image.is_dir() { return Err(format!( @@ -47,19 +52,22 @@ fn machine_image(relative: &str) -> ScenarioResult { } /// One watchdog state directory, bootstrapped from `image` at `block`, driven -/// through the production `sequencer-watchdog` wrapper. -struct Watchdog { +/// through the production `sequencer-watchdog` wrapper. Its checkpoints are +/// whole machines, so the directory goes when the watchdog is dropped. +pub(crate) struct Watchdog { + _dir: tempfile::TempDir, state_dir: PathBuf, image: PathBuf, block: u64, } impl Watchdog { - fn new(image: PathBuf, block: u64) -> ScenarioResult { - let state_dir = tempfile::tempdir() - .map_err(|err| format!("temp watchdog state dir: {err}"))? - .keep(); + pub(crate) fn new(image: PathBuf, block: u64) -> ScenarioResult { + let dir = tempfile::tempdir().map_err(|err| format!("temp watchdog state dir: {err}"))?; + // `init` creates the state directory, as it does after a wipe. + let state_dir = dir.path().join("state"); Ok(Self { + _dir: dir, state_dir, image, block, @@ -113,16 +121,23 @@ impl Watchdog { "CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK", self.block.to_string(), ); - let output = command - .output() + // Dropping the timed-out future kills the child. + command.kill_on_drop(true); + let output = tokio::time::timeout(WATCHDOG_COMMAND_TIMEOUT, command.output()) .await + .map_err(|_| { + format!( + "sequencer-watchdog {} ran past {WATCHDOG_COMMAND_TIMEOUT:?}", + args.join(" ") + ) + })? .map_err(|err| format!("failed to run sequencer-watchdog: {err}"))?; eprint!("{}", String::from_utf8_lossy(&output.stderr)); Ok(output) } /// Run a command that must exit with `code`; returns its stdout. - async fn expect( + pub(crate) async fn expect( &self, runtime: &ManagedSequencer, args: &[&str], @@ -140,14 +155,38 @@ impl Watchdog { Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } - async fn status(&self, runtime: &ManagedSequencer) -> ScenarioResult { + pub(crate) async fn status( + &self, + runtime: &ManagedSequencer, + ) -> ScenarioResult { let stdout = self.expect(runtime, &["status"], 0).await?; serde_json::from_str(&stdout).map_err(|err| format!("status JSON: {err}: {stdout}").into()) } - fn wipe(&self) -> ScenarioResult<()> { + /// The runbook's re-`init`: the state directory wiped, then bootstrapped + /// from `image` at `block`. + fn wiped_for(self, image: PathBuf, block: u64) -> ScenarioResult { std::fs::remove_dir_all(&self.state_dir) - .map_err(|err| format!("wipe watchdog state: {err}").into()) + .map_err(|err| format!("wipe watchdog state: {err}"))?; + Ok(Self { + image, + block, + ..self + }) + } +} + +/// A machine stored in a temporary directory, removed when this is dropped. +pub(crate) struct StoredMachine { + _dir: tempfile::TempDir, + pub(crate) path: PathBuf, +} + +impl StoredMachine { + pub(crate) fn new(name: &str) -> ScenarioResult { + let dir = tempfile::tempdir().map_err(|err| format!("stored machine dir: {err}"))?; + let path = dir.path().join(name); + Ok(Self { _dir: dir, path }) } } @@ -176,7 +215,7 @@ async fn sequencer_digest(runtime: &ManagedSequencer) -> ScenarioResult<(u64, St Ok((block, sha256)) } -fn head_block(status: &serde_json::Value) -> Option { +pub(crate) fn head_block(status: &serde_json::Value) -> Option { status["head"]["block"].as_u64() } @@ -276,12 +315,14 @@ pub async fn run_watchdog_divergence_drill_test( wait_for_non_genesis_finalized_state(finalized_url.as_str(), runtime, Duration::from_secs(60)) .await?; let (block, _) = sequencer_digest(runtime).await?; - let first_input_block = *runtime - .input_blocks() + let inputs = runtime + .raw_inputs() .await - .map_err(|err| format!("read InputBox inputs: {err}"))? + .map_err(|err| format!("read InputBox inputs: {err}"))?; + let first_input_block = inputs .first() - .ok_or("no InputBox inputs before the accepted checkpoint")?; + .ok_or("no InputBox inputs before the accepted checkpoint")? + .0; eprintln!( "[watchdog-harness] drill target block={block} first input block={first_input_block}" ); @@ -301,7 +342,8 @@ pub async fn run_watchdog_divergence_drill_test( eprintln!( "[watchdog-harness] drill 2/7: a wrong non-genesis bootstrap latches a state mismatch" ); - let watchdog = Watchdog::new(wrong, first_input_block)?; + let wrong_machine = wrong_machine_through(&wrong, &inputs, first_input_block).await?; + let watchdog = Watchdog::new(wrong_machine.path.clone(), first_input_block)?; watchdog.expect(runtime, &["init"], 0).await?; watchdog.expect(runtime, &["tick"], 2).await?; watchdog.expect(runtime, &["tick"], 2).await?; @@ -383,12 +425,7 @@ pub async fn run_watchdog_divergence_drill_test( watchdog.expect(runtime, &["tick"], 2).await?; eprintln!("[watchdog-harness] drill 7/7: re-initializing from the trusted image agrees"); - watchdog.wipe()?; - let fixed = Watchdog { - state_dir: watchdog.state_dir.clone(), - image: trusted, - block: 0, - }; + let fixed = watchdog.wiped_for(trusted, 0)?; fixed.expect(runtime, &["init"], 0).await?; fixed.expect(runtime, &["tick"], 0).await?; if fixed.status(runtime).await?["latched"] != false { @@ -397,14 +434,65 @@ pub async fn run_watchdog_divergence_drill_test( Ok(()) } +/// A stored machine of another application at the right input count: the +/// `wrong` template advanced by the Cartesi Machine CLI through every input up +/// to `block`. The guest takes inputs only in InputBox order, so a bootstrap +/// that skips inputs dies at the next one; this one agrees on the count and +/// differs in state, the runbook's bad-bootstrap case. +async fn wrong_machine_through( + wrong: &Path, + inputs: &[(u64, Vec)], + block: u64, +) -> ScenarioResult { + let stored = StoredMachine::new("machine")?; + let work_dir = stored + .path + .parent() + .ok_or("a stored machine lies in its directory")?; + let mut count = 0_usize; + for (index, (_, input)) in inputs.iter().take_while(|(b, _)| *b <= block).enumerate() { + std::fs::write(work_dir.join(format!("input-{index}.bin")), input) + .map_err(|err| format!("write input {index}: {err}"))?; + count = index + 1; + } + // No revert snapshots: the inputs are known to be accepted, and a + // rejection aborts the run, which fails the drill loud. Every output file + // is disabled, and the CLI runs in the work directory in case it writes + // another. + let status = Command::new("cartesi-machine") + .current_dir(work_dir) + .arg("--no-revert") + .arg(format!("--load={},sharing:none", wrong.display())) + .arg(format!( + "--cmio-advance-state=input:{},input_file_index_begin:0,input_file_index_end:{count},\ + output:,report:,outputs_merkle_root:,outputs_merkle_root_proof:", + work_dir.join("input-%i.bin").display() + )) + .arg(format!("--store={}", stored.path.display())) + .arg("--quiet") + .status() + .await + .map_err(|err| format!("cartesi-machine advance failed to start: {err}"))?; + if !status.success() { + return Err(format!("cartesi-machine advance exited with {status}").into()); + } + Ok(stored) +} + /// The canonical image's answer to the `state` inspect query, via the CLI. async fn cm_inspect_state(machine_image: &Path) -> ScenarioResult> { + cm_inspect(machine_image, b"state").await +} + +/// A stored machine's answer to an inspect query, via the CLI. +pub(crate) async fn cm_inspect(machine_image: &Path, query: &[u8]) -> ScenarioResult> { let work_dir = tempfile::tempdir().map_err(|err| format!("temp cm work dir: {err}"))?; let query_path = work_dir.path().join("inspect-query.bin"); let report_path = work_dir.path().join("inspect-report-0.bin"); - std::fs::write(query_path.as_path(), b"state") + std::fs::write(query_path.as_path(), query) .map_err(|err| format!("write inspect query: {err}"))?; let status = Command::new("cartesi-machine") + .current_dir(work_dir.path()) .arg("--no-revert") .arg(format!("--load={},sharing:none", machine_image.display())) .arg(format!( diff --git a/tests/fixtures/consensus_vectors_v1.json b/tests/fixtures/consensus_vectors_v1.json deleted file mode 100644 index 5e66191a..00000000 --- a/tests/fixtures/consensus_vectors_v1.json +++ /dev/null @@ -1,930 +0,0 @@ -{ - "about": "Consensus vectors pinned on Rust 1.95.0 / alloy-core 1.6.1, before the W3b toolchain and dependency moves. Never regenerated: on any later commit a mismatch is a finding to explain, not a reason to update this file. Bytes are 0x-prefixed hex; uint256 values are 0x-prefixed hex quantities.", - "eip712": { - "domain_encode_type": "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", - "domain_type_hash": "0x8b73c3c69bb8fe3d512ecc4cf759cc79239f7b179b0ffacaa9a75d522b39400f", - "encode_type": "UserOp(uint32 nonce,uint16 max_fee,bytes data)", - "type_hash": "0x794b86e861930e59e62ce17b915511462a70fd7518920d37d23766ce03bf6cb5", - "domains": [ - { - "name": "canonical_test", - "chain_id": 31337, - "verifying_contract": "0x0000000000000000000000000000000000000000", - "separator": "0x330487f13dc661ab8fe297ef0a70f998ab234b53e3ead2e450c80b22395571c3" - }, - { - "name": "sepolia_app", - "chain_id": 11155111, - "verifying_contract": "0x0123456789abcDEF0123456789abCDef01234567", - "separator": "0x2a2f994d3223f807b1de0a508576e48e49bef3fb0fb1eae8c9db0fac5c8e2b7e" - } - ], - "user_ops": [ - { - "name": "empty_data", - "domain": "canonical_test", - "nonce": 0, - "max_fee": 0, - "data": "0x", - "struct_hash": "0x0adb11d43608ba5495540efc8110e57e3bd16cdc5acfbb4b878978c891a26629", - "signing_hash": "0xcc60e745cef998b09a8e9658cda76670df34346af0f299ddfa0a5aa0641e3aae" - }, - { - "name": "four_bytes", - "domain": "canonical_test", - "nonce": 1, - "max_fee": 100, - "data": "0xdeadbeef", - "struct_hash": "0xda26caef39bdb65184b64ac6c7d6ed6273ac9119a6c418858d4b4786f4523e38", - "signing_hash": "0xdb014760580d2de614eae36089d0d09d7f8d87b9e03045573226c8ebcee48087" - }, - { - "name": "two_hundred_bytes", - "domain": "canonical_test", - "nonce": 2, - "max_fee": 200, - "data": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bbc2c9d0d7dee5ecf3fa01080f161d242b323940474e555c636a71787f868d949ba2a9b0b7bec5ccd3dae1e8eff6fd040b121920272e353c434a51585f666d74", - "struct_hash": "0x4e6e8192e33e744c4cfb3462263126735e45e6e0239b70fd38ea36267745bd49", - "signing_hash": "0x63f0fd4940e600f68a5cd0f7294351afa052fbec58f03b2b666ebdf94c85d4df" - }, - { - "name": "nonce_u32_max", - "domain": "canonical_test", - "nonce": 4294967295, - "max_fee": 1, - "data": "0x01", - "struct_hash": "0xc0ab5d4c2667fa8b0dc49bae86e056d3a6f10ba4b5e32cfb8bc08d78f3012872", - "signing_hash": "0x0a598fa1ccfbd54f376cb6902f6752d81a92c8ae2d54935cab86ab402794432d" - }, - { - "name": "max_fee_u16_max", - "domain": "canonical_test", - "nonce": 7, - "max_fee": 65535, - "data": "0x02", - "struct_hash": "0x601501fb53f1f11256b641bcc27cbb651803297acf6b55f2e2ae6563a4ec84da", - "signing_hash": "0x4eed33ceb9f6ba501a6c481e8563c13e27ac4a06d4dd9ea3250a2276f38ffdbe" - }, - { - "name": "four_bytes_sepolia_app", - "domain": "sepolia_app", - "nonce": 1, - "max_fee": 100, - "data": "0xdeadbeef", - "struct_hash": "0xda26caef39bdb65184b64ac6c7d6ed6273ac9119a6c418858d4b4786f4523e38", - "signing_hash": "0xe34abab31e6e31f4023ac0b4c8a87229765b805513fb7e8041f3d752437c4453" - } - ] - }, - "keccak256": [ - { - "name": "empty", - "input": "0x", - "digest": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - }, - { - "name": "abc", - "input": "0x616263", - "digest": "0x4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45" - }, - { - "name": "bytes_135", - "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6ad", - "digest": "0x00ef96af9cf4b24c7f269d922294444a197d0a33638c2e56634c57e892103a8f" - }, - { - "name": "bytes_136", - "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4", - "digest": "0x742061bcad767ed4c4f5883b1dcb1aad11afdcc140dc469d953759b127b9f9ed" - }, - { - "name": "bytes_137", - "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bb", - "digest": "0xe3371f61e770abf254c34239c3b0099ad90594507415bc81dd0a10b9692bbf2a" - }, - { - "name": "bytes_272", - "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bbc2c9d0d7dee5ecf3fa01080f161d242b323940474e555c636a71787f868d949ba2a9b0b7bec5ccd3dae1e8eff6fd040b121920272e353c434a51585f666d747b828990979ea5acb3bac1c8cfd6dde4ebf2f900070e151c232a31383f464d545b626970777e858c939aa1a8afb6bdc4cbd2d9e0e7eef5fc030a11181f262d343b424950575e656c", - "digest": "0xac141fd7b0a0ffcd2e967254d508da3ec616596493c36fa304425647d90e6de5" - } - ], - "signature": { - "private_key": "0x0101010101010101010101010101010101010101010101010101010101010101", - "address": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "domain": "canonical_test", - "user_op": { - "nonce": 0, - "max_fee": 0, - "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888" - }, - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "variants": [ - { - "name": "as_signed", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" - }, - { - "name": "v_0", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b800", - "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", - "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" - }, - { - "name": "v_1", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b801", - "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" - }, - { - "name": "v_2", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b802", - "guest": null, - "ingress": null - }, - { - "name": "v_27", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81b", - "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", - "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" - }, - { - "name": "v_28", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" - }, - { - "name": "v_35", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b823", - "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", - "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" - }, - { - "name": "v_36", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b824", - "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" - }, - { - "name": "v_255", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8ff", - "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", - "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" - }, - { - "name": "high_s_twin", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891b", - "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", - "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" - }, - { - "name": "high_s_same_v", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891c", - "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", - "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" - }, - { - "name": "sixty_four_bytes", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8", - "guest": null, - "ingress": null - }, - { - "name": "r_zero", - "signature": "0x00000000000000000000000000000000000000000000000000000000000000005d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "guest": null, - "ingress": null - }, - { - "name": "s_zero", - "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a00000000000000000000000000000000000000000000000000000000000000001c", - "guest": null, - "ingress": null - }, - { - "name": "r_at_order", - "signature": "0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd03641415d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "guest": null, - "ingress": null - } - ] - }, - "ssz": { - "spill_batch": { - "nonce": 0, - "frames": [ - { - "safe_block": 10, - "fee_price": 0, - "user_ops": [] - }, - { - "safe_block": 11, - "fee_price": 257, - "user_ops": [] - }, - { - "safe_block": 12, - "fee_price": 514, - "user_ops": [] - }, - { - "safe_block": 13, - "fee_price": 771, - "user_ops": [] - }, - { - "safe_block": 14, - "fee_price": 100, - "user_ops": [ - { - "nonce": 0, - "max_fee": 100, - "data": "0x0101000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x1c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b" - }, - { - "nonce": 1, - "max_fee": 101, - "data": "0x0102000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x20a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b" - }, - { - "nonce": 2, - "max_fee": 102, - "data": "0x0103000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x4ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b" - }, - { - "nonce": 3, - "max_fee": 103, - "data": "0x0104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c" - }, - { - "nonce": 4, - "max_fee": 104, - "data": "0x0105000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x3ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b" - }, - { - "nonce": 5, - "max_fee": 105, - "data": "0x0106000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0x59567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c" - }, - { - "nonce": 6, - "max_fee": 106, - "data": "0x0107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0xb526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c" - }, - { - "nonce": 7, - "max_fee": 107, - "data": "0x0108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0xd43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c" - }, - { - "nonce": 8, - "max_fee": 108, - "data": "0x0109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", - "signature": "0xa44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c" - } - ] - }, - { - "safe_block": 15, - "fee_price": 1285, - "user_ops": [] - }, - { - "safe_block": 16, - "fee_price": 1542, - "user_ops": [] - }, - { - "safe_block": 17, - "fee_price": 1799, - "user_ops": [] - }, - { - "safe_block": 18, - "fee_price": 2056, - "user_ops": [] - } - ], - "encoded": "0x00000000000000000c0000002400000032000000400000004e0000005c00000032050000400500004e0500005c0500000e0000000a0000000000000000000e0000000b0000000000000001010e0000000c0000000000000002020e0000000d0000000000000003030e0000000e00000000000000640024000000a80000002c010000b001000034020000b80200003c030000c0030000440400000000000064000e0000004300000001010000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888881c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b0100000065000e00000043000000010200000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888820a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b0200000066000e0000004300000001030000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888884ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b0300000067000e000000430000000104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c0400000068000e0000004300000001050000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888883ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b0500000069000e00000043000000010600000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888859567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c060000006a000e000000430000000107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888b526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c070000006b000e000000430000000108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888d43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c080000006c000e000000430000000109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888a44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c0e0000000f0000000000000005050e000000100000000000000006060e000000110000000000000007070e00000012000000000000000808" - } - }, - "wallet_outputs": { - "token": "0x95d0c8A7d11342299807A2Fc19ac44C2321cCc68", - "sender": "0x7777777777777777777777777777777777777777", - "recipient": "0x8888888888888888888888888888888888888888", - "cases": [ - { - "amount": "0x1", - "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc6800000000000000000000000077777777777777777777777777777777777777770000000000000000000000000000000000000000000000000000000000000001", - "transfer_notice": "0x98830f8d000000000000000000000000777777777777777777777777777777777777777700000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001", - "erc20_transfer": "0xa9059cbb00000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001" - }, - { - "amount": "0x10000000000000001", - "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc6800000000000000000000000077777777777777777777777777777777777777770000000000000000000000000000000000000000000000010000000000000001", - "transfer_notice": "0x98830f8d000000000000000000000000777777777777777777777777777777777777777700000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000010000000000000001", - "erc20_transfer": "0xa9059cbb00000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000010000000000000001" - }, - { - "amount": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000007777777777777777777777777777777777777777ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "transfer_notice": "0x98830f8d00000000000000000000000077777777777777777777777777777777777777770000000000000000000000008888888888888888888888888888888888888888ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "erc20_transfer": "0xa9059cbb0000000000000000000000008888888888888888888888888888888888888888ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - } - ] - }, - "fee": { - "max_exponent": 17101, - "table": [ - ["0x0200000000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0x0404000000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0x0818201000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0x1071c46707040100", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0x21f1f3e9e88a9fdd", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0x48642d6345d6b722", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0xa540db81e090d212", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], - ["0xb52e6267a9c41374", "0x0000000000000002", "0x0000000000000000", "0x0000000000000000"], - ["0x54f4292cc0341bbd", "0x0000000000000007", "0x0000000000000000", "0x0000000000000000"], - ["0xc18b645664e97742", "0x0000000000000035", "0x0000000000000000", "0x0000000000000000"], - ["0xb60b04f629f898ab", "0x0000000000000b49", "0x0000000000000000", "0x0000000000000000"], - ["0x4629a786bdd375fa", "0x00000000007f6ade", "0x0000000000000000", "0x0000000000000000"], - ["0x79e549182d192a6d", "0x00003f6b3526706c", "0x0000000000000000", "0x0000000000000000"], - ["0x7f3a4c7881787119", "0x64804eff3bb1395d", "0x000000000fb5f10e", "0x0000000000000000"], - ["0xe3d136d7ec593e8d", "0x50c3a57adca65b2a", "0x5822dda8c6c1ba0e", "0x00f6d38e711d40ba"] - ], - "fee_to_linear": [ - { - "exponent": 0, - "fixed": "0x10000000000000000", - "linear": "0x1" - }, - { - "exponent": 1, - "fixed": "0x10200000000000000", - "linear": "0x1" - }, - { - "exponent": 200, - "fixed": "0x4bdea1c0ae1fadf2f", - "linear": "0x4" - }, - { - "exponent": 17101, - "fixed": "0xff853cbad77d81420c3129ed78d03683ecdfc2c89dd67c9494e3ba8934ba46d3", - "linear": "0xff853cbad77d81420c3129ed78d03683ecdfc2c89dd67c94" - } - ] - }, - "evm_advance": { - "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000000", - "fields": { - "chain_id": "0xaa36a7", - "app_contract": "0x0123456789abcDEF0123456789abCDef01234567", - "msg_sender": "0x3332DE61a8BB9aC84893b2f552Fe81C9a6dC5419", - "block_number": "0x1234567", - "block_timestamp": "0x6553f100", - "prev_randao": "0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", - "index": "0x2a", - "payload": "0xaabb" - }, - "variants": [ - { - "name": "trailing_zero_word", - "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "accepted": true - }, - { - "name": "dirty_bytes_padding", - "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000001", - "accepted": true - }, - { - "name": "dirty_address_padding", - "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef01234567ff00000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000000", - "accepted": true - } - ] - }, - "guest": { - "chain_id": 31337, - "app_contract": "0x0000000000000000000000000000000000000000", - "senders": { - "portal": "0x3332DE61a8BB9aC84893b2f552Fe81C9a6dC5419", - "sequencer": "0xa0Ee7A142d267C1f36714E4a8F75612F20a79720" - }, - "first_input_envelope": "0x415bf3630000000000000000000000000000000000000000000000000000000000007a6900000000000000000000000000000000000000000000000000000000000000000000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc5419000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000004895d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8000000000000000000000000000000000000000000000000", - "payloads": { - "deposit_alice": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8", - "drain_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000", - "sig_as_signed_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "sig_high_s_same_v_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891c", - "sig_high_s_twin_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891b", - "sig_r_at_order_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd03641415d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "sig_r_zero_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d00000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888800000000000000000000000000000000000000000000000000000000000000005d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "sig_s_zero_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a00000000000000000000000000000000000000000000000000000000000000001c", - "sig_sixty_four_bytes_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8", - "sig_v_0_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b800", - "sig_v_1_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b801", - "sig_v_255_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8ff", - "sig_v_27_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81b", - "sig_v_28_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "sig_v_2_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b802", - "sig_v_35_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b823", - "sig_v_36_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b824", - "spill_batch": "0x00000000000000000c0000002400000032000000400000004e0000005c00000032050000400500004e0500005c0500000e0000000a0000000000000000000e0000000b0000000000000001010e0000000c0000000000000002020e0000000d0000000000000003030e0000000e00000000000000640024000000a80000002c010000b001000034020000b80200003c030000c0030000440400000000000064000e0000004300000001010000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888881c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b0100000065000e00000043000000010200000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888820a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b0200000066000e0000004300000001030000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888884ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b0300000067000e000000430000000104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c0400000068000e0000004300000001050000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888883ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b0500000069000e00000043000000010600000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888859567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c060000006a000e000000430000000107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888b526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c070000006b000e000000430000000108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888d43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c080000006c000e000000430000000109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888a44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c0e0000000f0000000000000005050e000000100000000000000006060e000000110000000000000007070e00000012000000000000000808", - "transfer_batch": "0x01000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", - "withdrawal_batch": "0x01000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e0000002f000000007d00000000000000000000000000000000000000000000000000000000000000f429270312f288812d5278931600bb8555b33979e672c3264680cbcd9fc5496b1fcea49af69cd866ea1fd806bce9947446cc9a9bec8b960e90f52f9ca6d930bd1c" - }, - "outputs": { - "deposit_notice_alice": { - "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" - }, - "spill_transfer_notice_1": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001" - }, - "spill_transfer_notice_2": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000002" - }, - "spill_transfer_notice_3": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000003" - }, - "spill_transfer_notice_4": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000004" - }, - "spill_transfer_notice_5": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000005" - }, - "spill_transfer_notice_6": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000006" - }, - "spill_transfer_notice_7": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000007" - }, - "spill_transfer_notice_8": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000008" - }, - "spill_transfer_notice_9": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000009" - }, - "transfer_notice_alice_bob": { - "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f10000000000000000000000008888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007d" - }, - "withdrawal_voucher_alice": { - "voucher": { - "destination": "0x95d0c8A7d11342299807A2Fc19ac44C2321cCc68", - "payload": "0xa9059cbb0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000007d", - "value": "0x0" - } - } - }, - "scenarios": [ - { - "name": "deposit", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "drain_batch" - } - ] - }, - { - "name": "transfer", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "drain_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "withdrawal", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "drain_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["withdrawal_voucher_alice"], - "payload": "withdrawal_batch" - } - ] - }, - { - "name": "spill_batch", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 20, - "from": "sequencer", - "outputs": [ - "deposit_notice_alice", - "spill_transfer_notice_1", - "spill_transfer_notice_2", - "spill_transfer_notice_3", - "spill_transfer_notice_4", - "spill_transfer_notice_5", - "spill_transfer_notice_6", - "spill_transfer_notice_7", - "spill_transfer_notice_8", - "spill_transfer_notice_9" - ], - "payload": "spill_batch" - } - ] - }, - { - "name": "signature_as_signed", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], - "payload": "sig_as_signed_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": [], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_0", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_v_0_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_1", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], - "payload": "sig_v_1_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": [], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_2", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_v_2_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_27", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_v_27_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_28", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], - "payload": "sig_v_28_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": [], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_35", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_v_35_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_36", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], - "payload": "sig_v_36_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": [], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_v_255", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_v_255_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_high_s_twin", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], - "payload": "sig_high_s_twin_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": [], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_high_s_same_v", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_high_s_same_v_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_sixty_four_bytes", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_sixty_four_bytes_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_r_zero", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_r_zero_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_s_zero", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_s_zero_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - }, - { - "name": "signature_r_at_order", - "steps": [ - { - "block": 10, - "from": "portal", - "outputs": [], - "payload": "deposit_alice" - }, - { - "block": 10, - "from": "sequencer", - "outputs": ["deposit_notice_alice"], - "payload": "sig_r_at_order_batch" - }, - { - "block": 11, - "from": "sequencer", - "outputs": ["transfer_notice_alice_bob"], - "payload": "transfer_batch" - } - ] - } - ] - } -} diff --git a/tests/fixtures/consensus_vectors_v2.json b/tests/fixtures/consensus_vectors_v2.json new file mode 100644 index 00000000..18bb30c7 --- /dev/null +++ b/tests/fixtures/consensus_vectors_v2.json @@ -0,0 +1,2796 @@ +{ + "about": "Consensus vectors of the canonical template with strict frames, written once and never regenerated: on any later commit a mismatch is a finding to explain, not a reason to update this file. The byte-level sections, eip712 through evm_advance, are carried unchanged from consensus_vectors_v1.json, which pinned them on Rust 1.95.0 / alloy-core 1.6.1. The guest section was computed once from the reference scheduler and the wallet, and the scheduler_only section from the reference scheduler alone. Bytes are 0x-prefixed hex; uint256 values are 0x-prefixed hex quantities.", + "eip712": { + "domain_encode_type": "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", + "domain_type_hash": "0x8b73c3c69bb8fe3d512ecc4cf759cc79239f7b179b0ffacaa9a75d522b39400f", + "encode_type": "UserOp(uint32 nonce,uint16 max_fee,bytes data)", + "type_hash": "0x794b86e861930e59e62ce17b915511462a70fd7518920d37d23766ce03bf6cb5", + "domains": [ + { + "name": "canonical_test", + "chain_id": 31337, + "verifying_contract": "0x0000000000000000000000000000000000000000", + "separator": "0x330487f13dc661ab8fe297ef0a70f998ab234b53e3ead2e450c80b22395571c3" + }, + { + "name": "sepolia_app", + "chain_id": 11155111, + "verifying_contract": "0x0123456789abcDEF0123456789abCDef01234567", + "separator": "0x2a2f994d3223f807b1de0a508576e48e49bef3fb0fb1eae8c9db0fac5c8e2b7e" + } + ], + "user_ops": [ + { + "name": "empty_data", + "domain": "canonical_test", + "nonce": 0, + "max_fee": 0, + "data": "0x", + "struct_hash": "0x0adb11d43608ba5495540efc8110e57e3bd16cdc5acfbb4b878978c891a26629", + "signing_hash": "0xcc60e745cef998b09a8e9658cda76670df34346af0f299ddfa0a5aa0641e3aae" + }, + { + "name": "four_bytes", + "domain": "canonical_test", + "nonce": 1, + "max_fee": 100, + "data": "0xdeadbeef", + "struct_hash": "0xda26caef39bdb65184b64ac6c7d6ed6273ac9119a6c418858d4b4786f4523e38", + "signing_hash": "0xdb014760580d2de614eae36089d0d09d7f8d87b9e03045573226c8ebcee48087" + }, + { + "name": "two_hundred_bytes", + "domain": "canonical_test", + "nonce": 2, + "max_fee": 200, + "data": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bbc2c9d0d7dee5ecf3fa01080f161d242b323940474e555c636a71787f868d949ba2a9b0b7bec5ccd3dae1e8eff6fd040b121920272e353c434a51585f666d74", + "struct_hash": "0x4e6e8192e33e744c4cfb3462263126735e45e6e0239b70fd38ea36267745bd49", + "signing_hash": "0x63f0fd4940e600f68a5cd0f7294351afa052fbec58f03b2b666ebdf94c85d4df" + }, + { + "name": "nonce_u32_max", + "domain": "canonical_test", + "nonce": 4294967295, + "max_fee": 1, + "data": "0x01", + "struct_hash": "0xc0ab5d4c2667fa8b0dc49bae86e056d3a6f10ba4b5e32cfb8bc08d78f3012872", + "signing_hash": "0x0a598fa1ccfbd54f376cb6902f6752d81a92c8ae2d54935cab86ab402794432d" + }, + { + "name": "max_fee_u16_max", + "domain": "canonical_test", + "nonce": 7, + "max_fee": 65535, + "data": "0x02", + "struct_hash": "0x601501fb53f1f11256b641bcc27cbb651803297acf6b55f2e2ae6563a4ec84da", + "signing_hash": "0x4eed33ceb9f6ba501a6c481e8563c13e27ac4a06d4dd9ea3250a2276f38ffdbe" + }, + { + "name": "four_bytes_sepolia_app", + "domain": "sepolia_app", + "nonce": 1, + "max_fee": 100, + "data": "0xdeadbeef", + "struct_hash": "0xda26caef39bdb65184b64ac6c7d6ed6273ac9119a6c418858d4b4786f4523e38", + "signing_hash": "0xe34abab31e6e31f4023ac0b4c8a87229765b805513fb7e8041f3d752437c4453" + } + ] + }, + "keccak256": [ + { + "name": "empty", + "input": "0x", + "digest": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + }, + { + "name": "abc", + "input": "0x616263", + "digest": "0x4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45" + }, + { + "name": "bytes_135", + "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6ad", + "digest": "0x00ef96af9cf4b24c7f269d922294444a197d0a33638c2e56634c57e892103a8f" + }, + { + "name": "bytes_136", + "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4", + "digest": "0x742061bcad767ed4c4f5883b1dcb1aad11afdcc140dc469d953759b127b9f9ed" + }, + { + "name": "bytes_137", + "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bb", + "digest": "0xe3371f61e770abf254c34239c3b0099ad90594507415bc81dd0a10b9692bbf2a" + }, + { + "name": "bytes_272", + "input": "0x030a11181f262d343b424950575e656c737a81888f969da4abb2b9c0c7ced5dce3eaf1f8ff060d141b222930373e454c535a61686f767d848b9299a0a7aeb5bcc3cad1d8dfe6edf4fb020910171e252c333a41484f565d646b727980878e959ca3aab1b8bfc6cdd4dbe2e9f0f7fe050c131a21282f363d444b525960676e757c838a91989fa6adb4bbc2c9d0d7dee5ecf3fa01080f161d242b323940474e555c636a71787f868d949ba2a9b0b7bec5ccd3dae1e8eff6fd040b121920272e353c434a51585f666d747b828990979ea5acb3bac1c8cfd6dde4ebf2f900070e151c232a31383f464d545b626970777e858c939aa1a8afb6bdc4cbd2d9e0e7eef5fc030a11181f262d343b424950575e656c", + "digest": "0xac141fd7b0a0ffcd2e967254d508da3ec616596493c36fa304425647d90e6de5" + } + ], + "signature": { + "private_key": "0x0101010101010101010101010101010101010101010101010101010101010101", + "address": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "domain": "canonical_test", + "user_op": { + "nonce": 0, + "max_fee": 0, + "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888" + }, + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "variants": [ + { + "name": "as_signed", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" + }, + { + "name": "v_0", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b800", + "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", + "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" + }, + { + "name": "v_1", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b801", + "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" + }, + { + "name": "v_2", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b802", + "guest": null, + "ingress": null + }, + { + "name": "v_27", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81b", + "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", + "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" + }, + { + "name": "v_28", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" + }, + { + "name": "v_35", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b823", + "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", + "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" + }, + { + "name": "v_36", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b824", + "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" + }, + { + "name": "v_255", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8ff", + "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", + "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" + }, + { + "name": "high_s_twin", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891b", + "guest": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", + "ingress": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1" + }, + { + "name": "high_s_same_v", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891c", + "guest": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", + "ingress": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd" + }, + { + "name": "sixty_four_bytes", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8", + "guest": null, + "ingress": null + }, + { + "name": "r_zero", + "signature": "0x00000000000000000000000000000000000000000000000000000000000000005d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "guest": null, + "ingress": null + }, + { + "name": "s_zero", + "signature": "0xdc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a00000000000000000000000000000000000000000000000000000000000000001c", + "guest": null, + "ingress": null + }, + { + "name": "r_at_order", + "signature": "0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd03641415d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "guest": null, + "ingress": null + } + ] + }, + "ssz": { + "spill_batch": { + "nonce": 0, + "frames": [ + { + "safe_block": 10, + "fee_price": 0, + "user_ops": [] + }, + { + "safe_block": 11, + "fee_price": 257, + "user_ops": [] + }, + { + "safe_block": 12, + "fee_price": 514, + "user_ops": [] + }, + { + "safe_block": 13, + "fee_price": 771, + "user_ops": [] + }, + { + "safe_block": 14, + "fee_price": 100, + "user_ops": [ + { + "nonce": 0, + "max_fee": 100, + "data": "0x0101000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x1c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b" + }, + { + "nonce": 1, + "max_fee": 101, + "data": "0x0102000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x20a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b" + }, + { + "nonce": 2, + "max_fee": 102, + "data": "0x0103000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x4ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b" + }, + { + "nonce": 3, + "max_fee": 103, + "data": "0x0104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c" + }, + { + "nonce": 4, + "max_fee": 104, + "data": "0x0105000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x3ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b" + }, + { + "nonce": 5, + "max_fee": 105, + "data": "0x0106000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0x59567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c" + }, + { + "nonce": 6, + "max_fee": 106, + "data": "0x0107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0xb526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c" + }, + { + "nonce": 7, + "max_fee": 107, + "data": "0x0108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0xd43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c" + }, + { + "nonce": 8, + "max_fee": 108, + "data": "0x0109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888", + "signature": "0xa44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c" + } + ] + }, + { + "safe_block": 15, + "fee_price": 1285, + "user_ops": [] + }, + { + "safe_block": 16, + "fee_price": 1542, + "user_ops": [] + }, + { + "safe_block": 17, + "fee_price": 1799, + "user_ops": [] + }, + { + "safe_block": 18, + "fee_price": 2056, + "user_ops": [] + } + ], + "encoded": "0x00000000000000000c0000002400000032000000400000004e0000005c00000032050000400500004e0500005c0500000e0000000a0000000000000000000e0000000b0000000000000001010e0000000c0000000000000002020e0000000d0000000000000003030e0000000e00000000000000640024000000a80000002c010000b001000034020000b80200003c030000c0030000440400000000000064000e0000004300000001010000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888881c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b0100000065000e00000043000000010200000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888820a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b0200000066000e0000004300000001030000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888884ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b0300000067000e000000430000000104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c0400000068000e0000004300000001050000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888883ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b0500000069000e00000043000000010600000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888859567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c060000006a000e000000430000000107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888b526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c070000006b000e000000430000000108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888d43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c080000006c000e000000430000000109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888a44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c0e0000000f0000000000000005050e000000100000000000000006060e000000110000000000000007070e00000012000000000000000808" + } + }, + "wallet_outputs": { + "token": "0x95d0c8A7d11342299807A2Fc19ac44C2321cCc68", + "sender": "0x7777777777777777777777777777777777777777", + "recipient": "0x8888888888888888888888888888888888888888", + "cases": [ + { + "amount": "0x1", + "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc6800000000000000000000000077777777777777777777777777777777777777770000000000000000000000000000000000000000000000000000000000000001", + "transfer_notice": "0x98830f8d000000000000000000000000777777777777777777777777777777777777777700000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001", + "erc20_transfer": "0xa9059cbb00000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001" + }, + { + "amount": "0x10000000000000001", + "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc6800000000000000000000000077777777777777777777777777777777777777770000000000000000000000000000000000000000000000010000000000000001", + "transfer_notice": "0x98830f8d000000000000000000000000777777777777777777777777777777777777777700000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000010000000000000001", + "erc20_transfer": "0xa9059cbb00000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000010000000000000001" + }, + { + "amount": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "deposit_notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000007777777777777777777777777777777777777777ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "transfer_notice": "0x98830f8d00000000000000000000000077777777777777777777777777777777777777770000000000000000000000008888888888888888888888888888888888888888ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "erc20_transfer": "0xa9059cbb0000000000000000000000008888888888888888888888888888888888888888ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + } + ] + }, + "fee": { + "max_exponent": 17101, + "table": [ + ["0x0200000000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0x0404000000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0x0818201000000000", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0x1071c46707040100", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0x21f1f3e9e88a9fdd", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0x48642d6345d6b722", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0xa540db81e090d212", "0x0000000000000001", "0x0000000000000000", "0x0000000000000000"], + ["0xb52e6267a9c41374", "0x0000000000000002", "0x0000000000000000", "0x0000000000000000"], + ["0x54f4292cc0341bbd", "0x0000000000000007", "0x0000000000000000", "0x0000000000000000"], + ["0xc18b645664e97742", "0x0000000000000035", "0x0000000000000000", "0x0000000000000000"], + ["0xb60b04f629f898ab", "0x0000000000000b49", "0x0000000000000000", "0x0000000000000000"], + ["0x4629a786bdd375fa", "0x00000000007f6ade", "0x0000000000000000", "0x0000000000000000"], + ["0x79e549182d192a6d", "0x00003f6b3526706c", "0x0000000000000000", "0x0000000000000000"], + ["0x7f3a4c7881787119", "0x64804eff3bb1395d", "0x000000000fb5f10e", "0x0000000000000000"], + ["0xe3d136d7ec593e8d", "0x50c3a57adca65b2a", "0x5822dda8c6c1ba0e", "0x00f6d38e711d40ba"] + ], + "fee_to_linear": [ + { + "exponent": 0, + "fixed": "0x10000000000000000", + "linear": "0x1" + }, + { + "exponent": 1, + "fixed": "0x10200000000000000", + "linear": "0x1" + }, + { + "exponent": 200, + "fixed": "0x4bdea1c0ae1fadf2f", + "linear": "0x4" + }, + { + "exponent": 17101, + "fixed": "0xff853cbad77d81420c3129ed78d03683ecdfc2c89dd67c9494e3ba8934ba46d3", + "linear": "0xff853cbad77d81420c3129ed78d03683ecdfc2c89dd67c94" + } + ] + }, + "evm_advance": { + "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000000", + "fields": { + "chain_id": "0xaa36a7", + "app_contract": "0x0123456789abcDEF0123456789abCDef01234567", + "msg_sender": "0x3332DE61a8BB9aC84893b2f552Fe81C9a6dC5419", + "block_number": "0x1234567", + "block_timestamp": "0x6553f100", + "prev_randao": "0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "index": "0x2a", + "payload": "0xaabb" + }, + "variants": [ + { + "name": "trailing_zero_word", + "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "accepted": true + }, + { + "name": "dirty_bytes_padding", + "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef012345670000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000001", + "accepted": true + }, + { + "name": "dirty_address_padding", + "calldata": "0x415bf3630000000000000000000000000000000000000000000000000000000000aa36a70000000000000000000000000123456789abcdef0123456789abcdef01234567ff00000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc54190000000000000000000000000000000000000000000000000000000001234567000000000000000000000000000000000000000000000000000000006553f1000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000002aabb000000000000000000000000000000000000000000000000000000000000", + "accepted": true + } + ] + }, + "guest": { + "about": "Scenarios replayed from genesis on the devnet template, whose scheduler configuration is config. Each step is the next InputBox input: its sender (a name in senders), block, timestamp and payload (a name in payloads). Then what the scheduler made of it: outcome, and for a rejected batch reject, whose reason is a diagnostic. Then txs, the transactions it handed out in order: a direct by its input index, a user op whole, each with the batch nonce it was handed out under, and for a user op whether the executor ran it or skipped it for its nonce. Then outputs, by name in outputs, and scheduler, the machine's scheduler report after the input. A scenario ends with the machine's state report, or its keccak256 when it is long.", + "chain_id": 31337, + "app_contract": "0x0000000000000000000000000000000000000000", + "config": {"sequencer_address": "0xa0Ee7A142d267C1f36714E4a8F75612F20a79720", "max_wait_blocks": 7200, "fridge_bound": 16384, "max_direct_payload_bytes": 1024}, + "senders": { + "portal": "0x3332DE61a8BB9aC84893b2f552Fe81C9a6dC5419", + "sequencer": "0xa0Ee7A142d267C1f36714E4a8F75612F20a79720", + "stranger": "0x2222222222222222222222222222222222222222" + }, + "first_input_envelope": "0x415bf3630000000000000000000000000000000000000000000000000000000000007a6900000000000000000000000000000000000000000000000000000000000000000000000000000000000000003332de61a8bb9ac84893b2f552fe81c9a6dc5419000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000006553f178000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000004895d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8000000000000000000000000000000000000000000000000", + "payloads": { + "at_clock_transfer_batch": "0x01000000000000000c000000040000000e0000000e000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "below_clock_transfer_batch": "0x01000000000000000c000000040000000e0000000c000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "cap_at_price_batch": "0x01000000000000000c000000040000000e0000000a000000000000006400040000000000000064000e00000043000000017d0000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888882f922833d55de6e0277cd86bf90f0a80d5e0479b8198da40632e4de0e3b0c93702814ceca19e6805bde7843e313cd3a486a1b6f1827c05f8f825c40c30f780571c", + "cap_below_price_batch": "0x00000000000000000c000000040000000e0000000a000000000000006400040000000000000063000e00000043000000017d0000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888886ca6463649490d8d95808e74e0b6d4f80abff8a792db81b2456e4373bcb658c41d53c001e60e9694e1d4eac610e624fdb56673cb72bae93c997c4a0a6c0cefa61c", + "deposit_alice": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8", + "deposit_alice_at_cap": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000001f4abababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab", + "deposit_alice_over_cap": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc681a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8ababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab", + "deposit_carol": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc683325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001f4", + "deposit_carol_late": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc683325a78425f17a7e487eb5666b2bfd93abb06c700000000000000000000000000000000000000000000000000000000000000007", + "deposit_dave": "0x95d0c8a7d11342299807a2fc19ac44c2321ccc68c48b812bb43401392c037381aca934f4069c0517000000000000000000000000000000000000000000000000000000000000012c", + "drain_7212_batch": "0x00000000000000000c000000040000000e0000002c1c0000000000000000", + "drain_at_11_batch": "0x00000000000000000c000000040000000e0000000b000000000000000000", + "drain_at_11_batch_1": "0x01000000000000000c000000040000000e0000000b000000000000000000", + "drain_at_14_batch": "0x00000000000000000c000000040000000e0000000e000000000000000000", + "drain_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000", + "empty_batch_0": "0x00000000000000000c000000", + "fee_at_max_batch": "0x00000000000000000c000000040000000e0000000a00000000000000cd42", + "fee_out_of_range_batch": "0x00000000000000000c00000008000000160000000e0000000a0000000000000000000e0000000a00000000000000ce420400000000000000ffff0e00000043000000017d0000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888888769761b394ba42acf8acd9928792e3c51dffd53d5903ec7b9244dcfbfa1760539a39c39398009060f664aefe0849e72febd0cff58273aeb85f51bc16f9c3b4a1c", + "frames_10_12_batch": "0x00000000000000000c00000008000000160000000e0000000a0000000000000000000e0000000c000000000000000000", + "late_drain_batch": "0x01000000000000000c000000040000000e0000000f000000000000000000040000000100000000000e000000430000000107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888a409fbee7ee1f870127cf7304d83e4adfd7612668af6acaa89a6b2d0b4e6d7c15441e2aa86f29da514977427ba0adc5e1d6ee835bee5ee187a52558cb197bd391c", + "non_monotonic_batch": "0x01000000000000000c00000008000000160000000e0000000b0000000000000000000e0000000a000000000000000000", + "oldest_fresh_transfer_batch": "0x00000000000000000c000000040000000e0000000c000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "queued_drain_batch": "0x00000000000000000c000000080000009e0000000e0000000b000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c0e0000000d000000000000000000080000008c0000000000000000000e000000430000000132000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888c5b29bcca8760ac7661852df736ddccbb866538c34364d2750b2cf7e5e6d41462dd6c628f4a0412c5a6a24fe53a1cd8afa6816b6f547cfd9078ca99c943a86501b0000000000000e000000430000000164000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f14ba2ac96a5e8ac6c154f950a8729a64c6932603a6c77bc34b7eddc4fb42a1d601eee95cbf3acada3fa1d6327c7a7bb0da229df6adfe10c67eedf08a16589f0ac1c", + "sig_as_signed_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "sig_high_s_same_v_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891c", + "sig_high_s_twin_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46aa2c9b629f38cc5ff6b32eeafe41341e7c8f174f4edc67a40bc08bf048e654b891b", + "sig_r_at_order_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd03641415d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "sig_r_zero_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d00000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888800000000000000000000000000000000000000000000000000000000000000005d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "sig_s_at_order_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46afffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd03641411c", + "sig_s_zero_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a00000000000000000000000000000000000000000000000000000000000000001c", + "sig_sixty_four_bytes_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8", + "sig_v_0_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b800", + "sig_v_1_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b801", + "sig_v_255_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b8ff", + "sig_v_27_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81b", + "sig_v_28_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "sig_v_29_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81d", + "sig_v_2_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b802", + "sig_v_30_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81e", + "sig_v_31_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81f", + "sig_v_32_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b820", + "sig_v_33_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b821", + "sig_v_34_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b822", + "sig_v_35_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b823", + "sig_v_36_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b824", + "spill_batch": "0x00000000000000000c0000002400000032000000400000004e0000005c00000032050000400500004e0500005c0500000e0000000a0000000000000000000e0000000b0000000000000001010e0000000c0000000000000002020e0000000d0000000000000003030e0000000e00000000000000640024000000a80000002c010000b001000034020000b80200003c030000c0030000440400000000000064000e0000004300000001010000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888881c85ee3d07a0821ef7a776650c80189b0d8b4e13c8ef7943dfd90abef99b0c754e6c7720754748f50894437ebc9a597233397f26ddb00ec3e957ed07234626191b0100000065000e00000043000000010200000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888820a59bd6086eafac900a33a265bf2d6b8dbdbefc2a265bc17e98b62859a8235c42547b3f35a1ff4522fe113ea2bcb5232767a10a8f648536490dd7c4e719670b1b0200000066000e0000004300000001030000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888884ac96b0e87ae8e1a387a16e2f3e9a48791ba8150162a24650537e7806f9608046283b5a4fa7678c42e1b86fd6083b15b33fd9d03727b0d9c5925eca77d7cccb91b0300000067000e000000430000000104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888162abcc045983f3559a5665003391d9a67a587ffb90ccd84341591808ec0166a54d4c2b9ec89b29c8fdf2009acab2f3c2698e123ff5d3a9b497f8be9f86949191c0400000068000e0000004300000001050000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888883ebd3564052098f1834c001930dd350d6b24e2777774d89d0f5c0cd9e6584ad65b5d971d93abc25053ab73f974b93cb5afd587e13b24576901a194e43092fd281b0500000069000e00000043000000010600000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888859567102b241f0243c8db87fd2fb2f4fb7760034aa4a96ca033b72dd81df800877ba4332099d1f319ab8f207b11ad6f55aefb8bf982343cc93c532b8df315af11c060000006a000e000000430000000107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888b526da98cf904b35df6db767d1f07d36fd0a027e8ca58c3cedbe64c46720fe4815de8686b8267fb4038df1307c3fa01a35b0c722d4054d40e42e9ffe60b0b5fc1c070000006b000e000000430000000108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888d43fb66cf0e5c18c8334a00ece9f5455b17f9aba8c45e712acbea097b4acbfc85ae6c8bcb5721f8519b8b94abcdac1f0678da4748b027fc0230403fde5ea98c91c080000006c000e000000430000000109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888a44a6cb4dbf77b359fab9c9d9cd8b298e4fdf7a1742575b3cf2a39563a2d83f50c2926072410c723f1c837cec9eb9e90edf9393bf29015ae625fd09be25b3cdb1c0e0000000f0000000000000005050e000000100000000000000006060e000000110000000000000007070e00000012000000000000000808", + "stale_transfer_batch": "0x00000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "stranger_direct": "0xdeadbeef", + "transfer_batch": "0x01000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c", + "undecodable_batch": "0xffeedd", + "underfunded_batch": "0x00000000000000000c000000040000000e0000000a00000000000000e803080000008c00000000000000e8030e000000430000000100000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888f0ae16d4b58bb867d172330bbd48d30ab762ce11852958c35955d72402cd23ce58c87a4e6733065689f124be50512e661d6131a2677c88f0be5cd7f2364698911c01000000e8030e0000004300000001010000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888884d24f160755d86c8776edca765d2c4268221cea85058709e0cf201ff3df0022a2d6f2323b891e24dbc8ca167df0ae70aa30f8f2a0ad403f333f2783b5d1dd5cd1c", + "withdrawal_batch": "0x01000000000000000c000000040000000e0000000a000000000000000000040000000000000000000e0000002f000000007d00000000000000000000000000000000000000000000000000000000000000f429270312f288812d5278931600bb8555b33979e672c3264680cbcd9fc5496b1fcea49af69cd866ea1fd806bce9947446cc9a9bec8b960e90f52f9ca6d930bd1c", + "wrong_nonce_ops_batch": "0x00000000000000000c000000040000000e0000000a0000000000000000000c00000090000000140100000100000000000e00000043000000017d00000000000000000000000000000000000000000000000000000000000000888888888888888888888888888888888888888884b07ab498edaeaae500ce6f45519787fba2ff52878c68e5a885a00929fb9ce9419889f8b9d6446472288c3e2282939a77b2436b30870f241b385c95ce9c99ad1c0000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c0000000000000e00000043000000017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888dc2528a0578bf931525b5770e77a7103ceb32ea315176ecca1b2386a3bc4f46a5d3649d60c733a0094cd11501becbe16f1bd67f1c18225fb03c99f8841d0f5b81c" + }, + "outputs": { + "deposit_notice_alice": { + "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + "deposit_notice_alice_500": { + "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000001f4" + }, + "deposit_notice_carol_500": { + "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000003325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001f4" + }, + "deposit_notice_carol_7": { + "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc680000000000000000000000003325a78425f17a7e487eb5666b2bfd93abb06c700000000000000000000000000000000000000000000000000000000000000007" + }, + "deposit_notice_dave_300": { + "notice": "0xd19f477d00000000000000000000000095d0c8a7d11342299807a2fc19ac44c2321ccc68000000000000000000000000c48b812bb43401392c037381aca934f4069c0517000000000000000000000000000000000000000000000000000000000000012c" + }, + "spill_transfer_notice_1": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000001" + }, + "spill_transfer_notice_2": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000002" + }, + "spill_transfer_notice_3": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000003" + }, + "spill_transfer_notice_4": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000004" + }, + "spill_transfer_notice_5": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000005" + }, + "spill_transfer_notice_6": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000006" + }, + "spill_transfer_notice_7": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000007" + }, + "spill_transfer_notice_8": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000008" + }, + "spill_transfer_notice_9": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000009" + }, + "transfer_notice_alice_bob": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f10000000000000000000000008888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007d" + }, + "transfer_notice_alice_bob_0": { + "notice": "0x98830f8d0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000000" + }, + "transfer_notice_carol_bob_50": { + "notice": "0x98830f8d0000000000000000000000003325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000032" + }, + "transfer_notice_carol_bob_7": { + "notice": "0x98830f8d0000000000000000000000003325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000007" + }, + "transfer_notice_dave_alice_100": { + "notice": "0x98830f8d000000000000000000000000c48b812bb43401392c037381aca934f4069c05170000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f10000000000000000000000000000000000000000000000000000000000000064" + }, + "withdrawal_voucher_alice": { + "voucher": { + "destination": "0x95d0c8A7d11342299807A2Fc19ac44C2321cCc68", + "payload": "0xa9059cbb0000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000007d", + "value": "0x0" + } + } + }, + "scenarios": [ + { + "name": "deposit", + "about": "A deposit queues, then a batch one block later drains it.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000001000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + { + "name": "transfer", + "about": "A signed transfer runs after the deposit that funds it.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "withdrawal", + "about": "A signed withdrawal emits its voucher.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "withdrawal_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x007d00000000000000000000000000000000000000000000000000000000000000"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["withdrawal_voucher_alice"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000b400000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036aa0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "spill_batch", + "about": "A batch of many frames and fee prices, one frame holding nine ops.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 20, + "timestamp": 1700000240, + "payload": "spill_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 100, "block": 14, "data": "0x0101000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 1, "fee": 100, "block": 14, "data": "0x0102000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 2, "fee": 100, "block": 14, "data": "0x0103000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 3, "fee": 100, "block": 14, "data": "0x0104000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 4, "fee": 100, "block": 14, "data": "0x0105000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 5, "fee": 100, "block": 14, "data": "0x0106000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 6, "fee": 100, "block": 14, "data": "0x0107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 7, "fee": 100, "block": 14, "data": "0x0108000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 8, "fee": 100, "block": 14, "data": "0x0109000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": [ + "deposit_notice_alice", + "spill_transfer_notice_1", + "spill_transfer_notice_2", + "spill_transfer_notice_3", + "spill_transfer_notice_4", + "spill_transfer_notice_5", + "spill_transfer_notice_6", + "spill_transfer_notice_7", + "spill_transfer_notice_8", + "spill_transfer_notice_9" + ], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 10, "next_batch_nonce": 1, "clock": 18, "first_queued_input": 2, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e80000000a000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003a98888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000002da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000121a642f0e3c3af545e7acbd38b07251b3990914f109000000" + }, + { + "name": "signature_as_signed", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_as_signed_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_0", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_0_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000001c286de05cd43e12cd2887fa74e2325f0c6e7a4cd00000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f101000000c286de05cd43e12cd2887fa74e2325f0c6e7a4cd01000000" + }, + { + "name": "signature_v_1", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_1_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_2", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_2_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_27", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_27_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000001c286de05cd43e12cd2887fa74e2325f0c6e7a4cd00000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f101000000c286de05cd43e12cd2887fa74e2325f0c6e7a4cd01000000" + }, + { + "name": "signature_v_28", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_28_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_35", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_35_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000001c286de05cd43e12cd2887fa74e2325f0c6e7a4cd00000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f101000000c286de05cd43e12cd2887fa74e2325f0c6e7a4cd01000000" + }, + { + "name": "signature_v_36", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_36_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_255", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_255_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000001c286de05cd43e12cd2887fa74e2325f0c6e7a4cd00000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f101000000c286de05cd43e12cd2887fa74e2325f0c6e7a4cd01000000" + }, + { + "name": "signature_high_s_twin", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_high_s_twin_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_high_s_same_v", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_high_s_same_v_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0xC286de05CD43E12cD2887FA74E2325F0C6E7A4Cd", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000001c286de05cd43e12cd2887fa74e2325f0c6e7a4cd00000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f101000000c286de05cd43e12cd2887fa74e2325f0c6e7a4cd01000000" + }, + { + "name": "signature_sixty_four_bytes", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_sixty_four_bytes_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_r_zero", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_r_zero_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_s_zero", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_s_zero_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_r_at_order", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_r_at_order_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_29", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_29_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_30", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_30_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_31", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_31_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_32", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_32_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_33", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_33_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_v_34", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_v_34_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "signature_s_at_order", + "about": "The signature variant rides in a batch after a deposit, then the signer's own op at the same nonce follows.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "sig_s_at_order_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "safe_block_at_inclusion_block", + "about": "A frame at its batch's own inclusion block is rejected; the same batch one block later drains.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 10, + "timestamp": 1700000120, + "payload": "drain_batch", + "outcome": "batch_rejected", + "reject": {"reason": "safe_block_not_below_inclusion_block"}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000001000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + { + "name": "first_frame_below_clock", + "about": "After a frame raises the clock, a batch whose first frame lies below it is rejected and one at it is accepted.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "portal", + "block": 14, + "timestamp": 1700000168, + "payload": "deposit_carol", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "sequencer", + "block": 15, + "timestamp": 1700000180, + "payload": "drain_at_14_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice", "deposit_notice_carol_500"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 14, "first_queued_input": 3, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 16, + "timestamp": 1700000192, + "payload": "below_clock_transfer_batch", + "outcome": "batch_rejected", + "reject": {"reason": "first_frame_below_clock", "safe_block": 12, "clock": 14}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 14, "first_queued_input": 4, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 16, + "timestamp": 1700000192, + "payload": "at_clock_transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 14, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 5, "l2_tx_count": 3, "next_batch_nonce": 2, "clock": 14, "first_queued_input": 5, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000001c01000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a3325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001f48888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "stale_batch", + "about": "The backstop drains a deposit before a stale batch is skipped; a batch exactly one block inside the window is accepted at the kept nonce.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 7210, + "timestamp": 1700086520, + "payload": "stale_transfer_batch", + "outcome": "batch_stale", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 7211, + "timestamp": 1700086532, + "payload": "oldest_fresh_transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 12, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 12, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "wrong_batch_nonce", + "about": "A batch at the wrong nonce changes nothing; the right one drains and runs its op; the next skips the spent op nonce.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "transfer_batch", + "outcome": "batch_rejected", + "reject": {"reason": "wrong_nonce", "expected": 0, "got": 1}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "sig_as_signed_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "transfer_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "wrong_nonce"} + ], + "outputs": [], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 4, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "fee_price_out_of_range", + "about": "A tail frame priced above the maximum exponent rejects the whole batch before its first frame drains; a frame at the maximum is accepted.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "fee_out_of_range_batch", + "outcome": "batch_rejected", + "reject": {"reason": "fee_price_out_of_range"}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "fee_at_max_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000001000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + { + "name": "backstop_drains_queued_directs", + "about": "Directs queued for the whole window execute when a later input arrives, even an undecodable batch or another direct.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_carol", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 3} + }, + { + "from": "sequencer", + "block": 7211, + "timestamp": 1700086532, + "payload": "undecodable_batch", + "outcome": "batch_rejected", + "reject": {"reason": "decode_failed"}, + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice", "deposit_notice_carol_500"], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 2, "next_batch_nonce": 0, "clock": 11, "first_queued_input": 2, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 7212, + "timestamp": 1700086544, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 2, "batch_nonce": 0}], + "outputs": [], + "scheduler": {"l1_input_count": 5, "l2_tx_count": 3, "next_batch_nonce": 0, "clock": 12, "first_queued_input": 4, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 7213, + "timestamp": 1700086556, + "payload": "drain_7212_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 4, "batch_nonce": 0}], + "outputs": [], + "scheduler": {"l1_input_count": 6, "l2_tx_count": 4, "next_batch_nonce": 1, "clock": 7212, "first_queued_input": 6, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000b400000004000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e83325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001f4" + }, + { + "name": "direct_over_payload_cap", + "about": "A direct one byte over the payload cap never joins the fridge but takes its input index; one at the cap joins and executes.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice_over_cap", + "outcome": "direct_dropped", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 0} + }, + { + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_alice_at_cap", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "drain_at_11_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 1, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice_500"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 11, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000001000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000001f4" + }, + { + "name": "fee_cap_below_price", + "about": "An op whose max fee is below its frame's price is never handed out and leaves its nonce unspent; the same nonce at a covering cap executes.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "cap_below_price_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "cap_at_price_batch", + "outcome": "batch_accepted", + "txs": [ + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 100, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003698888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000021a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "wrong_nonce_op", + "about": "Ops at a nonce other than the sender's next are handed out and skipped; the one at the next nonce executes.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "wrong_nonce_ops_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 1, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "wrong_nonce"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 10, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "wrong_nonce"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000036a8888888888888888888888888888888888888888000000000000000000000000000000000000000000000000000000000000007da0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000000011a642f0e3c3af545e7acbd38b07251b3990914f101000000" + }, + { + "name": "underfunded_op", + "about": "An op whose sender cannot pay the price is charged what the sender has and executes; the next op pays nothing and its transfer fails.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "underfunded_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 1000, "block": 10, "data": "0x0100000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 1, "fee": 1000, "block": 10, "data": "0x0101000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": ["deposit_notice_alice", "transfer_notice_alice_bob_0"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 3, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000e800000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1000000000000000000000000000000000000000000000000000000000000000088888888888888888888888888888888888888880000000000000000000000000000000000000000000000000000000000000000a0ee7a142d267c1f36714e4a8f75612f20a7972000000000000000000000000000000000000000000000000000000000000003e81a642f0e3c3af545e7acbd38b07251b3990914f102000000" + }, + { + "name": "directs_queue_then_drain_with_ops", + "about": "Directs stay queued across several inputs, then each frame drains those at or below its safe block before its ops; a later direct waits above the clock.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_carol", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 3} + }, + { + "from": "portal", + "block": 13, + "timestamp": 1700000156, + "payload": "deposit_dave", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 4} + }, + { + "from": "portal", + "block": 15, + "timestamp": 1700000180, + "payload": "deposit_carol_late", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 5, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 5} + }, + { + "from": "sequencer", + "block": 16, + "timestamp": 1700000192, + "payload": "queued_drain_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 0, "batch_nonce": 0}, + {"direct": 1, "batch_nonce": 0}, + {"user_op": {"sender": "0x1a642f0E3c3aF545E7AcBD38b07251B3990914F1", "nonce": 0, "fee": 0, "block": 11, "data": "0x017d000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"direct": 2, "batch_nonce": 0}, + {"direct": 3, "batch_nonce": 0}, + {"user_op": {"sender": "0x3325a78425F17a7E487Eb5666b2bFd93aBb06c70", "nonce": 0, "fee": 0, "block": 13, "data": "0x0132000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 0, "fate": "executed"}, + {"user_op": {"sender": "0xc48B812bB43401392c037381AcA934F4069C0517", "nonce": 0, "fee": 0, "block": 13, "data": "0x0164000000000000000000000000000000000000000000000000000000000000001a642f0e3c3af545e7acbd38b07251b3990914f1"}, "batch_nonce": 0, "fate": "executed"} + ], + "outputs": [ + "deposit_notice_alice", + "deposit_notice_carol_500", + "transfer_notice_alice_bob", + "deposit_notice_dave_300", + "transfer_notice_carol_bob_50", + "transfer_notice_dave_alice_100" + ], + "scheduler": {"l1_input_count": 6, "l2_tx_count": 7, "next_batch_nonce": 1, "clock": 13, "first_queued_input": 4, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 17, + "timestamp": 1700000204, + "payload": "late_drain_batch", + "outcome": "batch_accepted", + "txs": [ + {"direct": 4, "batch_nonce": 1}, + {"user_op": {"sender": "0x3325a78425F17a7E487Eb5666b2bFd93aBb06c70", "nonce": 1, "fee": 0, "block": 15, "data": "0x0107000000000000000000000000000000000000000000000000000000000000008888888888888888888888888888888888888888"}, "batch_nonce": 1, "fate": "executed"} + ], + "outputs": ["deposit_notice_carol_7", "transfer_notice_carol_bob_7"], + "scheduler": {"l1_input_count": 7, "l2_tx_count": 9, "next_batch_nonce": 2, "clock": 15, "first_queued_input": 7, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000005001000009000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003ce3325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001c0888888888888888888888888888888888888888800000000000000000000000000000000000000000000000000000000000000b6a0ee7a142d267c1f36714e4a8f75612f20a797200000000000000000000000000000000000000000000000000000000000000004c48b812bb43401392c037381aca934f4069c051700000000000000000000000000000000000000000000000000000000000000c71a642f0e3c3af545e7acbd38b07251b3990914f1010000003325a78425f17a7e487eb5666b2bfd93abb06c7002000000c48b812bb43401392c037381aca934f4069c051701000000" + }, + { + "name": "rejected_batch_after_drain", + "about": "Right after a frame drains, a replayed batch and a batch with decreasing safe blocks are rejected and drain nothing; the next valid batch drains the rest.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_carol", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 1, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "drain_batch", + "outcome": "batch_rejected", + "reject": {"reason": "wrong_nonce", "expected": 1, "got": 0}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 1, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "non_monotonic_batch", + "outcome": "batch_rejected", + "reject": {"reason": "non_monotonic_safe_blocks"}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 5, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 1, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 14, + "timestamp": 1700000168, + "payload": "drain_at_11_batch_1", + "outcome": "batch_accepted", + "txs": [{"direct": 1, "batch_nonce": 1}], + "outputs": ["deposit_notice_carol_500"], + "scheduler": {"l1_input_count": 6, "l2_tx_count": 2, "next_batch_nonce": 2, "clock": 11, "first_queued_input": 6, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c000000b400000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e83325a78425f17a7e487eb5666b2bfd93abb06c7000000000000000000000000000000000000000000000000000000000000001f4" + }, + { + "name": "empty_batch_at_any_age", + "about": "An empty batch is accepted at any age and spends its nonce; the backstop runs before it.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 11000, + "timestamp": 1700132000, + "payload": "empty_batch_0", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 0} + }, + { + "from": "sequencer", + "block": 11001, + "timestamp": 1700132012, + "payload": "empty_batch_0", + "outcome": "batch_rejected", + "reject": {"reason": "wrong_nonce", "expected": 1, "got": 0}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 3, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000001000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + { + "name": "later_frame_at_inclusion_block", + "about": "A batch whose first frame lies below its inclusion block but whose last frame sits at it is rejected; the same batch one block later drains both directs.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "frames_10_12_batch", + "outcome": "batch_rejected", + "reject": {"reason": "safe_block_not_below_inclusion_block"}, + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "frames_10_12_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 2, "next_batch_nonce": 1, "clock": 12, "first_queued_input": 4, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000002000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + }, + { + "name": "backstop_runs_before_a_dropped_direct", + "about": "The backstop runs before a direct the cap drops: the overdue direct executes though the arriving one never joins, and a later batch drains the rest.", + "steps": [ + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 1, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 15, + "timestamp": 1700000180, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 2, "l2_tx_count": 0, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "portal", + "block": 7210, + "timestamp": 1700086520, + "payload": "deposit_alice_over_cap", + "outcome": "direct_dropped", + "txs": [{"direct": 0, "batch_nonce": 0}], + "outputs": ["deposit_notice_alice"], + "scheduler": {"l1_input_count": 3, "l2_tx_count": 1, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 1, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 7212, + "timestamp": 1700086544, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "outputs": [], + "scheduler": {"l1_input_count": 4, "l2_tx_count": 1, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 1, "fridge_len": 2} + }, + { + "from": "sequencer", + "block": 7215, + "timestamp": 1700086580, + "payload": "drain_7212_batch", + "outcome": "batch_accepted", + "txs": [{"direct": 1, "batch_nonce": 0}, {"direct": 3, "batch_nonce": 0}], + "outputs": [], + "scheduler": {"l1_input_count": 5, "l2_tx_count": 3, "next_batch_nonce": 1, "clock": 7212, "first_queued_input": 5, "fridge_len": 0} + } + ], + "state": "0x3332de61a8bb9ac84893b2f552fe81c9a6dc541995d0c8a7d11342299807a2fc19ac44c2321ccc68a0ee7a142d267c1f36714e4a8f75612f20a797204c0000008000000003000000000000001a642f0e3c3af545e7acbd38b07251b3990914f100000000000000000000000000000000000000000000000000000000000003e8" + } + ] + }, + "scheduler_only": { + "about": "Scenarios for the scheduler alone, under config, whose fridge bound of 2 lets a few inputs overflow it; the wallet's bound of 16384 keeps overflow out of the guest scenarios. Senders, payloads, chain_id and app_contract are the guest section's. A step is as in a guest scenario without fates, outputs or l2_tx_count, and scheduler is the scheduler's state after the input. A resume case gives an exported state and the re-supplied range, each input with its InputBox index; result is resumed or refused, refusal names the reference's reason as a diagnostic, and then continues a resumed scheduler from the state's l1_input_count.", + "config": {"fridge_bound": 2, "max_direct_payload_bytes": 1024, "max_wait_blocks": 1200, "sequencer_address": "0xa0Ee7A142d267C1f36714E4a8F75612F20a79720"}, + "scenarios": [ + { + "name": "overflow_drains_the_oldest_whole_block", + "about": "A direct arriving at a full fridge drains every direct of the oldest block, more than it needs, and raises the clock; a frame at the raised clock is accepted.", + "steps": [ + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 1, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "portal", + "block": 10, + "timestamp": 1700000120, + "payload": "deposit_alice", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 2, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "scheduler": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 2, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 12, + "timestamp": 1700000144, + "payload": "drain_batch", + "outcome": "batch_accepted", + "txs": [], + "scheduler": {"l1_input_count": 4, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 2, "fridge_len": 1} + } + ] + }, + { + "name": "overflow_never_drains_the_arriving_block", + "about": "A direct at the block of every queued direct joins a full fridge, so the bound is soft by one block; the next block's direct drains them all.", + "steps": [ + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 1, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 2, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 3} + }, + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}, {"direct": 2, "batch_nonce": 0}], + "scheduler": {"l1_input_count": 4, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 3, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 5, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 3, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 3, "batch_nonce": 0}, {"direct": 4, "batch_nonce": 0}], + "scheduler": {"l1_input_count": 6, "next_batch_nonce": 0, "clock": 11, "first_queued_input": 5, "fridge_len": 1} + } + ] + }, + { + "name": "overflow_raises_the_clock_past_a_frame", + "about": "The clock an overflow drain raises rejects a batch whose first frame lies below it; a frame at it is accepted and drains nothing newer.", + "steps": [ + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 1, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 2, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "scheduler": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 11, "first_queued_input": 2, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "drain_batch", + "outcome": "batch_rejected", + "reject": {"reason": "first_frame_below_clock", "safe_block": 10, "clock": 11}, + "txs": [], + "scheduler": {"l1_input_count": 4, "next_batch_nonce": 0, "clock": 11, "first_queued_input": 2, "fridge_len": 1} + }, + { + "from": "sequencer", + "block": 14, + "timestamp": 1700000168, + "payload": "drain_at_11_batch", + "outcome": "batch_accepted", + "txs": [], + "scheduler": {"l1_input_count": 5, "next_batch_nonce": 1, "clock": 11, "first_queued_input": 2, "fridge_len": 1} + } + ] + }, + { + "name": "dropped_direct_never_overflows", + "about": "A direct over the payload cap arriving at a full fridge drains nothing; the next admitted direct does.", + "steps": [ + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 1, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 10, + "timestamp": 1700000120, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 2, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_alice_over_cap", + "outcome": "direct_dropped", + "txs": [], + "scheduler": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 0, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 0, "batch_nonce": 0}, {"direct": 1, "batch_nonce": 0}], + "scheduler": {"l1_input_count": 4, "next_batch_nonce": 0, "clock": 10, "first_queued_input": 3, "fridge_len": 1} + } + ] + } + ], + "resume": [ + { + "name": "resume_rebuilds_the_fridge", + "about": "The range's admitted directs refill the fridge, skipping a dropped direct; a frame drains them, and they count toward the bound.", + "state": {"l1_input_count": 4, "next_batch_nonce": 1, "clock": 10, "first_queued_input": 1, "fridge_len": 2}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + }, + { + "index": 2, + "from": "portal", + "block": 12, + "timestamp": 1700000144, + "payload": "deposit_alice_over_cap" + }, + { + "index": 3, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "resumed", + "then": [ + { + "from": "sequencer", + "block": 13, + "timestamp": 1700000156, + "payload": "drain_at_11_batch_1", + "outcome": "batch_accepted", + "txs": [{"direct": 1, "batch_nonce": 1}], + "scheduler": {"l1_input_count": 5, "next_batch_nonce": 2, "clock": 11, "first_queued_input": 3, "fridge_len": 1} + }, + { + "from": "stranger", + "block": 13, + "timestamp": 1700000156, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 6, "next_batch_nonce": 2, "clock": 11, "first_queued_input": 3, "fridge_len": 2} + }, + { + "from": "stranger", + "block": 14, + "timestamp": 1700000168, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [{"direct": 3, "batch_nonce": 2}], + "scheduler": {"l1_input_count": 7, "next_batch_nonce": 2, "clock": 12, "first_queued_input": 5, "fridge_len": 2} + } + ] + }, + { + "name": "resume_with_nothing_queued", + "about": "A state with nothing queued resumes from an empty range.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 3, "fridge_len": 0}, + "queued": [], + "result": "resumed", + "then": [ + { + "from": "stranger", + "block": 20, + "timestamp": 1700000240, + "payload": "stranger_direct", + "outcome": "direct_queued", + "txs": [], + "scheduler": {"l1_input_count": 4, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 3, "fridge_len": 1} + } + ] + }, + { + "name": "first_queued_after_count", + "about": "The first queued input lies past the input count.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 4, "fridge_len": 0}, + "queued": [], + "result": "refused", + "refusal": "first_queued_after_count" + }, + { + "name": "range_out_of_sequence", + "about": "The range does not start at the first queued input.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 2}, + "queued": [ + { + "index": 2, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "out_of_sequence" + }, + { + "name": "range_too_long", + "about": "The range runs to the input count and past it.", + "state": {"l1_input_count": 2, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 1}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + }, + { + "index": 2, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "beyond_input_count" + }, + { + "name": "range_too_short", + "about": "The range stops before the input count.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 2}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "missing_inputs" + }, + { + "name": "range_blocks_decrease", + "about": "A later input of the range lies at an earlier block.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 2}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + }, + { + "index": 2, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "block_decreases" + }, + { + "name": "first_queued_is_a_batch", + "about": "The first queued input is a batch.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 1}, + "queued": [ + { + "index": 1, + "from": "sequencer", + "block": 11, + "timestamp": 1700000132, + "payload": "empty_batch_0" + }, + { + "index": 2, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "first_queued_not_a_direct" + }, + { + "name": "first_queued_is_a_dropped_direct", + "about": "The first queued input is a direct over the payload cap.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 1}, + "queued": [ + { + "index": 1, + "from": "portal", + "block": 11, + "timestamp": 1700000132, + "payload": "deposit_alice_over_cap" + }, + { + "index": 2, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "first_queued_not_a_direct" + }, + { + "name": "queued_direct_at_the_clock", + "about": "A queued direct lies at the clock, so it should have executed.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 11, "first_queued_input": 1, "fridge_len": 2}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + }, + { + "index": 2, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "direct_not_above_clock" + }, + { + "name": "fridge_len_mismatch", + "about": "The range admits more directs than the state queues.", + "state": {"l1_input_count": 3, "next_batch_nonce": 0, "clock": 0, "first_queued_input": 1, "fridge_len": 1}, + "queued": [ + { + "index": 1, + "from": "stranger", + "block": 11, + "timestamp": 1700000132, + "payload": "stranger_direct" + }, + { + "index": 2, + "from": "stranger", + "block": 12, + "timestamp": 1700000144, + "payload": "stranger_direct" + } + ], + "result": "refused", + "refusal": "fridge_len_mismatch" + } + ] + } +} diff --git a/tests/fixtures/wallet_snapshot_empty.hex b/tests/fixtures/wallet_snapshot_empty.hex index 26ccd897..b882889d 100644 --- a/tests/fixtures/wallet_snapshot_empty.hex +++ b/tests/fixtures/wallet_snapshot_empty.hex @@ -1 +1 @@ -3332de61a8bb9ac84893b2f552fe81c9a6dc54191c7d4b196cb0c7b01d743fbc6116a902379c723879b88d2f598d8d001832d69f5201332a3f1e14c8540000005400000000000000000000000000000000000000 +3332de61a8bb9ac84893b2f552fe81c9a6dc54191c7d4b196cb0c7b01d743fbc6116a902379c723879b88d2f598d8d001832d69f5201332a3f1e14c84c0000004c0000000000000000000000 diff --git a/tests/harness/Cargo.toml b/tests/harness/Cargo.toml index a681a802..6b8ea853 100644 --- a/tests/harness/Cargo.toml +++ b/tests/harness/Cargo.toml @@ -19,8 +19,6 @@ futures-util = { workspace = true } k256 = { workspace = true } rusqlite = { workspace = true } reqwest = { workspace = true, features = ["json"] } -tar = { workspace = true } -toml = { workspace = true } sequencer-core = { path = "../../sequencer-core", features = ["test-support"] } sequencer-rust-client = { path = "../../sdk/rust-client" } serde = { workspace = true } diff --git a/tests/harness/src/lib.rs b/tests/harness/src/lib.rs index dd5461dc..5460c1b3 100644 --- a/tests/harness/src/lib.rs +++ b/tests/harness/src/lib.rs @@ -19,10 +19,10 @@ pub use replay::ReplayWalletApp; pub use rollups::{DEVNET_CHAIN_ID, DEVNET_SECONDS_PER_BLOCK, DevnetRollupsStack}; pub use sequencer::{ BatchCounts, BatchNonceFrontier, DEFAULT_DEVNET_SEQUENCER_BIN, DEFAULT_TEST_LOGS_DIR, - DEVNET_PREEMPTIVE_MARGIN_BLOCKS, ManagedSequencer, ManagedSequencerConfig, RecoveryCheckpoint, - RecoverySetupParams, RespawnAttemptOutcome, RespawnPolicy, StackChildExit, - default_c_wallet_sequencer_config, default_devnet_sequencer_config, - devnet_sequencer_config_no_faketime, + EXIT_TERMINAL, LandingHealth, ManagedSequencer, ManagedSequencerConfig, PublishedEra, + RebuiltHistoryRows, RunStart, SetupOutcome, StackChildExit, UnacceptedBatch, + batch_submitter_signer, default_c_wallet_sequencer_config, default_devnet_sequencer_config, + devnet_sequencer_config_no_faketime, encode_batch, is_terminal_abort, }; pub use wallet::{ TestSigner, WalletL1Client, WalletL2Client, address_from_signing_key, sign_user_op_hex, diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index e715bab8..e6631dbb 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -4,21 +4,24 @@ use alloy_primitives::{Address, U256}; use app_core::application::{WalletApp, WalletConfig}; use sequencer_core::api::WsTxMessage; -use sequencer_core::application::{ - Application, CanonicalState, execute_direct_input, execute_valid_user_op, -}; -use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::application::{Application, CanonicalState, Executed, execute}; +use sequencer_core::l2_tx::{Direct, L2Tx, UserOpTx}; use crate::HarnessResult; +use crate::util::io_other; pub struct ReplayWalletApp { app: WalletApp, + /// The application no longer keeps an L1 clock, so the mirror remembers + /// the block carried by the last message it applied. + last_applied_block: Option, } impl ReplayWalletApp { pub fn from_dump(prefix: &std::path::Path) -> HarnessResult { Ok(Self { app: WalletApp::from_dump(prefix)?, + last_applied_block: None, }) } @@ -29,21 +32,14 @@ impl ReplayWalletApp { pub fn devnet() -> Self { Self { app: WalletApp::new(WalletConfig::devnet()), + last_applied_block: None, } } pub fn apply(&mut self, message: WsTxMessage) -> HarnessResult<()> { - if let WsTxMessage::UserOp { sender, nonce, .. } = &message { - let sender = decode_address(sender.as_str()); - let expected = self.app.current_user_nonce(sender); - if *nonce != expected { - return Err(std::io::Error::other(format!( - "WS user-op nonce mismatch for {sender}: expected {expected}, got {nonce}" - )) - .into()); - } - } - apply_ws_message(&mut self.app, message) + let block = apply_ws_message(&mut self.app, message)?; + self.last_applied_block = Some(block); + Ok(()) } pub fn current_user_balance(&self, sender: Address) -> U256 { @@ -54,66 +50,105 @@ impl ReplayWalletApp { self.app.current_user_nonce(sender) } - pub fn executed_input_count(&self) -> u64 { - self.app.executed_input_count().get() + pub fn l2_tx_count(&self) -> u64 { + self.app.l2_tx_count().get() } - pub fn last_executed_safe_block(&self) -> u64 { - self.app.last_executed_safe_block() + /// The frame's safe block of the last applied user op, or the inclusion + /// block of the last applied direct; `None` until a message is applied. + pub fn last_applied_block(&self) -> Option { + self.last_applied_block } } -pub fn apply_ws_message(app: &mut A, message: WsTxMessage) -> HarnessResult<()> { - let expected = app.executed_input_count().get(); +/// Runs one feed message through the shared executor, as a replica would. +/// Returns the L1 block the message carries (the frame's safe block for a +/// user op, the inclusion block for a direct), so mirrors that need an L1 +/// clock can track it themselves. +pub fn apply_ws_message(app: &mut A, message: WsTxMessage) -> HarnessResult { + let expected = app.l2_tx_count().get(); if message.offset() != expected { - return Err(std::io::Error::other(format!( + return Err(io_other(format!( "WS history gap: expected input {expected}, got {}", message.offset() )) .into()); } - match message { + let tx = l2_tx_from_ws_message(&message)?; + let block = match &tx { + L2Tx::Direct(direct) => direct.block, + L2Tx::UserOp(op) => op.block, + }; + match execute(app, &tx)? { + Executed::Done { index, .. } => { + if index.get() != expected { + return Err(io_other(format!( + "executor reported index {}, expected {expected}", + index.get() + )) + .into()); + } + } + // The feed serves executed rows only, so a skip means the mirror and + // the sequencer disagree on a nonce. + Executed::Skipped(reason) => { + return Err(io_other(format!( + "WS input {expected} would be skipped by the mirror: {reason}" + )) + .into()); + } + } + Ok(block) +} + +/// Rebuilds the executor's record from a feed message; the wire names +/// `safe_block`, `block_number` and `block_timestamp` map to the record's +/// `block` and `timestamp`. +pub fn l2_tx_from_ws_message(message: &WsTxMessage) -> HarnessResult { + Ok(match message { WsTxMessage::DirectInput { sender, block_number, payload, + input_index, + block_timestamp, .. - } => { - execute_direct_input( - app, - &DirectInput { - sender: decode_address(sender.as_str()), - block_number, - payload: decode_hex_prefixed(payload.as_str()), - }, - )?; - } + } => L2Tx::Direct(Direct { + input_index: *input_index, + sender: decode_address(sender)?, + block: *block_number, + timestamp: *block_timestamp, + payload: decode_hex_prefixed(payload)?, + }), WsTxMessage::UserOp { sender, + nonce, fee, data, safe_block, .. - } => { - execute_valid_user_op( - app, - &ValidUserOp { - sender: decode_address(sender.as_str()), - fee, - data: decode_hex_prefixed(data.as_str()), - }, - safe_block, - )?; - } - } - Ok(()) + } => L2Tx::UserOp(UserOpTx { + sender: decode_address(sender)?, + nonce: *nonce, + fee: *fee, + block: *safe_block, + data: decode_hex_prefixed(data)?, + }), + }) } -fn decode_hex_prefixed(value: &str) -> Vec { - assert!(value.starts_with("0x"), "hex field must be 0x-prefixed"); - alloy_primitives::hex::decode(value).expect("decode hex") +fn decode_hex_prefixed(value: &str) -> HarnessResult> { + if !value.starts_with("0x") { + return Err(io_other(format!("WS hex field must be 0x-prefixed: {value}")).into()); + } + alloy_primitives::hex::decode(value) + .map_err(|err| io_other(format!("WS hex field {value}: {err}")).into()) } -fn decode_address(value: &str) -> Address { - Address::from_slice(&decode_hex_prefixed(value)) +fn decode_address(value: &str) -> HarnessResult

{ + let bytes = decode_hex_prefixed(value)?; + if bytes.len() != Address::len_bytes() { + return Err(io_other(format!("WS address has {} bytes", bytes.len())).into()); + } + Ok(Address::from_slice(&bytes)) } diff --git a/tests/harness/src/rollups.rs b/tests/harness/src/rollups.rs index ec375b9c..a880f372 100644 --- a/tests/harness/src/rollups.rs +++ b/tests/harness/src/rollups.rs @@ -156,6 +156,31 @@ impl DevnetRollupsStack { self.anvil.drop_all_pending_txs().await } + /// The account's ETH balance at the latest block. + pub async fn balance_of(&self, account: Address) -> HarnessResult { + let provider = ProviderBuilder::new() + .connect(self.anvil.endpoint.as_str()) + .await + .map_err(|err| io_other(format!("failed to connect anvil provider: {err}")))?; + provider.get_balance(account).await.map_err(|err| { + io_other(format!("failed to read the balance of {account}: {err}")).into() + }) + } + + /// Overwrite the account's ETH balance (`anvil_setBalance`). + pub async fn set_balance(&self, account: Address, balance: U256) -> HarnessResult<()> { + let provider = ProviderBuilder::new() + .connect(self.anvil.endpoint.as_str()) + .await + .map_err(|err| io_other(format!("failed to connect anvil provider: {err}")))?; + provider + .anvil_set_balance(account, balance) + .await + .map_err(|err| { + io_other(format!("failed to set the balance of {account}: {err}")).into() + }) + } + /// Wait until `sender` has a transaction in Anvil's mempool, i.e. its /// pending nonce is ahead of its latest one. pub async fn wait_for_pending_tx_from( @@ -300,12 +325,9 @@ impl ManagedAnvil { async fn mine_blocks(&self, block_count: u64) -> HarnessResult<()> { // L1 block-time coupling: each mined block advances the L1 // timestamp by `DEVNET_SECONDS_PER_BLOCK`, which - // `ManagedSequencer::advance_wall_and_mine` pairs with faketime. - // - // Without the explicit `interval`, anvil defaults to 1s/block — - // which then desyncs from faketime, making large advances trip - // spurious `L1ViewStale` even when wall clock and L1 should move - // together. See `ManagedSequencer::advance_wall_and_mine`. + // `ManagedSequencer::advance_wall_and_mine` pairs with faketime, so + // the safe head's age the sequencer reports follows the outage. + // Without the explicit `interval`, Anvil stamps 1 s per block. self.mine_blocks_with_interval(block_count, Some(DEVNET_SECONDS_PER_BLOCK)) .await } diff --git a/tests/harness/src/sequencer.rs b/tests/harness/src/sequencer.rs index 133bf7b1..da4cc41c 100644 --- a/tests/harness/src/sequencer.rs +++ b/tests/harness/src/sequencer.rs @@ -1,14 +1,16 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +use std::ffi::OsString; use std::fs::{self, OpenOptions}; -use std::io; +use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use std::time::Duration; use alloy::providers::{Provider, ProviderBuilder}; use alloy::rpc::types::Filter; +use alloy_primitives::{B256, U256}; use alloy_sol_types::SolEvent; use cartesi_rollups_contracts::input_box::InputBox::InputAdded; use sequencer_rust_client::SequencerClient; @@ -20,32 +22,51 @@ use crate::default_private_keys; use crate::paths; use crate::rollups::{DEVNET_CHAIN_ID, DEVNET_SECONDS_PER_BLOCK, DevnetRollupsStack}; use crate::util::{ - build_local_endpoint, io_other, path_as_str, send_graceful_terminate, timestamped_log_path, - wait_for_http_readiness, + Readiness, build_local_endpoint, io_other, path_as_str, send_graceful_terminate, + timestamped_log_path, wait_for_http_readiness, }; use crate::wallet::{TestSigner, WalletL1Client, WalletL2Client}; use crate::ws::WsClient; +/// A path under the data directory, with a file's length and keccak256, or +/// `None` for a directory. +pub type DataDirEntry = (String, Option<(u64, B256)>); + const DEFAULT_SEQUENCER_START_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(3); -/// Cadence at which the harness mines an L1 block during a recovery boot's -/// readiness wait when [`ManagedSequencer::set_mine_l1_during_boot`] is on. -/// One block per wall-clock second keeps the ordinary live-chain timestamp -/// coupled while still advancing Anvil's `safe` tag well within the recovery -/// readiness window. -const BOOT_L1_MINE_INTERVAL: Duration = Duration::from_secs(1); -/// Readiness budget for a *recovery* boot (mining-during-boot enabled). The startup -/// mempool flush polls `get_transaction_count(Safe)` once per L1 block time -/// (`safe_poll_interval = seconds_per_block`, 12 s here), so it cannot resolve -/// the stranded slot in under one poll cycle — well past the 10 s normal-boot -/// budget. Allow several cycles of slack so a recovery boot has room to flush, -/// cascade, and come up. Recovery boots are legitimately slow (exit-code class -/// 10: restart and expect recovery); this is the harness counterpart to that. -const RECOVERY_BOOT_START_TIMEOUT: Duration = Duration::from_secs(45); +/// Cadence at which the harness mines an L1 block while a setup runs when +/// [`ManagedSequencer::set_mine_l1_during_setup`] is on. One block per +/// wall-clock second keeps the live-chain timestamp coupled while still +/// advancing Anvil's `safe` tag well within [`FLUSHING_SETUP_TIMEOUT`]. +const SETUP_L1_MINE_INTERVAL: Duration = Duration::from_secs(1); +/// Budget for a setup that flushes stranded transactions: the flush polls +/// `get_transaction_count(Safe)` once per L1 block time (12 s here), so it +/// needs several poll cycles of slack. +const FLUSHING_SETUP_TIMEOUT: Duration = Duration::from_secs(45); +/// Budget for a setup from genesis: it syncs the chain and rebuilds the +/// history from its start, which on an aged devnet is hundreds of rows. +const GENESIS_SETUP_TIMEOUT: Duration = Duration::from_secs(30); +/// Budget for a setup from a stored machine: it loads the machine into the +/// emulator and answers two inspect queries before it rebuilds the history. +const BASE_SETUP_TIMEOUT: Duration = Duration::from_secs(120); const DEFAULT_SEQUENCER_RUST_LOG: &str = "info"; -/// Passed to `setup` and `run` as `--preemptive-margin-blocks`, so the e2e -/// zone math owns the value. -pub const DEVNET_PREEMPTIVE_MARGIN_BLOCKS: u64 = 300; +/// The data directory's name under the harness's root directory; each +/// [`ManagedSequencer::discard_data_dir`] moves on to the next numbered one. +const DATA_DIR_NAME: &str = "data"; +/// Anvil account the batch submitter signs with (NOT the deployer/funder, +/// account 0). Setup's escape check refuses while the submitter's wallet is +/// unsettled, so the submitter must not share a key with the deployer or with +/// an e2e wallet whose transactions are in flight. Must match +/// `DEVNET_SEQUENCER_ADDRESS` (app-core), which every spawn asserts. Account 9 +/// is the highest standard funded account, clear of the e2e wallets (0-3) and +/// the bench workload (indices 0..concurrency). +const DEVNET_SUBMITTER_ACCOUNT_INDEX: usize = 9; +/// The exit code of a command that must not be restarted (README "Running"): +/// among others, every start of `run` while a terminal fact holds. +pub const EXIT_TERMINAL: i32 = 30; +/// The signal a running sequencer raises on a terminal runtime fault, a +/// terminal fact included: it aborts instead of draining. +const SIGABRT: i32 = 6; pub const DEFAULT_DEVNET_SEQUENCER_BIN: &str = "target/debug/wallet-sequencer-devnet"; pub const DEFAULT_TEST_LOGS_DIR: &str = "tests/e2e/results"; @@ -53,7 +74,8 @@ pub const DEFAULT_TEST_LOGS_DIR: &str = "tests/e2e/results"; pub struct ManagedSequencerConfig { pub sequencer_bin: PathBuf, /// Optional genesis tool invoked as ` devnet` for - /// initial setup. Its output is deleted before `run` and never regenerated. + /// each setup without a base, whose `--state-file` it supplies. Its output + /// is deleted once setup ends, so `run` never depends on it. pub genesis_bin: Option, pub log_prefix: String, pub logs_dir: PathBuf, @@ -70,73 +92,98 @@ pub struct ManagedSequencerConfig { pub struct BatchCounts { pub total: u64, pub sealed: u64, - pub invalidated: u64, } /// Highest batch nonce sealed locally and highest accepted on safe L1. /// Returned by [`ManagedSequencer::batch_nonce_frontier`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BatchNonceFrontier { - /// Highest nonce among valid sealed batches. + /// Highest nonce among sealed batches. pub sealed: Option, - /// Highest nonce in `safe_accepted_batches`. + /// Highest accepted batch nonce (`batches.accepted_input` set). pub safe_accepted: Option, } -/// Drive the next respawn's `setup` phase into `--recovery` mode against a -/// captured checkpoint. Set via [`ManagedSequencer::set_recovery_setup`] before -/// wiping the DB and respawning. -#[derive(Debug, Clone)] -pub struct RecoverySetupParams { - /// `B` — the checkpoint's L1 inclusion block (`--checkpoint-block`). - pub checkpoint_block: u64, - /// The captured checkpoint dump dir (`--checkpoint-dump-dir`). - pub checkpoint_dump_dir: PathBuf, +/// The era a setup published, read from the `era` row. Returned by +/// [`ManagedSequencer::published_era`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PublishedEra { + /// The base's L2 tx count, where the era's history starts. + pub base_count: u64, + /// The stop block: where the rebuilt history ends. + pub stop_block: u64, + /// The L2 tx count at the end of the stop block. + pub rebuilt_history_end: u64, + /// InputBox index of the oldest direct still queued at the stop block, + /// or the input count there when none is queued. + pub first_queued_input: u64, + /// The scheduler's next batch nonce at the stop block (the anchor). + pub next_batch_nonce: u64, } -/// A checkpoint captured from a running sequencer's finalized snapshot, ready to -/// drive `setup --recovery`. Returned by -/// [`ManagedSequencer::capture_finalized_checkpoint`]. -#[derive(Debug, Clone)] -pub struct RecoveryCheckpoint { - /// The copied dump dir (lives outside the DB/`dumps` reset scope, so it - /// survives [`ManagedSequencer::reset_database`]). - pub dir: PathBuf, - /// `B` — the finalized snapshot's L1 inclusion block. - pub checkpoint_block: u64, - /// `N` — the snapshot's recorded resume nonce (`info.toml next_batch_nonce`). - pub resume_nonce: u64, +/// The rows setup's rebuild stored, and how many carry a count. Only an +/// executed transaction is stored, with its count, so the two agree. +/// Returned by [`ManagedSequencer::rebuilt_history_rows`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RebuiltHistoryRows { + pub total: u64, + pub executed: u64, } -/// Outcome of a single [`ManagedSequencer::respawn_and_watch`] attempt. +/// How a `setup` child ended. Setup prints its summary on stdout; the log +/// file gets stdout and stderr both. #[derive(Debug)] -pub enum RespawnAttemptOutcome { - /// The child came up and stayed alive for the requested stabilization - /// window. - Stable, - /// `respawn()` itself returned `Err` — the child exited during bootstrap - /// before HTTP became ready. Typically surfaces - /// `RecoveryError::Refuse(...)` from the startup decision table. - RespawnFailed(String), - /// `respawn()` returned `Ok` but the child exited within the - /// stabilization window. Typically surfaces a danger-detector worker - /// exit from the first post-boot poll. - ExitedPostRespawn(std::process::ExitStatus), +pub struct SetupOutcome { + pub status: ExitStatus, + pub stdout: String, } -impl RespawnAttemptOutcome { - pub fn is_stable(&self) -> bool { - matches!(self, Self::Stable) +impl SetupOutcome { + pub fn code(&self) -> Option { + self.status.code() + } + + pub fn success(&self) -> bool { + self.status.success() } } -/// Parameters for [`ManagedSequencer::respawn_until_stable`]. See that -/// method's doc for how `advance_per_retry` interacts with the restart cycle. -#[derive(Debug, Clone)] -pub struct RespawnPolicy { - pub max_attempts: u32, - pub stabilization: Duration, - pub advance_per_retry: Option, +/// How a start of `run` ended: it reached readiness, or the process exited +/// first. Returned by [`ManagedSequencer::try_respawn`]. +#[derive(Debug)] +pub enum RunStart { + Ready, + Exited(ExitStatus), +} + +/// Whether a running sequencer stopped on a terminal runtime fault: it +/// aborts (SIGABRT) instead of draining. +pub fn is_terminal_abort(status: &ExitStatus) -> bool { + use std::os::unix::process::ExitStatusExt; + status.signal() == Some(SIGABRT) +} + +/// The landing reading `GET /healthz` serves. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)] +pub struct LandingHealth { + /// `H + 1 - s` for the oldest unaccepted batch: its age at the earliest + /// block it can still land in, void at `max_wait_blocks`. `None` when + /// every batch is accepted or before the first reading. + pub landing_lag_blocks: Option, + pub max_wait_blocks: u64, + /// Wall clock minus the safe block's L1 timestamp; `None` before the + /// first reading. + pub safe_head_age_seconds: Option, +} + +/// The oldest batch L1 has not accepted, closed or open. Returned by +/// [`ManagedSequencer::oldest_unaccepted_batch`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnacceptedBatch { + pub nonce: u64, + /// Its first frame's safe block: the batch is stale on chain once it + /// lands this many blocks plus the window later. + pub first_frame_safe_block: u64, } /// Which child in the devnet stack exited during [`ManagedSequencer::observe_stack_for`]. @@ -151,10 +198,22 @@ pub struct ManagedSequencer { child: Child, shutdown_timeout: Duration, sequencer_bin: PathBuf, + genesis_bin: Option, log_prefix: String, logs_dir: PathBuf, - _data_dir: TempDir, + /// Holds the data directories and the faketime rc file. The data + /// directory is a child of it, never the root itself, because setup + /// refuses a directory that holds anything. + _root_dir: TempDir, data_dir_path: PathBuf, + /// Data directories discarded so far; names the next one. + discarded_data_dirs: u32, + /// When `true`, the next [`Self::respawn`] runs `setup` before `run`: the + /// data directory is new. Cleared once a setup succeeds there. + setup_pending: bool, + /// The `--base` machine directory setup starts from; `None` is genesis. + /// See [`Self::set_setup_base`]. + setup_base: Option, endpoint: String, internal_endpoint: String, log_path: PathBuf, @@ -173,19 +232,14 @@ pub struct ManagedSequencer { /// Path to the file libfaketime re-reads for its offset. `None` when /// faketime is disabled. faketime_rc_path: Option, - /// Internal cumulative forward-offset tracker for - /// [`Self::advance_wall_and_mine`]. Not touched by - /// [`Self::set_faketime_offset`]. + /// The cumulative forward offset [`Self::advance_wall_and_mine`] wrote. cumulative_offset_secs: u64, - /// When `true`, [`Self::respawn`] mines L1 blocks concurrently with the - /// post-boot readiness wait so a recovery boot's mempool flush can make - /// progress. See [`Self::set_mine_l1_during_boot`]. Persists across - /// respawns like the other overrides. - mine_l1_during_boot: bool, - /// When `Some`, [`Self::respawn`] runs the `setup` phase in `--recovery` - /// mode against this checkpoint (and mines L1 concurrently with setup so - /// its flush can settle). See [`Self::set_recovery_setup`]. - recovery_setup: Option, + /// When `true`, every setup mines L1 blocks while it runs so its flush + /// can make progress. See [`Self::set_mine_l1_during_setup`]. + mine_l1_during_setup: bool, + /// The submitter's balance while [`Self::hold_batch_submissions`] holds + /// it at zero; restored by [`Self::release_batch_submissions`]. + held_submitter_balance: Option, /// When `Some(secs)`, passes `--max-batch-open-seconds` to the `run` phase /// so the inclusion lane seals the open batch promptly (the default is 2h). /// Used by tests that need a batch to close + be promoted. Persists across @@ -241,13 +295,14 @@ impl ManagedSequencer { let rollups = DevnetRollupsStack::spawn(log_prefix.as_str(), logs_dir.as_path()).await?; fs::create_dir_all(logs_dir.as_path())?; - let data_dir = TempDir::new() + let root_dir = TempDir::new() .map_err(|err| io_other(format!("failed to create temp data dir: {err}")))?; - let data_dir_path = data_dir.path().to_path_buf(); + // Setup creates the data directory itself. + let data_dir_path = root_dir.path().join(DATA_DIR_NAME); let (libfaketime_path, faketime_rc_path) = if config.faketime { let libfaketime_path = find_libfaketime()?; - let faketime_rc_path = data_dir_path.join("faketime.rc"); + let faketime_rc_path = root_dir.path().join("faketime.rc"); fs::write(faketime_rc_path.as_path(), "+0\n") .map_err(|err| io_other(format!("create faketime rc file: {err}")))?; (Some(libfaketime_path), Some(faketime_rc_path)) @@ -255,42 +310,50 @@ impl ManagedSequencer { (None, None) }; + let child_env = ChildEnv { + bin: sequencer_bin.as_path(), + data_dir: data_dir_path.as_path(), + rollups: &rollups, + eth_rpc_url: rollups.l1_endpoint(), + faketime: libfaketime_path.as_deref().zip(faketime_rc_path.as_deref()), + }; + let log_path = timestamped_log_path(logs_dir.as_path(), log_prefix.as_str()); + let setup = SetupRequest { + chain_id: DEVNET_CHAIN_ID, + base: None, + dry_run: false, + genesis_bin: genesis_bin.as_deref(), + fee_oracle_fixed_log_gas_price: config.fee_oracle_fixed_log_gas_price, + }; + // A first setup from genesis finds a settled wallet, so its flush + // never waits on L1. + require_setup_success( + run_setup_process(&child_env, &setup, &log_path, false).await?, + &log_path, + )?; let SpawnedSequencerProcess { child, endpoint, internal_endpoint, - log_path, - } = spawn_sequencer_process( - sequencer_bin.as_path(), - log_prefix.as_str(), - logs_dir.as_path(), - data_dir_path.as_path(), - &rollups, - None, - None, - libfaketime_path.as_deref(), - faketime_rc_path.as_deref(), - // First boot has no stranded nonce to flush, so it never needs L1 - // to advance during readiness. - false, - // First boot is always genesis setup, never recovery. - None, - // Default batch-open deadline on first boot. - None, - config.fee_oracle_fixed_log_gas_price, - genesis_bin.as_deref(), - ) - .await?; + } = require_ready( + // The default batch-open deadline on first boot. + spawn_run_process(&child_env, &log_path, None).await?, + &log_path, + )?; Ok(Self { rollups, child, shutdown_timeout: DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT, sequencer_bin, + genesis_bin, log_prefix, logs_dir, - _data_dir: data_dir, + _root_dir: root_dir, data_dir_path, + discarded_data_dirs: 0, + setup_pending: false, + setup_base: None, endpoint, internal_endpoint, log_path, @@ -299,8 +362,8 @@ impl ManagedSequencer { faketime_rc_path, libfaketime_path, cumulative_offset_secs: 0, - mine_l1_during_boot: false, - recovery_setup: None, + mine_l1_during_setup: false, + held_submitter_balance: None, max_batch_open_seconds: None, fee_oracle_fixed_log_gas_price: config.fee_oracle_fixed_log_gas_price, }) @@ -321,109 +384,108 @@ impl ManagedSequencer { /// /// Used by chain-id mismatch tests to inject a mismatched chain id and /// assert that bootstrap refuses before silently pinning a wrong identity. - /// NOTE (post setup/run split): `--chain-id` now flows only to `setup`, - /// which is idempotent and early-returns once the `setup_complete` marker - /// exists — so the override only takes effect on a fresh or - /// [`Self::reset_database`]-d data dir. On an already-set-up DB, `setup` - /// is a no-op and `run` validates against the *pinned* chain id, so the - /// override is silently inert. Does not affect a running process. + /// `--chain-id` flows only to `setup`, which runs only in a new data + /// directory ([`Self::discard_data_dir`]); `run` validates against the + /// pinned chain id, so on a set-up directory the override is inert. Does + /// not affect a running process. pub fn set_chain_id_override(&mut self, chain_id: Option) { self.chain_id_override = chain_id; } - /// Make [`Self::respawn`] (this one and subsequent ones, until cleared) - /// mine L1 blocks concurrently with the post-boot readiness wait. - /// - /// Why this exists: a *recovery* boot whose persisted state names a wallet - /// nonce that L1 never accepted — e.g. a batch-submission tx was dropped - /// from the mempool ([`Self::drop_all_pending_txs`]) — runs the mempool - /// flush during startup. The flush submits a no-op at the stranded - /// wallet-nonce slot and blocks until that slot becomes *safe* - /// (`safe_wallet_nonce >= watermark + 1`, I14). Re-enabling auto-mining is not enough: - /// auto-mining lands the no-op tx but mints no *further* blocks, so Anvil's - /// `safe` tag never advances past it and the flush waits until the boot - /// hits the readiness timeout. In production the chain keeps producing - /// blocks throughout a slow recovery boot; an on-demand-mining devnet - /// produces none. This makes the harness mine a steady trickle for the - /// boot window (dropped the instant readiness is reached), reproducing that - /// drift so the flush — and the cascade behind it — can complete on the - /// first respawn. - /// - /// Only the tests that deliberately strand a submitted nonce before a - /// recovery respawn need this. Leave it off otherwise: every other boot - /// reaches readiness without new blocks, and mining empty blocks during an - /// unrelated boot would perturb L1-block-sensitive assertions. - pub fn set_mine_l1_during_boot(&mut self, enabled: bool) { - self.mine_l1_during_boot = enabled; - } - - /// Write a faketime offset to the rc file. Effective **immediately** for - /// the running sequencer (if any) and persists across respawns. The - /// libfaketime library re-reads the file on every time call (we pass - /// `FAKETIME_NO_CACHE=1`), so the next `SystemTime::now()` inside the - /// child sees the new offset. - /// - /// Format follows faketime's `-f` flag: `"+5h"`, `"-1h"`, `"+1d"`, or - /// `"+NNNs"` for absolute seconds. Passing `None` resets to `+0`. - /// See `man faketime` for advanced options (speed-up, interval mode). - /// - /// Does not mine L1 blocks — use [`Self::advance_wall_and_mine`] when you - /// want wall-clock and L1 to move together. - /// - /// Replaces any cumulative advance tracked by - /// [`Self::advance_wall_and_mine`], and resets its counter. - pub fn set_faketime_offset(&mut self, offset: Option) -> HarnessResult<()> { - let rc = self - .faketime_rc_path - .as_ref() - .ok_or_else(|| io_other("faketime is disabled for this ManagedSequencer"))?; - let s = offset.as_deref().unwrap_or("+0"); - fs::write(rc.as_path(), format!("{s}\n")) - .map_err(|err| io_other(format!("write faketime rc file: {err}")))?; - self.cumulative_offset_secs = 0; - Ok(()) + /// The machine directory the next setup starts from (`setup --base`); + /// `None` is genesis. Takes effect on the next setup only, which runs in a + /// new data directory ([`Self::discard_data_dir`]). Persists until changed. + pub fn set_setup_base(&mut self, base: Option) { + self.setup_base = base; } - /// Delete the sequencer DB file (and its `-wal` / `-shm` siblings) AND the - /// `dumps/` subtree, simulating a brand-new install — no pinned identity, - /// no batches, no safe-input rows, no genesis snapshot. Call while the - /// sequencer is stopped. (Clearing the DB but not `dumps/` would leave a - /// stale `genesis-*` dir; the next `setup` mints a fresh one and the old - /// one lingers until run's orphan sweep — harmless, but not "brand-new".) + /// Make every setup (until cleared) mine L1 blocks while it runs. /// - /// Distinct from "clear only the identity row": that would leave the DB - /// holding `batches` / `safe_inputs` / `l1_safe_head` rows from prior - /// boots, which `IdentityError::OrphanedState` then refuses on the next - /// `setup`. The right "first boot" simulation is to start from an empty DB. + /// Setup flushes the submitter's wallet and waits until the flushed + /// slots are *safe*. Auto-mining lands a flush transaction but mints no + /// further blocks, so on an on-demand-mining devnet Anvil's `safe` tag + /// never passes it; a live chain keeps producing blocks. Only a setup + /// that follows an instance with stranded submissions needs this: mining + /// during an unrelated setup would perturb block-sensitive assertions. + pub fn set_mine_l1_during_setup(&mut self, enabled: bool) { + self.mine_l1_during_setup = enabled; + } + + /// Move to a new, empty data directory, as an operator does for a new + /// deployment, after a failed setup, or for a recovery: the next + /// [`Self::respawn`] runs `setup` there before `run`. Call while the + /// sequencer is stopped. The old directory is left as it was, and nothing + /// runs against it again; it is removed when the harness drops. /// - /// Used by the no-cache-bootstrap and live-RPC chain-id-mismatch tests: - /// both want to exercise the no-cached-identity bootstrap path (next boot - /// re-runs `setup`). - pub fn reset_database(&self) -> HarnessResult<()> { - let db_path = self.data_dir_path.join("sequencer.db"); - for suffix in ["", "-wal", "-shm"] { - let path = db_path.with_extension(format!("db{suffix}")); - match fs::remove_file(path.as_path()) { - Ok(()) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => {} - Err(err) => return Err(io_other(format!("reset DB ({path:?}): {err}")).into()), - } + /// Setup refuses a directory that holds anything, so this is the only way + /// to set up again: clearing files inside a used directory would leave + /// whatever the clearing missed for setup to refuse. + pub fn discard_data_dir(&mut self) { + self.discarded_data_dirs += 1; + self.data_dir_path = self + ._root_dir + .path() + .join(format!("{DATA_DIR_NAME}-{}", self.discarded_data_dirs)); + self.setup_pending = true; + } + + /// Run `setup` in the current data directory without starting `run`, and + /// report how it ended instead of failing on a non-zero exit. A success + /// clears the pending setup, so the next [`Self::respawn`] only runs. Used + /// to assert setup's exit codes, such as its refusal of a set-up directory. + pub async fn run_setup(&mut self) -> HarnessResult { + let log_path = timestamped_log_path(self.logs_dir.as_path(), self.log_prefix.as_str()); + let outcome = run_setup_process( + &self.child_env(), + &self.setup_request(false), + &log_path, + self.mine_l1_during_setup, + ) + .await?; + if outcome.success() { + self.setup_pending = false; } - let dumps_dir = self.data_dir_path.join("dumps"); - match fs::remove_dir_all(dumps_dir.as_path()) { - Ok(()) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => {} - Err(err) => return Err(io_other(format!("reset dumps ({dumps_dir:?}): {err}")).into()), + Ok(outcome) + } + + /// `setup --dry-run` against the configured base: it needs no key and + /// never touches the data directory, so it may run at any time, with the + /// sequencer up or down. + pub async fn setup_dry_run(&self) -> HarnessResult { + let log_path = timestamped_log_path(self.logs_dir.as_path(), self.log_prefix.as_str()); + run_setup_process( + &self.child_env(), + &self.setup_request(true), + &log_path, + false, + ) + .await + } + + fn child_env(&self) -> ChildEnv<'_> { + ChildEnv { + bin: self.sequencer_bin.as_path(), + data_dir: self.data_dir_path.as_path(), + rollups: &self.rollups, + eth_rpc_url: self + .l1_endpoint_override + .as_deref() + .unwrap_or_else(|| self.rollups.l1_endpoint()), + faketime: self + .libfaketime_path + .as_deref() + .zip(self.faketime_rc_path.as_deref()), } - Ok(()) } - /// Drive the next [`Self::respawn`]'s `setup` phase into `--recovery` against - /// `params` (and mine L1 concurrently with setup so its flush can settle). - /// Set this after capturing a checkpoint and wiping the DB. `None` restores - /// the normal (genesis) setup. Persists across respawns until cleared. - pub fn set_recovery_setup(&mut self, params: Option) { - self.recovery_setup = params; + fn setup_request(&self, dry_run: bool) -> SetupRequest<'_> { + SetupRequest { + chain_id: self.chain_id_override.unwrap_or(DEVNET_CHAIN_ID), + base: self.setup_base.as_deref(), + dry_run, + genesis_bin: self.genesis_bin.as_deref(), + fee_oracle_fixed_log_gas_price: self.fee_oracle_fixed_log_gas_price, + } } /// Pass `--max-batch-open-seconds` to the `run` phase on the next (and @@ -434,8 +496,9 @@ impl ManagedSequencer { self.max_batch_open_seconds = secs; } - /// The currently exportable canonical checkpoint's L1 block. A rebuilt - /// optimistic baseline returns `None` until a matching batch is accepted. + /// The currently exportable canonical checkpoint's L1 block. In a new era + /// it is the baseline's stop block until the era's first batch is + /// accepted. pub async fn finalized_inclusion_block(&self) -> HarnessResult> { let response = reqwest::Client::new() .get(format!( @@ -467,7 +530,7 @@ impl ManagedSequencer { .map_err(|err| io_other(format!("open DB read-only: {err}")))?; let fee: i64 = conn .query_row( - "SELECT fee FROM frames ORDER BY batch_index, frame_in_batch LIMIT 1", + "SELECT fee FROM frames ORDER BY batch_nonce, frame_in_batch LIMIT 1", [], |row| row.get(0), ) @@ -490,8 +553,8 @@ impl ManagedSequencer { let (frame_safe_block, persisted_safe_head): (i64, i64) = conn .query_row( "SELECT f.safe_block, h.block_number \ - FROM valid_open_batch b \ - JOIN frames f ON f.batch_index = b.batch_index \ + FROM open_batch b \ + JOIN frames f ON f.batch_nonce = b.nonce \ JOIN l1_safe_head h ON h.singleton_id = 0 \ ORDER BY f.frame_in_batch DESC LIMIT 1", [], @@ -511,98 +574,84 @@ impl ManagedSequencer { Ok((frame_safe_block, persisted_safe_head)) } - /// Download the leased accepted-checkpoint export, including its coherent - /// `checkpoint.toml` receipt, outside the paths cleared by reset_database. - pub async fn capture_finalized_checkpoint(&self) -> HarnessResult { - let response = reqwest::Client::new() - .get(format!("{}/finalized_snapshot", self.internal_endpoint())) - .timeout(Duration::from_secs(30)) - .send() - .await? - .error_for_status()?; - let inclusion_block: u64 = response - .headers() - .get("X-Inclusion-Block") - .ok_or_else(|| io_other("checkpoint export missing X-Inclusion-Block"))? - .to_str()? - .parse()?; - let bytes = response.bytes().await?; - let dst = self.data_dir_path.join("checkpoint"); - if dst.exists() { - fs::remove_dir_all(&dst)?; - } - fs::create_dir_all(&dst)?; - tar::Archive::new(bytes.as_ref()).unpack(&dst)?; - #[derive(serde::Deserialize)] - struct Receipt { - inclusion_block: u64, - next_batch_nonce: u64, - } - let receipt: Receipt = toml::from_str(&fs::read_to_string(dst.join("checkpoint.toml"))?)?; - let resume_nonce = read_info_next_batch_nonce(&dst)?; - assert_eq!( - receipt.inclusion_block, inclusion_block, - "checkpoint receipt/header boundary mismatch" - ); - assert_eq!( - receipt.next_batch_nonce, resume_nonce, - "checkpoint receipt/artifact nonce mismatch" - ); - Ok(RecoveryCheckpoint { - dir: dst, - checkpoint_block: inclusion_block, - resume_nonce, + /// The oldest batch L1 has not accepted, closed or open, and its first + /// frame's safe block, from one read-only snapshot; `None` when every + /// batch is accepted. + pub fn oldest_unaccepted_batch(&self) -> HarnessResult> { + use rusqlite::OptionalExtension; + let conn = self.read_only_connection()?; + let row: Option<(i64, i64)> = conn + .query_row( + "SELECT b.nonce, \ + (SELECT f.safe_block FROM frames f \ + WHERE f.batch_nonce = b.nonce \ + ORDER BY f.frame_in_batch LIMIT 1) \ + FROM batches b \ + WHERE b.accepted_input IS NULL \ + ORDER BY b.nonce LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|err| io_other(format!("read the oldest unaccepted batch: {err}")))?; + row.map(|(nonce, first_frame_safe_block)| { + Ok(UnacceptedBatch { + nonce: u64::try_from(nonce) + .map_err(|_| io_other(format!("negative batch nonce {nonce}")))?, + first_frame_safe_block: u64::try_from(first_frame_safe_block).map_err(|_| { + io_other(format!( + "negative first-frame safe block {first_frame_safe_block}" + )) + })?, + }) }) + .transpose() } - /// Rewrite the L1 safe-head observation to "unknown", simulating a DB - /// that has never successfully synced from L1. Call while the sequencer - /// is stopped. - /// - /// Used by the wall-clock-never-synced test: the danger check treats a - /// missing safe-head row as an unusable L1 view and refuses to - /// proceed. Deleting this row while the deployment identity is - /// populated lets us hit that branch without losing the pinned chain - /// ID / InputBox address (which would fail earlier in bootstrap, not - /// in the wall-clock fallback). - pub fn clear_l1_safe_head_observation(&self) -> HarnessResult<()> { - let db_path = self.data_dir_path.join("sequencer.db"); - let conn = rusqlite::Connection::open(db_path.as_path()) - .map_err(|err| io_other(format!("open DB: {err}")))?; - conn.execute("DELETE FROM l1_safe_head WHERE singleton_id = 0", []) - .map_err(|err| io_other(format!("reset L1 safe-head observation: {err}")))?; - Ok(()) - } - - /// Read-only snapshot of the `safe_accepted_batches` view: rows - /// recovered from the L1-side scheduler frontier (i.e., batches the - /// sequencer has *observed accepted on chain*). Returns `(count, - /// min_nonce)` — count is the row count, min_nonce is `MIN(nonce)` or - /// `None` if empty. - /// - /// Used by the nonce-0 recovery test to confirm a recovery batch (which reuses nonce 0) - /// actually lands and gets accepted on L1 — proving the - /// `populate_safe_accepted_batches` cursor handles - /// reused-nonce-after-cascade correctly. - pub fn count_safe_accepted_batches(&self) -> HarnessResult<(u64, Option)> { - let db_path = self.data_dir_path.join("sequencer.db"); - let conn = rusqlite::Connection::open_with_flags( - db_path.as_path(), - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .map_err(|err| io_other(format!("open DB read-only: {err}")))?; - - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM safe_accepted_batches", [], |row| { - row.get(0) - }) - .map_err(|err| io_other(format!("count safe_accepted_batches: {err}")))?; - let min_nonce: Option = conn - .query_row("SELECT MIN(nonce) FROM safe_accepted_batches", [], |row| { - row.get(0) - }) - .map_err(|err| io_other(format!("min nonce: {err}")))?; - Ok((count as u64, min_nonce.map(|n| n as u64))) + /// The accepted batches as `(nonce, accepted safe input)`, by nonce. + pub fn accepted_batches(&self) -> HarnessResult> { + let conn = self.read_only_connection()?; + let mut statement = conn + .prepare( + "SELECT nonce, accepted_input FROM batches \ + WHERE accepted_input IS NOT NULL ORDER BY nonce", + ) + .map_err(|err| io_other(format!("prepare accepted batches: {err}")))?; + let rows = statement + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) + .map_err(|err| io_other(format!("read accepted batches: {err}")))? + .collect::>>() + .map_err(|err| io_other(format!("read accepted batches: {err}")))?; + Ok(rows + .into_iter() + .map(|(nonce, input)| (nonce as u64, input as u64)) + .collect()) + } + + /// The causes the terminal-fault black box recorded, oldest first. A + /// terminal fact records none, at a refused start or at a runtime abort. + pub fn terminal_fault_causes(&self) -> HarnessResult> { + let conn = self.read_only_connection()?; + let mut statement = conn + .prepare("SELECT cause FROM terminal_faults ORDER BY fault_id") + .map_err(|err| io_other(format!("prepare terminal faults: {err}")))?; + let causes = statement + .query_map([], |row| row.get::<_, String>(0)) + .map_err(|err| io_other(format!("read terminal faults: {err}")))? + .collect::>>() + .map_err(|err| io_other(format!("read terminal faults: {err}")))?; + Ok(causes) + } + + /// The landing reading `GET /healthz` serves on the internal listener, + /// whatever its status code. + pub async fn landing_health(&self) -> HarnessResult { + let response = reqwest::Client::new() + .get(format!("{}/healthz", self.internal_endpoint())) + .timeout(Duration::from_secs(5)) + .send() + .await?; + Ok(response.json().await?) } /// Read-only; safe to call while the sequencer is running. @@ -614,17 +663,16 @@ impl ManagedSequencer { .map_err(|err| io_other(format!("open DB read-only: {err}")))?; let sealed: Option = conn + .query_row("SELECT MAX(nonce) FROM closed_batches", [], |row| { + row.get(0) + }) + .map_err(|err| io_other(format!("max sealed batch nonce: {err}")))?; + let safe_accepted: Option = conn .query_row( - "SELECT MAX(nonce) FROM batches \ - WHERE sealed_at_ms IS NOT NULL AND invalidated_at_ms IS NULL", + "SELECT MAX(nonce) FROM batches WHERE accepted_input IS NOT NULL", [], |row| row.get(0), ) - .map_err(|err| io_other(format!("max sealed batch nonce: {err}")))?; - let safe_accepted: Option = conn - .query_row("SELECT MAX(nonce) FROM safe_accepted_batches", [], |row| { - row.get(0) - }) .map_err(|err| io_other(format!("max safe-accepted batch nonce: {err}")))?; Ok(BatchNonceFrontier { sealed: sealed.map(|n| n as u64), @@ -632,10 +680,11 @@ impl ManagedSequencer { }) } - /// The batch-tree anchor nonce, read from the run DB read-only: `N'` after - /// cockroach recovery, `0` for a genesis deployment. Lets a recovery e2e - /// assert the rebuilt tree is rooted at the checkpoint's resume nonce (I16). - pub fn batch_tree_anchor(&self) -> HarnessResult { + /// The era's anchor nonce (`anchor_batch_nonce`), read from the run DB + /// read-only: the resume nonce of a rebuilt deployment, `0` for a genesis + /// one. The batch nonces are contiguous from it (I16). Fails when the era + /// is absent. + pub fn anchor_batch_nonce(&self) -> HarnessResult { let db_path = self.data_dir_path.join("sequencer.db"); let conn = rusqlite::Connection::open_with_flags( db_path.as_path(), @@ -644,81 +693,80 @@ impl ManagedSequencer { .map_err(|err| io_other(format!("open DB read-only: {err}")))?; let anchor: i64 = conn .query_row( - "SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0", + "SELECT anchor_batch_nonce FROM era WHERE singleton_id = 0", [], |row| row.get(0), ) - .map_err(|err| io_other(format!("read batch_tree_anchor: {err}")))?; + .map_err(|err| io_other(format!("read the era's anchor nonce: {err}")))?; Ok(anchor as u64) } - /// The rebuilt baseline's L1 stop block `C` (`history_state.base_safe_block`), - /// read from the run DB read-only. - pub fn baseline_safe_block(&self) -> HarnessResult { - let db_path = self.data_dir_path.join("sequencer.db"); - let conn = rusqlite::Connection::open_with_flags( - db_path.as_path(), - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .map_err(|err| io_other(format!("open DB read-only: {err}")))?; - let block: i64 = conn + /// The era setup published, from one read-only snapshot. + pub fn published_era(&self) -> HarnessResult { + let conn = self.read_only_connection()?; + let (base_count, stop_block, rebuilt_history_end, first_queued_input, next_batch_nonce): ( + i64, + i64, + i64, + i64, + i64, + ) = conn .query_row( - "SELECT base_safe_block FROM history_state WHERE singleton_id = 0", + "SELECT base_l2_tx_count, stop_block, rebuilt_history_end, \ + first_queued_input, anchor_batch_nonce \ + FROM era WHERE singleton_id = 0", [], - |row| row.get(0), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) - .map_err(|err| io_other(format!("read history_state: {err}")))?; - Ok(block as u64) + .map_err(|err| io_other(format!("read the published era: {err}")))?; + let unsigned = |name: &str, value: i64| { + u64::try_from(value).map_err(|_| io_other(format!("{name} is negative: {value}"))) + }; + Ok(PublishedEra { + base_count: unsigned("base count", base_count)?, + stop_block: unsigned("stop block", stop_block)?, + rebuilt_history_end: unsigned("rebuilt history end", rebuilt_history_end)?, + first_queued_input: unsigned("first queued input", first_queued_input)?, + next_batch_nonce: unsigned("next batch nonce", next_batch_nonce)?, + }) } - /// Nonces of this app's batch submissions from the devnet submitter included - /// in L1 blocks `(after_block, through_block]`, in L1 order. Read from the - /// InputBox logs, independent of any sequencer database. - pub async fn l1_batch_nonces( - &self, - after_block: u64, - through_block: u64, - ) -> HarnessResult> { - use alloy::contract::Event; - use alloy::sol_types::{SolCall, SolEvent}; - use cartesi_rollups_contracts::input_box::InputBox::InputAdded; - use cartesi_rollups_contracts::inputs::Inputs::EvmAdvanceCall; - - let provider = alloy::providers::ProviderBuilder::new() - .connect(self.l1_endpoint()) - .await - .map_err(|err| io_other(format!("failed to connect anvil provider: {err}")))?; - let mut logs: Vec<(InputAdded, alloy::rpc::types::Log)> = - Event::new_sol(&provider, &self.input_box_address()) - .from_block(after_block + 1) - .to_block(through_block) - .event(InputAdded::SIGNATURE) - .topic1(self.app_address().into_word()) - .query() - .await - .map_err(|err| io_other(format!("query InputAdded logs: {err}")))?; - logs.sort_by_key(|(_, log)| (log.block_number, log.log_index)); - - let mut nonces = Vec::new(); - for (event, _) in logs { - let advance = EvmAdvanceCall::abi_decode(&event.input) - .map_err(|err| io_other(format!("decode EvmAdvance: {err}")))?; - if advance.msgSender != app_core::application::DEVNET_SEQUENCER_ADDRESS { - continue; - } - let batch = - ::from_ssz_bytes(&advance.payload) - .map_err(|err| io_other(format!("decode batch: {err:?}")))?; - nonces.push(batch.nonce); - } - Ok(nonces) + /// How many rows setup's rebuild stored, and how many carry a count. + pub fn rebuilt_history_rows(&self) -> HarnessResult { + let conn = self.read_only_connection()?; + let (total, executed): (i64, i64) = conn + .query_row( + "SELECT COUNT(*), COUNT(l2_tx_count) FROM rebuilt_history", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|err| io_other(format!("count rebuilt history rows: {err}")))?; + Ok(RebuiltHistoryRows { + total: total as u64, + executed: executed as u64, + }) + } + + fn read_only_connection(&self) -> HarnessResult { + rusqlite::Connection::open_with_flags( + self.db_path().as_path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|err| io_other(format!("open DB read-only: {err}")).into()) } /// The canonical-divergence marker (I9/I15) from the run DB, or `None` - /// when the frontier is healthy. A recovery/resync e2e asserts this is `None` - /// to prove the content-identity check did NOT spuriously freeze the frontier - /// (e.g. a false-positive against below-anchor collapsed history) — otherwise - /// that silent-failure class is invisible at e2e level. + /// when the frontier is healthy: its `kind`, `foreign` or `mismatch`. A + /// rebuilt era asserts `None` to prove the content-identity check did not + /// spuriously freeze the frontier against its rebuilt history. pub fn canonical_divergence(&self) -> HarnessResult> { use rusqlite::OptionalExtension; let db_path = self.data_dir_path.join("sequencer.db"); @@ -736,7 +784,7 @@ impl ManagedSequencer { .map_err(|err| io_other(format!("read canonical_divergence: {err}")).into()) } - /// Snapshot of the `batches` table: `(total, sealed, invalidated)`. + /// Snapshot of the `batches` table: `(total, sealed)`. /// Reads the DB file read-only; safe to call while the sequencer is /// running. Useful for asserting that batch closure happened during a /// test segment (e.g., the sequencer kept processing through an outage). @@ -758,36 +806,22 @@ impl ManagedSequencer { |row| row.get(0), ) .map_err(|err| io_other(format!("count sealed batches: {err}")))?; - let invalidated: i64 = conn - .query_row( - "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", - [], - |row| row.get(0), - ) - .map_err(|err| io_other(format!("count invalidated batches: {err}")))?; - Ok(BatchCounts { total: total as u64, sealed: sealed as u64, - invalidated: invalidated as u64, }) } - /// Assert the schema-level tree invariants on the sequencer's DB. Runs + /// Assert the schema-level batch invariants on the sequencer's DB. Runs /// against the DB file read-only; safe to call whether the sequencer is /// running or stopped (SQLite WAL + read-only flag handles concurrent /// writers). /// /// Invariants checked: - /// 1. At most one `valid_open_batch` row (partial unique index - /// `ux_single_valid_tip` should guarantee this structurally — + /// 1. At most one `open_batch` row (partial unique index + /// `ux_single_open_batch` should guarantee this structurally — /// we verify it in case the index ever regressed). - /// 2. Every valid batch's `nonce` equals `parent.nonce + 1`, or 0 if - /// `parent_batch_index IS NULL`. - /// 3. Every `parent_batch_index` is NULL or references an existing - /// batch (FK-backed, verified explicitly for cross-DB-tool - /// portability). - /// 4. The nonces on the valid path form a contiguous `0..N` sequence. + /// 2. The batch nonces are contiguous from the era's anchor (I16). /// /// Panics with a specific violation message if any invariant fails. /// Harness-only check (no sequencer changes) that catches regressions @@ -800,99 +834,35 @@ impl ManagedSequencer { ) .map_err(|err| io_other(format!("open DB read-only: {err}")))?; - // The batch-tree anchor: the nonce the single parentless root carries - // (0 for a genesis deployment, N' for a cockroach-recovered one). The - // root/contiguity invariants below are stated relative to it, so a - // recovered tree (rooted at N') passes the same checks a genesis tree - // (rooted at 0) does. Mirrors `trg_enforce_nonce_contiguity`. Reuses the - // `batch_tree_anchor()` accessor (its own short-lived read-only handle) - // rather than re-issuing the query against `conn`. - let anchor = self.batch_tree_anchor()? as i64; + // The era's anchor: the first batch's nonce (0 for a genesis + // deployment, N' for a cockroach-recovered one). Mirrors + // `trg_batch_insert`. Reuses the `anchor_batch_nonce()` + // accessor (its own short-lived read-only handle) rather than + // re-issuing the query against `conn`. + let anchor = self.anchor_batch_nonce()? as i64; - // 1. At most one valid open batch. + // 1. At most one open batch. let open_count: i64 = conn - .query_row("SELECT COUNT(*) FROM valid_open_batch", [], |row| { - row.get(0) - }) - .map_err(|err| io_other(format!("count valid_open_batch: {err}")))?; + .query_row("SELECT COUNT(*) FROM open_batch", [], |row| row.get(0)) + .map_err(|err| io_other(format!("count open_batch: {err}")))?; if open_count > 1 { - panic!("schema invariant: more than one valid Tip ({open_count} rows)"); + panic!("schema invariant: more than one open batch ({open_count} rows)"); } - // 1b. At most one *valid* parentless root (the anchored root). - let valid_root_count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM valid_batches WHERE parent_batch_index IS NULL", - [], - |row| row.get(0), - ) - .map_err(|err| io_other(format!("count valid roots: {err}")))?; - if valid_root_count > 1 { - panic!( - "schema invariant: more than one valid parentless root ({valid_root_count} rows)" - ); - } - - // 2. Nonce contiguity via parent. + // 2. The nonces are contiguous from the anchor. let mut stmt = conn - .prepare( - "SELECT b.batch_index, b.parent_batch_index, b.nonce, p.nonce \ - FROM batches b LEFT JOIN batches p ON p.batch_index = b.parent_batch_index", - ) - .map_err(|err| io_other(format!("prepare nonce-check: {err}")))?; - let rows: Vec<(i64, Option, i64, Option)> = stmt - .query_map([], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) - }) - .map_err(|err| io_other(format!("query nonce-check: {err}")))? - .collect::>() - .map_err(|err| io_other(format!("collect nonce-check: {err}")))?; - for (bi, parent, nonce, parent_nonce) in &rows { - match (parent, parent_nonce) { - (None, _) => { - if *nonce != anchor { - panic!( - "schema invariant: batch {bi} has NULL parent but nonce {nonce} (expected anchor {anchor})" - ); - } - } - (Some(p), None) => { - panic!( - "schema invariant: batch {bi}'s parent {p} doesn't exist (FK violation)" - ); - } - (Some(_), Some(pn)) => { - if *nonce != pn + 1 { - panic!( - "schema invariant: batch {bi} nonce={nonce}, expected parent.nonce+1 = {}", - pn + 1 - ); - } - } - } - } - - // 3. Valid-path nonce uniqueness and contiguity. - let mut stmt = conn - .prepare("SELECT nonce FROM valid_batches ORDER BY nonce ASC") - .map_err(|err| io_other(format!("prepare valid-nonces: {err}")))?; - let valid_nonces: Vec = stmt + .prepare("SELECT nonce FROM batches ORDER BY nonce ASC") + .map_err(|err| io_other(format!("prepare batch nonces: {err}")))?; + let nonces: Vec = stmt .query_map([], |row| row.get::<_, i64>(0)) - .map_err(|err| io_other(format!("query valid-nonces: {err}")))? + .map_err(|err| io_other(format!("query batch nonces: {err}")))? .collect::>() - .map_err(|err| io_other(format!("collect valid-nonces: {err}")))?; - for pair in valid_nonces.windows(2) { - if pair[0] == pair[1] { - panic!( - "schema invariant: duplicate valid nonce {} in {valid_nonces:?}", - pair[0] - ); - } - } - for (i, &n) in valid_nonces.iter().enumerate() { + .map_err(|err| io_other(format!("collect batch nonces: {err}")))?; + for (i, &n) in nonces.iter().enumerate() { if n != anchor + i as i64 { panic!( - "schema invariant: valid nonces not contiguous from anchor {anchor}: {valid_nonces:?}" + "schema invariant: batch nonces are not contiguous from the anchor \ + {anchor}: {nonces:?}" ); } } @@ -930,6 +900,63 @@ impl ManagedSequencer { self.data_dir_path.join("sequencer.db") } + /// The names in the data directory, sorted; empty while it does not + /// exist. Setup's refusals before its first write leave at most the lock + /// file here. + pub fn data_dir_entries(&self) -> HarnessResult> { + let entries = match fs::read_dir(self.data_dir_path.as_path()) { + Ok(entries) => entries, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(io_other(format!("read data dir: {err}")).into()), + }; + let mut names = entries + .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) + .collect::, _>>()?; + names.sort(); + Ok(names) + } + + /// Every file and directory under the data directory by relative path, + /// sorted, a file with its length and the keccak256 of its bytes; empty + /// while the directory does not exist. Two equal readings mean nothing was + /// added, removed or rewritten. + pub fn data_dir_contents(&self) -> HarnessResult> { + let root = self.data_dir_path.as_path(); + let mut contents = Vec::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + let entries = match fs::read_dir(dir.as_path()) { + Ok(entries) => entries, + Err(err) if err.kind() == std::io::ErrorKind::NotFound && dir == root => { + return Ok(Vec::new()); + } + Err(err) => { + return Err(io_other(format!("read {}: {err}", dir.display())).into()); + } + }; + for entry in entries { + let path = entry?.path(); + let name = path + .strip_prefix(root) + .map_err(|err| io_other(format!("{}: {err}", path.display())))? + .to_string_lossy() + .into_owned(); + if path.is_dir() { + contents.push((name, None)); + pending.push(path); + continue; + } + let bytes = fs::read(path.as_path()) + .map_err(|err| io_other(format!("read {}: {err}", path.display())))?; + let len = + u64::try_from(bytes.len()).map_err(|_| io_other("file length overflows"))?; + contents.push((name, Some((len, alloy_primitives::keccak256(&bytes))))); + } + } + contents.sort(); + Ok(contents) + } + /// Live bytes in the database: pages in use, including pages still in the /// WAL and excluding free pages. File sizes show neither. Read while the /// sequencer runs, so the WAL index is there for the read-only connection. @@ -959,6 +986,17 @@ impl ManagedSequencer { /// L1 blocks of the application's InputBox inputs, in input order. pub async fn input_blocks(&self) -> HarnessResult> { + Ok(self + .raw_inputs() + .await? + .into_iter() + .map(|(block, _)| block) + .collect()) + } + + /// The application's InputBox inputs, in input order: each one's L1 block + /// and its `EvmAdvance` encoding, the bytes a machine receives. + pub async fn raw_inputs(&self) -> HarnessResult)>> { let provider = ProviderBuilder::new() .connect(self.l1_endpoint()) .await @@ -972,7 +1010,26 @@ impl ManagedSequencer { .get_logs(&filter) .await .map_err(|err| io_other(format!("failed to read InputAdded logs: {err}")))?; - Ok(logs.iter().filter_map(|log| log.block_number).collect()) + // Callers index inputs by position, so the log order must be the + // InputBox order. + (0_u64..) + .zip(&logs) + .map(|(position, log)| { + let block = log + .block_number + .ok_or_else(|| io_other("InputAdded log without a block number"))?; + let event = InputAdded::decode_log_data(&log.inner.data) + .map_err(|err| io_other(format!("failed to decode InputAdded: {err}")))?; + if event.index != U256::from(position) { + return Err(io_other(format!( + "InputAdded log {position} carries index {}", + event.index + )) + .into()); + } + Ok((block, event.input.to_vec())) + }) + .collect() } pub fn erc20_portal_address(&self) -> Address { @@ -1010,10 +1067,10 @@ impl ManagedSequencer { } /// Toggle Anvil's auto-mining mode. When disabled, txs accumulate in - /// the mempool until an explicit mine or re-enable. Used to hold a - /// sequencer's batch-submission tx out of a block while the chain - /// advances, reproducing the "delayed inclusion" fault that the - /// scheduler handles by skipping past-stale batches. + /// the mempool until an explicit mine or re-enable: a sequencer's + /// batch-submission tx waits there, where [`Self::drop_all_pending_txs`] + /// can lose it. `anvil_mine` includes pending txs, so hold submissions + /// across mining with [`Self::hold_batch_submissions`] instead. pub async fn set_automine(&self, enabled: bool) -> HarnessResult<()> { self.rollups.set_automine(enabled).await } @@ -1048,12 +1105,7 @@ impl ManagedSequencer { /// call (or a post-respawn first call) sees the shifted clock. /// - Anvil mines `duration.as_secs() / DEVNET_SECONDS_PER_BLOCK` blocks. /// - /// **Cumulative**: calling with `1h` twice totals `+2h`, not `+1h`. Use - /// [`Self::set_faketime_offset`] to jump to a specific offset or reset. - /// - /// Tests that need decoupled wall-clock vs L1 (e.g., the `saturating_sub` - /// backward-jump test) should use [`Self::set_faketime_offset`] and - /// [`Self::mine_l1_blocks`] directly. + /// **Cumulative**: calling with `1h` twice totals `+2h`, not `+1h`. pub async fn advance_wall_and_mine(&mut self, duration: Duration) -> HarnessResult<()> { let secs = duration.as_secs(); let blocks = secs / DEVNET_SECONDS_PER_BLOCK; @@ -1072,9 +1124,8 @@ impl ManagedSequencer { /// exit handle. /// /// - Returns `Ok(None)` if the child is still alive when `grace` - /// elapses. The internal `wait()` future is dropped, so subsequent - /// calls to [`Self::wait_for_exit`] / [`Self::respawn_and_watch`] - /// still work. + /// elapses. The internal `wait()` future is dropped, so a later + /// [`Self::wait_for_exit`] still works. /// - Returns `Ok(Some(status))` if the child exited inside the /// window. The exit status is captured and the child is reaped; /// the caller shouldn't call `wait_for_exit` afterwards (it would @@ -1133,9 +1184,9 @@ impl ManagedSequencer { /// process's exit status. Times out after `timeout` to avoid hanging /// tests when the process refuses to exit. /// - /// Used by tests that expect the sequencer to detect a condition - /// (e.g., wall-clock danger) and self-exit with a non-zero status. - /// After this returns, call [`Self::respawn`] to start a fresh process. + /// Used by tests that expect the sequencer to stop on its own, such as + /// on a terminal fact. After this returns, call [`Self::respawn`] or + /// [`Self::try_respawn`] to start a fresh process. pub async fn wait_for_exit( &mut self, timeout: Duration, @@ -1151,84 +1202,6 @@ impl ManagedSequencer { Ok(status) } - /// Respawn the sequencer and watch the child for `stabilization` to - /// confirm it stays alive. Classifies the outcome so tests can model an - /// orchestrator restart cycle without re-deriving the failure modes. - /// - /// There are two distinct "unstable" shapes the sequencer can take: - /// - The child dies during bootstrap (before HTTP readiness), which - /// makes `respawn()` itself return `Err`. Canonical cause: - /// `RecoveryError::Refuse(...)` from the startup decision table - /// when L1 is unreachable and the persisted state looks stalled. - /// - The child comes up (HTTP ready, bootstrap passed), then one of - /// the internal tasks returns a fatal error and the process exits. - /// Canonical cause: danger-detector worker exit when the first - /// post-boot poll sees a batch past `danger_threshold`. - /// - /// The race between bootstrap-finishes and submitter-first-tick is - /// short (the poll interval is 5s by default, but the first tick runs - /// immediately), so both cases can surface for a single logical event — - /// tests should generally treat either as "not stable" and retry. - /// - /// Callers must ensure the previous child is already reaped (via - /// [`Self::stop`] or [`Self::wait_for_exit`]) — same rule as - /// [`Self::respawn`]. - pub async fn respawn_and_watch( - &mut self, - stabilization: Duration, - ) -> HarnessResult { - if let Err(err) = self.respawn().await { - return Ok(RespawnAttemptOutcome::RespawnFailed(err.to_string())); - } - tokio::select! { - wait_result = self.child.wait() => { - let status = wait_result - .map_err(|err| io_other(format!("child.wait(): {err}")))?; - Ok(RespawnAttemptOutcome::ExitedPostRespawn(status)) - } - _ = tokio::time::sleep(stabilization) => { - Ok(RespawnAttemptOutcome::Stable) - } - } - } - - /// Loop [`Self::respawn_and_watch`] until the sequencer stays up for - /// `policy.stabilization`, or `policy.max_attempts` is reached. Returns - /// the full sequence of attempts. - /// - /// The restart-loop convergence story: an aged Tip in the danger zone - /// (not yet past-stale) auto-closes on respawn, and the resulting closed - /// batch is in the danger zone, so the detector exits with `DangerDetected`. - /// Startup recovery's cascade fires at `MAX_WAIT_BLOCKS`, not at the - /// danger threshold — so the loop only converges once enough *additional* - /// L1 blocks have aged the batch past `MAX_WAIT_BLOCKS`. In production - /// the orchestrator restart itself takes seconds, during which real L1 - /// blocks are produced; `advance_per_retry` simulates that drift. Tests - /// that expect a short hiccup to self-heal (no danger involved) should - /// leave `advance_per_retry` unset. - /// - /// The loop always returns Ok — assert on the final attempt's outcome - /// to decide pass/fail in the test body. - pub async fn respawn_until_stable( - &mut self, - policy: RespawnPolicy, - ) -> HarnessResult> { - let mut outcomes = Vec::with_capacity(policy.max_attempts as usize); - for attempt in 0..policy.max_attempts { - let outcome = self.respawn_and_watch(policy.stabilization).await?; - let stable = outcome.is_stable(); - outcomes.push(outcome); - if stable { - break; - } - let is_last = attempt + 1 == policy.max_attempts; - if let Some(advance) = policy.advance_per_retry.filter(|_| !is_last) { - self.advance_wall_and_mine(advance).await?; - } - } - Ok(outcomes) - } - /// Kill the sequencer process. Anvil stays running, so `mine_l1_blocks()` still works. pub async fn stop(&mut self) -> HarnessResult<()> { self.shutdown_child().await @@ -1254,38 +1227,143 @@ impl ManagedSequencer { /// Respawn the sequencer process using the same data directory and Anvil instance. /// /// Honors any `l1_endpoint_override` set via [`Self::set_l1_endpoint_override`] - /// and the faketime offset in the rc file (see [`Self::set_faketime_offset`] / - /// [`Self::advance_wall_and_mine`]). + /// and the faketime offset [`Self::advance_wall_and_mine`] wrote. + /// + /// Runs `setup` first only when the data directory is new + /// ([`Self::discard_data_dir`]): setup refuses a directory that holds + /// anything, so a set-up directory goes straight to `run`. A failed setup + /// stays pending, and its retry meets whatever the failure left behind, + /// as an operator's would. A `run` that exits before readiness is an + /// error; [`Self::try_respawn`] reports its status instead. pub async fn respawn(&mut self) -> HarnessResult<()> { - let SpawnedSequencerProcess { - child, - endpoint, - internal_endpoint, - log_path, - } = spawn_sequencer_process( - self.sequencer_bin.as_path(), - self.log_prefix.as_str(), - self.logs_dir.as_path(), - self.data_dir_path.as_path(), - &self.rollups, - self.l1_endpoint_override.as_deref(), - self.chain_id_override, - self.libfaketime_path.as_deref(), - self.faketime_rc_path.as_deref(), - self.mine_l1_during_boot, - self.recovery_setup.as_ref(), - self.max_batch_open_seconds, - self.fee_oracle_fixed_log_gas_price, - None, - ) - .await?; - self.child = child; - self.endpoint = endpoint; - self.internal_endpoint = internal_endpoint; + match self.try_respawn().await? { + RunStart::Ready => Ok(()), + RunStart::Exited(status) => Err(io_other(format!( + "sequencer exited before readiness: status={status} (see {})", + self.log_path.display() + )) + .into()), + } + } + + /// [`Self::respawn`], reporting a `run` that exits before readiness by + /// its exit status, so a refused start is asserted by its code. The + /// exited process becomes the managed child. + pub async fn try_respawn(&mut self) -> HarnessResult { + let log_path = timestamped_log_path(self.logs_dir.as_path(), self.log_prefix.as_str()); + if self.setup_pending { + require_setup_success( + run_setup_process( + &self.child_env(), + &self.setup_request(false), + &log_path, + self.mine_l1_during_setup, + ) + .await?, + &log_path, + )?; + self.setup_pending = false; + } + let started = + spawn_run_process(&self.child_env(), &log_path, self.max_batch_open_seconds).await?; self.log_path = log_path; + match started { + RunProcess::Ready(SpawnedSequencerProcess { + child, + endpoint, + internal_endpoint, + }) => { + self.child = child; + self.endpoint = endpoint; + self.internal_endpoint = internal_endpoint; + Ok(RunStart::Ready) + } + RunProcess::Exited(child, status) => { + self.child = child; + Ok(RunStart::Exited(status)) + } + } + } + + /// Hold the batch submitter's transactions back while the sequencer + /// runs: its balance goes to zero, so every send is refused for + /// insufficient funds and the poster retries on its next tick. Nothing + /// reaches the mempool, so mining does not land it either. + pub async fn hold_batch_submissions(&mut self) -> HarnessResult<()> { + if self.held_submitter_balance.is_some() { + return Err(io_other("batch submissions are already held").into()); + } + let submitter = app_core::application::DEVNET_SEQUENCER_ADDRESS; + let balance = self.rollups.balance_of(submitter).await?; + self.rollups.set_balance(submitter, U256::ZERO).await?; + self.held_submitter_balance = Some(balance); Ok(()) } + /// Restore the balance [`Self::hold_batch_submissions`] took away. + pub async fn release_batch_submissions(&mut self) -> HarnessResult<()> { + let balance = self + .held_submitter_balance + .take() + .ok_or_else(|| io_other("batch submissions are not held"))?; + self.rollups + .set_balance(app_core::application::DEVNET_SEQUENCER_ADDRESS, balance) + .await + } + + /// Broadcast `payload` to the application's InputBox from `signer` without + /// waiting for it to be mined; returns its hash. With auto-mining off it + /// stays in the mempool, as a zombie of an old instance would. + pub async fn broadcast_input_as( + &self, + signer: &TestSigner, + payload: Vec, + ) -> HarnessResult { + use cartesi_rollups_contracts::input_box::InputBox; + let provider = signer.connect_l1_provider(self.l1_endpoint()).await?; + let pending = InputBox::new(self.input_box_address(), &provider) + .addInput(self.app_address(), payload.into()) + .send() + .await + .map_err(|err| io_other(format!("failed to send InputBox input: {err}")))?; + Ok(*pending.tx_hash()) + } + + /// Whether the transaction `hash` is in a block. + pub async fn transaction_mined(&self, hash: B256) -> HarnessResult { + let provider = ProviderBuilder::new() + .connect(self.l1_endpoint()) + .await + .map_err(|err| io_other(format!("failed to connect L1 provider: {err}")))?; + let receipt = provider + .get_transaction_receipt(hash) + .await + .map_err(|err| io_other(format!("failed to read receipt {hash}: {err}")))?; + Ok(receipt.is_some()) + } + + /// Send `payload` to the application's InputBox from `signer`, as any + /// account can; returns the block that holds it. Inputs from the batch + /// submitter's key are batches, whatever process signs them. + pub async fn add_input_as(&self, signer: &TestSigner, payload: Vec) -> HarnessResult { + use cartesi_rollups_contracts::input_box::InputBox; + let provider = signer.connect_l1_provider(self.l1_endpoint()).await?; + let receipt = InputBox::new(self.input_box_address(), &provider) + .addInput(self.app_address(), payload.into()) + .send() + .await + .map_err(|err| io_other(format!("failed to send InputBox input: {err}")))? + .get_receipt() + .await + .map_err(|err| io_other(format!("failed to confirm InputBox input: {err}")))?; + if !receipt.status() { + return Err(io_other("InputBox input reverted").into()); + } + receipt + .block_number + .ok_or_else(|| io_other("InputBox receipt has no block number").into()) + } + pub async fn restart(&mut self) -> HarnessResult<()> { self.stop().await?; self.respawn().await @@ -1305,19 +1383,13 @@ impl ManagedSequencer { self.data_dir_path.join("sequencer.db"), rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, )?; - let (era, generation): (Vec, i64) = conn.query_row( - "SELECT era_id, recovery_generation FROM history_state", - [], - |r| Ok((r.get(0)?, r.get(1)?)), - )?; + let era: Vec = + conn.query_row("SELECT era_id FROM era WHERE singleton_id = 0", [], |r| { + r.get(0) + })?; Ok(sequencer_rust_client::HistoryClaim { - version: sequencer_rust_client::HistoryVersion { - era_id: sequencer_core::history::EraId::try_from(era.as_slice())?, - recovery_generation: sequencer_core::history::RecoveryGeneration::new( - u64::try_from(generation)?, - ), - }, - next_input: sequencer_rust_client::ExecutedInputCount::new(next_input), + era_id: sequencer_core::history::EraId::try_from(era.as_slice())?, + next_input: sequencer_rust_client::L2TxCount::new(next_input), }) } @@ -1326,6 +1398,17 @@ impl ManagedSequencer { WsClient::connect(&client, self.history_claim(next_input)?).await } + /// Subscribe as a client of an older era holding state at `count` does + /// after a recovery: adopt the current era at its own count through the + /// SDK, then follow the feed from there. + pub async fn ws_adopting(&self, count: u64) -> HarnessResult { + let client = self.sequencer_client()?; + let claim = client + .adopt_current_era(sequencer_rust_client::L2TxCount::new(count)) + .await?; + WsClient::connect(&client, claim).await + } + pub async fn wallet_l1(&self, signer: TestSigner) -> HarnessResult { WalletL1Client::connect( self.l1_endpoint(), @@ -1353,8 +1436,8 @@ impl ManagedSequencer { rollups_result } - /// A client for the internal listener, where the feed lives. - fn sequencer_client(&self) -> HarnessResult { + /// A client for the internal listener, where the feed and `/history` live. + pub fn sequencer_client(&self) -> HarnessResult { SequencerClient::new_with_timeout(self.internal_endpoint.clone(), Duration::from_secs(5)) .map_err(|err| io_other(format!("failed to create sequencer client: {err}")).into()) } @@ -1386,41 +1469,49 @@ impl ManagedSequencer { use alloy_primitives::Address; -/// The timing flags the e2e zone math depends on, passed explicitly so an -/// exported `CARTESI_SEQUENCER_*` variable cannot shift them. -fn timing_args() -> [String; 4] { +/// The block time the harness mines outages at, passed explicitly so an +/// exported `CARTESI_SEQUENCER_*` variable cannot shift it. +fn timing_args() -> [String; 2] { [ - "--preemptive-margin-blocks".into(), - DEVNET_PREEMPTIVE_MARGIN_BLOCKS.to_string(), "--seconds-per-block".into(), DEVNET_SECONDS_PER_BLOCK.to_string(), ] } +/// The batch submitter's signer (Anvil account 9). A transaction it signs to +/// the InputBox is a batch, whichever process sends it. +pub fn batch_submitter_signer() -> HarnessResult { + TestSigner::from_private_key_hex(&devnet_submitter()?.0) +} + +/// A batch's InputBox payload: its SSZ encoding, with no envelope or tag. +pub fn encode_batch(batch: &sequencer_core::batch::Batch) -> Vec { + ssz::Encode::as_ssz_bytes(batch) +} + struct SpawnedSequencerProcess { child: Child, endpoint: String, internal_endpoint: String, - log_path: PathBuf, } -/// Read `next_batch_nonce` out of a dump dir's `info.toml` (the resume nonce -/// `N`). The file is the simple `key = value` form the sequencer writes. -fn read_info_next_batch_nonce(dump_dir: &Path) -> HarnessResult { - let info_path = dump_dir.join("info.toml"); - let content = fs::read_to_string(info_path.as_path()) - .map_err(|err| io_other(format!("read {info_path:?}: {err}")))?; - for line in content.lines() { - if let Some((key, value)) = line.split_once(" = ") - && key.trim() == "next_batch_nonce" - { - return value - .trim() - .parse::() - .map_err(|err| io_other(format!("parse next_batch_nonce: {err}")).into()); - } +/// How a `run` child's start ended. +enum RunProcess { + Ready(SpawnedSequencerProcess), + /// The child exited before readiness, with this status. + Exited(Child, ExitStatus), +} + +/// A start a spawn depends on must reach readiness. +fn require_ready(started: RunProcess, log_path: &Path) -> HarnessResult { + match started { + RunProcess::Ready(process) => Ok(process), + RunProcess::Exited(_, status) => Err(io_other(format!( + "sequencer exited before readiness: status={status} (see {})", + log_path.display() + )) + .into()), } - Err(io_other(format!("info.toml missing next_batch_nonce: {info_path:?}")).into()) } /// `(page_count - freelist_count) * page_size`, read in one statement so all @@ -1440,297 +1531,326 @@ fn logical_db_bytes_at(db_path: &Path) -> HarnessResult { u64::try_from(bytes).map_err(|_| io_other(format!("negative logical DB size: {bytes}")).into()) } -#[allow(clippy::too_many_arguments)] -async fn spawn_sequencer_process( - sequencer_bin: &Path, - log_prefix: &str, - logs_dir: &Path, - data_dir: &Path, - rollups: &DevnetRollupsStack, - l1_endpoint_override: Option<&str>, - chain_id_override: Option, - libfaketime_path: Option<&Path>, - faketime_rc_path: Option<&Path>, - mine_l1_during_boot: bool, - recovery: Option<&RecoverySetupParams>, - max_batch_open_seconds: Option, +/// What every child the harness starts shares: the binary, the stack it +/// talks to, its data directory and its clock. +struct ChildEnv<'a> { + bin: &'a Path, + data_dir: &'a Path, + rollups: &'a DevnetRollupsStack, + eth_rpc_url: &'a str, + /// The libfaketime library and the rc file it re-reads; `None` runs on + /// the real clock. + faketime: Option<(&'a Path, &'a Path)>, +} + +impl ChildEnv<'_> { + fn command(&self) -> HarnessResult { + let mut command = Command::new(self.bin); + // `kill_on_drop`: a `ManagedSequencer` normally reaps its child via + // `shutdown()` / `stop()`, but an error path that drops the struct + // without either (e.g. a benchmark returning early on a misconfigured + // run) would otherwise leave the child running while the owning + // `TempDir` deletes the data dir — the child's next DB open then + // fails `ENOENT` and it exits with a confusing `unable to open + // database file` that masks the real failure. `child` is declared + // before `_root_dir`, so on drop the SIGKILL lands before the dir is + // removed. + command + .kill_on_drop(true) + .env_remove("CARTESI_SEQUENCER_STATE_FILE") + .env("RUST_LOG", DEFAULT_SEQUENCER_RUST_LOG); + // libfaketime is applied via env vars (not the `faketime` wrapper + // binary), which the file-based FAKETIME_TIMESTAMP_FILE mechanism + // reads on every time call (FAKETIME_NO_CACHE=1) so tests can shift + // the clock at runtime. + if let Some((lib, rc)) = self.faketime { + apply_faketime_env(&mut command, lib, rc)?; + } + Ok(command) + } +} + +/// One `setup` invocation: the base it starts from and whether it only +/// reports. +struct SetupRequest<'a> { + chain_id: u64, + /// The stored machine to rebuild from; `None` is genesis. + base: Option<&'a Path>, + dry_run: bool, + /// The C host's genesis tool, run for a setup from genesis. + genesis_bin: Option<&'a Path>, fee_oracle_fixed_log_gas_price: Option, - genesis_bin: Option<&Path>, -) -> HarnessResult { - let (endpoint, http_addr) = build_local_endpoint()?; - let (internal_endpoint, internal_http_addr) = build_local_endpoint()?; - let log_path = timestamped_log_path(logs_dir, log_prefix); - - // Dedicated batch submitter = anvil account 9 (NOT the deployer/funder, - // account 0). `setup`'s detection gate refuses when the submitter's wallet - // nonce is unsettled, so the submitter must not share a key with the - // deployer or with an e2e wallet whose transactions are in flight. Account - // 9 starts at nonce 0. Must match `DEVNET_SEQUENCER_ADDRESS` (app-core); an - // assert below pins that. Account 9 is the highest standard funded anvil - // account, so it stays clear of the e2e wallets (0–3) and the bench - // workload (indices 0..concurrency). - const DEVNET_SUBMITTER_ACCOUNT_INDEX: usize = 9; - let batch_submitter_key = default_private_keys()[DEVNET_SUBMITTER_ACCOUNT_INDEX].clone(); - // `setup` takes the submitter *address* (it never signs); `run` takes the - // key. Derive the address from the key so the harness configures both - // consistently. - let submitter_address = TestSigner::from_private_key_hex(&batch_submitter_key)?.address(); - // `assert_eq!`, not `debug_assert_eq!`: benches build the harness `--release`, - // where a debug-assert is a no-op — drift between app-core and the harness - // submitter must fail fast in every build. Cost is one comparison per spawn. +} + +/// The harness's batch submitter: its private key (hex) and address. +fn devnet_submitter() -> HarnessResult<(String, Address)> { + let key = default_private_keys()[DEVNET_SUBMITTER_ACCOUNT_INDEX].clone(); + let address = TestSigner::from_private_key_hex(&key)?.address(); + // `assert_eq!`, not `debug_assert_eq!`: benches build the harness + // `--release`, where a debug-assert is a no-op — drift between app-core + // and the harness submitter must fail fast in every build. assert_eq!( - submitter_address, + address, app_core::application::DEVNET_SEQUENCER_ADDRESS, "harness submitter (anvil account {DEVNET_SUBMITTER_ACCOUNT_INDEX}) must equal \ DEVNET_SEQUENCER_ADDRESS — keep app-core and the harness in sync" ); - let batch_submitter_address = submitter_address.to_string(); - let eth_rpc_url = l1_endpoint_override.unwrap_or_else(|| rollups.l1_endpoint()); - - let chain_id = chain_id_override.unwrap_or(DEVNET_CHAIN_ID); - let bin = path_as_str(sequencer_bin)?.to_owned(); + Ok((key, address)) +} - let genesis_dir = if let Some(genesis_bin) = genesis_bin { - let dir = TempDir::new()?; - let mut command = Command::new(genesis_bin); - command - .kill_on_drop(true) - .arg(dir.path().join("state")) - .arg("devnet"); - let output = tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, command.output()) - .await - .map_err(|_| { - io_other(format!( - "genesis tool '{}' timed out", - genesis_bin.display() - )) - })? - .map_err(|err| { - io_other(format!( - "failed to run genesis tool '{}': {err}", - genesis_bin.display() - )) - })?; - if !output.status.success() { - return Err(io_other(format!( - "genesis tool '{}' failed: status={}: {}", - genesis_bin.display(), - output.status, - String::from_utf8_lossy(&output.stderr) - )) - .into()); - } - Some(dir) +/// The arguments of one `setup` run. Setup takes the submitter key, the same +/// way `run` does, because it flushes the submitter's wallet; the dry run +/// flushes nothing and gets no key. The genesis state goes only to a setup +/// without a base. +fn setup_args( + request: &SetupRequest<'_>, + data_dir: &Path, + eth_rpc_url: &str, + app_address: Address, + submitter: (&str, Address), + genesis_state: Option<&Path>, +) -> Vec { + let mut args: Vec = Vec::new(); + if request.base.is_none() + && let Some(state) = genesis_state + { + args.extend(["--state-file".into(), state.into()]); + } + args.extend([ + "setup".into(), + "--data-dir".into(), + data_dir.into(), + "--eth-rpc-url".into(), + eth_rpc_url.into(), + "--chain-id".into(), + request.chain_id.to_string().into(), + "--app-address".into(), + app_address.to_string().into(), + "--batch-submitter-address".into(), + submitter.1.to_string().into(), + ]); + args.extend(timing_args().map(OsString::from)); + if let Some(base) = request.base { + args.extend(["--base".into(), base.into()]); + } + if request.dry_run { + args.push("--dry-run".into()); } else { - None - }; + args.extend(["--batch-submitter-private-key".into(), submitter.0.into()]); + } + args +} - // libfaketime is applied via env vars (not the `faketime` wrapper binary), - // which the file-based FAKETIME_TIMESTAMP_FILE mechanism reads on every - // time call (FAKETIME_NO_CACHE=1) so tests can shift the clock at runtime. - let open_log = |append: bool| -> HarnessResult { - Ok(OpenOptions::new() - .create(true) - .write(true) - .truncate(!append) - .append(append) - .open(log_path.as_path())?) +/// Run `setup` to completion and report how it ended. Its summary (stdout) +/// is returned and, with stderr, appended to `log_path`. With +/// `mine_l1_during_setup`, L1 advances while setup runs, so a flush of an +/// old instance's stranded transactions can reach safe depth on an +/// on-demand-mining devnet. +async fn run_setup_process( + env: &ChildEnv<'_>, + request: &SetupRequest<'_>, + log_path: &Path, + mine_l1_during_setup: bool, +) -> HarnessResult { + let (submitter_key, submitter_address) = devnet_submitter()?; + // The genesis tool runs only for a setup from genesis; its output is + // removed when setup ends, so neither `run` nor a later recovery may + // depend on it. + let genesis_dir = match (request.base, request.genesis_bin) { + (None, Some(genesis_bin)) => Some(run_genesis_tool(genesis_bin).await?), + _ => None, + }; + let genesis_state = genesis_dir.as_ref().map(|dir| dir.path().join("state")); + + let mut command = env.command()?; + command + .args(setup_args( + request, + env.data_dir, + env.eth_rpc_url, + env.rollups.app_address(), + (submitter_key.as_str(), submitter_address), + genesis_state.as_deref(), + )) + // Unknown chains require an explicit setup-time source. Normal Anvil + // e2e pins zero; the dedicated fee test overrides it. + .env( + "CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE", + request + .fee_oracle_fixed_log_gas_price + .unwrap_or(0) + .to_string(), + ) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let timeout = if request.base.is_some() { + BASE_SETUP_TIMEOUT + } else if mine_l1_during_setup { + FLUSHING_SETUP_TIMEOUT + } else { + GENESIS_SETUP_TIMEOUT }; - // Phase A — `setup` (run once; idempotent on re-spawns, so it is a cheap - // no-op once the DB is set up, and needs L1 only on the first run / after - // a DB reset). Run to completion as a blocking step; a bootstrap refusal - // (e.g. a chain-id mismatch) surfaces here as a non-zero exit, just as it - // surfaced from the monolithic boot before the split. - let mut setup_cmd = Command::new(&bin); - setup_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); - if let Some(dir) = &genesis_dir { - setup_cmd.arg("--state-file").arg(dir.path().join("state")); - } - if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { - apply_faketime_env(&mut setup_cmd, lib, rc)?; - } - let setup_log = open_log(false)?; - let setup_log_err = setup_log.try_clone()?; - setup_cmd - .arg("setup") - .arg("--data-dir") - .arg(path_as_str(data_dir)?) - .arg("--eth-rpc-url") - .arg(eth_rpc_url) - .arg("--chain-id") - .arg(chain_id.to_string()) - .arg("--app-address") - .arg(rollups.app_address().to_string()) - .arg("--batch-submitter-address") - .arg(&batch_submitter_address) - .args(timing_args()); - if let Some(recovery) = recovery { - // Recovery setup rebuilds the wiped DB from a checkpoint: it signs (the - // flush), so it takes the key, plus the checkpoint block + dump dir. - setup_cmd - .arg("--recovery") - .arg("--checkpoint-block") - .arg(recovery.checkpoint_block.to_string()) - .arg("--checkpoint-dump-dir") - .arg(path_as_str(&recovery.checkpoint_dump_dir)?) - .arg("--batch-submitter-private-key") - .arg(&batch_submitter_key); - } - setup_cmd - .env("RUST_LOG", DEFAULT_SEQUENCER_RUST_LOG) - .stdout(Stdio::from(setup_log)) - .stderr(Stdio::from(setup_log_err)); - // Unknown chains require an explicit setup-time source. Normal Anvil e2e - // pins zero; the dedicated fee test overrides it before identity exists. - setup_cmd.env( - "CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE", - fee_oracle_fixed_log_gas_price.unwrap_or(0).to_string(), - ); - let mut setup_child = setup_cmd - .spawn() - .map_err(|err| io_other(format!("failed to spawn `setup` for '{bin}': {err}")))?; - // Bound the setup phase so a stalled/lagging RPC during the initial sync - // can't hang the harness indefinitely. Recovery setup additionally flushes - // the wallet nonce, which can block until a stranded slot becomes safe — so - // mine L1 concurrently (nothing else does on an on-demand devnet) and give - // it the generous recovery-boot budget. Genesis setup never flushes. - let setup_status = if recovery.is_some() { - let deadline = std::time::Instant::now() + RECOVERY_BOOT_START_TIMEOUT; - loop { - if let Some(status) = setup_child - .try_wait() - .map_err(|err| io_other(format!("poll recovery `setup`: {err}")))? - { - break status; - } - if std::time::Instant::now() >= deadline { - let _ = setup_child.start_kill(); - let _ = setup_child.wait().await; - return Err(io_other(format!( - "recovery `setup` timed out after {:?} (see {})", - RECOVERY_BOOT_START_TIMEOUT, - log_path.display() - )) - .into()); - } - let _ = rollups.mine_l1_block_at_wall_clock().await; - tokio::time::sleep(BOOT_L1_MINE_INTERVAL).await; + // Dropping the output future on timeout kills the child (`kill_on_drop`). + let output = command.output(); + let output = if mine_l1_during_setup { + let miner = mine_forever(env.rollups); + tokio::select! { + output = tokio::time::timeout(timeout, output) => output, + _ = miner => unreachable!("the setup miner loops forever; only setup resolves"), } } else { - match tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, setup_child.wait()).await { - Ok(status) => status?, - Err(_) => { - let _ = setup_child.start_kill(); - let _ = setup_child.wait().await; - return Err(io_other(format!( - "sequencer `setup` timed out after {:?} (see {})", - DEFAULT_SEQUENCER_START_TIMEOUT, - log_path.display() - )) - .into()); - } - } + tokio::time::timeout(timeout, output).await }; - if !setup_status.success() { - return Err(io_other(format!( - "sequencer `setup` failed: status={setup_status} (see {})", - log_path.display() - )) - .into()); - } + let output = output + .map_err(|_| { + io_other(format!( + "sequencer `setup` timed out after {timeout:?} (see {})", + log_path.display() + )) + })? + .map_err(|err| { + io_other(format!( + "failed to run `setup` for '{}': {err}", + env.bin.display() + )) + })?; + + let mut log = OpenOptions::new() + .create(true) + .append(true) + .open(log_path)?; + // Tracing writes to stdout; stderr holds what escaped it, such as clap's + // usage errors. + log.write_all(&output.stdout)?; + log.write_all(&output.stderr)?; if let Some(dir) = genesis_dir { - // Neither startup nor later recovery may depend on the original source. dir.close() .map_err(|err| io_other(format!("remove initial genesis after setup: {err}")))?; } + Ok(SetupOutcome { + status: output.status, + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + }) +} - // Phase B — `run` (re-spawned on every restart; reads identity from the - // setup DB). - let mut run_cmd = Command::new(&bin); - run_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); - if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { - apply_faketime_env(&mut run_cmd, lib, rc)?; +/// A setup that a spawn depends on must succeed. +fn require_setup_success(outcome: SetupOutcome, log_path: &Path) -> HarnessResult<()> { + if outcome.success() { + return Ok(()); } - let run_log = open_log(true)?; + Err(io_other(format!( + "sequencer `setup` failed: status={} (see {})", + outcome.status, + log_path.display() + )) + .into()) +} + +/// Write the C host's genesis state into a new temporary directory. +async fn run_genesis_tool(genesis_bin: &Path) -> HarnessResult { + let dir = TempDir::new()?; + let mut command = Command::new(genesis_bin); + command + .kill_on_drop(true) + .arg(dir.path().join("state")) + .arg("devnet"); + let output = tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, command.output()) + .await + .map_err(|_| { + io_other(format!( + "genesis tool '{}' timed out", + genesis_bin.display() + )) + })? + .map_err(|err| { + io_other(format!( + "failed to run genesis tool '{}': {err}", + genesis_bin.display() + )) + })?; + if !output.status.success() { + return Err(io_other(format!( + "genesis tool '{}' failed: status={}: {}", + genesis_bin.display(), + output.status, + String::from_utf8_lossy(&output.stderr) + )) + .into()); + } + Ok(dir) +} + +/// Mine one block per [`SETUP_L1_MINE_INTERVAL`] until dropped, as a live +/// chain advances while setup flushes. Transient RPC hiccups are non-fatal — +/// the next tick retries; a genuine mining failure shows as the caller's +/// timeout. +async fn mine_forever(rollups: &DevnetRollupsStack) { + loop { + tokio::time::sleep(SETUP_L1_MINE_INTERVAL).await; + let _ = rollups.mine_l1_block_at_wall_clock().await; + } +} + +/// Start `run` on a set-up data directory and wait for its public listener, +/// or for the process to exit first. Its output is appended to `log_path`. +async fn spawn_run_process( + env: &ChildEnv<'_>, + log_path: &Path, + max_batch_open_seconds: Option, +) -> HarnessResult { + let (endpoint, http_addr) = build_local_endpoint()?; + let (internal_endpoint, internal_http_addr) = build_local_endpoint()?; + let (submitter_key, _) = devnet_submitter()?; + + let run_log = OpenOptions::new() + .create(true) + .append(true) + .open(log_path)?; let run_log_err = run_log.try_clone()?; - // `kill_on_drop`: a `ManagedSequencer` normally reaps its child via - // `shutdown()` / `stop()`, but an error path that drops the struct without - // either (e.g. a benchmark returning early on a misconfigured run) would - // otherwise leave the child running while the owning `TempDir` deletes the - // data dir — the child's next DB open then fails `ENOENT` and it exits with - // a confusing `unable to open database file` that masks the real failure. - // `child` is declared before `_data_dir`, so on drop the SIGKILL lands - // before the dir is removed. + let mut run_cmd = env.command()?; run_cmd - .kill_on_drop(true) .arg("run") .arg("--http-addr") .arg(http_addr) .arg("--internal-http-addr") .arg(internal_http_addr) .arg("--data-dir") - .arg(path_as_str(data_dir)?) + .arg(env.data_dir) .arg("--eth-rpc-url") - .arg(eth_rpc_url) + .arg(env.eth_rpc_url) .arg("--batch-submitter-private-key") - .arg(&batch_submitter_key) + .arg(&submitter_key) .args(timing_args()); if let Some(secs) = max_batch_open_seconds { run_cmd .arg("--max-batch-open-seconds") .arg(secs.to_string()); } - run_cmd.env("RUST_LOG", DEFAULT_SEQUENCER_RUST_LOG); run_cmd .stdout(Stdio::from(run_log)) .stderr(Stdio::from(run_log_err)); let mut child = run_cmd.spawn().map_err(|err| { io_other(format!( "failed to spawn sequencer binary '{}': {err}", - sequencer_bin.display() + env.bin.display() )) })?; - if mine_l1_during_boot { - // A recovery boot blocks in the mempool flush until the stranded - // wallet-nonce slot becomes safe, which needs L1 to keep producing - // blocks. On an on-demand-mining devnet nothing produces them, so mine - // a trickle concurrently with the readiness wait — mirroring a live - // chain that advances during a slow boot. The miner is dropped the - // instant readiness resolves, so L1 only advances for the boot window. - let miner = async { - loop { - tokio::time::sleep(BOOT_L1_MINE_INTERVAL).await; - // Transient RPC hiccups mid-boot are non-fatal — the next tick - // retries. A genuine mining failure manifests as the boot - // timing out, reported by the readiness arm. - let _ = rollups.mine_l1_block_at_wall_clock().await; - } - }; - tokio::select! { - res = wait_for_http_readiness( - endpoint.as_str(), - &mut child, - RECOVERY_BOOT_START_TIMEOUT, - ) => res?, - _ = miner => unreachable!("boot miner loops forever; only readiness resolves"), - } - } else { - wait_for_http_readiness( - endpoint.as_str(), - &mut child, - DEFAULT_SEQUENCER_START_TIMEOUT, - ) - .await?; + match wait_for_http_readiness( + endpoint.as_str(), + &mut child, + DEFAULT_SEQUENCER_START_TIMEOUT, + ) + .await? + { + Readiness::Ready => Ok(RunProcess::Ready(SpawnedSequencerProcess { + child, + endpoint, + internal_endpoint, + })), + Readiness::Exited(status) => Ok(RunProcess::Exited(child, status)), } - - Ok(SpawnedSequencerProcess { - child, - endpoint, - internal_endpoint, - log_path, - }) } /// Configure the child process env to preload libfaketime and point it at @@ -1851,7 +1971,85 @@ fn find_libfaketime_in_dirs(lib_dirs: &[PathBuf], candidates: &[&str]) -> Option mod tests { use std::fs; - use super::{candidate_libfaketime_dirs, find_libfaketime_in_dirs, logical_db_bytes_at}; + use std::ffi::OsString; + use std::path::Path; + + use alloy_primitives::Address; + + use super::{ + SetupRequest, candidate_libfaketime_dirs, find_libfaketime_in_dirs, logical_db_bytes_at, + setup_args, + }; + + fn args_of(base: Option<&Path>, dry_run: bool, genesis_state: Option<&Path>) -> Vec { + let request = SetupRequest { + chain_id: 31337, + base, + dry_run, + genesis_bin: None, + fee_oracle_fixed_log_gas_price: None, + }; + setup_args( + &request, + Path::new("/data"), + "http://127.0.0.1:8545", + Address::repeat_byte(0xaa), + ("0xkey", Address::repeat_byte(0x99)), + genesis_state, + ) + .into_iter() + .map(|arg: OsString| arg.into_string().expect("utf-8 arg")) + .collect() + } + + fn value_after<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + let at = args.iter().position(|arg| arg == flag)?; + args.get(at + 1).map(String::as_str) + } + + /// Setup flushes the submitter's wallet, so it gets the key `run` gets; + /// the genesis state is the host's option, before the subcommand. + #[test] + fn a_genesis_setup_takes_the_key_and_the_genesis_state() { + let args = args_of(None, false, Some(Path::new("/genesis/state"))); + assert_eq!(&args[..3], ["--state-file", "/genesis/state", "setup"]); + assert_eq!( + value_after(&args, "--batch-submitter-private-key"), + Some("0xkey") + ); + assert_eq!(value_after(&args, "--data-dir"), Some("/data")); + assert!(!args.iter().any(|arg| arg == "--base" || arg == "--dry-run")); + } + + /// A setup from a stored machine builds its application from the + /// machine's state, so the genesis state is not passed. + #[test] + fn a_base_setup_names_the_machine_and_no_genesis_state() { + let args = args_of( + Some(Path::new("/machines/base")), + false, + Some(Path::new("/genesis/state")), + ); + assert_eq!(args[0], "setup"); + assert_eq!(value_after(&args, "--base"), Some("/machines/base")); + assert!( + args.iter() + .any(|arg| arg == "--batch-submitter-private-key") + ); + assert!(!args.iter().any(|arg| arg == "--state-file")); + } + + /// The dry run flushes nothing, so it never sees the key. + #[test] + fn a_dry_run_gets_no_key() { + let args = args_of(Some(Path::new("/machines/base")), true, None); + assert!(args.iter().any(|arg| arg == "--dry-run")); + assert!( + !args + .iter() + .any(|arg| arg.contains("private-key") || arg == "0xkey") + ); + } /// The logical size counts pages still in the WAL and drops freed pages, /// so it falls after a DELETE while the file, never vacuumed, does not. diff --git a/tests/harness/src/util.rs b/tests/harness/src/util.rs index ecc8edf0..25bc8c67 100644 --- a/tests/harness/src/util.rs +++ b/tests/harness/src/util.rs @@ -47,21 +47,25 @@ pub(crate) fn timestamped_log_path(logs_dir: &Path, prefix: &str) -> PathBuf { logs_dir.join(format!("{prefix}-{ts}.log")) } +/// How a child's wait for its HTTP listener ended. +pub(crate) enum Readiness { + Ready, + /// The child exited first, with this status. + Exited(std::process::ExitStatus), +} + pub(crate) async fn wait_for_http_readiness( endpoint: &str, child: &mut Child, timeout: Duration, -) -> HarnessResult<()> { +) -> HarnessResult { let deadline = tokio::time::Instant::now() + timeout; loop { if let Some(status) = child.try_wait()? { - return Err(io_other(format!( - "sequencer exited before readiness: status={status}" - )) - .into()); + return Ok(Readiness::Exited(status)); } if http_endpoint_is_ready(endpoint).await { - return Ok(()); + return Ok(Readiness::Ready); } if tokio::time::Instant::now() >= deadline { return Err(io_other(format!( diff --git a/tests/harness/src/wallet.rs b/tests/harness/src/wallet.rs index 6f6bcb64..2ad98268 100644 --- a/tests/harness/src/wallet.rs +++ b/tests/harness/src/wallet.rs @@ -247,9 +247,9 @@ impl WalletL2Client { } /// Override the next user-op nonce this client will sign. A fresh client - /// starts at 0; after a state-preserving recovery (where the sender's nonce - /// carries over) a test sets this to the sender's continuing nonce so its - /// post-recovery ops are accepted. + /// starts at 0; after a restart or a new era, where the sender's nonce + /// carries over, a test sets this to the sender's continuing nonce so its + /// later ops are accepted. pub fn set_next_nonce(&mut self, nonce: u32) { self.next_nonce = nonce; } diff --git a/watchdog/tests/e2e.lua b/watchdog/tests/e2e.lua index d481e2ab..e67bc602 100644 --- a/watchdog/tests/e2e.lua +++ b/watchdog/tests/e2e.lua @@ -19,6 +19,7 @@ local incident = require("watchdog.incident") local json = require("watchdog.json").new() local l1_mod = require("watchdog.l1") local machine = require("watchdog.machine").new() +local main = require("watchdog.main") local replay = require("watchdog.replay") local store_mod = require("watchdog.store") local support = require("watchdog.tests.support") @@ -462,6 +463,88 @@ test("replay re-derives the digest a tick agreed on", function() eq(result.input_count, 3) end) +-- ── Stored machines as a recovery base ──────────────────────────────────── + +--- Where a stored machine rests, read from its registers in the order of the +--- sequencer's base loader (`read_machine_export`, sequencer/src/machine.rs): +--- `setup --base` takes only "rx-accepted", a manual RX_ACCEPTED yield. +local function rest(dir) + local m = cartesi.new() + m:load(dir, {}, cartesi.SHARING_NONE) + local halted, yielded = m:read_reg("iflags_H") ~= 0, m:read_reg("iflags_Y") ~= 0 + local dev, cmd = m:read_reg("htif_tohost_dev"), m:read_reg("htif_tohost_cmd") + local reason = m:read_reg("htif_tohost_reason") + m:destroy() + if halted then + return "halted" + elseif not yielded then + return "not at a yield" + elseif dev == cartesi.HTIF_DEV_YIELD and cmd == cartesi.HTIF_YIELD_CMD_MANUAL and reason == RX_ACCEPTED then + return "rx-accepted" + end + return string.format("yield dev %d cmd %d reason %d", dev, cmd, reason) +end + +local EXCEPTION_REST = string.format("yield dev %d cmd %d reason %d", cartesi.HTIF_DEV_YIELD, + cartesi.HTIF_YIELD_CMD_MANUAL, TX_EXCEPTION) + +test("every checkpoint rests at an accept yield, after a rejected last input too", function() + local cfg, store, l1 = deployment(guest_inputs("hello", "world!", "reject")) + eq(rest(store:head().dir), "rx-accepted") + eq(run_tick(cfg, store, l1, { block = 3, sha256 = digest({ "hello", "world!" }) }).kind, "agreed") + eq(store:head().input_count, 3) + eq(rest(store:head().dir), "rx-accepted") +end) + +test("incident/canonical rests at an accept yield only for a state mismatch", function() + local cfg, store, l1 = deployment(guest_inputs("hello", "world!")) + local wrong = guest_state({ "hello", "WORLD!" }) + eq(run_tick(cfg, store, l1, { block = 2, sha256 = machine.sha256(wrong), bytes = wrong }).kind, "diverged") + eq(incident.marker(store).kind, "state_mismatch") + eq(rest(store:path("incident", "canonical")), "rx-accepted") + + for _, case in ipairs({ + { payload = "halt", rest = "halted" }, + { payload = "exception", rest = EXCEPTION_REST }, + }) do + local dead_cfg, dead_store, dead_l1 = deployment(guest_inputs("hello", case.payload)) + eq(run_tick(dead_cfg, dead_store, dead_l1, { block = 2, sha256 = digest({ "hello" }) }).kind, "diverged") + eq(incident.marker(dead_store).kind, "canonical_machine_dead") + eq(rest(dead_store:path("incident", "canonical")), case.rest) + end +end) + +--- `sequencer-watchdog replay` as an operator runs it; returns the exit +--- code, the decoded result, and the `--out` directory. +local function replay_command(inputs, to_block) + local cfg, store, l1 = deployment(inputs) + local out = support.tmpdir() .. "/replayed" + local printed = {} + local code = main.run({ "replay", "--from", cfg.bootstrap_dir, "--from-block", "0", + "--to-block", tostring(to_block), "--out", out }, { + env = { CARTESI_WATCHDOG_STATE_DIR = store.dir }, + factory = { machine = function() return machine end, l1 = function() return l1 end }, + stdout = { write = function(_, ...) printed[#printed + 1] = table.concat({ ... }) end }, + }) + return code, printed[1] and json.decode(printed[1]), out +end + +test("replay --out rests at an accept yield unless the replay stopped", function() + local code, result, out = replay_command(guest_inputs("hello", "world!", "reject"), 3) + eq(code, 0) + eq(result.input_count, 3) + eq(result.sha256, digest({ "hello", "world!" })) + eq(rest(out), "rx-accepted") + + -- A stop is a result, not a failure: the command exits 0 with `stop` set + -- and leaves the stopped machine at --out. + code, result, out = replay_command(guest_inputs("hello", "halt", "world!"), 3) + eq(code, 0) + eq(result.stop.status, "halted") + eq(result.sha256, nil) + eq(rest(out), "halted") +end) + test("init refuses a bootstrap machine the tick cannot run in place", function() local image = support.tmpdir() .. "/image" machine.clone(require_image(GUEST_IMAGE), image)