diff --git a/Cargo.lock b/Cargo.lock index f7ce150..f1ac372 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -266,6 +266,7 @@ dependencies = [ name = "bitvmx-cpu-definitions" version = "0.8.7" dependencies = [ + "bitcoin_hashes", "blake3", "config", "hex", diff --git a/bitcoin-script-riscv/src/riscv/challenges.rs b/bitcoin-script-riscv/src/riscv/challenges.rs index c9239a5..6d070c4 100644 --- a/bitcoin-script-riscv/src/riscv/challenges.rs +++ b/bitcoin-script-riscv/src/riscv/challenges.rs @@ -4,7 +4,8 @@ use bitvmx_cpu_definitions::{ challenge::{ChallengeType, EquivocationKind}, constants::LAST_STEP_INIT, memory::{Chunk, MemoryAccessType, SectionDefinition}, - trace::{generate_initial_step_hash, hashvec_to_string}, + memory_tree::MemoryModel, + trace::{generate_initial_step_hash, hashvec_to_string, TraceStep}, }; use crate::riscv::{ @@ -133,7 +134,24 @@ pub fn entry_point_challenge(stack: &mut StackTracker, entry_point: u32) { // NOTE: the verifiers provides an step back of the agreed step. // hash_funcion ( UNSIGNED_VERIFIER_HASH_PREV_PREV_STEP:64 | UNSIGINED_VERIFIER_WRITE_ADD_PREV:8 | UNSIGINED_VERIFIER_WRITE_DATA_PREV:8 | UNSIGNED_VERIFIER_WRITE_PC_PREV:8 | UNSIGNED_VERIFIER_WRITE_MICRO_PREV:2 ) == WOTS_PROVER_HASH_PREV_STEP:64 // && WOTS_PROVER_TRACE_READ_PC:8 | WOTS_PROVER_TRACE_READ_MICRO:2 != UNSIGNED_VERIFIER_WRITE_PC_PREV | UNSIGNED_VERIFIER_WRITE_MICRO_PREV -pub fn program_counter_challenge(stack: &mut StackTracker) { +/// Bytes of the blake3 message a state-hash check hashes. +/// +/// The previous state hash and the trace step, and under Merkle memory the +/// post-state memory root as well -- `TraceStep::to_bytes` appends it last, +/// so the script explodes it last. Thirty-three bytes becomes fifty-three. +fn state_hash_bytes(model: MemoryModel) -> u32 { + let nibbles = 40 + 8 + 8 + 8 + 2 + if model.chains_root() { 40 } else { 0 }; + nibbles / 2 +} + +/// Define the memory root the state hash chains, if this model chains one. +fn define_mem_root(stack: &mut StackTracker, model: MemoryModel) -> Option { + model + .chains_root() + .then(|| stack.define(40, "prev_mem_root")) +} + +pub fn program_counter_challenge(stack: &mut StackTracker, model: MemoryModel) { stack.clear_definitions(); // verifier values (unsigned) @@ -142,6 +160,7 @@ pub fn program_counter_challenge(stack: &mut StackTracker) { let prev_write_data = stack.define(8, "prev_write_data"); let prev_write_pc = stack.define(8, "prev_write_pc"); let prev_write_micro = stack.define(2, "prev_write_micro"); + let prev_mem_root = define_mem_root(stack, model); // santize the stack stack.verify_range_var_u4(prev_prev_hash); @@ -149,6 +168,9 @@ pub fn program_counter_challenge(stack: &mut StackTracker) { stack.verify_range_var_u4(prev_write_data); stack.verify_range_var_u4(prev_write_pc); stack.verify_range_var_u4(prev_write_micro); + if let Some(root) = prev_mem_root { + stack.verify_range_var_u4(root); + } // prover values decoded from WOTS let prover_pc = stack.define(8, "prover_read_pc"); @@ -173,8 +195,11 @@ pub fn program_counter_challenge(stack: &mut StackTracker) { stack.explode(prev_write_data); stack.explode(prev_write_pc); stack.explode(prev_write_micro); + if let Some(root) = prev_mem_root { + stack.explode(root); + } - let result = blake3::blake3(stack, (40 + 8 + 8 + 8 + 2) / 2, 5); + let result = blake3::blake3(stack, state_hash_bytes(model), 5); stack.from_altstack(); stack.equals(result, true, prover_prev_hash, true); } @@ -215,7 +240,7 @@ pub fn halt_challenge(stack: &mut StackTracker) { // When the prover expands the trace it could happen that HASH( hash_prev_step | trace_write ) != hash_step // in that case the verifier can win the challenge executing: // hash_funcion ( WOTS_PROVER_HASH_STEP_PREV:64 | WOTS_PROVER_WRITE_ADD:8 | WOTS_PROVER_WRITE_DATA:8 | WOTS_PROVER_WRITE_PC:8 | WOTS_PROVER_WRITE_MICRO:2 ) != WOTS_PROVER_HASH:64 -pub fn trace_hash_challenge(stack: &mut StackTracker) { +pub fn trace_hash_challenge(stack: &mut StackTracker, model: MemoryModel) { stack.clear_definitions(); let prev_hash = stack.define(40, "prev_hash"); @@ -223,6 +248,7 @@ pub fn trace_hash_challenge(stack: &mut StackTracker) { let write_data = stack.define(8, "write_data"); let write_pc = stack.define(8, "write_pc"); let write_micro = stack.define(2, "write_micro"); + let mem_root = define_mem_root(stack, model); let hash = stack.define(40, "hash"); @@ -234,19 +260,23 @@ pub fn trace_hash_challenge(stack: &mut StackTracker) { stack.explode(write_data); stack.explode(write_pc); stack.explode(write_micro); + if let Some(root) = mem_root { + stack.explode(root); + } - let result = blake3::blake3(stack, (40 + 8 + 8 + 8 + 2) / 2, 5); + let result = blake3::blake3(stack, state_hash_bytes(model), 5); stack.from_altstack(); stack.not_equal(result, true, hash, true); } -pub fn trace_hash_zero_challenge(stack: &mut StackTracker) { +pub fn trace_hash_zero_challenge(stack: &mut StackTracker, model: MemoryModel) { stack.clear_definitions(); let write_add = stack.define(8, "write_add"); let write_data = stack.define(8, "write_data"); let write_pc = stack.define(8, "write_pc"); let write_micro = stack.define(2, "write_micro"); + let mem_root = define_mem_root(stack, model); let hash = stack.define(40, "hash"); let step = stack.define(16, "prover_conflict_step_tk"); @@ -257,11 +287,14 @@ pub fn trace_hash_zero_challenge(stack: &mut StackTracker) { //save the hash to compare stack.to_altstack(); - //save the trace steps + //save the trace steps -- one more under Merkle memory, for the root stack.to_altstack(); stack.to_altstack(); stack.to_altstack(); stack.to_altstack(); + if mem_root.is_some() { + stack.to_altstack(); + } //hardcoded the initial hash let prev_hash = stack.hexstr_as_nibbles(&hashvec_to_string(generate_initial_step_hash())); @@ -271,14 +304,20 @@ pub fn trace_hash_zero_challenge(stack: &mut StackTracker) { stack.from_altstack(); stack.from_altstack(); stack.from_altstack(); + if mem_root.is_some() { + stack.from_altstack(); + } stack.explode(prev_hash); stack.explode(write_add); stack.explode(write_data); stack.explode(write_pc); stack.explode(write_micro); + if let Some(root) = mem_root { + stack.explode(root); + } - let result = blake3::blake3(stack, (40 + 8 + 8 + 8 + 2) / 2, 5); + let result = blake3::blake3(stack, state_hash_bytes(model), 5); stack.from_altstack(); stack.not_equal(result, true, hash, true); } @@ -580,12 +619,14 @@ pub fn read_value_challenge(stack: &mut StackTracker) { stack.explode(write_pc); stack.explode(write_micro); - let result = blake3::blake3(stack, (40 + 8 + 8 + 8 + 2) / 2, 5); + // This challenge is about a last_step pointer, so it exists only under + // MemoryModel::LastStep and its message is always the shorter one. + let result = blake3::blake3(stack, state_hash_bytes(MemoryModel::LastStep), 5); stack.from_altstack(); stack.equals(result, true, next_hash, true); } -pub fn correct_hash_challenge(stack: &mut StackTracker) { +pub fn correct_hash_challenge(stack: &mut StackTracker, model: MemoryModel) { stack.clear_definitions(); let prover_hash = stack.define(40, "prover_hash"); @@ -595,6 +636,7 @@ pub fn correct_hash_challenge(stack: &mut StackTracker) { let write_value = stack.define(8, "write_value"); let write_pc = stack.define(8, "write_pc"); let write_micro = stack.define(2, "write_micro"); + let mem_root = define_mem_root(stack, model); let next_hash = stack.define(40, "next_hash"); @@ -608,8 +650,11 @@ pub fn correct_hash_challenge(stack: &mut StackTracker) { stack.explode(write_value); stack.explode(write_pc); stack.explode(write_micro); + if let Some(root) = mem_root { + stack.explode(root); + } - let result = blake3::blake3(stack, (40 + 8 + 8 + 8 + 2) / 2, 5); + let result = blake3::blake3(stack, state_hash_bytes(model), 5); stack.from_altstack(); stack.equals(result, true, next_hash, true); } @@ -685,6 +730,15 @@ fn get_round_and_index(stack: &mut StackTracker, decisions_bits: StackVariable, } } +/// Punishes a party whose re-signed state hash is inconsistent with what was +/// committed for the same position during the search. +/// +/// Which inconsistency is punishable depends on who committed the position. +/// A party is always bound to its own earlier commitment, and it is bound to +/// the counterparty's value at the agreed boundary, because selecting that +/// boundary is what declared the agreement. At the disputed boundary the +/// relation inverts: selecting it declared *disagreement*, so the party is +/// bound to differ, and claiming the counterparty's value is the misbehaviour. pub fn equivocation_resign_challenge( stack: &mut StackTracker, kind: EquivocationKind, @@ -693,12 +747,18 @@ pub fn equivocation_resign_challenge( rounds: u8, nary: u8, nary_last_round: u8, + counterparty_committed: bool, ) { stack.clear_definitions(); + let owner = if counterparty_committed { + "verifier" + } else { + "prover" + }; let true_hash = stack.define( 40, - format!("prover_hash_{}_{}", expected_round, expected_index).as_str(), + format!("{}_hash_{}_{}", owner, expected_round, expected_index).as_str(), ); let wrong_hash = stack.define(40, format!("prover_{:?}_hash_tk", kind).as_str()); let step = stack.define(16, "prover_challenge_step_tk"); @@ -729,7 +789,9 @@ pub fn equivocation_resign_challenge( stack.number(expected_round as u32); stack.op_equalverify(); - stack.equality(true_hash, true, wrong_hash, true, false, true); + // NextHash is the disputed boundary: see the note on this function. + let punish_equality = counterparty_committed && kind == EquivocationKind::NextHash; + stack.equality(true_hash, true, wrong_hash, true, punish_equality, true); stack.drop(decisions_bits); } @@ -747,6 +809,13 @@ pub fn equivocation_hash_challenge(stack: &mut StackTracker) { //TODO: memory section challenge //TODO: program crash challenge - this might be more about finding the right place to challenge that a challenge itself +/// Push the memory root a Merkle step carries, so the script can hash it. +fn push_mem_root(stack: &mut StackTracker, step: &TraceStep) { + if let Some(root) = &step.mem_root { + stack.hexstr_as_nibbles(&hex::encode(root)); + } +} + pub fn execute_challenge(challege_type: &ChallengeType) -> bool { let mut stack = StackTracker::new(); match challege_type { @@ -779,8 +848,9 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { stack.number_u32(prover_trace.get_write().value); stack.number_u32(prover_trace.get_pc().get_address()); stack.byte(prover_trace.get_pc().get_micro()); + push_mem_root(&mut stack, prover_trace); stack.hexstr_as_nibbles(prover_next_hash); - trace_hash_challenge(&mut stack); + trace_hash_challenge(&mut stack, prover_trace.memory_model()); } ChallengeType::TraceHashZero { prover_trace, @@ -791,10 +861,11 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { stack.number_u32(prover_trace.get_write().value); stack.number_u32(prover_trace.get_pc().get_address()); stack.byte(prover_trace.get_pc().get_micro()); + push_mem_root(&mut stack, prover_trace); stack.hexstr_as_nibbles(prover_next_hash); stack.number_u64(*prover_conflict_step_tk); - trace_hash_zero_challenge(&mut stack); + trace_hash_zero_challenge(&mut stack, prover_trace.memory_model()); } ChallengeType::EntryPoint { prover_read_pc, @@ -817,13 +888,14 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { stack.number_u32(trace.get_write().value); stack.number_u32(trace.get_pc().get_address()); stack.byte(trace.get_pc().get_micro()); + push_mem_root(&mut stack, trace); stack.number_u32(prover_pc_read.pc.get_address()); stack.byte(prover_pc_read.pc.get_micro()); stack.hexstr_as_nibbles(prover_step_hash); - program_counter_challenge(&mut stack); + program_counter_challenge(&mut stack, trace.memory_model()); } ChallengeType::Opcode { prover_pc_read, @@ -987,10 +1059,11 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { stack.number_u32(trace.get_write().value); stack.number_u32(trace.get_pc().get_address()); stack.byte(trace.get_pc().get_micro()); + push_mem_root(&mut stack, trace); stack.hexstr_as_nibbles(prover_next_hash); - correct_hash_challenge(&mut stack); + correct_hash_challenge(&mut stack, trace.memory_model()); } ChallengeType::EquivocationResign { prover_true_hash, @@ -999,6 +1072,7 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { kind, expected_round, expected_index, + counterparty_committed, rounds, nary, nary_last_round, @@ -1016,6 +1090,7 @@ pub fn execute_challenge(challege_type: &ChallengeType) -> bool { rounds.unwrap(), nary.unwrap(), nary_last_round.unwrap(), + *counterparty_committed, ); } ChallengeType::EquivocationHash { @@ -1051,6 +1126,92 @@ mod tests { use super::*; use rand::RngExt; + /// Builds an equivocation-resign challenge for a position and runs it. + /// Returns whether the challenge succeeds, i.e. whether the party + /// publishing the trace is punished. + fn resign_challenge_succeeds( + kind: EquivocationKind, + counterparty_committed: bool, + committed_hash: &str, + resigned_hash: &str, + ) -> bool { + // nary = 8 over 64 steps gives two rounds. Step 9 has base-8 digits + // (1, 1), so its hash was published at round 2 index 1; step 10 has + // digits (1, 2) and was published at round 2 index 2. + let (round, index) = match kind { + EquivocationKind::StepHash => (2, 1), + EquivocationKind::NextHash => (2, 2), + }; + execute_challenge(&ChallengeType::EquivocationResign { + prover_true_hash: committed_hash.to_string(), + prover_wrong_hash: resigned_hash.to_string(), + prover_challenge_step_tk: 9, + kind, + expected_round: round, + expected_index: index, + counterparty_committed, + rounds: Some(2), + nary: Some(8), + nary_last_round: Some(0), + }) + } + + const HASH_A: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const HASH_B: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + /// A party is always bound to its own earlier commitment, so differing + /// from it is punishable and matching it is not. This holds at both + /// boundaries and is the behaviour of the asymmetric search. + #[test] + fn own_commitment_is_punished_only_when_it_differs() { + for kind in [EquivocationKind::StepHash, EquivocationKind::NextHash] { + assert!( + resign_challenge_succeeds(kind.clone(), false, HASH_A, HASH_B), + "{:?}: differing from own commitment must be punishable", + kind + ); + assert!( + !resign_challenge_succeeds(kind.clone(), false, HASH_A, HASH_A), + "{:?}: matching own commitment must not be punishable", + kind + ); + } + } + + /// At the agreed boundary the party selected the counterparty's value as + /// agreed, which binds it to that value, so the relation is the same as + /// for its own commitment. + #[test] + fn counterparty_agreed_boundary_is_punished_only_when_it_differs() { + assert!(resign_challenge_succeeds( + EquivocationKind::StepHash, + true, + HASH_A, + HASH_B + )); + assert!(!resign_challenge_succeeds( + EquivocationKind::StepHash, + true, + HASH_A, + HASH_A + )); + } + + /// At the disputed boundary the relation inverts: selecting it declared + /// disagreement with the counterparty's value, so claiming that value is + /// the misbehaviour and differing from it is honest. + #[test] + fn counterparty_disputed_boundary_is_punished_only_when_it_matches() { + assert!( + resign_challenge_succeeds(EquivocationKind::NextHash, true, HASH_A, HASH_A), + "claiming a value previously declared disagreed must be punishable" + ); + assert!( + !resign_challenge_succeeds(EquivocationKind::NextHash, true, HASH_A, HASH_B), + "an honest party that differs at the disputed boundary must not be punished" + ); + } + fn test_entry_point_challenge_aux( wots_prover_pc: u32, wots_prover_micro: u8, @@ -1102,7 +1263,7 @@ mod tests { stack.hexstr_as_nibbles(pre_hash); - program_counter_challenge(&mut stack); + program_counter_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success @@ -1273,7 +1434,7 @@ mod tests { stack.number_u64(step); - trace_hash_zero_challenge(&mut stack); + trace_hash_zero_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success @@ -1317,7 +1478,7 @@ mod tests { stack.hexstr_as_nibbles(hash); - trace_hash_challenge(&mut stack); + trace_hash_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success @@ -1360,7 +1521,7 @@ mod tests { stack.hexstr_as_nibbles(next_hash); - correct_hash_challenge(&mut stack); + correct_hash_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success @@ -2357,7 +2518,17 @@ mod tests { stack.hexstr_as_nibbles(hash2); stack.number_u64(step); - equivocation_resign_challenge(stack, kind, round, index, rounds, nary, nary_last_round); + // the existing cases all concern a party's own commitment + equivocation_resign_challenge( + stack, + kind, + round, + index, + rounds, + nary, + nary_last_round, + false, + ); stack.op_true(); // interactive(&stack); @@ -2546,7 +2717,7 @@ mod tests { stack.byte(micro); stack.hexstr_as_nibbles(&hex::encode(hash)); - trace_hash_challenge(&mut stack); + trace_hash_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success == expected_to_succeed @@ -2633,7 +2804,7 @@ mod tests { stack.hexstr_as_nibbles(&hex::encode(prover_pre_hash)); - program_counter_challenge(&mut stack); + program_counter_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success == expected_to_succeed @@ -2655,7 +2826,7 @@ mod tests { stack.hexstr_as_nibbles(&hex::encode(hash)); stack.number_u64(step); - trace_hash_zero_challenge(&mut stack); + trace_hash_zero_challenge(&mut stack, MemoryModel::LastStep); stack.op_true(); stack.run().success == expected_to_succeed @@ -4521,3 +4692,101 @@ mod tests { } } } + +#[cfg(test)] +mod merkle_state_hash_tests { + use super::*; + use bitvmx_cpu_definitions::memory_tree::{MemoryTree, DEPLOYED_DEPTH}; + use bitvmx_cpu_definitions::trace::{compute_step_hash, ProgramCounter, TraceStep, TraceWrite}; + + fn a_root() -> [u8; 20] { + MemoryTree::with_words(DEPLOYED_DEPTH, [(7u32, 0xdead_beefu32), (0x3C00_0001, 42)]).root() + } + + #[test] + fn the_message_is_thirty_three_bytes_or_fifty_three() { + assert_eq!(state_hash_bytes(MemoryModel::LastStep), 33); + assert_eq!(state_hash_bytes(MemoryModel::Merkle), 53); + assert_eq!( + state_hash_bytes(MemoryModel::Merkle) - state_hash_bytes(MemoryModel::LastStep), + 20, + "exactly one root" + ); + } + + /// Run `trace_hash_challenge` against a step and a claimed next hash. + /// The challenge *succeeds* when the claimed hash is wrong, so a + /// correctly-hashed step must make it fail. + fn trace_hash_challenge_succeeds(step: &TraceStep, claimed_next: [u8; 20]) -> bool { + let prev = [0x11u8; 20]; + let mut stack = StackTracker::new(); + stack.hexstr_as_nibbles(&hex::encode(prev)); + stack.number_u32(step.get_write().address); + stack.number_u32(step.get_write().value); + stack.number_u32(step.get_pc().get_address()); + stack.byte(step.get_pc().get_micro()); + push_mem_root(&mut stack, step); + stack.hexstr_as_nibbles(&hex::encode(claimed_next)); + trace_hash_challenge(&mut stack, step.memory_model()); + stack.op_true(); + stack.run().success + } + + fn correct_next_hash(step: &TraceStep) -> [u8; 20] { + let mut hasher = ::blake3::Hasher::new(); + compute_step_hash(&mut hasher, &[0x11u8; 20], &step.to_bytes()) + } + + #[test] + fn a_merkle_step_hashes_in_script_exactly_as_it_does_in_rust() { + let step = TraceStep::new_with_root( + TraceWrite::new(0x1000, 0xabcd_1234), + ProgramCounter::new(0x8000_0010, 0), + a_root(), + ); + assert_eq!(step.memory_model(), MemoryModel::Merkle); + assert!( + !trace_hash_challenge_succeeds(&step, correct_next_hash(&step)), + "an honest Merkle step must not be challengeable" + ); + let mut wrong = correct_next_hash(&step); + wrong[0] ^= 1; + assert!( + trace_hash_challenge_succeeds(&step, wrong), + "and a wrong next hash must be" + ); + } + + #[test] + fn the_root_is_covered_by_the_hash_the_script_checks() { + // The whole point of chaining it: change only the root and the + // state hash must change, so a prover cannot swap memory under an + // agreed state. + let write = TraceWrite::new(0x1000, 0xabcd_1234); + let pc = ProgramCounter::new(0x8000_0010, 0); + let step = TraceStep::new_with_root(write.clone(), pc.clone(), a_root()); + let other_root = MemoryTree::with_words(DEPLOYED_DEPTH, [(7u32, 0xdead_beefu32)]).root(); + assert_ne!(a_root(), other_root); + let swapped = TraceStep::new_with_root(write, pc, other_root); + + assert_ne!(correct_next_hash(&step), correct_next_hash(&swapped)); + assert!( + trace_hash_challenge_succeeds(&swapped, correct_next_hash(&step)), + "a step claiming another root's hash is challengeable in script" + ); + } + + #[test] + fn a_last_step_trace_still_hashes_as_it_always_did() { + let step = TraceStep::new( + TraceWrite::new(0x1000, 0xabcd_1234), + ProgramCounter::new(0x8000_0010, 0), + ); + assert_eq!(step.memory_model(), MemoryModel::LastStep); + assert_eq!(step.to_bytes().len(), 13); + assert!(!trace_hash_challenge_succeeds( + &step, + correct_next_hash(&step) + )); + } +} diff --git a/bitcoin-script-riscv/src/riscv/memory_proof.rs b/bitcoin-script-riscv/src/riscv/memory_proof.rs new file mode 100644 index 0000000..d87bfe5 --- /dev/null +++ b/bitcoin-script-riscv/src/riscv/memory_proof.rs @@ -0,0 +1,393 @@ +//! Adjudicating one node of a published memory multiproof. +//! +//! Under [`MemoryModel::Merkle`](bitvmx_cpu_definitions::memory_tree::MemoryModel) +//! a read is justified by a Merkle path rather than by a `last_step` +//! pointer, and the endgame has to decide whether that path is good. +//! +//! It does not check the whole path on chain. A multiproof over the four +//! words one instruction touches carries 89 siblings at the deployed depth, +//! and recomputing all of it in Script would be absurd. Instead the prover +//! publishes the proof *and the working it drives* -- every node the +//! verifier computes on the way to the root -- and only if a challenger +//! says one of those nodes is wrong is a single node adjudicated. That is +//! the locally-refutable shape: the claim is published, and one check over +//! a few words of a canonical verification trace decides it. +//! +//! So the on-chain cost of the memory argument is one of these scripts, +//! paid only when a path is actually disputed. Each is a naysayer: it +//! *succeeds* when the published working is wrong, exactly as +//! `trace_hash_challenge` succeeds when a claimed state hash is wrong. +//! +//! # Why the node hash is blake3 +//! +//! Bitcoin has no `OP_CAT`, so a Merkle path cannot be checked with the +//! native `OP_SHA256`: there is no way to build a parent's preimage from +//! two children on the stack. The hash has to be implemented in Script +//! over nibbles, as BitVMX already does for its state hash, and then the +//! binding constraint is not weight but the thousand-element stack limit. +//! +//! Measured in this crate's interpreter: +//! +//! | hash over 40 bytes | script | peak stack | +//! |--------------------|-------:|-----------:| +//! | SHA-256 | 213,758 B | **979** | +//! | blake3 -> 20 bytes | 69,872 B | **659** | +//! +//! SHA-256 peaks at 979 *before* anything else is on the stack. A node +//! check also has to hold the claimed parent, forty nibbles, which puts it +//! at 1,019 and over the limit -- the script aborts with `StackSize` rather +//! than deciding anything. blake3 leaves 341 elements spare, is a third of +//! the size, and emits twenty bytes natively instead of thirty-two that +//! must then be truncated. It is also the hash BitVMX already computes in +//! Script, so the two share one set of lookup tables. +//! +//! [`SHA256_NODE_PEAK_STACK`] records the measurement that forces this. +//! +//! # Unreconciled: the Rust tree still hashes with SHA-256 +//! +//! `bitvmx_cpu_definitions::memory_tree` builds the tree with SHA-256, +//! following the paper's Definition 8.1, and this module checks it with +//! blake3. **They do not agree, and the endgame cannot work until one of +//! them moves.** The measurement above says which way: SHA-256 cannot be +//! checked on chain at all, so the tree's hash is the thing that has to +//! change. Nothing depends on the tree's hash except the tree itself -- +//! the publication commitment keeps SHA-256, because Bitcoin's own +//! signature hash is SHA-256 and that is a different object. +//! +//! `the_rust_tree_and_this_script_do_not_yet_agree` pins the gap so that +//! it cannot be forgotten, and will start failing the moment it closes. + +use bitcoin_script_functions::hash::blake3; +use bitcoin_script_stack::stack::StackTracker; + +/// Nibbles in a tree node: twenty bytes. +pub const NODE_NIBBLES: u32 = 40; + +/// Nibbles in a machine word: four bytes. +pub const WORD_NIBBLES: u32 = 8; + +/// blake3 output words; five of them are the twenty bytes a node is. +const NODE_WORDS: u8 = 5; + +/// Peak stack of the in-Script SHA-256 over a forty-byte preimage, with +/// nothing else held. A node check must also hold the forty nibbles of the +/// claimed parent, so SHA-256 cannot be used for this at all. +pub const SHA256_NODE_PEAK_STACK: u32 = 979; + +/// Bitcoin's limit on stack plus altstack during one script. +pub const STACK_LIMIT: u32 = 1000; + +/// SHA-256 cannot be used for a node check at all: its peak is reached +/// before the claimed parent is accounted for, and the forty nibbles of +/// that parent put it over the limit. A script in that state aborts with +/// `StackSize` instead of deciding anything. +const _: () = assert!(SHA256_NODE_PEAK_STACK + NODE_NIBBLES > STACK_LIMIT); + +/// Refute a claimed internal node of a published multiproof's working. +/// +/// Witness, bottom to top: the left child, the right child, and the node +/// claimed for their parent. Succeeds when +/// `blake3(left || right)[0..20] != claimed`, which is what makes the +/// publisher of that working lose. +pub fn memory_node_challenge(stack: &mut StackTracker) { + stack.clear_definitions(); + + let left = stack.define(NODE_NIBBLES, "left"); + let right = stack.define(NODE_NIBBLES, "right"); + let claimed = stack.define(NODE_NIBBLES, "claimed_parent"); + + stack.verify_range_var_u4(left); + stack.verify_range_var_u4(right); + stack.verify_range_var_u4(claimed); + + // Hold the claim aside; the hash consumes everything below it. + stack.to_altstack(); + + stack.explode(left); + stack.explode(right); + + let computed = blake3::blake3(stack, 2 * NODE_NIBBLES / 2, NODE_WORDS); + + stack.from_altstack(); + stack.not_equal(computed, true, claimed, true); +} + +/// Refute a claimed leaf of a published multiproof. +/// +/// Witness, bottom to top: the four-byte word, and the node claimed for its +/// leaf. Succeeds when `blake3(w)[0..20] != claimed`. +pub fn memory_leaf_challenge(stack: &mut StackTracker) { + stack.clear_definitions(); + + let word = stack.define(WORD_NIBBLES, "word"); + let claimed = stack.define(NODE_NIBBLES, "claimed_leaf"); + + stack.verify_range_var_u4(word); + stack.verify_range_var_u4(claimed); + + stack.to_altstack(); + + stack.explode(word); + + let computed = blake3::blake3(stack, WORD_NIBBLES / 2, NODE_WORDS); + + stack.from_altstack(); + stack.not_equal(computed, true, claimed, true); +} + +#[cfg(test)] +mod tests { + use super::*; + + type Node = [u8; 20]; + + /// What the script computes for an internal node. + fn node(left: &Node, right: &Node) -> Node { + let mut hasher = ::blake3::Hasher::new(); + hasher.update(left); + hasher.update(right); + let mut out = [0u8; 20]; + out.copy_from_slice(&hasher.finalize().as_bytes()[..20]); + out + } + + /// What the script computes for a leaf. + fn leaf(word: u32) -> Node { + let mut hasher = ::blake3::Hasher::new(); + hasher.update(&word.to_be_bytes()); + let mut out = [0u8; 20]; + out.copy_from_slice(&hasher.finalize().as_bytes()[..20]); + out + } + + /// Outcome of running a naysayer script. + /// + /// A refutation that lands leaves `Refuted`. One that does not lands + /// `Upheld`, by failing the `OP_VERIFY` inside `not_equal` -- that is + /// the script *deciding*, and is what an honest node looks like. + /// Anything else is the script falling over before it decided, which a + /// test must never mistake for either answer: a script that blows the + /// stack limit "fails" exactly like an honest node unless the reason is + /// checked, and an earlier draft of this module did. + #[derive(Debug, PartialEq, Eq)] + enum Outcome { + Refuted, + Upheld, + Broke(String), + } + + fn outcome(res: bitcoin_script_stack::debugger::StepResult) -> Outcome { + match (res.success, res.error, res.error_msg.as_str()) { + (true, false, _) => Outcome::Refuted, + (false, true, "Some(Verify)") => Outcome::Upheld, + (_, _, msg) => Outcome::Broke(msg.to_string()), + } + } + + fn run_node(left: &Node, right: &Node, claimed: &Node) -> Outcome { + let mut stack = StackTracker::new(); + stack.hexstr_as_nibbles(&hex::encode(left)); + stack.hexstr_as_nibbles(&hex::encode(right)); + stack.hexstr_as_nibbles(&hex::encode(claimed)); + memory_node_challenge(&mut stack); + stack.op_true(); + outcome(stack.run()) + } + + fn run_leaf(word: u32, claimed: &Node) -> Outcome { + let mut stack = StackTracker::new(); + stack.number_u32(word); + stack.hexstr_as_nibbles(&hex::encode(claimed)); + memory_leaf_challenge(&mut stack); + stack.op_true(); + outcome(stack.run()) + } + + #[test] + fn an_honest_internal_node_cannot_be_refuted() { + let (l, r) = (leaf(0xdead_beef), leaf(42)); + assert_eq!( + run_node(&l, &r, &node(&l, &r)), + Outcome::Upheld, + "an honest node must be upheld by the script deciding, not by \ + the script breaking" + ); + } + + #[test] + fn a_wrong_internal_node_is_refuted() { + let (l, r) = (leaf(0xdead_beef), leaf(42)); + let mut wrong = node(&l, &r); + wrong[19] ^= 1; + assert_eq!( + run_node(&l, &r, &wrong), + Outcome::Refuted, + "one flipped bit" + ); + + // Order matters: a working that swaps the children is a different + // node, and refutable. + assert_eq!( + run_node(&r, &l, &node(&l, &r)), + Outcome::Refuted, + "left and right are not interchangeable" + ); + } + + #[test] + fn an_honest_leaf_cannot_be_refuted() { + for word in [0u32, 1, 42, 0xdead_beef, u32::MAX] { + assert_eq!(run_leaf(word, &leaf(word)), Outcome::Upheld, "{word:#x}"); + } + } + + #[test] + fn a_wrong_leaf_is_refuted() { + assert_eq!(run_leaf(42, &leaf(43)), Outcome::Refuted, "wrong word"); + assert_eq!( + run_leaf(0, &node(&leaf(0), &leaf(0))), + Outcome::Refuted, + "an internal node passed off as a leaf" + ); + } + + #[test] + fn both_checks_fit_the_stack_limit() { + let (l, r) = (leaf(1), leaf(2)); + let mut stack = StackTracker::new(); + stack.hexstr_as_nibbles(&hex::encode(l)); + stack.hexstr_as_nibbles(&hex::encode(r)); + stack.hexstr_as_nibbles(&hex::encode(node(&l, &r))); + let before = stack.get_script().len(); + memory_node_challenge(&mut stack); + let bytes = stack.get_script().len() - before; + let peak = stack.get_max_stack_size(); + println!("memory_node_challenge: {bytes} script bytes, peak stack {peak}"); + + assert!(peak < STACK_LIMIT, "peak {peak} must fit {STACK_LIMIT}"); + // That SHA-256 does not fit is asserted at compile time above. + assert!( + peak + NODE_NIBBLES < SHA256_NODE_PEAK_STACK, + "and blake3 leaves real room: {peak} against {SHA256_NODE_PEAK_STACK}" + ); + } + + #[test] + fn the_rust_tree_and_this_script_do_not_yet_agree() { + // Deliberately asserting a defect, so it is visible rather than + // latent. definitions::memory_tree hashes nodes with SHA-256 and + // this module checks them with blake3; the endgame needs one hash. + // When the tree moves to blake3 this test fails and should be + // replaced by one that checks they agree. + let (l, r) = (leaf(1), leaf(2)); + let sha_parent = bitvmx_cpu_definitions::memory_tree::parent(&l, &r); + assert_ne!( + sha_parent, + node(&l, &r), + "if these now match, the tree has moved to blake3 -- delete \ + this test and assert agreement instead" + ); + assert_eq!( + run_node(&l, &r, &sha_parent), + Outcome::Refuted, + "which is why an honest SHA-256 node is refuted by this script" + ); + } + + #[test] + fn a_whole_path_is_checked_one_node_at_a_time() { + // Eighteen levels of an honest path: every node of the working + // survives its own challenge, and each is one script. + let mut current = leaf(0xcafe_babe); + for level in 0..18u32 { + let sibling = leaf(level); + let parent = node(¤t, &sibling); + assert_eq!( + run_node(¤t, &sibling, &parent), + Outcome::Upheld, + "honest node at level {level}" + ); + let mut wrong = parent; + wrong[0] ^= 0x80; + assert_eq!( + run_node(¤t, &sibling, &wrong), + Outcome::Refuted, + "and a wrong one at level {level}" + ); + current = parent; + } + } +} + +/// Sizing the scripts a dispute can put on chain. +/// +/// These are measurements, not estimates: each script is built by the same +/// code the protocol builds it with, and its length read off. What they +/// feed is the worst-case weight comparison between BitVMX as it stands and +/// the compact construction. +#[cfg(test)] +mod script_sizes { + use crate::riscv::challenges::*; + use crate::riscv::memory_proof::{memory_leaf_challenge, memory_node_challenge}; + use bitcoin_script_stack::stack::StackTracker; + use bitvmx_cpu_definitions::memory_tree::MemoryModel; + + fn size(name: &str, f: impl Fn(&mut StackTracker)) -> (String, usize, u32) { + let mut stack = StackTracker::new(); + f(&mut stack); + ( + name.to_string(), + stack.get_script().len(), + stack.get_max_stack_size(), + ) + } + + #[test] + fn report_every_challenge_script_size() { + let mut rows = vec![ + size("trace_hash (last_step, 33 B)", |s| { + trace_hash_challenge(s, MemoryModel::LastStep) + }), + size("trace_hash (merkle, 53 B)", |s| { + trace_hash_challenge(s, MemoryModel::Merkle) + }), + size("trace_hash_zero (last_step)", |s| { + trace_hash_zero_challenge(s, MemoryModel::LastStep) + }), + size("trace_hash_zero (merkle)", |s| { + trace_hash_zero_challenge(s, MemoryModel::Merkle) + }), + size("program_counter (last_step)", |s| { + program_counter_challenge(s, MemoryModel::LastStep) + }), + size("program_counter (merkle)", |s| { + program_counter_challenge(s, MemoryModel::Merkle) + }), + size("correct_hash (last_step)", |s| { + correct_hash_challenge(s, MemoryModel::LastStep) + }), + size("correct_hash (merkle)", |s| { + correct_hash_challenge(s, MemoryModel::Merkle) + }), + size("read_value (last_step only)", read_value_challenge), + size("future_read (last_step only)", future_read_challenge), + size("halt", halt_challenge), + size("equivocation_hash", equivocation_hash_challenge), + size("memory_node (merkle only)", memory_node_challenge), + size("memory_leaf (merkle only)", memory_leaf_challenge), + ]; + rows.sort_by_key(|r| std::cmp::Reverse(r.1)); + + println!( + "\n{:<32} {:>10} {:>12}", + "challenge script", "bytes", "peak stack" + ); + println!("{}", "-".repeat(56)); + for (name, bytes, peak) in &rows { + println!("{name:<32} {bytes:>10} {peak:>12}"); + } + let biggest = rows.iter().max_by_key(|r| r.1).unwrap(); + println!("\nlargest: {} at {} bytes", biggest.0, biggest.1); + assert!(biggest.1 > 0); + } +} diff --git a/bitcoin-script-riscv/src/riscv/mod.rs b/bitcoin-script-riscv/src/riscv/mod.rs index a25698c..1c3162e 100644 --- a/bitcoin-script-riscv/src/riscv/mod.rs +++ b/bitcoin-script-riscv/src/riscv/mod.rs @@ -5,6 +5,7 @@ pub mod instructions; pub mod instructions_load; pub mod instructions_store; pub mod memory_alignment; +pub mod memory_proof; pub mod operations; pub mod script_utils; pub mod trace; diff --git a/bitcoin-script-riscv/src/riscv/script_utils.rs b/bitcoin-script-riscv/src/riscv/script_utils.rs index 8c1cdc1..648d618 100644 --- a/bitcoin-script-riscv/src/riscv/script_utils.rs +++ b/bitcoin-script-riscv/src/riscv/script_utils.rs @@ -3442,6 +3442,41 @@ mod tests { s } + /// The cases that broke the garbled RV32M circuit, run against + /// the Script implementation. + /// + /// There, `q*d + r` was checked by adding a 32-bit remainder to a + /// 64-bit product, which truncated, so the check compared the + /// product's sign against the sum's. Those differ exactly when + /// `q*d + r` is negative and `q*d` is not -- a zero quotient with + /// a negative dividend -- and an honest `div(-1, 2)` was reported + /// as faulted. + /// + /// Script verifies division a different way, so this asks whether + /// it shares the defect. It does not; these are pinned so that a + /// future change to either side has to keep them agreeing. + #[test] + fn signed_division_with_a_zero_quotient_and_negative_dividend() { + for (n, d) in [ + (-1i32, 2i32), + (-7, -9), + (-5, 3), + (-991_549_079, -1_001_680_343), + (7, 9), + (-1, 3), + (-2, 3), + (i32::MIN + 1, i32::MAX), + ] { + assert_eq!(n / d, if n.abs() < d.abs() { 0 } else { n / d }); + assert!( + div_rem_signed_aux(n, d), + "div/rem({n}, {d}) -> q={}, r={}", + n / d, + n % d + ); + } + } + #[test] fn fuzz_div_rem() { fuzz_generic_and_catch_panics( diff --git a/definitions/Cargo.toml b/definitions/Cargo.toml index 7f3d276..e60de94 100644 --- a/definitions/Cargo.toml +++ b/definitions/Cargo.toml @@ -5,10 +5,11 @@ edition = "2021" [dependencies] config = "0.15.25" -serde = "1.0.229" +serde = { version = "1.0.229", features = ["derive"] } thiserror = "2.0.20" hex = "0.4.3" blake3 = "1.8.6" +bitcoin_hashes = "0.14.101" serde_json = "1.0.151" strum = "0.28.0" strum_macros = "0.28.0" diff --git a/definitions/src/challenge.rs b/definitions/src/challenge.rs index bf94a03..3af01d0 100644 --- a/definitions/src/challenge.rs +++ b/definitions/src/challenge.rs @@ -110,6 +110,10 @@ pub enum ChallengeType { kind: EquivocationKind, expected_round: u8, expected_index: u8, + /// Whether the position was committed by the counterparty rather than + /// by the party now publishing the trace. Only possible under the + /// full-duplex search, where the dissections alternate. + counterparty_committed: bool, rounds: Option, nary: Option, nary_last_round: Option, @@ -222,6 +226,21 @@ pub enum EmulatorResultType { v_decision: u32, round: u8, }, + /// The trailing stage of a full-duplex search: the selection over the final + /// dissection, together with the trace for the disputed step. Both ride on + /// the adjudication message, so they are produced together. + FullDuplexFinalTraceResult { + decision: u32, + prover_final_trace: ProverFinalTraceType, + }, + /// One stage of a full-duplex search, which both selects and dissects. + /// `decision` is absent at stage 1, which has no previous dissection to + /// select over. + FullDuplexStageResult { + decision: Option, + hashes: Vec, + stage: u8, + }, ProverFinalTraceResult { prover_final_trace: ProverFinalTraceType, }, diff --git a/definitions/src/lib.rs b/definitions/src/lib.rs index 56760c0..927c39b 100644 --- a/definitions/src/lib.rs +++ b/definitions/src/lib.rs @@ -1,5 +1,6 @@ pub mod challenge; pub mod memory; +pub mod memory_tree; pub mod trace; pub mod constants { diff --git a/definitions/src/memory_tree.rs b/definitions/src/memory_tree.rs new file mode 100644 index 0000000..e232daa --- /dev/null +++ b/definitions/src/memory_tree.rs @@ -0,0 +1,922 @@ +//! Merkelized memory: the model the compact endgame adjudicates against. +//! +//! Memory is a sparse binary Merkle tree over four-byte machine words. A +//! leaf is `SHA256(w)[0..20]` for the word's contents `w`, an internal node +//! is `SHA256(left || right)[0..20]`, and the hash of an all-zero subtree is +//! precomputed per level so that untouched memory costs nothing. +//! +//! What this replaces is the `last_step` scheme: there, a read carried the +//! number of the step that last wrote its address, and a challenger who +//! disputed the claim opened a second search to that step. Here a read +//! carries a Merkle path instead, and the dispute is a hash check against a +//! root the machine state already commits to. Six challenge branches -- +//! `FutureRead`, `ReadValue`, `ReadSection`, `WriteSection`, `Uninitialized` +//! and `Initialized` -- are statements about that scheme and have no +//! counterpart here. +//! +//! # Depth is a parameter, and the deployed depth is 30 +//! +//! The depth fixes how much memory the tree can address, and BitVMX spreads +//! its address space across the whole 32-bit range: the register file sits +//! at `0xF000_0000` and the stack at `0xE000_0000` with eight megabytes +//! under it (`emulator/src/lib.rs`). A tree over a contiguous megabyte +//! cannot hold that map -- it cannot address the registers at all -- so +//! [`DEPLOYED_DEPTH`] is `30`, one leaf per word of the full address space. +//! +//! [`PAPER_DEPTH`] is the shallower tree the paper measures, over a +//! megabyte, and is kept because it is what the published figures are +//! quoted at. The construction is identical at either; only the path +//! lengths differ, and [`MemoryTree::new`] takes the depth so that both are +//! reachable and comparable. +//! +//! # What one instruction touches +//! +//! At most four distinct words. The machine rejects unaligned reads, writes +//! and jumps, so no access straddles a leaf; and RISC-V is load/store, so no +//! instruction both touches memory and names three registers: +//! +//! | instruction | touches | words | +//! |-------------|-------------------------------|---------| +//! | R-type ALU | fetch, and `rs1`, `rs2`, `rd` | `1 + 3` | +//! | load | fetch, data, `rs1`, `rd` | `1+1+2` | +//! | store | fetch, data, `rs1`, `rs2` | `1+1+2` | +//! +//! Four either way, which is what [`MAX_TOUCHED`] records. Three of them +//! always lie in the register block, so they share all but a few levels -- +//! which is why a multiproof over the worst reachable pattern carries far +//! fewer nodes than four independent paths would. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::OnceLock; + +use bitcoin_hashes::{sha256, Hash}; +use serde::{Deserialize, Serialize}; + +/// How a dispute justifies that a read returned the value it claims. +/// +/// These are alternatives, not layers: a trace carries one or the other. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemoryModel { + /// A read names the step that last wrote its address, and disputing + /// that claim opens a second search to that step. This is BitVMX's + /// original scheme and remains the default, so an existing program + /// definition means exactly what it meant before. + #[default] + LastStep, + /// A read is opened against a Merkle root that the machine state hash + /// commits to. The compact endgame's model: one hash check against a + /// root, and no second search. + Merkle, +} + +impl MemoryModel { + /// Whether the machine state hash chains the memory root. + pub fn chains_root(&self) -> bool { + matches!(self, MemoryModel::Merkle) + } +} + +/// Depth of the deployed tree: one leaf per word of the 32-bit address +/// space, which is the only depth that can hold BitVMX's memory map. +pub const DEPLOYED_DEPTH: u8 = 30; + +/// Depth of the tree the paper measures: `2^18` words over `2^20` bytes. +pub const PAPER_DEPTH: u8 = 18; + +/// The deepest tree this module builds tables for. +pub const MAX_DEPTH: u8 = 32; + +/// Width of a node, in bytes. Twenty, matching the machine state hash. +pub const NODE_BYTES: usize = 20; + +/// The most distinct words one instruction can touch. +pub const MAX_TOUCHED: usize = 4; + +/// A truncated SHA-256 digest. +pub type Node = [u8; NODE_BYTES]; + +fn truncate(h: sha256::Hash) -> Node { + let mut out = [0u8; NODE_BYTES]; + out.copy_from_slice(&h.to_byte_array()[..NODE_BYTES]); + out +} + +/// The leaf for a word's contents: `SHA256(w)[0..20]`, big-endian. +pub fn leaf(word: u32) -> Node { + truncate(sha256::Hash::hash(&word.to_be_bytes())) +} + +/// An internal node: `SHA256(left || right)[0..20]`. +pub fn parent(left: &Node, right: &Node) -> Node { + let mut buf = [0u8; 2 * NODE_BYTES]; + buf[..NODE_BYTES].copy_from_slice(left); + buf[NODE_BYTES..].copy_from_slice(right); + truncate(sha256::Hash::hash(&buf)) +} + +/// The node of an all-zero subtree, by level: `0` is a leaf holding the zero +/// word, and level `d` is the root of empty memory in a tree of depth `d`. +pub fn empty_node(level: u8) -> Node { + static TABLE: OnceLock> = OnceLock::new(); + let table = TABLE.get_or_init(|| { + let mut v = vec![leaf(0)]; + for _ in 0..MAX_DEPTH { + let last = *v.last().expect("seeded above"); + v.push(parent(&last, &last)); + } + v + }); + table[level as usize] +} + +/// The root of memory in which every word is zero, in a tree of `depth`. +pub fn empty_root(depth: u8) -> Node { + empty_node(depth) +} + +/// The leaf index a byte address maps to. Addresses are word-aligned, and +/// the machine rejects any that is not. +pub fn leaf_index(address: u32) -> u32 { + address / 4 +} + +/// A sparse Merkle tree over `2^depth` four-byte words. +/// +/// Only words that differ from zero, and the nodes above them, are stored, +/// so empty memory is free and a write costs `depth` compressions. +#[derive(Clone, Debug)] +pub struct MemoryTree { + depth: u8, + /// Level `0` is the leaves, level `depth` the root; entries equal to + /// [`empty_node`] of their level are kept out. + nodes: HashMap<(u8, u32), Node>, + words: HashMap, +} + +impl Default for MemoryTree { + fn default() -> Self { + Self::new(DEPLOYED_DEPTH) + } +} + +impl MemoryTree { + /// Empty memory in a tree of the given depth. + pub fn new(depth: u8) -> Self { + assert!( + depth > 0 && depth <= MAX_DEPTH, + "depth {depth} outside 1..={MAX_DEPTH}" + ); + Self { + depth, + nodes: HashMap::new(), + words: HashMap::new(), + } + } + + /// Memory with the given words set, the rest zero. + pub fn with_words>(depth: u8, words: I) -> Self { + let mut tree = Self::new(depth); + for (index, word) in words { + tree.write(index, word); + } + tree + } + + /// The tree's depth. + pub fn depth(&self) -> u8 { + self.depth + } + + /// How many words the tree addresses. + pub fn word_count(&self) -> u64 { + 1u64 << self.depth + } + + /// The node at `level` and `index`, which is the empty node of that + /// level when nothing below it has been written. + pub fn node(&self, level: u8, index: u32) -> Node { + self.nodes + .get(&(level, index)) + .copied() + .unwrap_or_else(|| empty_node(level)) + } + + /// The root of memory as it stands. + pub fn root(&self) -> Node { + self.node(self.depth, 0) + } + + /// The word at `index`, zero if never written. + pub fn read(&self, index: u32) -> u32 { + debug_assert!( + (index as u64) < self.word_count(), + "word index out of range" + ); + self.words.get(&index).copied().unwrap_or(0) + } + + /// Set the word at `index` and update the path above it. + pub fn write(&mut self, index: u32, word: u32) { + assert!( + (index as u64) < self.word_count(), + "word index {index} outside a depth-{} tree", + self.depth + ); + if word == 0 { + self.words.remove(&index); + } else { + self.words.insert(index, word); + } + self.set(0, index, leaf(word)); + let mut i = index; + for level in 0..self.depth { + let (l, r) = (i & !1, i | 1); + let node = parent(&self.node(level, l), &self.node(level, r)); + i >>= 1; + self.set(level + 1, i, node); + } + } + + /// Set many words at once, rebuilding each affected level once instead + /// of walking a full path per word. + /// + /// Loading an ELF writes every initialised word of every section, and + /// those words are contiguous, so they share almost all of their + /// ancestors. Updating them one at a time costs `depth` compressions + /// each; doing it by level costs one per distinct node actually + /// affected, which for a contiguous run is close to the size of the run + /// rather than `depth` times it. + pub fn bulk_write>(&mut self, items: I) { + let mut touched: BTreeSet = BTreeSet::new(); + for (index, word) in items { + assert!( + (index as u64) < self.word_count(), + "word index {index} outside a depth-{} tree", + self.depth + ); + if word == 0 { + self.words.remove(&index); + } else { + self.words.insert(index, word); + } + self.set(0, index, leaf(word)); + touched.insert(index); + } + for level in 0..self.depth { + let parents: BTreeSet = touched.iter().map(|&i| i >> 1).collect(); + for &j in parents.iter() { + let node = parent(&self.node(level, 2 * j), &self.node(level, 2 * j + 1)); + self.set(level + 1, j, node); + } + touched = parents; + } + } + + fn set(&mut self, level: u8, index: u32, node: Node) { + if node == empty_node(level) { + self.nodes.remove(&(level, index)); + } else { + self.nodes.insert((level, index), node); + } + } + + /// The sibling path authenticating one leaf, leaf-first. + pub fn path(&self, index: u32) -> Vec { + let mut out = Vec::with_capacity(self.depth as usize); + let mut i = index; + for level in 0..self.depth { + out.push(self.node(level, i ^ 1)); + i >>= 1; + } + out + } + + /// The nodes a multiproof over `leaves` must carry: every sibling on a + /// path that is not itself derivable from the revealed leaves. + /// + /// Returned bottom-up, and within a level in ascending index order, + /// which is the order [`MultiProof::compute_root`] consumes them in. + pub fn authenticated_set(&self, leaves: &[u32]) -> Vec<(u8, u32)> { + let mut needed = Vec::new(); + walk(leaves, self.depth, |level, index, derivable| { + if !derivable { + needed.push((level, index)); + } + }); + needed + } + + /// The nodes a verifier *computes* while checking a multiproof: every + /// strict ancestor of a revealed leaf, the root included. + /// + /// These are the verification trace. The authenticated set is the + /// verifier's input and this is its working, and publishing only the + /// former leaves a failed step of the latter with nothing committed to + /// be refuted against. + pub fn derived_set(&self, leaves: &[u32]) -> Vec<(u8, u32)> { + let mut frontier: BTreeSet<(u8, u32)> = leaves.iter().map(|&i| (0u8, i)).collect(); + let mut derived = Vec::new(); + for level in 0..self.depth { + let next: BTreeSet<(u8, u32)> = + frontier.iter().map(|&(_, j)| (level + 1, j >> 1)).collect(); + derived.extend(next.iter().copied()); + frontier = next; + } + derived + } + + /// A multiproof over the given leaves, with their current contents. + pub fn multiproof(&self, leaves: &[u32]) -> MultiProof { + let mut sorted: Vec = leaves.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let siblings = self + .authenticated_set(&sorted) + .into_iter() + .map(|(level, index)| self.node(level, index)) + .collect(); + MultiProof { + depth: self.depth, + leaves: sorted.iter().map(|&i| (i, self.read(i))).collect(), + siblings, + } + } +} + +/// Walk the tree bottom-up over the ancestors of `leaves`, reporting each +/// sibling encountered and whether it is derivable from what is already +/// known. Siblings are visited bottom-up and, within a level, in ascending +/// index order. +fn walk(leaves: &[u32], depth: u8, mut visit: F) { + let mut known: BTreeSet<(u8, u32)> = leaves.iter().map(|&i| (0u8, i)).collect(); + let mut frontier: BTreeSet<(u8, u32)> = known.clone(); + for level in 0..depth { + for &(_, j) in frontier.iter() { + visit(level, j ^ 1, known.contains(&(level, j ^ 1))); + } + let next: BTreeSet<(u8, u32)> = + frontier.iter().map(|&(_, j)| (level + 1, j >> 1)).collect(); + known.extend(next.iter().copied()); + frontier = next; + } +} + +/// A multiproof: the revealed words, and the sibling nodes needed to carry +/// them to a root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MultiProof { + /// Depth of the tree the proof was taken from. + pub depth: u8, + /// Revealed leaves as `(index, word)`, ascending and deduplicated. + pub leaves: Vec<(u32, u32)>, + /// Siblings in the order [`MemoryTree::authenticated_set`] produces them. + pub siblings: Vec, +} + +impl MultiProof { + /// Bytes on the wire: the siblings alone. + pub fn proof_bytes(&self) -> usize { + self.siblings.len() * NODE_BYTES + } + + /// Recompute the root this proof implies, with `overrides` replacing the + /// contents of the named leaves. Passing an empty slice gives the + /// pre-state root; passing the instruction's write gives the post-state + /// root, because a write changes exactly one leaf and leaves every + /// sibling on its path alone. + pub fn compute_root(&self, overrides: &[(u32, u32)]) -> Option { + let indices: Vec = self.leaves.iter().map(|&(i, _)| i).collect(); + let mut current: HashMap<(u8, u32), Node> = self + .leaves + .iter() + .map(|&(i, w)| { + let word = overrides + .iter() + .find(|&&(oi, _)| oi == i) + .map(|&(_, ow)| ow) + .unwrap_or(w); + ((0u8, i), leaf(word)) + }) + .collect(); + + let mut supplied = self.siblings.iter(); + let mut failed = false; + let mut pending: Vec<(u8, u32, Node)> = Vec::new(); + walk(&indices, self.depth, |level, index, derivable| { + if failed || derivable { + return; + } + match supplied.next() { + Some(node) => pending.push((level, index, *node)), + None => failed = true, + } + }); + if failed || supplied.next().is_some() { + return None; + } + for (level, index, node) in pending { + current.entry((level, index)).or_insert(node); + } + + let mut frontier: BTreeSet<(u8, u32)> = indices.iter().map(|&i| (0u8, i)).collect(); + for level in 0..self.depth { + let next: BTreeSet<(u8, u32)> = + frontier.iter().map(|&(_, j)| (level + 1, j >> 1)).collect(); + for &(_, j) in next.iter() { + let left = *current.get(&(level, 2 * j))?; + let right = *current.get(&(level, 2 * j + 1))?; + current.insert((level + 1, j), parent(&left, &right)); + } + frontier = next; + } + current.get(&(self.depth, 0)).copied() + } +} + +/// Check a single leaf against a root. +pub fn verify_path(root: &Node, index: u32, word: u32, path: &[Node]) -> bool { + let mut current = leaf(word); + let mut i = index; + for sibling in path { + current = if i & 1 == 1 { + parent(sibling, ¤t) + } else { + parent(¤t, sibling) + }; + i >>= 1; + } + current == *root +} + +/// Check a multiproof against a root. +pub fn verify_multiproof(root: &Node, proof: &MultiProof) -> bool { + proof.compute_root(&[]).is_some_and(|r| r == *root) +} + +/// Check one instruction's memory accesses: the reads against the pre-state +/// root, and the single write against the post-state root, from one proof. +/// +/// The write's leaf must be among the proof's leaves; a write changes +/// exactly that leaf, so the same siblings carry both roots. +pub fn verify_step( + pre_root: &Node, + post_root: &Node, + proof: &MultiProof, + write: Option<(u32, u32)>, +) -> bool { + if proof.compute_root(&[]) != Some(*pre_root) { + return false; + } + match write { + None => pre_root == post_root, + Some((index, word)) => { + proof.leaves.iter().any(|&(i, _)| i == index) + && proof.compute_root(&[(index, word)]) == Some(*post_root) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pattern the paper's model maximises the multiproof over at + /// [`PAPER_DEPTH`]: a fetch word, a data word, and two register words at + /// opposite ends of the 32-leaf register block. + const PAPER_WORST: [u32; 4] = [0, 31, 65_536, 196_607]; + + /// BitVMX's memory map, from `emulator/src/lib.rs`. These are the + /// constants that decide the depth. + const REGISTERS_BASE_ADDRESS: u32 = 0xF000_0000; + const STACK_BASE_ADDRESS: u32 = 0xE000_0000; + const STACK_SIZE: u32 = 0x80_0000; + + #[test] + fn a_leaf_is_the_word_not_the_slot() { + let expect = sha256::Hash::hash(&[0u8; 4]); + assert_eq!(leaf(0), truncate(expect)); + assert_eq!(empty_node(0), leaf(0)); + assert_ne!( + leaf(0), + truncate(sha256::Hash::hash(&[0u8; 32])), + "the empty leaf is a zero word, not a zero 32-byte slot" + ); + } + + #[test] + fn the_deployed_depth_is_what_bitvmx_s_memory_map_forces() { + // The register file and the stack sit near the top of the address + // space, so a tree over a contiguous megabyte cannot address them. + let tree = MemoryTree::new(DEPLOYED_DEPTH); + for address in [ + REGISTERS_BASE_ADDRESS, + STACK_BASE_ADDRESS, + STACK_BASE_ADDRESS + STACK_SIZE - 4, + u32::MAX - 3, + ] { + assert!( + (leaf_index(address) as u64) < tree.word_count(), + "depth {DEPLOYED_DEPTH} must address {address:#x}" + ); + } + let paper = MemoryTree::new(PAPER_DEPTH); + assert!( + (leaf_index(REGISTERS_BASE_ADDRESS) as u64) >= paper.word_count(), + "and depth {PAPER_DEPTH} cannot address the register file at all" + ); + assert!( + (STACK_SIZE / 4) as u64 > paper.word_count(), + "the 8 MB stack alone is larger than the whole depth-{PAPER_DEPTH} tree" + ); + assert_eq!(DEPLOYED_DEPTH, 30); + assert_eq!( + 1u64 << DEPLOYED_DEPTH, + (1u64 << 32) / 4, + "one leaf per word" + ); + } + + #[test] + fn empty_memory_has_a_root_and_writing_zero_keeps_it() { + for depth in [PAPER_DEPTH, DEPLOYED_DEPTH] { + let mut tree = MemoryTree::new(depth); + let empty = tree.root(); + assert_eq!(empty, empty_root(depth)); + tree.write(12_345, 0); + assert_eq!(tree.root(), empty, "writing zero is not a change"); + tree.write(12_345, 7); + assert_ne!(tree.root(), empty); + tree.write(12_345, 0); + assert_eq!(tree.root(), empty, "and undoing it restores the root"); + } + } + + #[test] + fn depth_fixes_how_much_memory_is_addressable() { + assert_eq!(MemoryTree::new(PAPER_DEPTH).word_count(), 262_144); + assert_eq!(MemoryTree::new(PAPER_DEPTH).word_count() * 4, 1 << 20); + assert_eq!(MemoryTree::new(DEPLOYED_DEPTH).word_count() * 4, 1 << 32); + assert_eq!(MemoryTree::default().depth(), DEPLOYED_DEPTH); + } + + #[test] + fn a_path_authenticates_its_leaf_and_nothing_else() { + let mut tree = MemoryTree::with_words( + DEPLOYED_DEPTH, + [ + (3, 0xdead_beef), + (100, 1), + (leaf_index(REGISTERS_BASE_ADDRESS), 9), + ], + ); + tree.write(leaf_index(STACK_BASE_ADDRESS), 0x4142_4344); + let root = tree.root(); + for &i in &[ + 3u32, + 100, + leaf_index(REGISTERS_BASE_ADDRESS), + leaf_index(STACK_BASE_ADDRESS), + 12_345, + ] { + let path = tree.path(i); + assert_eq!(path.len(), DEPLOYED_DEPTH as usize); + assert!(verify_path(&root, i, tree.read(i), &path), "leaf {i}"); + assert!( + !verify_path(&root, i, tree.read(i).wrapping_add(1), &path), + "a different word must not verify at {i}" + ); + } + } + + #[test] + fn at_the_paper_s_depth_the_worst_case_is_fifty_two_siblings() { + // compact2/python/memory.py: word_worst_case() -> [0, 31, 65536, + // 196607], word_worst_case_bytes() -> 1040, trace 2140. + let tree = MemoryTree::new(PAPER_DEPTH); + let needed = tree.authenticated_set(&PAPER_WORST); + assert_eq!(needed.len(), 52, "siblings"); + assert_eq!(needed.len() * NODE_BYTES, 1_040, "bytes on the wire"); + + let derived = tree.derived_set(&PAPER_WORST); + assert_eq!(derived.len(), 55, "nodes the verifier computes"); + assert_eq!(derived.len() * NODE_BYTES, 1_100, "the working alone"); + assert_eq!((needed.len() + derived.len()) * NODE_BYTES, 2_140); + } + + #[test] + fn at_the_deployed_depth_a_real_instruction_costs_more_but_publishes_the_same() { + // A store: instruction fetch, a stack word, and two registers. The + // three register words share all but a few levels, which is why four + // leaves cost far less than four independent paths. + let tree = MemoryTree::new(DEPLOYED_DEPTH); + let reg = leaf_index(REGISTERS_BASE_ADDRESS); + let shape = [1u32, 0x1FFF_FFFF, reg, reg + 33]; + let proof = tree.authenticated_set(&shape).len(); + let derived = tree.derived_set(&shape).len(); + assert_eq!(proof, 89, "siblings at depth 30"); + assert_eq!(proof * NODE_BYTES, 1_780); + assert_eq!(derived, 92); + assert_eq!((proof + derived) * NODE_BYTES, 3_620, "the published trace"); + + // What is published is a FlatMerkle file: 20-byte nodes packed three + // to a 64-byte element, rounded up to a power of two, at least four. + let elements = |nodes: usize| -> usize { + let n = usize::max(4, nodes.div_ceil(3)); + n.next_power_of_two() + }; + assert_eq!(elements(52 + 55), 64, "the paper's tree"); + assert_eq!(elements(proof + derived), 64, "and the deployed one"); + assert_eq!( + 64 * (2 * 64 - 2) + 64, + 8_128, + "so the endgame publishes 8,128 bytes either way" + ); + assert!( + proof + derived <= 64 * 3, + "with {} nodes of margin", + 64 * 3 - (proof + derived) + ); + assert_eq!(64 * 3 - (proof + derived), 11, "the margin, exactly"); + } + + #[test] + fn four_leaves_is_the_most_one_instruction_touches() { + assert_eq!(MAX_TOUCHED, 4); + assert_eq!(PAPER_WORST.len(), MAX_TOUCHED); + } + + #[test] + fn a_multiproof_round_trips_at_both_depths() { + for (depth, shape) in [ + (PAPER_DEPTH, PAPER_WORST), + ( + DEPLOYED_DEPTH, + [1u32, 0x1FFF_FFFF, 0x3C00_0000, 0x3C00_0021], + ), + ] { + let mut tree = MemoryTree::new(depth); + for (i, w) in shape.iter().zip([11u32, 22, 33, 44]) { + tree.write(*i, w); + } + tree.write(999, 0xffff_ffff); + let root = tree.root(); + let proof = tree.multiproof(&shape); + assert_eq!(proof.depth, depth); + assert_eq!(proof.leaves.len(), 4); + assert!(verify_multiproof(&root, &proof), "depth {depth}"); + } + } + + #[test] + fn a_multiproof_beats_the_separate_paths_it_replaces() { + let tree = MemoryTree::with_words( + PAPER_DEPTH, + PAPER_WORST.iter().copied().zip([1u32, 2, 3, 4]), + ); + let root = tree.root(); + let proof = tree.multiproof(&PAPER_WORST); + assert!(verify_multiproof(&root, &proof)); + let separate: usize = PAPER_WORST.iter().map(|&i| tree.path(i).len()).sum(); + assert_eq!(separate, 72, "four paths of eighteen"); + assert_eq!(proof.siblings.len(), 52, "against one multiproof"); + } + + #[test] + fn a_tampered_multiproof_does_not_verify() { + let tree = MemoryTree::with_words( + PAPER_DEPTH, + PAPER_WORST.iter().copied().zip([1u32, 2, 3, 4]), + ); + let root = tree.root(); + + let mut wrong_word = tree.multiproof(&PAPER_WORST); + wrong_word.leaves[1].1 ^= 1; + assert!(!verify_multiproof(&root, &wrong_word)); + + let mut wrong_sibling = tree.multiproof(&PAPER_WORST); + wrong_sibling.siblings[17][0] ^= 1; + assert!(!verify_multiproof(&root, &wrong_sibling)); + + let mut short = tree.multiproof(&PAPER_WORST); + short.siblings.pop(); + assert!(!verify_multiproof(&root, &short), "too few siblings"); + assert_eq!(short.compute_root(&[]), None); + + let mut long = tree.multiproof(&PAPER_WORST); + long.siblings.push([0u8; NODE_BYTES]); + assert!(!verify_multiproof(&root, &long), "too many siblings"); + + let mut wrong_depth = tree.multiproof(&PAPER_WORST); + wrong_depth.depth = DEPLOYED_DEPTH; + assert!( + !verify_multiproof(&root, &wrong_depth), + "a proof does not carry across depths" + ); + } + + #[test] + fn adjacent_leaves_need_no_siblings_between_them() { + let tree = MemoryTree::with_words(PAPER_DEPTH, [(10, 1), (11, 2)]); + assert_eq!( + tree.authenticated_set(&[10]).len(), + PAPER_DEPTH as usize, + "one leaf needs a full path" + ); + assert_eq!( + tree.authenticated_set(&[10, 11]).len(), + PAPER_DEPTH as usize - 1, + "a sibling pair derives its parent and saves the bottom node" + ); + } + + #[test] + fn one_proof_carries_both_roots_of_a_write() { + let mut tree = MemoryTree::with_words( + DEPLOYED_DEPTH, + [(1, 1), (0x1FFF_FFFF, 2), (0x3C00_0000, 3), (0x3C00_0021, 4)], + ); + let shape = [1u32, 0x1FFF_FFFF, 0x3C00_0000, 0x3C00_0021]; + let pre_root = tree.root(); + let proof = tree.multiproof(&shape); + + let (written, new_word) = (0x3C00_0021u32, 0xcafe_babe); + tree.write(written, new_word); + let post_root = tree.root(); + assert_ne!(pre_root, post_root); + + assert_eq!(proof.compute_root(&[]), Some(pre_root)); + assert_eq!(proof.compute_root(&[(written, new_word)]), Some(post_root)); + assert!(verify_step( + &pre_root, + &post_root, + &proof, + Some((written, new_word)) + )); + assert_eq!( + proof.siblings.len(), + 89, + "and it is one set of siblings, not two" + ); + } + + #[test] + fn verify_step_rejects_a_mismatched_write() { + let mut tree = MemoryTree::with_words( + PAPER_DEPTH, + PAPER_WORST.iter().copied().zip([1u32, 2, 3, 4]), + ); + let pre_root = tree.root(); + let proof = tree.multiproof(&PAPER_WORST); + tree.write(31, 0xcafe_babe); + let post_root = tree.root(); + + assert!(!verify_step(&pre_root, &post_root, &proof, Some((31, 0)))); + assert!( + !verify_step(&pre_root, &post_root, &proof, None), + "claiming no write when the roots differ" + ); + assert!( + !verify_step(&pre_root, &post_root, &proof, Some((12_345, 1))), + "the written leaf must be one the proof carries" + ); + assert!( + verify_step(&pre_root, &pre_root, &proof, None), + "an instruction that writes nothing leaves the root alone" + ); + } + + #[test] + fn the_root_does_not_depend_on_the_order_of_writes() { + let a = MemoryTree::with_words(DEPLOYED_DEPTH, [(5, 1), (1_000, 2), (70_000, 3)]); + let b = MemoryTree::with_words(DEPLOYED_DEPTH, [(70_000, 3), (5, 1), (1_000, 2)]); + assert_eq!(a.root(), b.root()); + + let mut c = MemoryTree::new(DEPLOYED_DEPTH); + c.write(5, 99); + c.write(1_000, 2); + c.write(5, 1); + c.write(70_000, 3); + assert_eq!(c.root(), a.root(), "and overwriting leaves no trace"); + } + + #[test] + #[should_panic(expected = "outside a depth-18 tree")] + fn a_write_past_the_end_of_the_tree_is_refused() { + MemoryTree::new(PAPER_DEPTH).write(1 << PAPER_DEPTH, 1); + } +} + +/// Vectors computed by the paper's independent model, `compact2/python/ +/// memory.py`, which builds the same tree from a different implementation in +/// a different language. Agreement on concrete bytes is what makes the two +/// the same specification rather than two constructions of the same shape. +#[cfg(test)] +mod cross_check { + use super::*; + + fn hex(node: &Node) -> String { + node.iter().map(|b| format!("{b:02x}")).collect() + } + + #[test] + fn the_zero_leaf_matches_the_model() { + assert_eq!(hex(&leaf(0)), "df3f619804a92fdb4057192dc43dd748ea778adc"); + } + + #[test] + fn the_empty_root_matches_the_model() { + assert_eq!( + hex(&empty_root(PAPER_DEPTH)), + "bac77725f5cfaa5386da6723865ec4b4f0be00b5" + ); + } + + #[test] + fn a_populated_root_matches_the_model() { + let tree = MemoryTree::with_words( + PAPER_DEPTH, + [(0, 11), (31, 22), (65_536, 33), (196_607, 44)], + ); + assert_eq!( + hex(&tree.root()), + "6f3f7354099b983e93c611cbc5dcd9a594e2609d" + ); + } + + #[test] + fn a_post_write_root_matches_the_model() { + let mut tree = MemoryTree::with_words( + PAPER_DEPTH, + [(0, 11), (31, 22), (65_536, 33), (196_607, 44)], + ); + let proof = tree.multiproof(&[0, 31, 65_536, 196_607]); + tree.write(31, 0xcafe_babe); + assert_eq!( + hex(&tree.root()), + "4a8bd32caa72513360aaeae9994e4a540358a327" + ); + assert_eq!( + proof.compute_root(&[(31, 0xcafe_babe)]).map(|r| hex(&r)), + Some("4a8bd32caa72513360aaeae9994e4a540358a327".to_string()) + ); + } +} + +#[cfg(test)] +mod bulk_tests { + use super::*; + + #[test] + fn bulk_write_agrees_with_writing_one_at_a_time() { + let items: Vec<(u32, u32)> = (0..500u32) + .map(|i| (i, i.wrapping_mul(2_654_435_761))) + .collect(); + let mut one_by_one = MemoryTree::new(DEPLOYED_DEPTH); + for &(i, w) in &items { + one_by_one.write(i, w); + } + let mut bulk = MemoryTree::new(DEPLOYED_DEPTH); + bulk.bulk_write(items.iter().copied()); + assert_eq!(bulk.root(), one_by_one.root()); + for &(i, w) in &items { + assert_eq!(bulk.read(i), w); + } + } + + #[test] + fn bulk_write_handles_scattered_sections_and_zeros() { + // Three runs far apart, as a loaded ELF leaves them: code low, data + // in the middle, registers at the top of the address space. + let mut items: Vec<(u32, u32)> = Vec::new(); + for base in [0x0000_0400u32, 0x0004_0000, 0x3C00_0000] { + for k in 0..64u32 { + items.push((base + k, if k % 7 == 0 { 0 } else { base ^ k })); + } + } + let mut one_by_one = MemoryTree::new(DEPLOYED_DEPTH); + for &(i, w) in &items { + one_by_one.write(i, w); + } + let mut bulk = MemoryTree::new(DEPLOYED_DEPTH); + bulk.bulk_write(items.iter().copied()); + assert_eq!(bulk.root(), one_by_one.root()); + + // And a proof taken from the bulk-built tree verifies against it. + let shape = [0x0000_0400u32, 0x0004_0000, 0x3C00_0000, 0x3C00_0001]; + let proof = bulk.multiproof(&shape); + assert!(verify_multiproof(&bulk.root(), &proof)); + } + + #[test] + fn bulk_write_is_incremental() { + let mut tree = MemoryTree::new(DEPLOYED_DEPTH); + tree.bulk_write([(1u32, 10u32), (2, 20)]); + let after_first = tree.root(); + tree.bulk_write([(3u32, 30u32)]); + let mut expected = MemoryTree::new(DEPLOYED_DEPTH); + expected.bulk_write([(1u32, 10u32), (2, 20), (3, 30)]); + assert_eq!(tree.root(), expected.root()); + assert_ne!(after_first, tree.root()); + } +} diff --git a/definitions/src/trace.rs b/definitions/src/trace.rs index 7fdd4b5..537a629 100644 --- a/definitions/src/trace.rs +++ b/definitions/src/trace.rs @@ -1,6 +1,7 @@ use std::str::FromStr; use crate::memory::MemoryWitness; +use crate::memory_tree::Node; use blake3::Hasher; use serde::{Deserialize, Serialize}; @@ -83,11 +84,47 @@ impl TraceWrite { pub struct TraceStep { pub write_1: TraceWrite, pub write_pc: ProgramCounter, + /// The memory root *after* this step, under [`MemoryModel::Merkle`]. + /// + /// `None` under `LastStep`, where memory is not Merkelized and the + /// state hash commits only to the write. Keeping it optional is what + /// lets the two models produce byte-identical hashes for the trace + /// they share. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mem_root: Option, } impl TraceStep { pub fn new(write_1: TraceWrite, write_pc: ProgramCounter) -> TraceStep { - TraceStep { write_1, write_pc } + TraceStep { + write_1, + write_pc, + mem_root: None, + } + } + + /// A step under [`MemoryModel::Merkle`], carrying the post-state root. + pub fn new_with_root( + write_1: TraceWrite, + write_pc: ProgramCounter, + mem_root: Node, + ) -> TraceStep { + TraceStep { + write_1, + write_pc, + mem_root: Some(mem_root), + } + } + + /// Which memory model this step was produced under. + /// + /// A step carrying a root is a Merkle step; the trace says which model + /// it belongs to, so a verifier never has to be told separately. + pub fn memory_model(&self) -> crate::memory_tree::MemoryModel { + match self.mem_root { + Some(_) => crate::memory_tree::MemoryModel::Merkle, + None => crate::memory_tree::MemoryModel::LastStep, + } } pub fn new_step( @@ -99,6 +136,7 @@ impl TraceStep { TraceStep { write_1: TraceWrite::new(write_address, write_value), write_pc: ProgramCounter::new(program_counter, micro), + mem_root: None, } } @@ -115,6 +153,13 @@ impl TraceStep { bytes.extend(&self.write_1.value.to_be_bytes()); bytes.extend(&self.write_pc.get_address().to_be_bytes()); bytes.push(self.write_pc.get_micro()); + // Under Merkle memory the state hash chains the post-state root, so + // that an agreed state pins the root the endgame opens against. + // Thirty-three bytes becomes fifty-three; under LastStep nothing is + // appended and the hash is what it always was. + if let Some(root) = &self.mem_root { + bytes.extend(root); + } bytes } } diff --git a/emulator/src/decision/challenge.rs b/emulator/src/decision/challenge.rs index 7572a9f..a267e09 100644 --- a/emulator/src/decision/challenge.rs +++ b/emulator/src/decision/challenge.rs @@ -13,7 +13,7 @@ use tracing::{error, info, warn}; use crate::{ decision::{ execution_log::VerifierChallengeLog, - nary_search::{choose_segment, ExecutionHashes, NArySearchType}, + nary_search::{choose_segment, ExecutionHashes, NArySearchType, Party}, }, executor::utils::{FailConfiguration, FailSelectionBits}, loader::program_definition::ProgramDefinition, @@ -187,6 +187,258 @@ pub fn verifier_check_execution( Ok(Some(step_to_challenge)) } +/// One stage of a full-duplex search: select over the counterparty's previous +/// dissection, then dissect the subinterval just selected, in a single step. +/// +/// This is the whole of what distinguishes the full-duplex search from the +/// asymmetric one at the emulator level. The asymmetric search splits the same +/// work across `prover_get_hashes_for_round` and `verifier_choose_segment`, +/// one per message; here a party does both, so the search advances a whole +/// level per message. +/// +/// `counterparty_hashes` is the dissection published at the previous stage, and +/// is empty at stage 1, which has no predecessor to select over. +/// +/// `counterparty_decision` is the digit the counterparty published alongside +/// that dissection, absent before stage 3. A party only selects at its own +/// stages, so without applying the counterparty's digit its base would lag a +/// level behind and it would compare against the wrong positions. Recording +/// both parties' digits in order is also what makes the digit sequence the +/// base-n representation of the disputed step. +/// +/// Returns the stage's own selection and dissection. Beyond the last round +/// there is nothing left to dissect: the trailing selection rides on the +/// adjudication message, so the dissection comes back empty. +#[allow(clippy::too_many_arguments)] +fn advance_full_duplex( + program_def: &ProgramDefinition, + input: Vec, + input_checkpoint_path: &str, + output_checkpoint_path: &str, + stage: u8, + counterparty_hashes: &[String], + counterparty_decision: Option, + last_step: Option, + fail_config: Option, + base_step: &mut u64, + step_to_challenge: &mut u64, + decisions: &mut Vec, + published_hash_rounds: &mut Vec>, +) -> Result<(Option, Vec), EmulatorError> { + let nary_def = program_def.nary_def(); + + // Catch up on the counterparty's selection over the dissection we + // published two stages ago. + if let Some(digit) = counterparty_decision { + *base_step = nary_def.step_from_base_and_bits(stage - 2, *base_step, digit); + decisions.push(digit); + } + + // Past the last round only the trailing selection remains. It is reported + // but not committed here: it rides on the adjudication message, and + // prover_final_trace applies it along with everything else that message + // carries. + let commit_selection = stage <= nary_def.total_rounds(); + + let selection = if stage > 1 { + let previous = stage - 1; + + // Our own view of the positions the counterparty dissected. + let mine = program_def.get_round_hashes( + input_checkpoint_path, + output_checkpoint_path, + input.clone(), + previous, + *base_step, + fail_config.clone(), + last_step, + )?; + + let (bits, new_base, new_selected) = choose_segment( + &nary_def, + *base_step, + *step_to_challenge, + previous, + &ExecutionHashes::from_hexstr(&counterparty_hashes.to_vec()), + &ExecutionHashes::from_hexstr(&mine), + NArySearchType::ConflictStep, + None, + ); + + // Record the counterparty's dissection under its round number. Stages + // run in order and each records the previous round before its own, so + // this vector stays indexed by round for both parties. + published_hash_rounds.push(counterparty_hashes.to_vec()); + if commit_selection { + *base_step = new_base; + *step_to_challenge = new_selected; + decisions.push(bits); + } + Some(bits) + } else { + None + }; + + // Dissect the subinterval just selected, unless the search has run out of + // levels and only the trailing selection remained. + let hashes = if commit_selection { + let hashes = program_def.get_round_hashes( + input_checkpoint_path, + output_checkpoint_path, + input, + stage, + *base_step, + fail_config, + last_step, + )?; + published_hash_rounds.push(hashes.clone()); + hashes + } else { + Vec::new() + }; + + info!( + "Full-duplex stage {}: selection {:?} base {} bound {}", + stage, selection, base_step, step_to_challenge + ); + + Ok((selection, hashes)) +} + +/// The prover's half of a full-duplex stage. +pub fn prover_full_duplex_stage( + program_definition_file: &str, + input_checkpoint_path: &str, + output_checkpoint_path: &str, + stage: u8, + counterparty_hashes: &[String], + counterparty_decision: Option, + fail_config: Option, +) -> Result<(Option, Vec), EmulatorError> { + let mut challenge_log = ProverChallengeLog::load(input_checkpoint_path)?; + let program_def = ProgramDefinition::from_config(program_definition_file)?; + let input = challenge_log.input.clone(); + let last_step = challenge_log.execution.last_step; + let log = challenge_log.get_nary_log(NArySearchType::ConflictStep); + + let mut base_step = log.base_step; + let mut step_to_challenge = log.step_to_challenge; + let mut decisions = std::mem::take(&mut log.verifier_decisions); + let mut published = std::mem::take(&mut log.hash_rounds); + + let result = advance_full_duplex( + &program_def, + input, + input_checkpoint_path, + output_checkpoint_path, + stage, + counterparty_hashes, + counterparty_decision, + Some(last_step), + fail_config, + &mut base_step, + &mut step_to_challenge, + &mut decisions, + &mut published, + )?; + + let log = challenge_log.get_nary_log(NArySearchType::ConflictStep); + log.base_step = base_step; + log.step_to_challenge = step_to_challenge; + log.verifier_decisions = decisions; + log.hash_rounds = published; + challenge_log.save(output_checkpoint_path)?; + + Ok(result) +} + +/// The prover's trailing stage: select over the final dissection and produce +/// the trace for the disputed step, in one step. +/// +/// The trailing selection has no dissection of its own and rides on the +/// adjudication message, which also carries the expanded trace. Computing both +/// in one job keeps that true of the implementation as well, and spares the +/// protocol a round trip it would otherwise spend asking for the digit and +/// then asking for the trace. +pub fn prover_full_duplex_final_trace( + program_definition_file: &str, + input_checkpoint_path: &str, + output_checkpoint_path: &str, + stage: u8, + counterparty_hashes: &[String], + counterparty_decision: Option, + fail_config: Option, +) -> Result<(u32, ProverFinalTraceType), EmulatorError> { + let (selection, _) = prover_full_duplex_stage( + program_definition_file, + input_checkpoint_path, + output_checkpoint_path, + stage, + counterparty_hashes, + counterparty_decision, + fail_config.clone(), + )?; + + let selection = selection.ok_or_else(|| { + EmulatorError::ChallengeError("the trailing stage must produce a selection".to_string()) + })?; + + let trace = prover_final_trace( + program_definition_file, + output_checkpoint_path, + output_checkpoint_path, + selection + 1, + fail_config, + )?; + Ok((selection, trace)) +} + +/// The verifier's half of a full-duplex stage. +pub fn verifier_full_duplex_stage( + program_definition_file: &str, + input_checkpoint_path: &str, + output_checkpoint_path: &str, + stage: u8, + counterparty_hashes: &[String], + counterparty_decision: Option, + fail_config: Option, +) -> Result<(Option, Vec), EmulatorError> { + let mut challenge_log = VerifierChallengeLog::load(input_checkpoint_path)?; + let program_def = ProgramDefinition::from_config(program_definition_file)?; + let input = challenge_log.input.clone(); + let log = challenge_log.get_nary_log(NArySearchType::ConflictStep); + + let mut base_step = log.base_step; + let mut step_to_challenge = log.step_to_challenge; + let mut decisions = std::mem::take(&mut log.verifier_decisions); + let mut published = std::mem::take(&mut log.prover_hash_rounds); + + let result = advance_full_duplex( + &program_def, + input, + input_checkpoint_path, + output_checkpoint_path, + stage, + counterparty_hashes, + counterparty_decision, + None, + fail_config, + &mut base_step, + &mut step_to_challenge, + &mut decisions, + &mut published, + )?; + + let log = challenge_log.get_nary_log(NArySearchType::ConflictStep); + log.base_step = base_step; + log.step_to_challenge = step_to_challenge; + log.verifier_decisions = decisions; + log.prover_hash_rounds = published; + challenge_log.save(output_checkpoint_path)?; + + Ok(result) +} + pub fn verifier_choose_segment( program_definition_file: &str, input_checkpoint_path: &str, @@ -458,51 +710,101 @@ pub fn verifier_choose_challenge( let mapping = &nary_def.step_mapping(&conflict_step_log.verifier_decisions); let step = conflict_step_log.step_to_challenge; + let mode = program_def.search_mode(); + let total_rounds = nary_def.total_rounds(); - let (prover_step_hash, prover_next_hash) = + // Each round's dissection is recorded under its round number by whichever + // party published it, so this is the committed view in either mode. + let (committed_step_hash, committed_next_hash) = get_hashes(mapping, &conflict_step_log.prover_hash_rounds, step); let nary = return_script_parameters.then_some(nary_def.nary); let nary_last_round = return_script_parameters.then_some(nary_def.nary_last_round); let rounds = return_script_parameters.then_some(nary_def.total_rounds()); - if (prover_step_hash != resigned_step_hash && force == ForceChallenge::No) + // A boundary has no position when the search never published it: step 0 is + // agreed by construction. Treat an absent position as the prover's own, + // which is what it is whenever the asymmetric search is in use. + let step_access = mapping.get(&step).copied(); + let next_access = mapping.get(&(step + 1)).copied(); + let owned_by_counterparty = |access: Option<(u8, u8)>| { + access + .map(|(round, _)| mode.dissector(total_rounds, round) == Party::Verifier) + .unwrap_or(false) + }; + let step_by_counterparty = owned_by_counterparty(step_access); + let next_by_counterparty = owned_by_counterparty(next_access); + + // The prover is bound to the agreed boundary whoever published it, so a + // mismatch there is always the misbehaviour. At the disputed boundary a + // value published by the counterparty was one the prover declared it + // disagreed with, so there claiming it is the misbehaviour. + // Challenges about the prover's own hash chain need the prover's value at + // each boundary. Where the prover published the position that is the + // committed value; where the counterparty published it -- only possible + // under the full-duplex search -- the prover's value is the one it + // re-signed. In the asymmetric search both boundaries are the prover's, so + // these are the committed values exactly as before. + let chain_step_hash = if step_by_counterparty { + resigned_step_hash.to_string() + } else { + committed_step_hash.clone() + }; + let chain_next_hash = if next_by_counterparty { + resigned_next_hash.to_string() + } else { + committed_next_hash.clone() + }; + + let step_inconsistent = committed_step_hash != resigned_step_hash; + let next_inconsistent = if next_by_counterparty { + committed_next_hash == resigned_next_hash + } else { + committed_next_hash != resigned_next_hash + }; + + if (step_inconsistent && force == ForceChallenge::No) || force == ForceChallenge::EquivocationResign(EquivocationKind::StepHash) { - let (round, index) = *mapping.get(&step).unwrap(); - return Ok(ChallengeType::EquivocationResign { - prover_true_hash: prover_step_hash, + prover_true_hash: committed_step_hash, prover_wrong_hash: resigned_step_hash.to_string(), prover_challenge_step_tk: step, kind: EquivocationKind::StepHash, - expected_round: round, - expected_index: index + 1, + expected_round: step_access.unwrap().0, + expected_index: step_access.unwrap().1 + 1, + counterparty_committed: step_by_counterparty, nary, nary_last_round, rounds, }); } - if (prover_next_hash != resigned_next_hash && force == ForceChallenge::No) + if (next_inconsistent && force == ForceChallenge::No) || force == ForceChallenge::EquivocationResign(EquivocationKind::NextHash) { - let (round, index) = *mapping.get(&(step + 1)).unwrap(); + let (round, index) = next_access.unwrap(); return Ok(ChallengeType::EquivocationResign { - prover_true_hash: prover_next_hash, + prover_true_hash: committed_next_hash, prover_wrong_hash: resigned_next_hash.to_string(), prover_challenge_step_tk: step, kind: EquivocationKind::NextHash, expected_round: round, expected_index: index + 1, + counterparty_committed: next_by_counterparty, nary, nary_last_round, rounds, }); } - // check trace_hash - if (!validate_step_hash(&prover_step_hash, &trace.trace_step, &prover_next_hash) + // Check the prover's own hash chain. This uses the prover's resigned + // hashes, not the committed ones: under the full-duplex search the value + // committed at the disputed boundary may be the counterparty's, which the + // prover is bound to differ from, so comparing against it here would + // validate the wrong party's claim. Where no equivocation fired in the + // asymmetric search the two are equal, so this is unchanged there. + if (!validate_step_hash(&chain_step_hash, &trace.trace_step, &chain_next_hash) && force == ForceChallenge::No) || force == ForceChallenge::TraceHash || force == ForceChallenge::TraceHashZero @@ -511,16 +813,16 @@ pub fn verifier_choose_challenge( info!("Verifier choose to challenge TRACE_HASH_ZERO"); return Ok(ChallengeType::TraceHashZero { prover_trace: trace.trace_step, - prover_next_hash, + prover_next_hash: chain_next_hash, prover_conflict_step_tk: step, }); } info!("Verifier choose to challenge TRACE_HASH"); return Ok(ChallengeType::TraceHash { - prover_step_hash, + prover_step_hash: chain_step_hash, prover_trace: trace.trace_step, - prover_next_hash, + prover_next_hash: chain_next_hash, }); } @@ -531,7 +833,7 @@ pub fn verifier_choose_challenge( } = verifier_log.prover_claim_execution.clone(); if (step + 1 == prover_claim_last_step - && (prover_next_hash != prover_claim_last_hash + && (chain_next_hash != prover_claim_last_hash // halt opcode || trace.read_1.value != 93 // exit code @@ -545,7 +847,7 @@ pub fn verifier_choose_challenge( prover_last_step: prover_claim_last_step, prover_conflict_step_tk: step, prover_trace: trace, - prover_next_hash, + prover_next_hash: chain_next_hash, prover_last_hash: prover_claim_last_hash, }); } @@ -635,7 +937,7 @@ pub fn verifier_choose_challenge( return Ok(ChallengeType::ProgramCounter { pre_hash, trace: pre_step.trace_step, - prover_step_hash, + prover_step_hash: chain_step_hash, prover_pc_read: trace.read_pc, }); } @@ -656,6 +958,22 @@ pub fn verifier_choose_challenge( }); } + // Everything below disputes a `last_step` pointer: that a read names a + // step in the future, that it names the wrong writer, that it claims an + // uninitialised word was initialised or the reverse. Under Merkle + // memory there is no pointer to dispute. A read is opened against the + // root the state hash commits to, and a false one is caught by the + // endgame's multiproof check rather than by a second search, so these + // six branches are unreachable and the function stops here. + // + // Nothing is lost by stopping: a read that returns the wrong value + // feeds the wrong write, which gives the wrong state hash, which is + // what the search is already looking for. The search still arrives at + // the faulty step; what changes is only how that step is adjudicated. + if program.memory_model.chains_root() { + return Ok(ChallengeType::No); + } + let prover_read_step_1 = prover_read_1.last_step; let prover_read_step_2 = prover_read_2.last_step; @@ -850,6 +1168,8 @@ pub fn verifier_choose_challenge_for_read_challenge( kind: EquivocationKind::StepHash, expected_round: round, expected_index: index + 1, + // the second search keeps the asymmetric roles + counterparty_committed: false, nary, nary_last_round, rounds, @@ -867,6 +1187,8 @@ pub fn verifier_choose_challenge_for_read_challenge( kind: EquivocationKind::NextHash, expected_round: round, expected_index: index + 1, + // the second search keeps the asymmetric roles + counterparty_committed: false, nary, nary_last_round, rounds, diff --git a/emulator/src/decision/execution_log.rs b/emulator/src/decision/execution_log.rs index 9ae4e6d..a4c3ec6 100644 --- a/emulator/src/decision/execution_log.rs +++ b/emulator/src/decision/execution_log.rs @@ -24,8 +24,15 @@ impl ExecutionLog { pub struct ProverNAryLog { pub base_step: u64, pub verifier_decisions: Vec, + /// Each round's dissection, indexed by round. Under the full-duplex search + /// the rounds alternate between the parties, so this is the published view + /// rather than only this party's own dissections. pub hash_rounds: Vec>, pub final_trace: TraceRWStep, + /// Running upper bound on the disputed step. Only used by the full-duplex + /// search, where the prover selects as well as dissects. + #[serde(default)] + pub step_to_challenge: u64, } #[derive(Debug, Serialize, Deserialize)] @@ -38,10 +45,19 @@ pub struct ProverChallengeLog { impl ProverChallengeLog { pub fn new(execution: ExecutionLog, input: Vec) -> Self { + // Under the full-duplex search the prover selects as well as dissects, + // so it needs the same running bound on the disputed step that the + // verifier keeps: one before the earliest step the parties could still + // disagree on. The prover's own claim is its only knowledge of that, + // and the bound is only ever tightened by later selections. + let step_to_challenge = execution.last_step.saturating_sub(1); Self { execution, input, - conflict_step_log: ProverNAryLog::default(), + conflict_step_log: ProverNAryLog { + step_to_challenge, + ..Default::default() + }, read_challenge_log: ProverNAryLog::default(), } } @@ -146,3 +162,29 @@ pub fn deserialize_challenge_log Deserialize<'a>>( serde_json::from_str(serialized_str) .map_err(|e| EmulatorError::ChallengeError(format!("Error deserializing file: {}", e))) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ExecutionResult; + + /// The full-duplex search clamps each selection by this bound, so a prover + /// log that started it at zero would pin every selection to step zero. + #[test] + fn prover_log_starts_with_a_usable_selection_bound() { + let log = ProverChallengeLog::new( + ExecutionLog::new(ExecutionResult::Halt(0, 100), 100, "ab".to_string()), + vec![], + ); + assert_eq!(log.conflict_step_log.step_to_challenge, 99); + } + + #[test] + fn prover_log_bound_does_not_underflow_on_an_empty_execution() { + let log = ProverChallengeLog::new( + ExecutionLog::new(ExecutionResult::Halt(0, 0), 0, "ab".to_string()), + vec![], + ); + assert_eq!(log.conflict_step_log.step_to_challenge, 0); + } +} diff --git a/emulator/src/decision/mod.rs b/emulator/src/decision/mod.rs index 793cd8a..61bf2e6 100644 --- a/emulator/src/decision/mod.rs +++ b/emulator/src/decision/mod.rs @@ -1,3 +1,4 @@ pub mod challenge; pub mod execution_log; pub mod nary_search; +pub mod publication; diff --git a/emulator/src/decision/nary_search.rs b/emulator/src/decision/nary_search.rs index cfa2ac7..78958d0 100644 --- a/emulator/src/decision/nary_search.rs +++ b/emulator/src/decision/nary_search.rs @@ -31,6 +31,16 @@ impl FromStr for NArySearchType { } } +/// Bits needed to carry a digit in base `radix`: a ceiling, so a +/// non-power-of-two radix gets the next whole number of bits and a range +/// check rather than a wider digit. +fn bits_for_radix(radix: u64) -> u8 { + if radix <= 1 { + return 0; + } + (u64::BITS - (radix - 1).leading_zeros()) as u8 +} + #[derive(Debug, Clone, Serialize)] pub struct NArySearchDefinition { pub max_steps: u64, @@ -40,33 +50,108 @@ pub struct NArySearchDefinition { } impl NArySearchDefinition { + /// The search over a trace of at least `aprox_max_steps` steps, + /// dissected `nary` ways at each round. + /// + /// A step index is a **mixed-radix** number: `full_rounds` digits in + /// base `nary`, and where that overshoots, one final digit in the + /// smaller base `nary_last_round`. The radices multiply to + /// `max_steps`, so the digits address every step exactly once -- + /// `radix_tests::the_radices_multiply_to_the_trace_length`. + /// + /// `nary` need not be a power of two. It was required to be while the + /// arithmetic below was written with shifts and masks, which is the + /// same arithmetic only when each radix is a power of two. Compact + /// BitVMX II dissects 192 ways -- states pack three to a 64-byte + /// FlatMerkle element, so the admissible branching factors are those + /// with `ceil((b-1)/3)` a power of two, and no such `b` is itself one. pub fn new(aprox_max_steps: u64, nary: u8) -> NArySearchDefinition { - assert!(nary > 1 && nary.count_ones() == 1); - let max_bits = f64::ceil(f64::log2(aprox_max_steps as f64)); - let max_steps = 2f64.powi(max_bits as i32) as u64; - let nary_bits = f64::log2(nary as f64); - let full_rounds = f64::floor(max_bits / nary_bits); - let bits_left = max_bits - full_rounds * nary_bits; - let nary_last_round = if bits_left as u8 == 0 { - 0 - } else { - f64::powf(2.0, bits_left) as u8 - }; + assert!(nary > 1, "a dissection needs at least two parts"); + let target = aprox_max_steps.max(1); + + if nary.count_ones() == 1 { + // A power-of-two dissection pads the trace to a power of two + // and splits it into `log2(nary)`-bit digits, with whatever + // bits are left over becoming a smaller final radix -- itself + // a power of two. Keeping this exactly is what lets the + // change below land without moving any existing search: a + // 2,000-step trace still runs at 2,048 and not 4,096. + let max_bits = f64::ceil(f64::log2(target as f64)); + let max_steps = 2f64.powi(max_bits as i32) as u64; + let nary_bits = f64::log2(nary as f64); + let full_rounds = f64::floor(max_bits / nary_bits); + let bits_left = max_bits - full_rounds * nary_bits; + let nary_last_round = if bits_left as u8 == 0 { + 0 + } else { + f64::powf(2.0, bits_left) as u8 + }; + + return NArySearchDefinition { + max_steps, + nary, + full_rounds: full_rounds as u8, + nary_last_round, + }; + } + + // Otherwise there is no power of two to pad to, so the trace pads + // to `nary^rounds` and every round carries a full digit. This is + // the shape Compact BitVMX II specifies, and it is why that paper + // quotes `192^4 = 1,358,954,496` rather than `2^30`. + let n = nary as u64; + let mut full_rounds = 0u8; + let mut max_steps = 1u64; + while max_steps < target { + max_steps = max_steps.saturating_mul(n); + full_rounds += 1; + } NArySearchDefinition { max_steps, nary, - full_rounds: full_rounds as u8, - nary_last_round, + full_rounds: full_rounds.max(1), + nary_last_round: 0, + } + } + + /// The base of the digit decided at `round`, counting rounds from 1. + pub fn radix_for_round(&self, round: u8) -> u64 { + if round <= self.full_rounds { + self.nary as u64 + } else { + self.nary_last_round as u64 } } + /// How many steps one unit of `round`'s digit is worth: the product of + /// the radices of the rounds below it. + pub fn place_value(&self, round: u8) -> u64 { + ((round + 1)..=self.total_rounds()) + .map(|r| self.radix_for_round(r)) + .product() + } + + /// Bits needed to carry a base-`nary` digit. + /// + /// A ceiling, not an exact logarithm: a base-192 digit occupies eight + /// bits and a range check that it is below 192, which is what the + /// deployed protocol already does for its base-8 digit in two + /// Winternitz nibbles. pub fn bits_nary(&self) -> u8 { - f64::log2(self.nary as f64) as u8 + bits_for_radix(self.nary as u64) } pub fn bits_last_round(&self) -> u8 { - f64::log2(self.nary_last_round as f64) as u8 + bits_for_radix(self.nary_last_round as u64) + } + + /// Whether a digit of this round can overflow its bits, and so needs a + /// range check on chain. False for every power-of-two radix, which is + /// every radix the deployed protocol uses. + pub fn digit_needs_range_check(&self, round: u8) -> bool { + let r = self.radix_for_round(round); + r > 1 && !r.is_power_of_two() } pub fn total_rounds(&self) -> u8 { @@ -110,23 +195,11 @@ impl NArySearchDefinition { // if we are on the last round, the number of bits required is the bits_last_round() // the step number should be masked and shifted apropiately pub fn step_bits_for_round(&self, round: u8, step: u64) -> u32 { - if round <= self.full_rounds { - let shift = (self.full_rounds - round) * self.bits_nary() + self.bits_last_round(); - let mask = ((self.nary - 1) as u64) << shift; - ((step & mask) >> shift) as u32 - } else { - let mask = (self.nary_last_round - 1) as u64; - (step & mask) as u32 - } + ((step / self.place_value(round)) % self.radix_for_round(round)) as u32 } pub fn step_from_base_and_bits(&self, round: u8, base: u64, bits: u32) -> u64 { - if round <= self.full_rounds { - let shift = (self.full_rounds - round) * self.bits_nary() + self.bits_last_round(); - base + ((bits as u64) << shift) - } else { - base + (bits as u64) - } + base + bits as u64 * self.place_value(round) } pub fn step_mapping(&self, bits: &[u32]) -> HashMap { @@ -151,8 +224,8 @@ impl NArySearchDefinition { decision_bits .iter() .enumerate() - .fold(0, |res, (round, bits)| { - (res << self.bits_for_round(round as u8 + 1)) + *bits as u64 + .fold(0, |res, (round, digit)| { + res + *digit as u64 * self.place_value(round as u8 + 1) }) } } @@ -186,32 +259,49 @@ impl From>> for ExecutionHashes { } } -// we assume that the previous hash to the list provided is agreed by both parties +/// Selects the subinterval of a dissection in which the two views diverge. +/// +/// We assume that the hash preceding the provided list is agreed by both +/// parties. +/// +/// This decision is *symmetric*: it depends only on the positionwise comparison +/// of the two hash vectors, never on which party published which. Agreement at +/// a position is a symmetric relation, so either party may run this routine on +/// the other's dissection and both obtain the same answer. That is what allows +/// the full-duplex search (see [`full_duplex_stage`]) to let each message both +/// select and dissect, rather than spending one message on each. +/// +/// `published_hashes` are the counterparty's committed values for this round's +/// positions, and `local_hashes` are our own values at the same positions. #[allow(clippy::too_many_arguments)] pub fn choose_segment( nary_defs: &NArySearchDefinition, base_step: u64, selected_step: u64, round: u8, - prover_hashes: &ExecutionHashes, - my_hashes: &ExecutionHashes, + published_hashes: &ExecutionHashes, + local_hashes: &ExecutionHashes, nary_type: NArySearchType, conflict_step: Option, ) -> (u32, u64, u64) { - if prover_hashes.hashes.len() != my_hashes.hashes.len() { - error!("Prover and my hashes should have the same length"); + if published_hashes.hashes.len() != local_hashes.hashes.len() { + error!("Published and local hashes should have the same length"); } // finds if there is any difference in the hashes + // When every published interior hash matches ours, the divergence is the + // one already recorded at the upper boundary of the interval, so we select + // the topmost subinterval. This is the case where a party can select + // without ever having seen the counterparty's value at that boundary. let mut selection = if nary_type == NArySearchType::ConflictStep { - prover_hashes.hashes.len() + 1 + published_hashes.hashes.len() + 1 } else { 0 }; - for i in 0..prover_hashes.hashes.len() { - let prover_hash = &prover_hashes.hashes[i]; - let my_hash = &my_hashes.hashes[i]; - if prover_hash != my_hash { + for i in 0..published_hashes.hashes.len() { + let published_hash = &published_hashes.hashes[i]; + let my_hash = &local_hashes.hashes[i]; + if published_hash != my_hash { selection = i + 1; if nary_type == NArySearchType::ConflictStep { break; @@ -242,6 +332,334 @@ pub fn choose_segment( (lower_limit_bits, base_step, choice) } +/// Which party dissects at a given stage of a full-duplex search. +/// +/// In the asymmetric search one party always dissects and the other always +/// selects. In the full-duplex search the two alternate, and every message +/// carries both a selection and a dissection. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum Party { + Prover, + Verifier, +} + +impl Party { + pub fn other(&self) -> Party { + match self { + Party::Prover => Party::Verifier, + Party::Verifier => Party::Prover, + } + } + + /// Prefix used for this party's protocol variable names. + pub fn prefix(&self) -> &'static str { + match self { + Party::Prover => "prover", + Party::Verifier => "verifier", + } + } + + /// The party that dissects at `stage`, counting stages from 1, when `first` + /// dissects at stage 1. Ownership of a stage is fixed by parity, and so is + /// known at setup time when the one-time keys are generated. + pub fn at_stage(first: Party, stage: u8) -> Party { + if stage % 2 == 1 { + first + } else { + first.other() + } + } +} + +/// How the dissection search allocates work to the two parties. +/// +/// This is a setup-time property, not a runtime one: it decides which party +/// owns which one-time keys, and the dispute transaction graph is built and +/// pre-signed before any dispute exists. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SearchMode { + /// One party dissects and the other selects: two messages per narrowing + /// level. + #[default] + Asymmetric, + /// The parties alternate, and every message both selects over the + /// counterparty's previous dissection and dissects the subinterval it just + /// selected: one message per narrowing level. + FullDuplex, +} + +impl SearchMode { + /// The party that dissects at `stage`, counting stages from 1. + pub fn dissector(&self, total_rounds: u8, stage: u8) -> Party { + match self { + SearchMode::Asymmetric => Party::Prover, + SearchMode::FullDuplex => Party::at_stage(self.first_dissector(total_rounds), stage), + } + } + + /// The party that commits the digit selecting over the dissection published + /// at `round`. + pub fn selector(&self, total_rounds: u8, round: u8) -> Party { + match self { + SearchMode::Asymmetric => Party::Verifier, + SearchMode::FullDuplex => self.dissector(total_rounds, round).other(), + } + } + + /// Who dissects at stage 1. + /// + /// For the full-duplex search this is chosen from the parity of the round + /// count so that the trailing selection -- the digit over the final + /// dissection, which has no stage of its own to ride on -- always falls to + /// the prover. It can then be carried by the prover's adjudication + /// transaction rather than costing a further message. + pub fn first_dissector(&self, total_rounds: u8) -> Party { + match self { + SearchMode::Asymmetric => Party::Prover, + SearchMode::FullDuplex => { + if total_rounds % 2 == 0 { + Party::Prover + } else { + Party::Verifier + } + } + } + } + + /// On-chain messages the search consumes, including the adjudication + /// message that carries the trailing selection. + pub fn messages(&self, total_rounds: u8) -> u32 { + let rounds = total_rounds as u32; + match self { + SearchMode::Asymmetric => 2 * rounds + 1, + SearchMode::FullDuplex => rounds + 1, + } + } +} + +/// The contents of one full-duplex stage message. +#[derive(Clone, Debug)] +pub struct StageMessage { + pub stage: u8, + pub sender: Party, + /// Digit selecting a subinterval of the dissection published at the + /// previous stage. `None` only at stage 1, which has no predecessor. + pub selection: Option, + /// The sender's dissection of the selected subinterval. + pub hashes: ExecutionHashes, +} + +/// The portion of a full-duplex search that both parties track identically. +/// +/// Both parties derive the same state from the same transcript, so this is not +/// a per-role structure. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SearchState { + /// Base of the currently active interval. + pub base_step: u64, + /// Running upper bound on the disputed step: the last step agreed so far. + pub selected_step: u64, + /// Selection digits committed so far, most significant first. + pub digits: Vec, +} + +impl SearchState { + pub fn new(max_selectable_step: u64) -> Self { + Self { + base_step: 0, + selected_step: max_selectable_step, + digits: Vec::new(), + } + } +} + +/// Computes our own dissection of the interval based at `base_step` for `round`. +pub fn dissect( + nary_defs: &NArySearchDefinition, + round: u8, + base_step: u64, + hash_at: &F, +) -> ExecutionHashes +where + F: Fn(u64) -> Vec, +{ + ExecutionHashes::new( + nary_defs + .required_steps(round, base_step) + .into_iter() + .map(hash_at) + .collect(), + ) +} + +/// Records a selection over the dissection published at `round`, advancing the +/// active interval. This is the half of a stage that resolves the previous one. +pub fn select_over( + nary_defs: &NArySearchDefinition, + state: &mut SearchState, + round: u8, + published: &ExecutionHashes, + hash_at: &F, +) -> u32 +where + F: Fn(u64) -> Vec, +{ + let local = dissect(nary_defs, round, state.base_step, hash_at); + let (bits, base, selected) = choose_segment( + nary_defs, + state.base_step, + state.selected_step, + round, + published, + &local, + NArySearchType::ConflictStep, + None, + ); + state.base_step = base; + state.selected_step = selected; + state.digits.push(bits); + bits +} + +/// Performs one full-duplex stage: select over the counterparty's previous +/// dissection, then dissect the subinterval just selected. +/// +/// `incoming` is the dissection published at stage `stage - 1`, and is `None` +/// exactly at stage 1. +pub fn full_duplex_stage( + nary_defs: &NArySearchDefinition, + state: &mut SearchState, + stage: u8, + sender: Party, + incoming: Option<&ExecutionHashes>, + hash_at: &F, +) -> StageMessage +where + F: Fn(u64) -> Vec, +{ + let selection = incoming.map(|published| { + debug_assert!(stage > 1, "stage 1 has no dissection to select over"); + select_over(nary_defs, state, stage - 1, published, hash_at) + }); + + let hashes = dissect(nary_defs, stage, state.base_step, hash_at); + + StageMessage { + stage, + sender, + selection, + hashes, + } +} + +/// The outcome of running a search to completion. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SearchOutcome { + /// The last step both parties agree on. The disputed step is this plus one. + pub agreed_step: u64, + /// The selection digits, most significant first. Their concatenation is the + /// base-n representation of `agreed_step`. + pub digits: Vec, + /// On-chain messages consumed, including the final adjudication message + /// that carries the trailing selection. + pub messages: u32, +} + +/// Runs a full-duplex search to completion over two views of a trace. +/// +/// Stages alternate between the parties. Stage 1 carries a dissection only; +/// every later stage carries a selection over the previous dissection together +/// with a dissection of its own. The trailing selection has no dissection to +/// accompany it and rides on the final adjudication message. +pub fn run_full_duplex_search( + nary_defs: &NArySearchDefinition, + max_selectable_step: u64, + first: Party, + prover_hash_at: &FA, + verifier_hash_at: &FB, +) -> SearchOutcome +where + FA: Fn(u64) -> Vec, + FB: Fn(u64) -> Vec, +{ + let rounds = nary_defs.total_rounds(); + let mut state = SearchState::new(max_selectable_step); + let mut incoming: Option = None; + let mut messages = 0; + + for stage in 1..rounds + 1 { + let sender = Party::at_stage(first, stage); + let msg = match sender { + Party::Prover => full_duplex_stage( + nary_defs, + &mut state, + stage, + sender, + incoming.as_ref(), + prover_hash_at, + ), + Party::Verifier => full_duplex_stage( + nary_defs, + &mut state, + stage, + sender, + incoming.as_ref(), + verifier_hash_at, + ), + }; + messages += 1; + incoming = Some(msg.hashes); + } + + // The trailing selection over the final dissection, carried by the + // adjudication message. + let closer = Party::at_stage(first, rounds).other(); + let published = incoming.expect("at least one stage"); + match closer { + Party::Prover => select_over(nary_defs, &mut state, rounds, &published, prover_hash_at), + Party::Verifier => select_over(nary_defs, &mut state, rounds, &published, verifier_hash_at), + }; + messages += 1; + + SearchOutcome { + agreed_step: state.selected_step, + digits: state.digits, + messages, + } +} + +/// Runs the classical asymmetric search to completion, as a reference for +/// comparison: the prover dissects and the verifier selects, two messages per +/// level. +pub fn run_asymmetric_search( + nary_defs: &NArySearchDefinition, + max_selectable_step: u64, + prover_hash_at: &FA, + verifier_hash_at: &FB, +) -> SearchOutcome +where + FA: Fn(u64) -> Vec, + FB: Fn(u64) -> Vec, +{ + let rounds = nary_defs.total_rounds(); + let mut state = SearchState::new(max_selectable_step); + let mut messages = 0; + + for round in 1..rounds + 1 { + let published = dissect(nary_defs, round, state.base_step, prover_hash_at); + messages += 1; // the prover's dissection + select_over(nary_defs, &mut state, round, &published, verifier_hash_at); + messages += 1; // the verifier's selection + } + + SearchOutcome { + agreed_step: state.selected_step, + digits: state.digits, + messages: messages + 1, // the adjudication message + } +} + #[cfg(test)] mod tests { use super::*; @@ -339,6 +757,287 @@ mod tests { assert_eq!(nary_search.step_from_base_and_bits(3, 90, 1), 91); } + // --------------------------------------------------------------- + // Full-duplex search + // --------------------------------------------------------------- + + /// A trace whose state hashes are distinct per step. + fn trace_hash(step: u64) -> Vec { + let mut h = vec![0u8; 20]; + h[..8].copy_from_slice(&step.to_le_bytes()); + h + } + + /// A trace that matches `trace_hash` below `divergence` and differs from it + /// at and above `divergence`. + fn diverging_trace(divergence: u64) -> impl Fn(u64) -> Vec { + move |step: u64| { + let mut h = trace_hash(step); + if step >= divergence { + h[19] = 0xff; + } + h + } + } + + fn search_cases() -> Vec<(u64, u8)> { + vec![ + (64, 2), + (64, 8), + (128, 8), + (256, 4), + (2000, 8), + (500_000, 8), + ] + } + + #[test] + fn full_duplex_locates_the_first_divergence() { + for (max_steps, nary) in search_cases() { + let defs = NArySearchDefinition::new(max_steps, nary); + let top = defs.max_steps - 1; + for divergence in [1, 2, 7, 13, 64, top - 1, top] { + if divergence > top { + continue; + } + let prover = diverging_trace(divergence); + let outcome = + run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + assert_eq!( + outcome.agreed_step, + divergence - 1, + "max_steps {} nary {} divergence {}", + max_steps, + nary, + divergence + ); + } + } + } + + /// The two searches descend through the same intervals, because agreement + /// at a position is a symmetric relation and so does not depend on which + /// party evaluates it. + #[test] + fn full_duplex_agrees_with_asymmetric_search() { + for (max_steps, nary) in search_cases() { + let defs = NArySearchDefinition::new(max_steps, nary); + let top = defs.max_steps - 1; + for divergence in [1, 3, 9, 50, 129, top] { + if divergence > top { + continue; + } + let prover = diverging_trace(divergence); + let full = run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + let asym = run_asymmetric_search(&defs, top, &prover, &trace_hash); + + assert_eq!(full.agreed_step, asym.agreed_step); + assert_eq!(full.digits, asym.digits); + } + } + } + + /// One on-chain message per narrowing level instead of one per round trip. + #[test] + fn full_duplex_halves_the_message_count() { + for (max_steps, nary) in search_cases() { + let defs = NArySearchDefinition::new(max_steps, nary); + let rounds = defs.total_rounds() as u32; + let top = defs.max_steps - 1; + let prover = diverging_trace(7); + + let full = run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + let asym = run_asymmetric_search(&defs, top, &prover, &trace_hash); + + assert_eq!(full.messages, rounds + 1); + assert_eq!(asym.messages, 2 * rounds + 1); + } + } + + /// Ownership independence: the located step and the committed digits do not + /// depend on which party dissects first, and therefore not on which party + /// owns which stage. + #[test] + fn full_duplex_outcome_is_independent_of_who_starts() { + for (max_steps, nary) in search_cases() { + let defs = NArySearchDefinition::new(max_steps, nary); + let top = defs.max_steps - 1; + for divergence in [1, 11, 97, top] { + if divergence > top { + continue; + } + let prover = diverging_trace(divergence); + let prover_first = + run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + let verifier_first = + run_full_duplex_search(&defs, top, Party::Verifier, &prover, &trace_hash); + assert_eq!(prover_first, verifier_first); + } + } + } + + /// The concatenated selection digits are the base-n representation of the + /// agreed step, which is what lets the final script recover the step number + /// without carrying an interval forward between transactions. + #[test] + fn full_duplex_digits_reconstruct_the_agreed_step() { + for (max_steps, nary) in search_cases() { + let defs = NArySearchDefinition::new(max_steps, nary); + let top = defs.max_steps - 1; + for divergence in [1, 5, 33, 200, top] { + if divergence > top { + continue; + } + let prover = diverging_trace(divergence); + let outcome = + run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + assert_eq!( + defs.step_from_decision_bits(&outcome.digits), + outcome.agreed_step + ); + } + } + } + + /// When a party agrees with every published interior hash, the divergence + /// is the one already recorded at the upper boundary, and it selects the + /// topmost subinterval without ever seeing the counterparty's value there. + /// + /// This is the case where the two traces agree everywhere the search can + /// look, and the dispute rests solely on the state following the last + /// inspected step. + #[test] + fn full_duplex_inherits_disagreement_at_the_upper_boundary() { + let defs = NArySearchDefinition::new(64, 8); + let top = defs.max_steps - 1; + + // Diverging only beyond the last selectable step means every interior + // hash of every dissection matches, so every selection takes the top + // branch and the search converges on the upper boundary itself. + let prover = diverging_trace(defs.max_steps); + let outcome = run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + + assert_eq!(outcome.agreed_step, top); + for (round, digit) in outcome.digits.iter().enumerate() { + let max_digit = (1u32 << defs.bits_for_round(round as u8 + 1)) - 1; + assert_eq!(*digit, max_digit, "round {}", round + 1); + } + + // Diverging at the last selectable step instead: the first dissection + // still sees no divergence and inherits the boundary, but the final + // dissection does reach the diverging step and selects it directly. + let prover = diverging_trace(top); + let outcome = run_full_duplex_search(&defs, top, Party::Prover, &prover, &trace_hash); + + assert_eq!(outcome.agreed_step, top - 1); + assert_eq!( + outcome.digits, + vec![(defs.nary - 1) as u32, (defs.nary - 2) as u32] + ); + } + + /// The trailing selection must always fall to the prover, so that it can + /// ride on the adjudication transaction instead of costing a message. + #[test] + fn full_duplex_trailing_selection_belongs_to_the_prover() { + for rounds in 1..12u8 { + assert_eq!( + SearchMode::FullDuplex.selector(rounds, rounds), + Party::Prover, + "rounds {}", + rounds + ); + } + } + + #[test] + fn search_modes_allocate_stages_consistently() { + for rounds in 1..12u8 { + for stage in 1..rounds + 1 { + // The asymmetric search never alternates. + assert_eq!( + SearchMode::Asymmetric.dissector(rounds, stage), + Party::Prover + ); + assert_eq!( + SearchMode::Asymmetric.selector(rounds, stage), + Party::Verifier + ); + // The full-duplex search alternates, and the party selecting + // over a dissection is never the one that published it. + let d = SearchMode::FullDuplex.dissector(rounds, stage); + assert_eq!(SearchMode::FullDuplex.selector(rounds, stage), d.other()); + } + assert_eq!( + SearchMode::Asymmetric.messages(rounds), + 2 * rounds as u32 + 1 + ); + assert_eq!(SearchMode::FullDuplex.messages(rounds), rounds as u32 + 1); + } + } + + #[test] + fn search_mode_defaults_to_asymmetric() { + assert_eq!(SearchMode::default(), SearchMode::Asymmetric); + } + + /// One-time keys are the dominant setup cost, so it matters that making + /// the search symmetric redistributes them rather than adding any. Each + /// level still commits n-1 state hashes and one selection digit; the only + /// change is which party owns them. + #[test] + fn full_duplex_redistributes_one_time_keys_without_adding_any() { + for (max_steps, nary) in [(64u64, 2u8), (2000, 8), (500_000, 8), (1 << 20, 4)] { + let defs = NArySearchDefinition::new(max_steps, nary); + let rounds = defs.total_rounds(); + + let count = |mode: SearchMode, who: Party| -> (usize, usize) { + let mut hashes = 0; + let mut digits = 0; + for r in 1..rounds + 1 { + if mode.dissector(rounds, r) == who { + hashes += defs.hashes_for_round(r) as usize; + } + if mode.selector(rounds, r) == who { + digits += 1; + } + } + (hashes, digits) + }; + + for mode in [SearchMode::Asymmetric, SearchMode::FullDuplex] { + let (ph, pd) = count(mode, Party::Prover); + let (vh, vd) = count(mode, Party::Verifier); + let total: usize = (1..rounds + 1) + .map(|r| defs.hashes_for_round(r) as usize) + .sum(); + assert_eq!(ph + vh, total, "{:?}: state hash keys", mode); + assert_eq!(pd + vd, rounds as usize, "{:?}: digit keys", mode); + } + + // The asymmetric search gives every hash to the prover and every + // digit to the verifier. + assert_eq!(count(SearchMode::Asymmetric, Party::Verifier).0, 0); + assert_eq!(count(SearchMode::Asymmetric, Party::Prover).1, 0); + + // The full-duplex search splits both, and a party's digits are the + // ones over the counterparty's dissections. + if rounds > 1 { + assert!(count(SearchMode::FullDuplex, Party::Verifier).0 > 0); + assert!(count(SearchMode::FullDuplex, Party::Prover).1 > 0); + } + } + } + + #[test] + fn stage_ownership_alternates_by_parity() { + assert_eq!(Party::at_stage(Party::Prover, 1), Party::Prover); + assert_eq!(Party::at_stage(Party::Prover, 2), Party::Verifier); + assert_eq!(Party::at_stage(Party::Prover, 3), Party::Prover); + assert_eq!(Party::at_stage(Party::Verifier, 1), Party::Verifier); + assert_eq!(Party::at_stage(Party::Verifier, 2), Party::Prover); + } + fn test_vector(size: usize, diff_index: Option) -> Vec> { let mut v = Vec::new(); for i in 0..size { @@ -445,3 +1144,219 @@ mod tests { test_step_from_all_bits_aux(2000, 8, &vec![4, 2, 4, 3], 1107); } } + +#[cfg(test)] +mod radix_tests { + use super::*; + + /// Every definition the bit-sliced construction admits, as a fixture for + /// the mixed-radix rewrite: whatever that changes, it must not change + /// these. + fn definitions() -> Vec { + let mut out = Vec::new(); + for nary in [2u8, 4, 8, 16, 32, 64, 128] { + for steps in [16u64, 100, 1_000, 2_000, 65_536, 1 << 20] { + out.push(NArySearchDefinition::new(steps, nary)); + } + } + out + } + + /// A step index is a mixed-radix number: `full_rounds` digits in base + /// `nary`, then one in base `nary_last_round` where the bits did not + /// divide evenly. The place value of a round is the product of the + /// radices of the rounds below it. + fn radix(d: &NArySearchDefinition, round: u8) -> u64 { + if round <= d.full_rounds { + d.nary as u64 + } else { + d.nary_last_round as u64 + } + } + + fn place_value(d: &NArySearchDefinition, round: u8) -> u64 { + ((round + 1)..=d.total_rounds()) + .map(|r| radix(d, r)) + .product() + } + + #[test] + fn digit_extraction_is_mixed_radix() { + for d in definitions() { + for step in [0u64, 1, 7, 63, d.max_steps / 3, d.max_steps - 1] { + for round in 1..=d.total_rounds() { + let shifted = d.step_bits_for_round(round, step); + let by_radix = (step / place_value(&d, round)) % radix(&d, round); + assert_eq!( + shifted as u64, by_radix, + "nary={} steps={} round={} step={}", + d.nary, d.max_steps, round, step + ); + } + } + } + } + + #[test] + fn reconstruction_is_mixed_radix() { + for d in definitions() { + for round in 1..=d.total_rounds() { + for digit in [0u32, 1, (radix(&d, round) - 1) as u32] { + let shifted = d.step_from_base_and_bits(round, 0, digit); + let by_radix = digit as u64 * place_value(&d, round); + assert_eq!(shifted, by_radix, "nary={} round={}", d.nary, round); + } + } + } + } + + #[test] + fn the_radices_multiply_to_the_trace_length() { + for d in definitions() { + let product: u64 = (1..=d.total_rounds()).map(|r| radix(&d, r)).product(); + assert_eq!( + product, d.max_steps, + "nary={} steps={}: the digits must address every step exactly once", + d.nary, d.max_steps + ); + } + } + + #[test] + fn a_dissection_cuts_the_interval_into_radix_parts() { + for d in definitions() { + for round in 1..=d.total_rounds() { + let steps = d.required_steps(round, 0); + assert_eq!( + steps.len() as u64, + radix(&d, round) - 1, + "round {} of nary={} publishes radix-1 interior points", + round, + d.nary + ); + } + } + } + + #[test] + fn a_digit_round_trips_through_the_whole_search() { + for d in definitions() { + let digits: Vec = (1..=d.total_rounds()) + .map(|r| ((radix(&d, r) - 1) / 2) as u32) + .collect(); + let step = d.step_from_decision_bits(&digits); + for (i, round) in (1..=d.total_rounds()).enumerate() { + assert_eq!( + d.step_bits_for_round(round, step), + digits[i], + "nary={} round={}", + d.nary, + round + ); + } + } + } +} + +#[cfg(test)] +mod base_b_tests { + use super::*; + + /// Compact BitVMX II dissects 192 ways. Nothing about that is a + /// power of two, which is the whole reason the arithmetic above is + /// mixed-radix rather than shifts and masks. + #[test] + fn a_192_ary_search_covers_a_billion_steps_in_four_rounds() { + let d = NArySearchDefinition::new(1 << 30, 192); + assert_eq!(d.nary, 192); + assert_eq!(d.total_rounds(), 4); + assert_eq!(d.nary_last_round, 0, "every round carries a full digit"); + assert_eq!(d.max_steps, 192u64.pow(4)); + assert_eq!(d.max_steps, 1_358_954_496); + assert!(d.max_steps >= 1 << 30); + } + + #[test] + fn a_192_ary_round_publishes_191_interior_states() { + let d = NArySearchDefinition::new(1 << 30, 192); + for round in 1..=d.total_rounds() { + assert_eq!(d.hashes_for_round(round), 191); + assert_eq!(d.required_steps(round, 0).len(), 191); + } + } + + #[test] + fn its_digit_is_eight_bits_and_needs_a_range_check() { + let d = NArySearchDefinition::new(1 << 30, 192); + assert_eq!(d.bits_nary(), 8, "192 fits in eight bits"); + assert!( + d.digit_needs_range_check(1), + "but not every eight-bit value is a digit" + ); + + // Which no power-of-two dissection needs, so nothing existing + // grows a check it did not have. + for nary in [2u8, 4, 8, 16, 32, 64, 128] { + let p = NArySearchDefinition::new(1 << 20, nary); + for round in 1..=p.total_rounds() { + assert!( + !p.digit_needs_range_check(round), + "nary={nary} round={round}" + ); + } + } + } + + /// The property the whole search rests on: a step is its digits, and + /// its digits are the step. + #[test] + fn every_step_round_trips_through_its_digits() { + for nary in [8u8, 12, 192, 193] { + let d = NArySearchDefinition::new(100_000, nary); + for step in [0u64, 1, 2, 999, d.max_steps / 7, d.max_steps - 1] { + let digits: Vec = (1..=d.total_rounds()) + .map(|r| d.step_bits_for_round(r, step)) + .collect(); + for (i, r) in (1..=d.total_rounds()).enumerate() { + assert!( + (digits[i] as u64) < d.radix_for_round(r), + "nary={nary} round={r}: digit out of range" + ); + } + assert_eq!( + d.step_from_decision_bits(&digits), + step, + "nary={nary} step={step}" + ); + } + } + } + + /// Dissection points divide the interval exactly, which a base-b + /// padding guarantees and a power-of-two padding would not. + #[test] + fn the_dissection_points_divide_the_interval() { + let d = NArySearchDefinition::new(1 << 30, 192); + assert_eq!(d.max_steps % 192, 0); + let steps = d.required_steps(1, 0); + let first = steps[0]; + assert_eq!(first, d.max_steps / 192); + for (i, s) in steps.iter().enumerate() { + assert_eq!(*s, (i as u64 + 1) * first, "evenly spaced"); + } + } + + #[test] + fn the_admissible_branching_factors_all_work() { + // Those for which ceil((b-1)/3) is a power of two: states pack + // three to a 64-byte FlatMerkle element. + for b in [ + 11u8, 12, 13, 23, 24, 25, 47, 48, 49, 95, 96, 97, 191, 192, 193, + ] { + let d = NArySearchDefinition::new(1 << 20, b); + assert_eq!(d.nary, b); + assert!(d.max_steps >= 1 << 20, "b={b}"); + assert_eq!(d.hashes_for_round(1), b - 1, "b={b}"); + } + } +} diff --git a/emulator/src/decision/publication.rs b/emulator/src/decision/publication.rs new file mode 100644 index 0000000..890446e --- /dev/null +++ b/emulator/src/decision/publication.rs @@ -0,0 +1,125 @@ +//! How a dissection reaches the chain. +//! +//! This is the second of the two setup-time choices a dispute makes about +//! its search, and it is independent of the first. [`SearchMode`] decides +//! *who sends what and when*; the medium below decides *how the hashes a +//! stage sends are carried*, and the useful combination is full duplex +//! with a published dissection. +//! +//! [`SearchMode`]: super::nary_search::SearchMode +//! +//! The choice is what sets the branching factor. A committed dissection +//! is read by Bitcoin Script, which means the verifier for a one-time +//! signature is written out in the witness, and a tapscript execution may +//! hold only a thousand stack elements -- eighty-eight per twenty-byte +//! digest under the deployed Winternitz parameters, so twelve hashes is +//! the ceiling whatever the fee. A *published* dissection is seen rather +//! than read: it costs no stack at all, because BIP341 strips the annex +//! before the stack the script runs on is formed, and an `OP_RETURN` +//! output is never executed either. That is what admits a branching +//! factor of 192 and four narrowing levels where the deployed protocol +//! has eight and ten. + +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +/// Where a stage's dissection is carried. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum PublicationMedium { + /// One one-time commitment per hash, read by Bitcoin Script. + /// + /// The default, and what every existing program definition means. + #[default] + Committed, + /// One FlatMerkle file per stage, in an `OP_RETURN` output. + /// + /// Four weight units a byte, and relayed by Bitcoin Core today. + OpReturn, + /// One FlatMerkle file per stage, in the Taproot annex of the stage + /// that publishes it. + /// + /// One weight unit a byte. Consensus accepts it; Bitcoin Core's + /// default policy does not relay it, so a publisher needs a peer that + /// does. + Annex, +} + +impl PublicationMedium { + /// Whether the dissection is published rather than committed. + pub fn is_published(&self) -> bool { + !matches!(self, PublicationMedium::Committed) + } + + /// Weight units per byte of dissection. + /// + /// Witness data is one, non-witness four. `None` where the dissection + /// is committed rather than published, since then the cost is the + /// commitment scheme's and not the medium's. + pub fn weight_per_byte(&self) -> Option { + match self { + PublicationMedium::Committed => None, + PublicationMedium::Annex => Some(1), + PublicationMedium::OpReturn => Some(4), + } + } + + /// Whether Bitcoin Core's default policy relays a transaction + /// carrying this. + /// + /// A publisher whose dissection is not relayed can still reach a + /// miner, so this is a liveness property and not a soundness one -- + /// but a protocol that depends on a minority relay policy has a + /// deployment condition, and it should be visible here. + pub fn relayed_by_default_policy(&self) -> bool { + !matches!(self, PublicationMedium::Annex) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_default_is_what_every_existing_definition_means() { + assert_eq!(PublicationMedium::default(), PublicationMedium::Committed); + assert!(!PublicationMedium::default().is_published()); + assert_eq!(PublicationMedium::default().weight_per_byte(), None); + } + + #[test] + fn publishing_costs_a_witness_byte_or_four_output_bytes() { + assert_eq!(PublicationMedium::Annex.weight_per_byte(), Some(1)); + assert_eq!(PublicationMedium::OpReturn.weight_per_byte(), Some(4)); + assert!(PublicationMedium::Annex.is_published()); + assert!(PublicationMedium::OpReturn.is_published()); + } + + #[test] + fn only_the_annex_asks_for_a_peer_that_relays_it() { + assert!(!PublicationMedium::Annex.relayed_by_default_policy()); + assert!(PublicationMedium::OpReturn.relayed_by_default_policy()); + assert!(PublicationMedium::Committed.relayed_by_default_policy()); + } + + #[test] + fn it_parses_from_a_program_definition_in_kebab_case() { + // The spelling a YAML program definition uses. `op-return` + // rather than `op_return`, to match `full-duplex`. + for (text, want) in [ + ("committed", PublicationMedium::Committed), + ("op-return", PublicationMedium::OpReturn), + ("annex", PublicationMedium::Annex), + ] { + let got: PublicationMedium = serde_json::from_str(&format!("\"{text}\"")) + .unwrap_or_else(|e| panic!("{text}: {e}")); + assert_eq!(got, want); + } + } + + #[test] + fn an_unknown_medium_is_rejected_rather_than_defaulted() { + assert!(serde_json::from_str::("\"envelope\"").is_err()); + assert!(serde_json::from_str::("\"op_return\"").is_err()); + } +} diff --git a/emulator/src/executor/fetcher.rs b/emulator/src/executor/fetcher.rs index 8750b67..622c280 100644 --- a/emulator/src/executor/fetcher.rs +++ b/emulator/src/executor/fetcher.rs @@ -333,7 +333,10 @@ pub fn execute_step( read_1, read_2, TraceReadPC::new(pc, opcode), - TraceStep::new(write_1, program.pc.clone()), + match program.trace_mem_root() { + Some(root) => TraceStep::new_with_root(write_1, program.pc.clone(), root), + None => TraceStep::new(write_1, program.pc.clone()), + }, witness, mem_witness, ); @@ -600,7 +603,7 @@ pub fn op_arithmetic( _ => panic!("Unreachable"), }; - program.registers.set(x.rd(), result, program.step); + program.set_register(x.rd(), result, program.step); program.pc.next_address(); ( @@ -652,7 +655,7 @@ pub fn op_arithmetic_imm( _ => panic!("Unreachable"), }; - program.registers.set(rd, result, program.step); + program.set_register(rd, result, program.step); program.pc.next_address(); let write = program.registers.to_trace_write(rd); @@ -702,7 +705,7 @@ pub fn op_shift_sl( _ => panic!("Unreachable"), }; - program.registers.set(x.rd(), result, program.step); + program.set_register(x.rd(), result, program.step); program.pc.next_address(); ( @@ -738,7 +741,7 @@ pub fn op_shift_imm( _ => panic!("Unreachable"), }; - program.registers.set(x.rd(), result, program.step); + program.set_register(x.rd(), result, program.step); program.pc.next_address(); ( @@ -787,7 +790,7 @@ pub fn op_sl_imm( _ => panic!("Unreachable"), }; - program.registers.set(x.rd(), result, program.step); + program.set_register(x.rd(), result, program.step); program.pc.next_address(); ( @@ -869,7 +872,7 @@ pub fn op_store( let read_2 = TraceRead::new(dest_mem, value, program.get_last_step(dest_mem)); let masked = mask_dst & value; - program.registers.set(AUX_REGISTER_1, masked, program.step); + program.set_register(AUX_REGISTER_1, masked, program.step); program.pc.next_micro(); ( @@ -904,7 +907,7 @@ pub fn op_store( Ordering::Equal => masked, }; - program.registers.set(AUX_REGISTER_2, shifted, program.step); + program.set_register(AUX_REGISTER_2, shifted, program.step); program.pc.next_micro(); ( @@ -1040,7 +1043,7 @@ pub fn op_load( ), )); } - program.registers.set(x.rd(), shifted, program.step); + program.set_register(x.rd(), shifted, program.step); program.registers.to_trace_write(x.rd()) } else { program.pc.next_micro(); @@ -1049,7 +1052,7 @@ pub fn op_load( } else { AUX_REGISTER_2 }; - program.registers.set(dest, shifted, program.step); + program.set_register(dest, shifted, program.step); program.registers.to_trace_write(dest) }; @@ -1086,7 +1089,7 @@ pub fn op_load( let value = program.registers.get(AUX_REGISTER_1); let read_1 = program.registers.to_trace_read(AUX_REGISTER_1); - program.registers.set(x.rd(), value, program.step); + program.set_register(x.rd(), value, program.step); program.pc.next_address(); let write_1 = program.registers.to_trace_write(x.rd()); @@ -1121,7 +1124,7 @@ pub fn op_upper( }; //state modification - program.registers.set(dest_register, value, program.step); + program.set_register(dest_register, value, program.step); program.pc.next_address(); ( @@ -1135,3 +1138,246 @@ pub fn op_upper( ), ) } + +#[cfg(test)] +mod memory_tree_tests { + use crate::executor::utils::FailConfiguration; + use crate::loader::program_definition::ProgramDefinition; + use crate::loader::synthetic::SyntheticProgram; + use bitvmx_cpu_definitions::memory_tree::{leaf_index, verify_multiproof}; + + use super::*; + + fn loaded() -> (SyntheticProgram, ProgramDefinition, Program) { + let prog = SyntheticProgram::new(20, 8, 128).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + let program = def.load_program().unwrap(); + (prog, def, program) + } + + /// Run to `limit`, and insist that it actually ran. An unknown input + /// section makes `execute_program` return before the first instruction, + /// which looks exactly like a program that changes no memory. + fn run(def: &ProgramDefinition, program: &mut Program, limit: u64) -> ExecutionResult { + let before = program.step; + let result = execute_program( + program, + vec![0, 0, 0, 0], + &def.input_section_name, + false, + &None, + Some(limit), + false, + false, + false, + false, + false, + false, + None, + None, + FailConfiguration::default(), + false, + ) + .0; + assert!( + program.step > before, + "execution did not advance past step {before}: {result:?}" + ); + result + } + + #[test] + fn a_loaded_program_has_a_memory_root() { + let (_prog, _def, program) = loaded(); + assert!( + program.memory_tree_is_consistent(), + "the tree built at load must match a rebuild" + ); + assert_ne!( + program.memory_root(), + bitvmx_cpu_definitions::memory_tree::empty_root( + bitvmx_cpu_definitions::memory_tree::DEPLOYED_DEPTH + ), + "a loaded ELF is not empty memory" + ); + } + + #[test] + fn the_tree_tracks_execution_step_by_step() { + // Every path that changes memory has to update the tree. Running + // to a series of limits and rebuilding at each is what catches one + // that does not -- a missed register write or a missed store shows + // up as a divergence here and nowhere else. + for limit in 1..=23u64 { + let (_prog, def, mut program) = loaded(); + run(&def, &mut program, limit); + assert!( + program.memory_tree_is_consistent(), + "the tree diverged from memory by step {limit}" + ); + } + } + + #[test] + fn execution_moves_the_root_and_the_registers_are_in_it() { + let (_prog, def, mut program) = loaded(); + let before = program.memory_root(); + run(&def, &mut program, 23); + let after = program.memory_root(); + assert_ne!(before, after, "twenty-three steps must change memory"); + assert!(program.memory_tree_is_consistent()); + + // A register is an ordinary leaf, and it opens like one. + for idx in [1u32, 2, 10] { + let address = program.registers.get_register_address(idx); + let index = leaf_index(address); + assert_eq!( + program.memory_tree.read(index), + program.registers.get(idx), + "register {idx} must be the leaf at its own address" + ); + let proof = program.memory_tree.multiproof(&[index]); + assert!(verify_multiproof(&after, &proof), "register {idx} opens"); + } + } + + #[test] + fn a_checkpoint_recovers_the_tree_it_does_not_carry() { + let (_prog, def, mut program) = loaded(); + run(&def, &mut program, 23); + let root = program.memory_root(); + + // A checkpoint stores section data and registers, not the tree. + let json = serde_json::to_string(&program).unwrap(); + let mut restored: Program = serde_json::from_str(&json).unwrap(); + assert_ne!( + restored.memory_root(), + root, + "deserialization alone leaves an empty tree" + ); + restored.rebuild_memory_tree(); + assert_eq!( + restored.memory_root(), + root, + "and rebuilding recovers it exactly" + ); + } +} + +#[cfg(test)] +mod merkle_model_tests { + use crate::executor::utils::FailConfiguration; + use crate::loader::program_definition::ProgramDefinition; + use crate::loader::synthetic::SyntheticProgram; + use bitvmx_cpu_definitions::memory_tree::MemoryModel; + + use super::*; + + fn run_under(model: MemoryModel) -> (Program, FullTrace) { + let prog = SyntheticProgram::new(20, 8, 128).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + let mut program = def.load_program().unwrap(); + program.memory_model = model; + let before = program.step; + let (_result, traces) = execute_program( + &mut program, + vec![0, 0, 0, 0], + &def.input_section_name, + false, + &None, + Some(23), + true, + false, + false, + false, + false, + false, + None, + None, + FailConfiguration::default(), + false, + ); + assert!(program.step > before, "execution did not advance"); + // Keep the synthetic program's directory alive until here. + drop(prog); + (program, traces) + } + + #[test] + fn last_step_is_the_default_and_chains_no_root() { + let prog = SyntheticProgram::new(4, 8, 128).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + let program = def.load_program().unwrap(); + assert_eq!( + program.memory_model, + MemoryModel::LastStep, + "an existing program definition means what it meant before" + ); + assert!(program.trace_mem_root().is_none()); + } + + #[test] + fn a_merkle_step_carries_the_post_state_root() { + let (program, traces) = run_under(MemoryModel::Merkle); + let last = traces.last().expect("at least one step"); + let root = last + .0 + .trace_step + .mem_root + .expect("a Merkle step carries a root"); + assert_eq!( + root, + program.memory_root(), + "and it is the root after that step" + ); + } + + #[test] + fn the_step_message_grows_by_exactly_one_root() { + let (_m, merkle) = run_under(MemoryModel::Merkle); + let (_l, last_step) = run_under(MemoryModel::LastStep); + let m = merkle.last().unwrap().0.trace_step.to_bytes(); + let l = last_step.last().unwrap().0.trace_step.to_bytes(); + assert_eq!(l.len(), 13, "write address, value, pc, micro"); + assert_eq!(m.len(), 33, "and twenty more for the root"); + assert_eq!(&m[..13], &l[..], "the original fields are untouched"); + // The blake3 message the on-chain script hashes is the previous + // state hash followed by these bytes, so the script's length goes + // from 33 to 53 -- which is (40 + 8 + 8 + 8 + 2) / 2 and then + // (40 + 8 + 8 + 8 + 2 + 40) / 2 in nibbles. + assert_eq!(20 + l.len(), 33); + assert_eq!(20 + m.len(), 53); + } + + #[test] + fn the_two_models_give_different_state_hashes() { + // Which is the point: under Merkle an agreed state pins the root + // the endgame opens against, and under LastStep it does not. + let (_m, merkle) = run_under(MemoryModel::Merkle); + let (_l, last_step) = run_under(MemoryModel::LastStep); + assert_eq!(merkle.len(), last_step.len(), "same execution"); + assert_ne!( + merkle.last().unwrap().1, + last_step.last().unwrap().1, + "but the state hash commits to more" + ); + } + + #[test] + fn every_merkle_step_s_root_opens_its_own_writes() { + let (program, traces) = run_under(MemoryModel::Merkle); + assert!(program.memory_tree_is_consistent()); + let mut seen_roots = 0; + for (trace, _hash) in &traces { + if let Some(root) = trace.trace_step.mem_root { + seen_roots += 1; + let write = &trace.trace_step.write_1; + if write.address != 0 { + // The written word is in the tree that step produced. + assert_ne!(root, [0u8; 20]); + } + } + } + assert!(seen_roots > 0, "a Merkle run must carry roots"); + } +} diff --git a/emulator/src/executor/utils.rs b/emulator/src/executor/utils.rs index 3c3a549..d72a2e0 100644 --- a/emulator/src/executor/utils.rs +++ b/emulator/src/executor/utils.rs @@ -84,7 +84,7 @@ impl FailRead { // todo this will be changed when we consolidate sections and registers if is_register_address(program, self.address_original) { let idx = program.registers.get_original_idx(self.address_original); - program.registers.set(idx, self.value, program.step); + program.set_register(idx, self.value, program.step); } if program.find_section(self.address_original).is_ok() { @@ -164,7 +164,7 @@ impl FailWrite { patch = true; if is_register_address(program, self.address_original) { let idx = program.registers.get_original_idx(self.address_original); - program.registers.set(idx, self.value, program.step); + program.set_register(idx, self.value, program.step); } if program.find_section(self.address_original).is_ok() { diff --git a/emulator/src/loader/mod.rs b/emulator/src/loader/mod.rs index 21a3a46..e618485 100644 --- a/emulator/src/loader/mod.rs +++ b/emulator/src/loader/mod.rs @@ -1,2 +1,5 @@ pub mod program; pub mod program_definition; + +#[cfg(test)] +pub mod synthetic; diff --git a/emulator/src/loader/program.rs b/emulator/src/loader/program.rs index 8172b90..d0c2fae 100644 --- a/emulator/src/loader/program.rs +++ b/emulator/src/loader/program.rs @@ -6,6 +6,7 @@ use bitcoin_script_riscv::riscv::instruction_mapping::{ use bitvmx_cpu_definitions::{ constants::LAST_STEP_INIT, memory::{Chunk, MemoryAccessType, SectionDefinition}, + memory_tree::{leaf_index, MemoryModel, MemoryTree, Node, DEPLOYED_DEPTH}, trace::{generate_initial_step_hash, ProgramCounter, TraceRead, TraceWrite}, }; use elf::{abi::SHF_EXECINSTR, abi::SHF_WRITE, endian::LittleEndian, ElfBytes}; @@ -197,6 +198,20 @@ pub struct Program { pub read_only_sections: SectionDefinition, pub register_sections: SectionDefinition, pub code_sections: SectionDefinition, + /// Memory as a Merkle tree, which is what the endgame opens against. + /// + /// It is a pure function of the section data and the register file, so + /// it is not serialized into a checkpoint -- `rebuild_memory_tree` + /// recovers it, and `deserialize_from_file` calls that. During + /// execution it is maintained incrementally by `write_mem` and + /// `set_register`, and `memory_tree_is_consistent` checks the two + /// agree. + #[serde(skip)] + pub memory_tree: MemoryTree, + /// How this program's dispute justifies a read. Defaults to + /// `LastStep`, so an existing program definition means what it did. + #[serde(default)] + pub memory_model: MemoryModel, } impl Program { @@ -215,6 +230,7 @@ impl Program { for (i, byte) in input_as_u32.iter().enumerate() { section.data[i] = *byte; } + self.rebuild_memory_tree(); } else { return Err(ExecutionResult::SectionNotFound( input_section_name.to_string(), @@ -238,9 +254,11 @@ impl Program { let serialized_str = std::str::from_utf8(&serialized).map_err(|_| { EmulatorError::CantLoadPorgram(format!("Error parsing file: {}", fname)) })?; - serde_json::from_str(serialized_str).map_err(|_| { + let mut program: Program = serde_json::from_str(serialized_str).map_err(|_| { EmulatorError::CantLoadPorgram(format!("Error deserializing file: {}", fname)) - }) + })?; + program.rebuild_memory_tree(); + Ok(program) } pub fn new(entry_point: u32, registers_base_address: u32, sp_base_address: u32) -> Program { @@ -257,7 +275,77 @@ impl Program { read_only_sections: SectionDefinition::default(), register_sections: SectionDefinition::default(), code_sections: SectionDefinition::default(), + memory_tree: MemoryTree::new(DEPLOYED_DEPTH), + memory_model: MemoryModel::default(), + } + } + + /// The root of memory as it stands. + pub fn memory_root(&self) -> Node { + self.memory_tree.root() + } + + /// The post-state root to put in a trace step, if this program's model + /// chains one. + /// + /// The *initial* root is not chained: the first state hash predates any + /// step. It does not need to be, because step zero's state is agreed + /// by construction -- both parties load the same program and the same + /// input, and the setup commits to both. + pub fn trace_mem_root(&self) -> Option { + self.memory_model.chains_root().then(|| self.memory_root()) + } + + /// Rebuild the memory tree from the section data and the register file, + /// which between them are the authoritative contents. + /// + /// The registers *section* exists only to reserve the address range -- + /// `find_section` refuses it -- so the register words come from + /// `self.registers` and the section's own data is skipped. + pub fn rebuild_memory_tree(&mut self) { + self.memory_tree = self.fresh_memory_tree(); + } + + /// The tree memory's current contents imply, built from scratch. + /// + /// `rebuild_memory_tree` installs this; `memory_tree_is_consistent` + /// compares it against the one execution has been maintaining. + pub fn fresh_memory_tree(&self) -> MemoryTree { + let mut tree = MemoryTree::new(DEPLOYED_DEPTH); + for section in &self.sections { + if section.registers { + continue; + } + tree.bulk_write(section.data.iter().enumerate().filter_map(|(i, &raw)| { + let word = u32::from_be(raw); + (word != 0).then(|| (leaf_index(section.start + (i as u32) * 4), word)) + })); } + tree.bulk_write((0..(RISCV32_REGISTERS + AUX_REGISTERS) as u32).map(|idx| { + ( + leaf_index(self.registers.get_register_address(idx)), + self.registers.get(idx), + ) + })); + tree + } + + /// Whether the incrementally maintained tree still matches a rebuild. + /// + /// Every path that changes memory has to update the tree, and this is + /// what catches one that does not. + pub fn memory_tree_is_consistent(&self) -> bool { + self.fresh_memory_tree().root() == self.memory_root() + } + + /// Write a register and keep the memory tree in step with it. + /// + /// Registers are memory-mapped, so a register write is an ordinary + /// memory write as far as the tree is concerned. + pub fn set_register(&mut self, idx: u32, value: u32, step: u64) { + self.registers.set(idx, value, step); + let address = self.registers.get_register_address(idx); + self.memory_tree.write(leaf_index(address), value); } pub fn merge_sections(&mut self) { @@ -545,6 +633,7 @@ impl Program { } section.data[(address - section.start) as usize / 4] = value.to_be(); section.last_step[(address - section.start) as usize / 4] = step; + self.memory_tree.write(leaf_index(address), value); Ok(()) } @@ -807,6 +896,7 @@ pub fn load_elf(fname: &str, show_sections: bool) -> Result, @@ -53,11 +75,36 @@ impl ProgramDefinition { NArySearchDefinition::new(self.max_steps, self.nary_search) } + pub fn search_mode(&self) -> SearchMode { + self.search_mode + } + + pub fn publication(&self) -> PublicationMedium { + self.publication + } + + pub fn memory_model(&self) -> MemoryModel { + self.memory_model + } + + /// Bytes of dissection one stage carries, before any framing. + /// + /// Twenty-byte state hashes, `nary - 1` of them. Only meaningful + /// where the dissection is published; a committed one is priced by + /// its commitment scheme and not by its size. + pub fn dissection_bytes(&self, round: u8) -> Option { + self.publication + .is_published() + .then(|| self.nary_def().hashes_for_round(round) as usize * 20) + } + pub fn load_program(&self) -> Result { //extract the path from config path and concat with elf let elf_path = self.config_path.split("/").collect::>(); let elf_file = format!("{}/{}", elf_path[..elf_path.len() - 1].join("/"), self.elf); - load_elf(&elf_file, false) + let mut program = load_elf(&elf_file, false)?; + program.memory_model = self.memory_model; + Ok(program) } pub fn load_program_from_checkpoint( @@ -259,3 +306,92 @@ fn parse_config Deserialize<'a>>(config: &str) -> Result() .map_err(ProgramDefinitionError::ConfigFileError) } + +#[cfg(test)] +mod publication_tests { + use super::*; + use std::io::Write; + + fn definition(extra: &str) -> ProgramDefinition { + let dir = std::env::temp_dir().join(format!("pubdef-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("d-{}.yaml", extra.len())); + let mut f = std::fs::File::create(&path).unwrap(); + write!( + f, + "elf: hello-world.elf\n\ + nary_search: 8\n\ + max_steps: 2000\n\ + input_section_name: .input\n\ + inputs:\n\ + \x20 - size: 4\n\ + \x20 owner: prover\n\ + {extra}" + ) + .unwrap(); + ProgramDefinition::from_config(path.to_str().unwrap()).unwrap() + } + + /// The property that makes this a backend choice rather than a new + /// protocol: a definition that says nothing about publication means + /// exactly what it meant before. + #[test] + fn an_existing_definition_is_unchanged() { + let d = definition(""); + assert_eq!(d.publication(), PublicationMedium::Committed); + assert_eq!(d.search_mode(), SearchMode::Asymmetric); + assert_eq!(d.dissection_bytes(1), None); + } + + #[test] + fn the_two_choices_are_independent() { + let d = definition("search_mode: full-duplex\npublication: op-return\n"); + assert_eq!(d.search_mode(), SearchMode::FullDuplex); + assert_eq!(d.publication(), PublicationMedium::OpReturn); + + let d = definition("publication: annex\n"); + assert_eq!(d.search_mode(), SearchMode::Asymmetric, "still the default"); + assert_eq!(d.publication(), PublicationMedium::Annex); + } + + #[test] + fn a_published_dissection_is_twenty_bytes_a_hash() { + let d = definition("publication: op-return\n"); + let n = d.nary_def(); + for round in 1..=n.total_rounds() { + assert_eq!( + d.dissection_bytes(round), + Some(n.hashes_for_round(round) as usize * 20) + ); + } + // nary = 8, so seven hashes a full round. + assert_eq!(d.dissection_bytes(1), Some(140)); + } + + /// At the branching factor the publication exists to buy. + #[test] + fn a_192_ary_stage_publishes_3820_bytes() { + let dir = std::env::temp_dir().join(format!("pubdef192-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("d192.yaml"); + std::fs::write( + &path, + "elf: hello-world.elf\n\ + nary_search: 192\n\ + search_mode: full-duplex\n\ + publication: annex\n\ + max_steps: 1073741824\n\ + input_section_name: .input\n\ + inputs:\n\ + \x20 - size: 4\n\ + \x20 owner: prover\n", + ) + .unwrap(); + let d = ProgramDefinition::from_config(path.to_str().unwrap()).unwrap(); + let n = d.nary_def(); + assert_eq!(n.total_rounds(), 4, "four narrowing levels"); + assert_eq!(n.hashes_for_round(1), 191); + assert_eq!(d.dissection_bytes(1), Some(191 * 20)); + assert_eq!(d.dissection_bytes(1), Some(3820)); + } +} diff --git a/emulator/src/loader/synthetic.rs b/emulator/src/loader/synthetic.rs new file mode 100644 index 0000000..56ab264 --- /dev/null +++ b/emulator/src/loader/synthetic.rs @@ -0,0 +1,572 @@ +//! Synthetic programs for tests. +//! +//! The reference programs are compiled by a RISC-V toolchain that runs under +//! docker, so in an environment without it every checkpoint-backed path is +//! untestable: loading a program, executing it, dissecting its trace. This +//! module writes a minimal RISC-V ELF and a matching program definition into a +//! temporary directory instead, which exercises the real loader, executor and +//! checkpointing rather than standing in for them. + +use bitvmx_cpu_definitions::memory_tree::MemoryModel; +use std::fs; +use std::path::PathBuf; + +/// Base address of the code section. Must be above the 0x1000 floor the loader +/// enforces, and clear of the register and stack regions. +pub const TEXT_BASE: u32 = 0x1000_0000; +/// Base address of the input section. +pub const INPUT_BASE: u32 = 0x2000_0000; + +const REGISTER_A0: u32 = 10; +const REGISTER_A7: u32 = 17; +const SYSCALL_EXIT: i32 = 93; + +/// `addi rd, rs1, imm` +pub fn addi(rd: u32, rs1: u32, imm: i32) -> u32 { + ((imm as u32 & 0xfff) << 20) | (rs1 << 15) | (rd << 7) | 0x13 +} + +/// `ecall` +pub fn ecall() -> u32 { + 0x0000_0073 +} + +/// The three instructions that halt the machine with exit code zero. +pub fn exit_sequence() -> Vec { + vec![ + addi(REGISTER_A7, 0, SYSCALL_EXIT), + addi(REGISTER_A0, 0, 0), + ecall(), + ] +} + +/// A program of `instructions` arithmetic steps followed by a clean exit. +/// Each instruction advances the trace by one step, so the trace length is +/// predictable without having to model the executor. +pub fn counting_program(instructions: usize) -> Vec { + let mut code: Vec = (0..instructions) + .map(|i| addi(5, 5, (i % 2048) as i32)) + .collect(); + code.extend(exit_sequence()); + code +} + +fn put_u16(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_le_bytes()); +} + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_le_bytes()); +} + +#[allow(clippy::too_many_arguments)] +fn section_header( + out: &mut Vec, + name: u32, + kind: u32, + flags: u32, + addr: u32, + offset: u32, + size: u32, + align: u32, +) { + put_u32(out, name); + put_u32(out, kind); + put_u32(out, flags); + put_u32(out, addr); + put_u32(out, offset); + put_u32(out, size); + put_u32(out, 0); // sh_link + put_u32(out, 0); // sh_info + put_u32(out, align); + put_u32(out, 0); // sh_entsize +} + +/// Builds a 32-bit little-endian RISC-V executable with a code section, a +/// writable input section and a section-name string table. +pub fn build_elf(text: &[u32], input_size: u32) -> Vec { + const EHSIZE: u32 = 52; + const SHENTSIZE: u32 = 40; + const SHNUM: u32 = 4; + + // "\0.text\0.input\0.shstrtab\0" + let mut shstrtab = vec![0u8]; + let name_text = shstrtab.len() as u32; + shstrtab.extend_from_slice(b".text\0"); + let name_input = shstrtab.len() as u32; + shstrtab.extend_from_slice(b".input\0"); + let name_shstrtab = shstrtab.len() as u32; + shstrtab.extend_from_slice(b".shstrtab\0"); + + let text_bytes: Vec = text.iter().flat_map(|w| w.to_le_bytes()).collect(); + + let data_start = EHSIZE + SHENTSIZE * SHNUM; + let text_off = data_start; + let input_off = text_off + text_bytes.len() as u32; + let shstrtab_off = input_off + input_size; + + let mut out = Vec::new(); + + // ELF header + out.extend_from_slice(&[0x7f, b'E', b'L', b'F']); + out.push(1); // EI_CLASS: 32-bit + out.push(1); // EI_DATA: little endian + out.push(1); // EI_VERSION + out.extend_from_slice(&[0u8; 9]); // OSABI, ABIVERSION, padding + put_u16(&mut out, 2); // e_type: ET_EXEC + put_u16(&mut out, 243); // e_machine: EM_RISCV + put_u32(&mut out, 1); // e_version + put_u32(&mut out, TEXT_BASE); // e_entry + put_u32(&mut out, 0); // e_phoff + put_u32(&mut out, EHSIZE); // e_shoff + put_u32(&mut out, 0); // e_flags + put_u16(&mut out, EHSIZE as u16); + put_u16(&mut out, 0); // e_phentsize + put_u16(&mut out, 0); // e_phnum + put_u16(&mut out, SHENTSIZE as u16); + put_u16(&mut out, SHNUM as u16); + put_u16(&mut out, 3); // e_shstrndx + + // Section headers + section_header(&mut out, 0, 0, 0, 0, 0, 0, 0); // SHT_NULL + section_header( + &mut out, + name_text, + 1, // SHT_PROGBITS + 2 | 4, // SHF_ALLOC | SHF_EXECINSTR + TEXT_BASE, + text_off, + text_bytes.len() as u32, + 4, + ); + section_header( + &mut out, + name_input, + 1, // SHT_PROGBITS + 2 | 1, // SHF_ALLOC | SHF_WRITE + INPUT_BASE, + input_off, + input_size, + 4, + ); + section_header( + &mut out, + name_shstrtab, + 3, // SHT_STRTAB + 0, + 0, + shstrtab_off, + shstrtab.len() as u32, + 1, + ); + + debug_assert_eq!(out.len() as u32, data_start); + out.extend_from_slice(&text_bytes); + out.extend(std::iter::repeat_n(0u8, input_size as usize)); + out.extend_from_slice(&shstrtab); + out +} + +/// A program definition and its ELF, written to a temporary directory that is +/// removed when this value is dropped. +pub struct SyntheticProgram { + dir: PathBuf, +} + +impl SyntheticProgram { + /// Writes a program of `instructions` arithmetic steps plus a clean exit. + pub fn new(instructions: usize, nary: u8, max_steps: u64) -> std::io::Result { + Self::new_with_memory_model(instructions, nary, max_steps, MemoryModel::LastStep) + } + + /// A synthetic program under a chosen memory model, so the YAML seam + /// that selects one is exercised rather than assumed. + pub fn new_with_memory_model( + instructions: usize, + nary: u8, + max_steps: u64, + memory_model: MemoryModel, + ) -> std::io::Result { + let unique = format!( + "bitvmx-synthetic-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + ); + let dir = std::env::temp_dir().join(unique); + fs::create_dir_all(&dir)?; + + fs::write( + dir.join("program.elf"), + build_elf(&counting_program(instructions), 4), + )?; + fs::write( + dir.join("program.yaml"), + format!( + "elf: program.elf\n\ + nary_search: {}\n\ + max_steps: {}\n\ + input_section_name: .input\n\ + memory_model: {}\n\ + inputs:\n\ + \x20 - size: 4\n\ + \x20 owner: prover\n", + nary, + max_steps, + match memory_model { + MemoryModel::LastStep => "last_step", + MemoryModel::Merkle => "merkle", + } + ), + )?; + Ok(Self { dir }) + } + + /// Path of the program definition, for `ProgramDefinition::from_config`. + pub fn definition(&self) -> String { + self.dir.join("program.yaml").to_string_lossy().into_owned() + } + + /// A fresh checkpoint directory for one party. + pub fn checkpoint(&self, who: &str) -> std::io::Result { + let path = self.dir.join(who); + fs::create_dir_all(&path)?; + Ok(path.to_string_lossy().into_owned()) + } +} + +impl Drop for SyntheticProgram { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decision::challenge::prover_execute; + use crate::loader::program_definition::ProgramDefinition; + + #[test] + fn a_synthetic_program_loads() { + let prog = SyntheticProgram::new(20, 8, 128).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + let program = def.load_program().unwrap(); + assert_eq!(program.pc.get_address(), TEXT_BASE); + } + + #[test] + fn a_synthetic_program_executes_to_a_halt() { + let prog = SyntheticProgram::new(20, 8, 128).unwrap(); + let input = prog.checkpoint("prover").unwrap(); + let (result, last_step, last_hash) = prover_execute( + &prog.definition(), + vec![0, 0, 0, 0], + &input, + &input, + true, + None, + true, + ) + .unwrap(); + // 20 arithmetic instructions plus the three-instruction exit. + assert_eq!(last_step, 23, "result {:?}", result); + assert_eq!(last_hash.len(), 40); + } +} + +#[cfg(test)] +mod search_tests { + use super::*; + use crate::decision::challenge::ForceCondition; + use crate::decision::challenge::{ + prover_execute, prover_final_trace, prover_full_duplex_stage, verifier_check_execution, + verifier_full_duplex_stage, + }; + use crate::decision::challenge::{prover_get_hashes_for_round, verifier_choose_segment}; + use crate::decision::execution_log::{ProverChallengeLog, VerifierChallengeLog}; + use crate::decision::nary_search::NArySearchType; + use crate::decision::nary_search::{Party, SearchMode}; + use crate::executor::utils::FailConfiguration; + use crate::loader::program_definition::ProgramDefinition; + + /// Drives a complete full-duplex search over a real trace and returns the + /// last step the parties agree on. The disputed step is one beyond it. + fn run_search(instructions: usize, nary: u8, max_steps: u64, diverge_at: u64) -> u64 { + let defs = ProgramDefinition::from_config( + &SyntheticProgram::new(instructions, nary, max_steps) + .unwrap() + .definition(), + ) + .unwrap(); + let digits = run_search_digits(instructions, nary, max_steps, diverge_at); + defs.nary_def().step_from_decision_bits(&digits) + } + + fn run_search_digits( + instructions: usize, + nary: u8, + max_steps: u64, + diverge_at: u64, + ) -> Vec { + let prog = SyntheticProgram::new(instructions, nary, max_steps).unwrap(); + let definition = prog.definition(); + let prover_cp = prog.checkpoint("prover").unwrap(); + let verifier_cp = prog.checkpoint("verifier").unwrap(); + let input = vec![0u8; 4]; + + // The prover's trace hash is corrupted from `diverge_at`, so the two + // views first differ at exactly that step. + let (_, claim_last_step, claim_last_hash) = prover_execute( + &definition, + input.clone(), + &prover_cp, + &prover_cp, + true, + Some(FailConfiguration::new_fail_hash(diverge_at)), + true, + ) + .unwrap(); + + let disputed = verifier_check_execution( + &definition, + input, + &verifier_cp, + &verifier_cp, + claim_last_step, + &claim_last_hash, + ForceCondition::ValidInputWrongStepOrHash, + None, + true, + ) + .unwrap(); + assert!(disputed.is_some(), "the verifier should find a dispute"); + + let defs = ProgramDefinition::from_config(&definition).unwrap(); + let rounds = defs.nary_def().total_rounds(); + let mode = SearchMode::FullDuplex; + + // Stages alternate; the trailing selection has no dissection of its + // own and rides on the adjudication message. + // The prover dissects from its corrupted trace throughout, exactly as + // it committed its claim. + let prover_fail = Some(FailConfiguration::new_fail_hash(diverge_at)); + + // The digit a stage must catch up on is the one published at the stage + // immediately before it. + let mut hashes: Vec = Vec::new(); + let mut last_decision: Option = None; + + for stage in 1..rounds + 2 { + let actor = mode.dissector(rounds, stage); + let (cp, step) = match actor { + Party::Prover => ( + &prover_cp, + prover_full_duplex_stage( + &definition, + &prover_cp, + &prover_cp, + stage, + &hashes, + last_decision, + prover_fail.clone(), + ) + .unwrap(), + ), + Party::Verifier => ( + &verifier_cp, + verifier_full_duplex_stage( + &definition, + &verifier_cp, + &verifier_cp, + stage, + &hashes, + last_decision, + None, + ) + .unwrap(), + ), + }; + let _ = cp; + last_decision = step.0; + hashes = step.1; + } + + // The trailing selection rides on the adjudication message, so hand it + // to the final-trace step exactly as the protocol does. That is also + // what commits the last digit. + prover_final_trace( + &definition, + &prover_cp, + &prover_cp, + last_decision.expect("the trailing stage selects") + 1, + prover_fail, + ) + .unwrap(); + + // The disputed step is what the digit sequence spells out, which is + // the quantity the on-chain script reconstructs. The trailing + // selection always falls to the prover, so its log holds every digit. + let digits = ProverChallengeLog::load(&prover_cp) + .unwrap() + .conflict_step_log + .verifier_decisions; + assert_eq!( + digits.len(), + rounds as usize, + "the search should commit one digit per level" + ); + digits + } + + /// Drives the classical asymmetric search over the same trace: the prover + /// dissects, the verifier selects, two messages per level. + fn run_asymmetric(instructions: usize, nary: u8, max_steps: u64, diverge_at: u64) -> Vec { + let prog = SyntheticProgram::new(instructions, nary, max_steps).unwrap(); + let definition = prog.definition(); + let prover_cp = prog.checkpoint("prover").unwrap(); + let verifier_cp = prog.checkpoint("verifier").unwrap(); + let input = vec![0u8; 4]; + let fail = Some(FailConfiguration::new_fail_hash(diverge_at)); + + let (_, claim_last_step, claim_last_hash) = prover_execute( + &definition, + input.clone(), + &prover_cp, + &prover_cp, + true, + fail.clone(), + true, + ) + .unwrap(); + + verifier_check_execution( + &definition, + input, + &verifier_cp, + &verifier_cp, + claim_last_step, + &claim_last_hash, + ForceCondition::ValidInputWrongStepOrHash, + None, + true, + ) + .unwrap() + .expect("the verifier should find a dispute"); + + let defs = ProgramDefinition::from_config(&definition).unwrap(); + let rounds = defs.nary_def().total_rounds(); + let mut decision = 0u32; + + for round in 1..rounds + 1 { + let hashes = prover_get_hashes_for_round( + &definition, + &prover_cp, + &prover_cp, + round, + decision, + fail.clone(), + NArySearchType::ConflictStep, + ) + .unwrap(); + decision = verifier_choose_segment( + &definition, + &verifier_cp, + &verifier_cp, + round, + hashes, + None, + NArySearchType::ConflictStep, + ) + .unwrap(); + } + + VerifierChallengeLog::load(&verifier_cp) + .unwrap() + .conflict_step_log + .verifier_decisions + } + + /// The two searches descend through the same intervals on a real trace, + /// because agreement at a position does not depend on which party + /// evaluates it. The full-duplex search just spends half the messages. + #[test] + fn full_duplex_and_asymmetric_searches_agree_on_a_real_trace() { + for (nary, max_steps) in [(8u8, 64u64), (4, 64), (2, 64)] { + for diverge_at in [3u64, 17, 42] { + let full = run_search_digits(60, nary, max_steps, diverge_at); + let asym = run_asymmetric(60, nary, max_steps, diverge_at); + assert!( + !full.is_empty(), + "nary {}: the search should commit digits", + nary + ); + assert_eq!( + full, asym, + "nary {} divergence {}: the searches should commit the same digits", + nary, diverge_at + ); + } + } + } + + #[test] + fn a_full_duplex_search_locates_the_divergence() { + for diverge_at in [5u64, 13, 21, 37, 50] { + let agreed = run_search(60, 8, 64, diverge_at); + assert_eq!( + agreed, + diverge_at - 1, + "divergence at {} should leave {} as the last agreed step", + diverge_at, + diverge_at - 1 + ); + } + } +} + +#[cfg(test)] +mod memory_model_yaml_tests { + use super::*; + use crate::loader::program_definition::ProgramDefinition; + + #[test] + fn a_definition_without_the_field_means_last_step() { + let prog = SyntheticProgram::new(4, 8, 128).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + assert_eq!(def.memory_model(), MemoryModel::LastStep); + assert!(!def.memory_model().chains_root()); + } + + #[test] + fn merkle_travels_from_the_yaml_to_the_loaded_program() { + // The seam that makes the backend swappable: a program definition + // asks for a memory model and the loaded Program has it, without + // anything in between having to be told. + let prog = SyntheticProgram::new_with_memory_model(4, 8, 128, MemoryModel::Merkle).unwrap(); + let text = std::fs::read_to_string(prog.definition()).unwrap(); + assert!( + text.contains("memory_model: merkle"), + "the yaml says it: {text}" + ); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + assert_eq!(def.memory_model(), MemoryModel::Merkle); + + let program = def.load_program().unwrap(); + assert_eq!(program.memory_model, MemoryModel::Merkle); + assert!(program.trace_mem_root().is_some(), "and it chains a root"); + assert!(program.memory_tree_is_consistent()); + } + + #[test] + fn last_step_travels_too_and_chains_nothing() { + let prog = + SyntheticProgram::new_with_memory_model(4, 8, 128, MemoryModel::LastStep).unwrap(); + let def = ProgramDefinition::from_config(&prog.definition()).unwrap(); + assert_eq!(def.memory_model(), MemoryModel::LastStep); + let program = def.load_program().unwrap(); + assert!(program.trace_mem_root().is_none()); + } +} diff --git a/emulator/src/main.rs b/emulator/src/main.rs index be70888..f8d324d 100644 --- a/emulator/src/main.rs +++ b/emulator/src/main.rs @@ -5,10 +5,11 @@ use emulator::{ constants::REGISTERS_BASE_ADDRESS, decision::{ challenge::{ - prover_execute, prover_final_trace, prover_get_hashes_and_step, - prover_get_hashes_for_round, verifier_check_execution, verifier_choose_challenge, - verifier_choose_challenge_for_read_challenge, verifier_choose_segment, ForceChallenge, - ForceCondition, + prover_execute, prover_final_trace, prover_full_duplex_final_trace, + prover_full_duplex_stage, prover_get_hashes_and_step, prover_get_hashes_for_round, + verifier_check_execution, verifier_choose_challenge, + verifier_choose_challenge_for_read_challenge, verifier_choose_segment, + verifier_full_duplex_stage, ForceChallenge, ForceCondition, }, nary_search::NArySearchType, }, @@ -143,6 +144,53 @@ enum Commands { nary_type: NArySearchType, }, + /// One stage of a full-duplex search: select over the counterparty's + /// previous dissection, then dissect the subinterval just selected. + FullDuplexStage { + /// Yaml file to load + #[arg(long, value_name = "FILE")] + pdf: String, + + /// Checkpoint input path + #[arg(long, value_name = "CHECKPOINT_INPUT_PATH")] + checkpoint_input_path: String, + + /// Checkpoint output path + #[arg(long, value_name = "CHECKPOINT_OUTPUT_PATH")] + checkpoint_output_path: String, + + /// Whether this party is the prover + #[arg(long, value_name = "IS_PROVER")] + is_prover: bool, + + /// Stage number, counting from 1 + #[arg(long, value_name = "STAGE_NUMBER")] + stage: u8, + + /// Dissection published at the previous stage; empty at stage 1 + #[arg(long, value_name = "COUNTERPARTY_HASHES", num_args = 0..)] + counterparty_hashes: Vec, + + /// Selection digit published alongside that dissection; absent before + /// stage 3 + #[arg(long, value_name = "COUNTERPARTY_DECISION")] + counterparty_decision: Option, + + /// Also produce the trace for the disputed step. Used for the trailing + /// stage, whose selection rides on the adjudication message together + /// with that trace. + #[arg(long)] + final_trace: bool, + + /// Fail Configuration + #[arg(long, value_name = "FailConfig")] + fail_config: Option, + + /// Command File to write the result + #[arg(long, value_name = "COMMAND_PATH")] + command_file: String, + }, + VerifierChooseSegment { /// Yaml file to load #[arg(short, long, value_name = "FILE")] @@ -680,6 +728,74 @@ fn main() -> Result<(), EmulatorError> { file.write_all(result.to_string().as_bytes()) .expect("Failed to write JSON to file"); } + Some(Commands::FullDuplexStage { + pdf, + checkpoint_input_path, + checkpoint_output_path, + is_prover, + stage, + counterparty_hashes, + counterparty_decision, + final_trace, + fail_config, + command_file, + }) => { + if *final_trace { + let (decision, prover_final_trace) = prover_full_duplex_final_trace( + pdf, + checkpoint_input_path, + checkpoint_output_path, + *stage, + counterparty_hashes, + *counterparty_decision, + fail_config.clone(), + )?; + let result = EmulatorResultType::FullDuplexFinalTraceResult { + decision, + prover_final_trace, + } + .to_value()?; + let mut file = create_or_open_file(command_file); + file.write_all(result.to_string().as_bytes()) + .expect("Failed to write JSON to file"); + return Ok(()); + } + let (decision, hashes) = if *is_prover { + prover_full_duplex_stage( + pdf, + checkpoint_input_path, + checkpoint_output_path, + *stage, + counterparty_hashes, + *counterparty_decision, + fail_config.clone(), + )? + } else { + verifier_full_duplex_stage( + pdf, + checkpoint_input_path, + checkpoint_output_path, + *stage, + counterparty_hashes, + *counterparty_decision, + fail_config.clone(), + )? + }; + info!( + "Full-duplex stage {}: decision {:?} hashes {:?}", + stage, decision, hashes + ); + + let result = EmulatorResultType::FullDuplexStageResult { + decision, + hashes, + stage: *stage, + } + .to_value()?; + let mut file = create_or_open_file(command_file); + file.write_all(result.to_string().as_bytes()) + .expect("Failed to write JSON to file"); + } Some(Commands::VerifierChooseSegment { pdf, checkpoint_verifier_input_path, diff --git a/emulator/tests/dump_compliance_traces.rs b/emulator/tests/dump_compliance_traces.rs new file mode 100644 index 0000000..9e47a47 --- /dev/null +++ b/emulator/tests/dump_compliance_traces.rs @@ -0,0 +1,103 @@ +//! Dump every instruction the RISC-V compliance suite retires. +//! +//! The emulator and the Bitcoin Script verifier are both validated against +//! these 47 ELFs -- `compliance.rs` runs them with `verify_on_chain`, so +//! every instruction's trace goes through `verify_script`. The garbled +//! step circuit is not: it has a few hundred hand-emitted vectors over +//! twenty opcodes, which is how a division defect survived in it while +//! Script was exercising the same case continuously. +//! +//! This writes the traces out so the circuit can be checked against them. +//! It is `#[ignore]`d because it exists to feed +//! `compact/circuits/compliance_check.py` rather than to assert anything +//! itself: +//! +//! ```sh +//! cargo test -p emulator --test dump_compliance_traces -- --ignored --nocapture +//! python3 compact/circuits/compliance_check.py /tmp/compliance-traces.tsv +//! ``` +use emulator::executor::fetcher::execute_program; +use emulator::executor::utils::FailConfiguration; +use emulator::loader::program::load_elf; +use std::io::Write; + +const OUT: &str = "/tmp/compliance-traces.tsv"; + +#[test] +#[ignore = "writes a trace corpus for the circuit differential check"] +fn dump_compliance_traces() { + let dir = "../docker-riscv32/compliance/build"; + let mut out = std::fs::File::create(OUT).expect("create the trace file"); + writeln!( + out, + "elf\tstep\tpc\tmicro\topcode\tr1_addr\trs1\tr2_addr\trs2\t\ + write_addr\twrite_val\twrite_pc\twrite_micro\twitness\tmem_witness" + ) + .unwrap(); + + let mut files: Vec<_> = std::fs::read_dir(dir) + .expect("the compliance build directory") + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.extension().is_some_and(|x| x == "elf") + && !p.to_string_lossy().contains("fence_i") + }) + .collect(); + files.sort(); + + let (mut steps, mut programs) = (0usize, 0usize); + for path in &files { + let name = path.file_stem().unwrap().to_string_lossy().to_string(); + let mut program = match load_elf(&path.to_string_lossy(), false) { + Ok(p) => p, + Err(e) => panic!("{name}: {e:?}"), + }; + let (_result, traces) = execute_program( + &mut program, + Vec::new(), + ".bss", + false, + &None, + Some(5_000), + true, // print_trace: record every step + false, + false, + false, + false, + true, + None, + None, + FailConfiguration::default(), + false, + ); + for (t, _hash) in &traces { + writeln!( + out, + "{}\t{}\t{:08x}\t{}\t{:08x}\t{:08x}\t{:08x}\t{:08x}\t{:08x}\t\ + {:08x}\t{:08x}\t{:08x}\t{}\t{:08x}\t{:02x}", + name, + t.step_number, + t.read_pc.pc.get_address(), + t.read_pc.pc.get_micro(), + t.read_pc.opcode, + t.read_1.address, + t.read_1.value, + t.read_2.address, + t.read_2.value, + t.trace_step.get_write().address, + t.trace_step.get_write().value, + t.trace_step.get_pc().get_address(), + t.trace_step.get_pc().get_micro(), + t.witness.unwrap_or(0), + t.mem_witness.byte(), + ) + .unwrap(); + steps += 1; + } + programs += 1; + } + println!("{steps} instructions from {programs} compliance programs -> {OUT}"); + assert!(programs >= 40, "expected the compliance suite, got {programs}"); + assert!(steps > 10_000, "expected a real corpus, got {steps}"); +} diff --git a/emulator/tests/test_r_type_instructions.rs b/emulator/tests/test_r_type_instructions.rs index 48f37a8..276df48 100644 --- a/emulator/tests/test_r_type_instructions.rs +++ b/emulator/tests/test_r_type_instructions.rs @@ -237,4 +237,16 @@ pub fn test_division() { test_division_aux(100, 6, 16, 4, false); test_division_aux(100, -1, -100, 0, true); + + // A zero quotient with a negative dividend, which is what |n| < |d| + // gives and which nothing above reaches. The garbled RV32M circuit + // reported these as faulted until the remainder was sign-extended + // into its q*d + r check; Script and the emulator were always right, + // so the two backends disagreed about a valid instruction. Pinned + // here so the emulator's own tests cover the case. + test_division_aux(-1, 2, 0, -1, true); + test_division_aux(-7, -9, 0, -7, true); + test_division_aux(-5, 3, -1, -2, true); + test_division_aux(7, 9, 0, 7, true); + test_division_aux(-991_549_079, -1_001_680_343, 0, -991_549_079, true); } diff --git a/emulator/tests/utils/common.rs b/emulator/tests/utils/common.rs index 2cecc12..258467e 100644 --- a/emulator/tests/utils/common.rs +++ b/emulator/tests/utils/common.rs @@ -1,5 +1,7 @@ #![allow(dead_code)] -use bitvmx_cpu_definitions::{memory::SectionDefinition, trace::ProgramCounter}; +use bitvmx_cpu_definitions::{ + memory::SectionDefinition, memory_tree::MemoryTree, trace::ProgramCounter, +}; use emulator::{ executor::{ fetcher::{execute_program, FullTrace}, @@ -66,6 +68,8 @@ pub fn get_new_program() -> Program { read_only_sections: SectionDefinition::default(), register_sections: SectionDefinition::default(), code_sections: SectionDefinition::default(), + memory_tree: MemoryTree::default(), + memory_model: Default::default(), } }