From 49d24c20ed6fc87d5f730a090cbe002c5f362870 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 00:29:30 +0200 Subject: [PATCH 01/10] feat: add atomic instructions Signed-off-by: Henry --- crates/parser/src/lib.rs | 1 - crates/parser/src/macros.rs | 1 + crates/parser/src/visit.rs | 70 +++++++++- crates/tinywasm/src/error.rs | 5 + .../src/interpreter/executor/instructions.rs | 2 + .../tinywasm/src/interpreter/executor/mod.rs | 68 ++++++++++ crates/tinywasm/src/shared.rs | 4 +- crates/tinywasm/tests/atomics.rs | 121 ++++++++++++++++++ .../tinywasm/tests/generated/wasm-threads.csv | 2 +- crates/types/src/instructions.rs | 90 +++++++++++++ 10 files changed, 357 insertions(+), 7 deletions(-) create mode 100644 crates/tinywasm/tests/atomics.rs diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 019e027e..c729f6bd 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -174,7 +174,6 @@ impl Parser { #[cfg(feature = "validate")] { let features = WasmFeatures::WASM3 - .difference(WasmFeatures::THREADS) .union(WasmFeatures::CUSTOM_PAGE_SIZES) .union(WasmFeatures::WIDE_ARITHMETIC) .union(WasmFeatures::COMPACT_IMPORTS); diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index fe1364bf..3779d5e0 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -203,6 +203,7 @@ pub(crate) mod visit { (@@function_references $($rest:tt)* ) => {}; (@@gc $($rest:tt)* ) => {}; (@@exceptions $($rest:tt)* ) => {}; + (@@threads $($rest:tt)* ) => {}; (@@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*)) => { fn $visit(&mut self $($(,_: $argty)*)?) -> Self::Output { diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index a362f846..844eda1c 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -11,9 +11,9 @@ use alloc::{ vec::Vec, }; use tinywasm_types::{ - BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, ImportKind, Instruction, MemoryType, Operand64, - Operand64Idx, Operand128, Operand128Idx, StorageType, TableDefinition, TagType, TypeSection, ValueCounts, - ValueLane, WasmFunctionData, + AtomicArg, AtomicOp, AtomicWidth, BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, ImportKind, + Instruction, MemoryType, Operand64, Operand64Idx, Operand128, Operand128Idx, StorageType, TableDefinition, TagType, + TypeSection, ValueCounts, ValueLane, WasmFunctionData, }; use wasmparser::{FunctionBody, OperatorsReader, OperatorsReaderAllocations, VisitSimdOperator}; @@ -497,6 +497,35 @@ pub(crate) fn process_operators_and_validate( Ok((instructions, builder.data, builder.uses_local_memory, validator.into_allocations(), reader.into_allocations())) } +macro_rules! atomic_visitors { + ($($op:ident, $inputs:tt => $outputs:tt: $(($visit:ident, $width:literal)),+;)*) => { + $(atomic_visitors!(@group $op, $inputs => $outputs: $(($visit, $width)),+);)* + }; + (@group $op:ident, $inputs:tt => $outputs:tt: $(($visit:ident, $width:literal)),+) => { + $(atomic_visitors!(@one $op, $inputs => $outputs: $visit, $width);)+ + }; + (@one $op:ident, [$($input:ident),*] => [$($output:ident),*]: $visit:ident, $width:literal) => { + fn $visit(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + if memarg.align != ($width as u32).trailing_zeros() as u8 { + return Err(crate::ParseError::Other("invalid atomic alignment".into())); + } + let address = self.metadata.memory_size(memarg.memory)?; + self.mark_memory(memarg.memory); + let memory = self.push128(Operand128::::new(memarg.offset, memarg.memory))?; + self.emit( + &[$(atomic_visitors!(@size $input, address)),*], + &[$(atomic_visitors!(@size $output, address)),*], + Instruction::Atomic(AtomicArg::new(memory, AtomicWidth::from_bytes($width), atomic_visitors!(@is64 [$($input),*] [$($output),*]), AtomicOp::$op)), + ) + } + }; + (@size Addr, $address:ident) => { $address }; + (@size $lane:ident, $address:ident) => { ValueLane::$lane }; + (@is64 [Addr, S64 $(, S64)*] $outputs:tt) => { true }; + (@is64 [Addr] [S64]) => { true }; + (@is64 $inputs:tt $outputs:tt) => { false }; +} + impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { type Output = Result<()>; @@ -506,6 +535,41 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { wasmparser::for_each_visit_operator!(impl_visit_operator); + fn visit_atomic_fence(&mut self) -> Self::Output { + self.emit(&[], &[], Instruction::AtomicFence) + } + + fn visit_memory_atomic_notify(&mut self, _: wasmparser::MemArg) -> Self::Output { + Err(crate::ParseError::UnsupportedOperator("memory.atomic.notify".into())) + } + fn visit_memory_atomic_wait32(&mut self, _: wasmparser::MemArg) -> Self::Output { + Err(crate::ParseError::UnsupportedOperator("memory.atomic.wait32".into())) + } + fn visit_memory_atomic_wait64(&mut self, _: wasmparser::MemArg) -> Self::Output { + Err(crate::ParseError::UnsupportedOperator("memory.atomic.wait64".into())) + } + + atomic_visitors! { + Load, [Addr] => [S32]: (visit_i32_atomic_load, 4), (visit_i32_atomic_load8_u, 1), (visit_i32_atomic_load16_u, 2); + Load, [Addr] => [S64]: (visit_i64_atomic_load, 8), (visit_i64_atomic_load8_u, 1), (visit_i64_atomic_load16_u, 2), (visit_i64_atomic_load32_u, 4); + Store, [Addr, S32] => []: (visit_i32_atomic_store, 4), (visit_i32_atomic_store8, 1), (visit_i32_atomic_store16, 2); + Store, [Addr, S64] => []: (visit_i64_atomic_store, 8), (visit_i64_atomic_store8, 1), (visit_i64_atomic_store16, 2), (visit_i64_atomic_store32, 4); + Add, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_add, 4), (visit_i32_atomic_rmw8_add_u, 1), (visit_i32_atomic_rmw16_add_u, 2); + Add, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_add, 8), (visit_i64_atomic_rmw8_add_u, 1), (visit_i64_atomic_rmw16_add_u, 2), (visit_i64_atomic_rmw32_add_u, 4); + Sub, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_sub, 4), (visit_i32_atomic_rmw8_sub_u, 1), (visit_i32_atomic_rmw16_sub_u, 2); + Sub, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_sub, 8), (visit_i64_atomic_rmw8_sub_u, 1), (visit_i64_atomic_rmw16_sub_u, 2), (visit_i64_atomic_rmw32_sub_u, 4); + And, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_and, 4), (visit_i32_atomic_rmw8_and_u, 1), (visit_i32_atomic_rmw16_and_u, 2); + And, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_and, 8), (visit_i64_atomic_rmw8_and_u, 1), (visit_i64_atomic_rmw16_and_u, 2), (visit_i64_atomic_rmw32_and_u, 4); + Or, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_or, 4), (visit_i32_atomic_rmw8_or_u, 1), (visit_i32_atomic_rmw16_or_u, 2); + Or, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_or, 8), (visit_i64_atomic_rmw8_or_u, 1), (visit_i64_atomic_rmw16_or_u, 2), (visit_i64_atomic_rmw32_or_u, 4); + Xor, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_xor, 4), (visit_i32_atomic_rmw8_xor_u, 1), (visit_i32_atomic_rmw16_xor_u, 2); + Xor, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_xor, 8), (visit_i64_atomic_rmw8_xor_u, 1), (visit_i64_atomic_rmw16_xor_u, 2), (visit_i64_atomic_rmw32_xor_u, 4); + Xchg, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_xchg, 4), (visit_i32_atomic_rmw8_xchg_u, 1), (visit_i32_atomic_rmw16_xchg_u, 2); + Xchg, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_xchg, 8), (visit_i64_atomic_rmw8_xchg_u, 1), (visit_i64_atomic_rmw16_xchg_u, 2), (visit_i64_atomic_rmw32_xchg_u, 4); + Cmpxchg, [Addr, S32, S32] => [S32]: (visit_i32_atomic_rmw_cmpxchg, 4), (visit_i32_atomic_rmw8_cmpxchg_u, 1), (visit_i32_atomic_rmw16_cmpxchg_u, 2); + Cmpxchg, [Addr, S64, S64] => [S64]: (visit_i64_atomic_rmw_cmpxchg, 8), (visit_i64_atomic_rmw8_cmpxchg_u, 1), (visit_i64_atomic_rmw16_cmpxchg_u, 2), (visit_i64_atomic_rmw32_cmpxchg_u, 4); + } + lowering_ops! { memory [Addr] => [S32] { visit_i32_load => I32Load [load(Instruction::LoadLocal32)], diff --git a/crates/tinywasm/src/error.rs b/crates/tinywasm/src/error.rs index 77856aff..cc26d103 100644 --- a/crates/tinywasm/src/error.rs +++ b/crates/tinywasm/src/error.rs @@ -131,6 +131,9 @@ pub enum Trap { max: usize, }, + /// An atomic memory access was not naturally aligned. + UnalignedAtomic, + /// An out-of-bounds table access occurred TableOutOfBounds { /// The offset of the access @@ -216,6 +219,7 @@ impl Trap { match self { Self::Unreachable => "unreachable", Self::MemoryOutOfBounds { .. } => "out of bounds memory access", + Self::UnalignedAtomic => "unaligned atomic", Self::TableOutOfBounds { .. } => "out of bounds table access", Self::ArrayOutOfBounds => "out of bounds array access", Self::DivisionByZero => "integer divide by zero", @@ -324,6 +328,7 @@ impl Display for Trap { Self::MemoryOutOfBounds { offset, len, max } => { write!(f, "out of bounds memory access: offset={offset}, len={len}, max={max}") } + Self::UnalignedAtomic => write!(f, "unaligned atomic"), Self::TableOutOfBounds { offset, len, max } => { write!(f, "out of bounds table access: offset={offset}, len={len}, max={max}") } diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 587d776b..c139af39 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -323,6 +323,8 @@ macro_rules! instruction_handlers { ArrayInitElem(idx) => executor.exec_array_init_elem(*idx)?, MemorySize(addr) => executor.exec_memory_size(*addr)?, MemoryGrow(addr) => executor.exec_memory_grow(*addr)?, + Atomic(arg) => executor.exec_atomic(*arg)?, + AtomicFence => crate::shared::fence(crate::shared::Ordering::SeqCst), // Bulk memory operations MemoryCopy(idx) => executor.exec_memory_copy(*idx)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index cb2b7f80..5697a6c0 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -1430,6 +1430,74 @@ impl<'store> Executor<'store> { TARGET::stack_push(&mut self.store.value_stack, cast(value)) } + fn exec_atomic(&mut self, arg: AtomicArg) -> Result<(), Trap> { + match arg.width() { + AtomicWidth::Bytes1 => self.exec_atomic_width::<1>(arg), + AtomicWidth::Bytes2 => self.exec_atomic_width::<2>(arg), + AtomicWidth::Bytes4 => self.exec_atomic_width::<4>(arg), + AtomicWidth::Bytes8 => self.exec_atomic_width::<8>(arg), + } + } + + fn exec_atomic_width(&mut self, arg: AtomicArg) -> Result<(), Trap> { + let op = arg.op(); + let value = if op == AtomicOp::Load { + 0 + } else if arg.is_64() { + u64::stack_pop(&mut self.store.value_stack) + } else { + u32::stack_pop(&mut self.store.value_stack) as u64 + }; + let expected = if op == AtomicOp::Cmpxchg { + if arg.is_64() { + u64::stack_pop(&mut self.store.value_stack) + } else { + u32::stack_pop(&mut self.store.value_stack) as u64 + } + } else { + 0 + }; + let memory = arg.memory.resolve(&self.func.data); + let mem = self.store.state.get_mem_mut(self.mem_addr(memory.memory())); + let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; + if addr % N != 0 { + return cold!(Err(Trap::UnalignedAtomic)); + } + if op == AtomicOp::Store { + let mut bytes = [0u8; N]; + bytes.copy_from_slice(&value.to_le_bytes()[..N]); + return cold_err!(mem.inner.write_fixed::(addr, &bytes)); + } + let bytes = cold_err!(mem.inner.read_fixed::(addr))?; + let mut padded = [0u8; 8]; + padded[..N].copy_from_slice(&bytes); + let old = u64::from_le_bytes(padded); + let mask = if N == 8 { u64::MAX } else { (1u64 << (N * 8)) - 1 }; + let next = match op { + AtomicOp::Load => None, + AtomicOp::Store => unreachable!(), + AtomicOp::Xchg => Some(value), + AtomicOp::Add => Some(old.wrapping_add(value)), + AtomicOp::Sub => Some(old.wrapping_sub(value)), + AtomicOp::And => Some(old & value), + AtomicOp::Or => Some(old | value), + AtomicOp::Xor => Some(old ^ value), + AtomicOp::Cmpxchg => (old == (expected & mask)).then_some(value), + }; + if let Some(next) = next { + let mut bytes = [0u8; N]; + bytes.copy_from_slice(&next.to_le_bytes()[..N]); + cold_err!(mem.inner.write_fixed::(addr, &bytes))?; + } + if arg.is_64() { + u64::stack_push(&mut self.store.value_stack, old)?; + } else { + u32::stack_push(&mut self.store.value_stack, old as u32)?; + } + Ok(()) + } + #[inline(always)] fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { let bytes = Value128::stack_pop(&mut self.store.value_stack).to_mem_bytes(); diff --git a/crates/tinywasm/src/shared.rs b/crates/tinywasm/src/shared.rs index 97f42b75..5fe65f2a 100644 --- a/crates/tinywasm/src/shared.rs +++ b/crates/tinywasm/src/shared.rs @@ -4,6 +4,6 @@ pub(crate) use alloc::rc::Rc as StoreShared; pub(crate) use tinywasm_types::Shared as StoreShared; #[cfg(not(feature = "portable-atomic"))] -pub(crate) use core::sync::atomic::{AtomicU32, Ordering}; +pub(crate) use core::sync::atomic::{AtomicU32, Ordering, fence}; #[cfg(feature = "portable-atomic")] -pub(crate) use portable_atomic::{AtomicU32, Ordering}; +pub(crate) use portable_atomic::{AtomicU32, Ordering, fence}; diff --git a/crates/tinywasm/tests/atomics.rs b/crates/tinywasm/tests/atomics.rs new file mode 100644 index 00000000..3aaaaa39 --- /dev/null +++ b/crates/tinywasm/tests/atomics.rs @@ -0,0 +1,121 @@ +use tinywasm::{Error, ModuleInstance, Store, Trap}; + +type TestResult = Result<(), Box>; + +#[test] +fn atomic_operations_on_ordinary_memory() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (memory (export "memory") 1) + (func (export "run") (result i64) + i32.const 12 + i32.const 255 + i32.atomic.store8 + i32.const 12 + i32.const 2 + i32.atomic.rmw8.add_u + drop + i32.const 12 + i32.const 1 + i32.const 7 + i32.atomic.rmw8.cmpxchg_u + drop + i32.const 16 + i64.const -1 + i64.atomic.store32 + i32.const 16 + i64.atomic.load32_u + atomic.fence)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert_eq!(instance.func::<(), i64>(&store, "run")?.call(&mut store, ())?, 0xffff_ffff); + let memory = instance.memory("memory")?; + assert_eq!(memory.read_vec(&store, 12, 1)?, [7]); + Ok(()) +} + +#[test] +fn atomic_alignment_and_bounds() -> TestResult { + let wasm = wat::parse_str( + r#" + (module (memory 1) + (func (export "load") (param i32) (result i32) + local.get 0 + i32.atomic.load offset=2)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let load = instance.func::(&store, "load")?; + assert!(matches!(load.call(&mut store, 0), Err(Error::Trap(Trap::UnalignedAtomic)))); + assert!(matches!(load.call(&mut store, 65534), Err(Error::Trap(Trap::MemoryOutOfBounds { .. })))); + + let invalid_alignment = wat::parse_str("(module (memory 1) (func i32.const 0 i32.atomic.load align=1 drop))")?; + assert!(tinywasm::parse_bytes(&invalid_alignment).is_err()); + Ok(()) +} + +#[test] +fn atomic_rmw_operations_wrap_at_byte_width() -> TestResult { + for (op, expected, updated) in [ + ("add", 0xff, 1), + ("sub", 0xff, 0xfd), + ("and", 0xff, 2), + ("or", 0xff, 0xff), + ("xor", 0xff, 0xfd), + ("xchg", 0xff, 2), + ] { + let wat = format!( + r#" + (module (memory 1) + (func (export "run") (result i32) + i32.const 0 + i32.const 255 + i32.atomic.store8 + i32.const 0 + i32.const 2 + i32.atomic.rmw8.{op}_u) + (func (export "read") (result i32) + i32.const 0 + i32.atomic.load8_u)) + "# + ); + let module = tinywasm::parse_bytes(&wat::parse_str(&wat)?)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert_eq!(instance.func::<(), i32>(&store, "run")?.call(&mut store, ())?, expected); + assert_eq!(instance.func::<(), i32>(&store, "read")?.call(&mut store, ())?, updated); + } + Ok(()) +} + +#[test] +fn narrow_cmpxchg_truncates_expected_and_preserves_old_value() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (memory (export "memory") 1) + (func (export "exchange") (param i64 i64) (result i64) + i32.const 4 + local.get 0 + local.get 1 + i64.atomic.rmw32.cmpxchg_u)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let memory = instance.memory("memory")?; + let exchange = instance.func::<(i64, i64), i64>(&store, "exchange")?; + memory.copy_from_slice(&mut store, 4, &0x89ab_cdef_u32.to_le_bytes())?; + + assert_eq!(exchange.call(&mut store, (0x1234_89ab_cdef, 7))?, 0x89ab_cdef); + assert_eq!(exchange.call(&mut store, (0x89ab_cdef, 9))?, 7); + assert_eq!(memory.read_vec(&store, 4, 4)?, 7u32.to_le_bytes()); + Ok(()) +} diff --git a/crates/tinywasm/tests/generated/wasm-threads.csv b/crates/tinywasm/tests/generated/wasm-threads.csv index 25d09b77..454014e7 100644 --- a/crates/tinywasm/tests/generated/wasm-threads.csv +++ b/crates/tinywasm/tests/generated/wasm-threads.csv @@ -1,3 +1,3 @@ 0.9.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] 0.10.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] -0.11.0-pre.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] +0.11.0,609,10,[{"name":"atomic.wast","passed":292,"failed":5},{"name":"exports.wast","passed":88,"failed":0},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":82,"failed":0}] diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index 3bc71a23..e45a5aba 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -9,6 +9,94 @@ use crate::{ /// Identifies the packed full-width memory operand layout. pub enum MemoryOperand {} +/// Operation performed by an atomic memory instruction. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(u8)] +pub enum AtomicOp { + Load = 0, + Store = 1, + Add = 2, + Sub = 3, + And = 4, + Or = 5, + Xor = 6, + Xchg = 7, + Cmpxchg = 8, +} + +/// Width of an atomic memory access, in bytes. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(u8)] +pub enum AtomicWidth { + Bytes1 = 1, + Bytes2 = 2, + Bytes4 = 4, + Bytes8 = 8, +} + +impl AtomicWidth { + /// Converts a supported byte width into an atomic width. + pub const fn from_bytes(bytes: u8) -> Self { + match bytes { + 1 => Self::Bytes1, + 2 => Self::Bytes2, + 4 => Self::Bytes4, + 8 => Self::Bytes8, + _ => panic!("invalid atomic width"), + } + } +} + +/// An atomic operation with its module-local memory operand and access width. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(C, packed)] +pub struct AtomicArg { + pub memory: Operand128Idx, + code: u8, +} + +impl AtomicArg { + /// Packs the operation, access width, and result type into one byte. + pub const fn new(memory: Operand128Idx, width: AtomicWidth, is_64: bool, op: AtomicOp) -> Self { + Self { memory, code: (op as u8) | ((width as u8).trailing_zeros() as u8) << 4 | (is_64 as u8) << 6 } + } + + /// Returns the access width. + pub const fn width(self) -> AtomicWidth { + match (self.code >> 4) & 3 { + 0 => AtomicWidth::Bytes1, + 1 => AtomicWidth::Bytes2, + 2 => AtomicWidth::Bytes4, + _ => AtomicWidth::Bytes8, + } + } + /// Returns whether the value is an i64. + pub const fn is_64(self) -> bool { + self.code & 64 != 0 + } + /// Returns the atomic operation. + pub fn op(self) -> AtomicOp { + match self.code & 15 { + 0 => AtomicOp::Load, + 1 => AtomicOp::Store, + 2 => AtomicOp::Add, + 3 => AtomicOp::Sub, + 4 => AtomicOp::And, + 5 => AtomicOp::Or, + 6 => AtomicOp::Xor, + 7 => AtomicOp::Xchg, + 8 => AtomicOp::Cmpxchg, + _ => unreachable!("invalid atomic operation"), + } + } +} + /// Identifies the packed compact memory operand layout. pub enum CompactMemoryOperand {} @@ -647,6 +735,8 @@ define_instructions! { I32Store8(Operand128Idx), I32Store16(Operand128Idx), I64Store8(Operand128Idx), I64Store16(Operand128Idx), I64Store32(Operand128Idx), MemorySize(MemAddr), MemoryGrow(MemAddr), + Atomic(AtomicArg), + AtomicFence, // > Constants Const32(i32), From 455ffd4f0c75162fc3580f4e8b96dbd672dd1695 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 14:46:43 +0200 Subject: [PATCH 02/10] feat: add shared memories Signed-off-by: Henry --- Cargo.toml | 2 +- crates/cli/src/wast_runner.rs | 9 +- crates/parser/src/conversion.rs | 1 + crates/tinywasm/src/func/context.rs | 6 + crates/tinywasm/src/imports.rs | 42 ++- crates/tinywasm/src/instance.rs | 79 ++++- .../tinywasm/src/interpreter/executor/mod.rs | 334 +++++++++--------- crates/tinywasm/src/reference/store.rs | 9 +- crates/tinywasm/src/store/memory/instance.rs | 55 ++- crates/tinywasm/src/store/memory/mod.rs | 6 +- crates/tinywasm/src/store/memory/shared.rs | 194 ++++++++++ crates/tinywasm/src/store/memory/vec.rs | 70 ++-- crates/tinywasm/src/store/mod.rs | 54 ++- crates/tinywasm/src/store/state.rs | 118 +++++++ .../tinywasm/tests/generated/wasm-threads.csv | 2 +- crates/tinywasm/tests/shared_memory.rs | 303 ++++++++++++++++ crates/types/src/archive.rs | 4 +- crates/types/src/lib.rs | 14 +- examples/threads.rs | 51 +++ 19 files changed, 1096 insertions(+), 257 deletions(-) create mode 100644 crates/tinywasm/src/store/memory/shared.rs create mode 100644 crates/tinywasm/tests/shared_memory.rs create mode 100644 examples/threads.rs diff --git a/Cargo.toml b/Cargo.toml index 94858af5..355ae6c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ unexpected_cfgs = { level = "warn", check-cfg = ["cfg(rust_analyzer)"] } [dev-dependencies] anyhow.workspace = true -tinywasm = { path = "crates/tinywasm", features = ["state"] } +tinywasm = { path = "crates/tinywasm", features = ["send", "state"] } tinywasm-wasi.workspace = true wat.workspace = true diff --git a/crates/cli/src/wast_runner.rs b/crates/cli/src/wast_runner.rs index 4ebdc6c9..a3f17e59 100644 --- a/crates/cli/src/wast_runner.rs +++ b/crates/cli/src/wast_runner.rs @@ -8,7 +8,9 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use log::{debug, error}; use tinywasm::types::{MemoryType, RefType, RefValue, TableType, WasmType, WasmValue}; -use tinywasm::{ExecProgress, Global, HostFunction, Imports, Memory, Module, ModuleInstance, Store, Table}; +use tinywasm::{ + ExecProgress, Global, HostFunction, Imports, Memory, MemoryShared, Module, ModuleInstance, Store, Table, +}; use wast::QuoteWat; use wast::core::{AbstractHeapType, NanPattern}; @@ -17,7 +19,7 @@ const TEST_MAX_SUSPENSIONS: u32 = 1000; // Older suites classify encodings that wasmparser accepts with the latest feature set as malformed. const ACCEPTED_MALFORMED_MESSAGES: &[&str] = &["integer representation too long", "zero byte expected", "zero flag expected"]; -const ACCEPTED_INVALID_MESSAGES: &[&str] = &["multiple memories"]; +const ACCEPTED_INVALID_MESSAGES: &[&str] = &["multiple memories", "multiple tables"]; macro_rules! float_value { ($pattern:expr, $float:ty, $variant:ident) => { @@ -172,6 +174,8 @@ impl WastRunner { let table64 = Table::try_new(store, TableType::new64(RefType::FUNCREF, 10, Some(20)), RefValue::Null.into())?; let memory = Memory::try_new(store, MemoryType::default().with_page_count_initial(1).with_page_count_max(Some(2)))?; + let shared_memory = + MemoryShared::try_new(MemoryType::default().with_page_count_initial(1).with_page_count_max(Some(2)))?; let global_i32 = Global::try_new(store, tinywasm::types::GlobalType::new(WasmType::I32, false), WasmValue::I32(666))?; let global_i64 = @@ -183,6 +187,7 @@ impl WastRunner { imports .define("spectest", "memory", memory) + .define("spectest", "shared_memory", shared_memory) .define("spectest", "table", table) .define("spectest", "table64", table64) .define("spectest", "global_i32", global_i32) diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 36455eee..25743229 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -109,6 +109,7 @@ pub(crate) fn convert_module_memory(memory: wasmparser::MemoryType) -> MemoryTyp memory.maximum, memory.page_size_log2.map(|x| 1 << x), ) + .with_shared(memory.shared) } pub(crate) fn convert_module_globals( diff --git a/crates/tinywasm/src/func/context.rs b/crates/tinywasm/src/func/context.rs index e5c183c3..27a38364 100644 --- a/crates/tinywasm/src/func/context.rs +++ b/crates/tinywasm/src/func/context.rs @@ -30,6 +30,12 @@ impl FuncContext<'_> { self.module().memory(name) } + /// Get a shared memory export. + #[cfg(feature = "std")] + pub fn memory_shared(&self, name: &str) -> Result { + self.module().memory_shared(name) + } + /// Get any exported extern value by name. pub fn extern_item(&self, name: &str) -> Result { self.module().extern_item(name) diff --git a/crates/tinywasm/src/imports.rs b/crates/tinywasm/src/imports.rs index 49234e6f..870d76bb 100644 --- a/crates/tinywasm/src/imports.rs +++ b/crates/tinywasm/src/imports.rs @@ -2,6 +2,8 @@ use alloc::collections::BTreeMap; use alloc::string::{String, ToString}; use alloc::vec::Vec; +#[cfg(feature = "std")] +use crate::MemoryShared; use crate::{Function, Global, HostFunction, LinkingError, Memory, Result, Table, Tag}; use tinywasm_types::*; @@ -16,6 +18,9 @@ pub enum Extern { Table(Table), /// A memory instance. Memory(Memory), + /// A shared memory, importable into independent stores. + #[cfg(feature = "std")] + MemoryShared(MemoryShared), /// A function import. Function(Function), /// A reusable host function definition. @@ -45,6 +50,13 @@ impl_conv! { Tag => Tag, } +#[cfg(feature = "std")] +impl From for Extern { + fn from(value: MemoryShared) -> Self { + Self::MemoryShared(value) + } +} + /// Imports for a module instance /// /// This is used to link a module instance to its imports @@ -188,6 +200,8 @@ impl Imports { crate::ExternItem::Func(value) => Extern::Function(value), crate::ExternItem::Global(value) => Extern::Global(value), crate::ExternItem::Memory(value) => Extern::Memory(value), + #[cfg(feature = "std")] + crate::ExternItem::MemoryShared(value) => Extern::MemoryShared(value), crate::ExternItem::Table(value) => Extern::Table(value), crate::ExternItem::Tag(value) => Extern::Tag(value), }) @@ -228,6 +242,7 @@ impl ResolvedImports { real_size: usize, ) -> Result<()> { Self::compare_types(import, &expected.arch(), &actual.arch())?; + Self::compare_types(import, &expected.shared(), &actual.shared())?; if actual.page_count_initial() > expected.page_count_initial() && actual.page_count_initial() > real_size as u64 { @@ -286,6 +301,16 @@ impl ResolvedImports { memory.0.validate_store(store)?; ExternVal::Memory(memory.0.addr) } + #[cfg(feature = "std")] + Extern::MemoryShared(memory) => { + let index = MemAddr::try_from(store.state.shared_memories.len()) + .map_err(|_| crate::Error::UnsupportedFeature("too many shared memories"))?; + if index >= crate::store::SHARED_MEM_BIT - 1 { + return Err(crate::Error::UnsupportedFeature("too many shared memories")); + } + store.state.shared_memories.push(memory.clone()); + ExternVal::Memory(index | crate::store::SHARED_MEM_BIT) + } Extern::Function(func) => { func.item.validate_store(store)?; ExternVal::Func(func.addr()) @@ -339,8 +364,21 @@ impl ResolvedImports { imports.tables.push(table_addr); } (ExternVal::Memory(memory_addr), ImportKind::Memory(ty)) => { - let mem = store.state.get_mem(memory_addr); - Self::compare_memory_types(import, &mem.kind, ty, mem.page_count)?; + #[cfg(feature = "std")] + let (kind, pages) = if memory_addr & crate::store::SHARED_MEM_BIT != 0 { + let memory = + &store.state.shared_memories[(memory_addr & !crate::store::SHARED_MEM_BIT) as usize]; + (memory.ty(), memory.page_count()) + } else { + let memory = store.state.get_mem(memory_addr); + (memory.kind, memory.page_count) + }; + #[cfg(not(feature = "std"))] + let (kind, pages) = { + let memory = store.state.get_mem(memory_addr); + (memory.kind, memory.page_count) + }; + Self::compare_memory_types(import, &kind, ty, pages)?; imports.memories.push(memory_addr); } (ExternVal::Func(func_addr), ImportKind::Function(ty)) => { diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index 7eda5017..9cce7b88 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -9,6 +9,8 @@ use crate::store::MemoryInstance; use crate::{ Error, Function, FunctionTyped, Global, Imports, Memory, Result, Store, StoreItem, Table, Tag, Trap, WasmValue, }; +#[cfg(feature = "std")] +use crate::{MemoryShared, store::SHARED_MEM_BIT}; /// A typed view over an exported extern value. pub enum ExternItem { @@ -16,6 +18,9 @@ pub enum ExternItem { Func(Function), /// Exported memory reference. Memory(Memory), + /// Exported shared memory. + #[cfg(feature = "std")] + MemoryShared(MemoryShared), /// Exported table reference. Table(Table), /// Exported global reference. @@ -57,6 +62,8 @@ struct ModuleInstanceInner { func_addrs: Box<[FuncAddr]>, table_addrs: Box<[TableAddr]>, mem_addrs: Box<[MemAddr]>, + #[cfg(feature = "std")] + shared_backings: Box<[(MemAddr, MemoryShared)]>, global_addrs: Box<[GlobalAddr]>, tag_addrs: Box<[TagAddr]>, elem_addrs: Box<[ElemAddr]>, @@ -187,6 +194,19 @@ impl ModuleInstance { } fn instantiate_inner(store: &mut Store, module: &Module, imports: &[crate::Extern]) -> Result { + if module.memory_types.iter().any(|ty| ty.shared() && ty.page_count_max_declared().is_none()) + || module.imports.iter().any(|import| { + matches!(import.kind, ImportKind::Memory(ty) if ty.shared() && ty.page_count_max_declared().is_none()) + }) + { + return Err(Error::UnsupportedFeature("shared memory requires a maximum")); + } + #[cfg(not(feature = "std"))] + if module.memory_types.iter().any(MemoryType::shared) + || module.imports.iter().any(|import| matches!(import.kind, ImportKind::Memory(ty) if ty.shared())) + { + return Err(Error::UnsupportedFeature("shared memory requires std")); + } let type_addrs = store.register_module_types(&module.types); let id = store.next_module_instance_id(); let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?; @@ -205,6 +225,14 @@ impl ModuleInstance { let elem_addrs = store.alloc_elements(&addrs.funcs, &addrs.globals, &module.elements, &type_addrs)?; let data_addrs = store.alloc_data(&module.data); + #[cfg(feature = "std")] + let shared_backings = addrs + .memories + .iter() + .filter(|&&addr| addr & SHARED_MEM_BIT != 0) + .map(|&addr| (addr, store.state.shared_memories[(addr & !SHARED_MEM_BIT) as usize].clone())) + .collect(); + let instance = ModuleInstanceInner { store_id: store.id(), id, @@ -212,6 +240,8 @@ impl ModuleInstance { func_addrs: addrs.funcs.into_boxed_slice(), table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), + #[cfg(feature = "std")] + shared_backings, global_addrs: addrs.globals.into_boxed_slice(), tag_addrs: addrs.tags.into_boxed_slice(), elem_addrs, @@ -295,7 +325,17 @@ impl ModuleInstance { ExternItem::Table(Table(StoreItem::new(self.0.store_id, self.resolve_table_addr(export.index)))) } ExternalKind::Memory => { - ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, self.resolve_mem_addr(export.index)))) + let addr = self.resolve_mem_addr(export.index); + #[cfg(feature = "std")] + if addr & SHARED_MEM_BIT != 0 { + ExternItem::MemoryShared( + self.shared_backing(addr).expect("exported shared memory has a backing"), + ) + } else { + ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr))) + } + #[cfg(not(feature = "std"))] + ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr))) } ExternalKind::Global => { ExternItem::Global(Global(StoreItem::new(self.0.store_id, self.resolve_global_addr(export.index)))) @@ -352,7 +392,13 @@ impl ModuleInstance { ExternVal::Func(addr) => { Ok(ExternItem::Func(Function { item: StoreItem::new(self.0.store_id, addr), module_id: self.id() })) } - ExternVal::Memory(addr) => Ok(ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr)))), + ExternVal::Memory(addr) => { + #[cfg(feature = "std")] + if addr & SHARED_MEM_BIT != 0 { + return Ok(ExternItem::MemoryShared(self.shared_backing(addr)?)); + } + Ok(ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr)))) + } ExternVal::Table(addr) => Ok(ExternItem::Table(Table(StoreItem::new(self.0.store_id, addr)))), ExternVal::Global(addr) => Ok(ExternItem::Global(Global(StoreItem::new(self.0.store_id, addr)))), ExternVal::Tag(addr) => Ok(ExternItem::Tag(Tag(StoreItem::new(self.0.store_id, addr)))), @@ -482,11 +528,32 @@ impl ModuleInstance { /// Get a memory export by name. pub fn memory(&self, name: &str) -> Result { match self.require_export(name)? { - ExternVal::Memory(mem_addr) => Ok(Memory(StoreItem::new(self.0.store_id, mem_addr))), + ExternVal::Memory(mem_addr) if mem_addr & crate::store::SHARED_MEM_BIT == 0 => { + Ok(Memory(StoreItem::new(self.0.store_id, mem_addr))) + } _ => cold!(Err(Error::Other(format!("Export is not a memory: {name}")))), } } + #[cfg(feature = "std")] + fn shared_backing(&self, addr: MemAddr) -> Result { + self.0 + .shared_backings + .iter() + .find(|(index, _)| *index == addr) + .map(|(_, backing)| backing.clone()) + .ok_or_else(|| Error::Other("invalid shared memory address".into())) + } + + /// Returns a shared memory export by name. + #[cfg(feature = "std")] + pub fn memory_shared(&self, name: &str) -> Result { + match self.require_export(name)? { + ExternVal::Memory(addr) if addr & SHARED_MEM_BIT != 0 => self.shared_backing(addr), + _ => Err(Error::Other(format!("Export is not a shared memory: {name}"))), + } + } + /// Get a memory by its module-local index. /// /// This exposes an internal module-owned memory directly and bypasses the @@ -496,7 +563,11 @@ impl ModuleInstance { #[cfg_attr(docsrs, doc(cfg(feature = "guest-debug")))] #[cfg(feature = "guest-debug")] pub fn memory_by_index(&self, memory_index: MemAddr) -> Result { - Ok(Memory(StoreItem::new(self.0.store_id, Self::index_addr(&self.0.mem_addrs, memory_index, "memory")?))) + let addr = Self::index_addr(&self.0.mem_addrs, memory_index, "memory")?; + if addr & crate::store::SHARED_MEM_BIT != 0 { + return Err(Error::UnsupportedFeature("use shared memory access for this index")); + } + Ok(Memory(StoreItem::new(self.0.store_id, addr))) } /// Get a table export by name. diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 5697a6c0..8a562164 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -867,20 +867,21 @@ impl<'store> Executor<'store> { let memarg = index.resolve(&self.func.data); let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); - let mem = self.store.state.get_mem_mut(mem_addr); - let addr = if mem.is_64bit() { - let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { - offset: usize::MAX, - len: N, - max: mem.inner.len(), - }))?; - cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? - } else { - let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? - }; - value.store_at(&mut mem.inner, addr) + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let addr = if kind.arch() == MemoryArch::I64 { + let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { + offset: usize::MAX, + len: N, + max: mem.len(), + }))?; + cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? + } else { + let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? + }; + value.store_at(&mut *mem, addr) + }) } fn exec_inc_memory_local, const N: usize>( @@ -891,22 +892,23 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); - let mem = self.store.state.get_mem_mut(mem_addr); - let addr = if mem.is_64bit() { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; - let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { - offset: usize::MAX, - len: N, - max: mem.inner.len(), - }))?; - cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? - } else { - let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? - }; + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let addr = if kind.arch() == MemoryArch::I64 { + let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { + offset: usize::MAX, + len: N, + max: mem.len(), + }))?; + cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? + } else { + let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? + }; - let value = cold_err!(T::load_at(&mem.inner, addr))?; - increment(value).store_at(&mut mem.inner, addr) + let value = cold_err!(T::load_at(&*mem, addr))?; + increment(value).store_at(&mut *mem, addr) + }) } fn exec_fma_store< @@ -921,10 +923,11 @@ impl<'store> Executor<'store> { let acc = T::stack_pop(&mut self.store.value_stack); let fma = acc + lhs * rhs; let mem_addr = self.mem_addr(m.mem_addr()); - let mem = self.store.state.get_mem_mut(mem_addr); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; - cold_err!(fma.store_at(&mut mem.inner, addr)) + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; + cold_err!(fma.store_at(&mut *mem, addr)) + }) } fn exec_load_local< @@ -942,28 +945,30 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); - let mem = self.store.state.get_mem(self.mem_addr(MemAddr::from(memarg.memory()))); - let base = if mem.is_64bit() { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; - cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { - offset: usize::MAX, - len: N, - max: mem.inner.len(), - })? - } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize - }; + let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = if kind.arch() == MemoryArch::I64 { + let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { + offset: usize::MAX, + len: N, + max: mem.len(), + })? + } else { + u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + }; - let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; - let value = cast(cold_err!(LOAD::load_at(&mem.inner, addr))?); + let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; + let value = cast(cold_err!(LOAD::load_at(&*mem, addr))?); - if SET_LOCAL { - TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); - } - if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value)?; - } - Ok(()) + if SET_LOCAL { + TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); + } + if !SET_LOCAL || TEE { + TARGET::stack_push(&mut self.store.value_stack, value)?; + } + Ok(()) + }) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { @@ -1262,24 +1267,24 @@ impl<'store> Executor<'store> { } fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { - let mem = self.store.state.get_mem(self.mem_addr(addr)); - match mem.is_64bit() { - true => i64::stack_push(&mut self.store.value_stack, mem.page_count as i64), - false => i32::stack_push(&mut self.store.value_stack, mem.page_count as i32), + let mem_addr = self.mem_addr(addr); + let (arch, pages) = self.store.state.memory_size(mem_addr); + match arch { + MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, pages as i64), + MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, pages as i32), } } fn exec_memory_grow(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); let limiter = self.store.engine.config().resource_limiter.as_deref(); - let mem = self.store.state.get_mem_mut(mem_addr); - let is_64bit = mem.is_64bit(); + let is_64bit = self.store.state.memory_type(mem_addr).arch() == MemoryArch::I64; let pages_delta = match is_64bit { true => i64::stack_pop(&mut self.store.value_stack), false => i64::from(i32::stack_pop(&mut self.store.value_stack)), }; - let size = mem.grow(pages_delta, limiter)?.unwrap_or(-1); + let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { true => i64::stack_push(&mut self.store.value_stack, size)?, false => i32::stack_push(&mut self.store.value_stack, size as i32)?, @@ -1294,29 +1299,20 @@ impl<'store> Executor<'store> { let src_mem = operand.b(); let dst_mem_addr = self.mem_addr(dst_mem); let src_mem_addr = self.mem_addr(src_mem); - let dst_arch = self.store.state.get_mem(dst_mem_addr).kind.arch(); - let src_arch = self.store.state.get_mem(src_mem_addr).kind.arch(); + let dst_arch = self.store.state.memory_type(dst_mem_addr).arch(); + let src_arch = self.store.state.memory_type(src_mem_addr).arch(); let len_arch = if dst_arch == MemoryArch::I32 || src_arch == MemoryArch::I32 { MemoryArch::I32 } else { MemoryArch::I64 }; let size = self.store.value_stack.pop_memory_operand(len_arch)?; let src = self.store.value_stack.pop_memory_operand(src_arch)?; let dst = self.store.value_stack.pop_memory_operand(dst_arch)?; - if dst_mem_addr == src_mem_addr { - // copy within the same memory - let mem = self.store.state.get_mem_mut(dst_mem_addr); - mem.copy_within(dst, src, size)?; - } else { - // copy between two memories - let (dst_memory, src_memory) = self.store.state.get_mems_mut(dst_mem_addr, src_mem_addr); - dst_memory.copy_from_memory(dst, src_memory, src, size)?; - } - Ok(()) + self.store.state.copy_memories(dst_mem_addr, dst, src_mem_addr, src, size) } fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); - let arch = self.store.state.get_mem(mem_addr).kind.arch(); + let arch = self.store.state.memory_type(mem_addr).arch(); let size = self.store.value_stack.pop_memory_operand(arch)?; let val = i32::stack_pop(&mut self.store.value_stack); let dst = self.store.value_stack.pop_memory_operand(arch)?; @@ -1326,18 +1322,19 @@ impl<'store> Executor<'store> { fn exec_memory_fill_const(&mut self, index: Operand128Idx) -> Result<(), Trap> { let operand = index.resolve(&self.func.data); let mem_addr = self.mem_addr(operand.memory()); - let arch = self.store.state.get_mem(mem_addr).kind.arch(); + let arch = self.store.state.memory_type(mem_addr).arch(); let dst = self.store.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, operand.byte(), operand.value() as u32 as usize) } fn exec_memory_fill_impl(&mut self, mem_addr: MemAddr, dst: usize, val: u8, size: usize) -> Result<(), Trap> { - let mem = self.store.state.get_mem_mut(mem_addr); - let max = mem.inner.len(); - if mem.inner.fill(dst, size, val).is_none() { - return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max })); - } - Ok(()) + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let max = mem.len(); + if mem.fill(dst, size, val).is_none() { + return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max })); + } + Ok(()) + }) } fn exec_memory_init(&mut self, index: Operand64Idx<(u32, u32)>) -> Result<(), Trap> { @@ -1347,32 +1344,33 @@ impl<'store> Executor<'store> { let size = u32::stack_pop(&mut self.store.value_stack) as usize; let offset = u32::stack_pop(&mut self.store.value_stack) as usize; let mem_addr = self.mem_addr(mem_index); - let arch = self.store.state.get_mem(mem_addr).kind.arch(); + let arch = self.store.state.memory_type(mem_addr).arch(); let dst = self.store.value_stack.pop_memory_operand(arch)?; - let data = &self.store.state.data[self.module.resolve_data_addr(data_index) as usize]; - let mem = &mut self.store.state.memories[mem_addr as usize]; - let data_len = data.data.as_ref().map_or(0, |d| d.len()); - let mem_len = mem.inner.len(); - if offset.checked_add(size).is_none_or(|end| end > data_len) { - return cold!(Err(Trap::MemoryOutOfBounds { offset, len: size, max: data_len })); - } - if dst.checked_add(size).is_none_or(|end| end > mem_len) { - return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); - } + let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let data_len = data.as_ref().map_or(0, |d| d.len()); + let mem_len = mem.len(); + if offset.checked_add(size).is_none_or(|end| end > data_len) { + return cold!(Err(Trap::MemoryOutOfBounds { offset, len: size, max: data_len })); + } + if dst.checked_add(size).is_none_or(|end| end > mem_len) { + return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); + } - if size == 0 { - return Ok(()); - } + if size == 0 { + return Ok(()); + } - let Some(data) = &data.data else { - return cold!(Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 })); - }; + let Some(data) = &data else { + return cold!(Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 })); + }; - if mem.inner.write_all(dst, &data[offset..offset + size]).is_none() { - return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); - } - Ok(()) + if mem.write_all(dst, &data[offset..offset + size]).is_none() { + return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); + } + Ok(()) + }) } fn exec_table_copy(&mut self, index: Operand64Idx<(u32, u32)>) -> Result<(), Trap> { @@ -1404,17 +1402,19 @@ impl<'store> Executor<'store> { arg: MemoryLaneArg, ) -> Result<(), Trap> { let m = arg.memory_arg_idx.resolve(&self.func.data); - let mem = self.store.state.get_mem(self.mem_addr(m.memory())); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; - let val = cold_err!(LOAD::load_at(&mem.inner, addr))?; - let offset = arg.lane as usize * LOAD_SIZE; - Value128::stack_update(&mut self.store.value_stack, |value| { - let mut bytes = value.to_mem_bytes(); - bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); - Value128(bytes) - }); - Ok(()) + let mem_addr = self.mem_addr(m.memory()); + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; + let val = cold_err!(LOAD::load_at(&*mem, addr))?; + let offset = arg.lane as usize * LOAD_SIZE; + Value128::stack_update(&mut self.store.value_stack, |value| { + let mut bytes = value.to_mem_bytes(); + bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); + Value128(bytes) + }); + Ok(()) + }) } #[inline(always)] @@ -1423,11 +1423,13 @@ impl<'store> Executor<'store> { m: Operand128, cast: impl Fn(LOAD) -> TARGET, ) -> Result<(), Trap> { - let mem = self.store.state.get_mem(self.mem_addr(m.memory())); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; - let value = cold_err!(LOAD::load_at(&mem.inner, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)) + let mem_addr = self.mem_addr(m.memory()); + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; + let value = cold_err!(LOAD::load_at(&*mem, addr))?; + TARGET::stack_push(&mut self.store.value_stack, cast(value)) + }) } fn exec_atomic(&mut self, arg: AtomicArg) -> Result<(), Trap> { @@ -1458,44 +1460,46 @@ impl<'store> Executor<'store> { 0 }; let memory = arg.memory.resolve(&self.func.data); - let mem = self.store.state.get_mem_mut(self.mem_addr(memory.memory())); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; - if addr % N != 0 { - return cold!(Err(Trap::UnalignedAtomic)); - } - if op == AtomicOp::Store { - let mut bytes = [0u8; N]; - bytes.copy_from_slice(&value.to_le_bytes()[..N]); - return cold_err!(mem.inner.write_fixed::(addr, &bytes)); - } - let bytes = cold_err!(mem.inner.read_fixed::(addr))?; - let mut padded = [0u8; 8]; - padded[..N].copy_from_slice(&bytes); - let old = u64::from_le_bytes(padded); - let mask = if N == 8 { u64::MAX } else { (1u64 << (N * 8)) - 1 }; - let next = match op { - AtomicOp::Load => None, - AtomicOp::Store => unreachable!(), - AtomicOp::Xchg => Some(value), - AtomicOp::Add => Some(old.wrapping_add(value)), - AtomicOp::Sub => Some(old.wrapping_sub(value)), - AtomicOp::And => Some(old & value), - AtomicOp::Or => Some(old | value), - AtomicOp::Xor => Some(old ^ value), - AtomicOp::Cmpxchg => (old == (expected & mask)).then_some(value), - }; - if let Some(next) = next { - let mut bytes = [0u8; N]; - bytes.copy_from_slice(&next.to_le_bytes()[..N]); - cold_err!(mem.inner.write_fixed::(addr, &bytes))?; - } - if arg.is_64() { - u64::stack_push(&mut self.store.value_stack, old)?; - } else { - u32::stack_push(&mut self.store.value_stack, old as u32)?; - } - Ok(()) + let mem_addr = self.mem_addr(memory.memory()); + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; + if addr % N != 0 { + return cold!(Err(Trap::UnalignedAtomic)); + } + if op == AtomicOp::Store { + let mut bytes = [0u8; N]; + bytes.copy_from_slice(&value.to_le_bytes()[..N]); + return cold_err!(mem.write_fixed::(addr, &bytes)); + } + let bytes = cold_err!(mem.read_fixed::(addr))?; + let mut padded = [0u8; 8]; + padded[..N].copy_from_slice(&bytes); + let old = u64::from_le_bytes(padded); + let mask = if N == 8 { u64::MAX } else { (1u64 << (N * 8)) - 1 }; + let next = match op { + AtomicOp::Load => None, + AtomicOp::Store => unreachable!(), + AtomicOp::Xchg => Some(value), + AtomicOp::Add => Some(old.wrapping_add(value)), + AtomicOp::Sub => Some(old.wrapping_sub(value)), + AtomicOp::And => Some(old & value), + AtomicOp::Or => Some(old | value), + AtomicOp::Xor => Some(old ^ value), + AtomicOp::Cmpxchg => (old == (expected & mask)).then_some(value), + }; + if let Some(next) = next { + let mut bytes = [0u8; N]; + bytes.copy_from_slice(&next.to_le_bytes()[..N]); + cold_err!(mem.write_fixed::(addr, &bytes))?; + } + if arg.is_64() { + u64::stack_push(&mut self.store.value_stack, old)?; + } else { + u32::stack_push(&mut self.store.value_stack, old as u32)?; + } + Ok(()) + }) } #[inline(always)] @@ -1507,11 +1511,12 @@ impl<'store> Executor<'store> { let val = U::from_mem_bytes(val_bytes); let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); - let mem = self.store.state.get_mem_mut(mem_addr); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; - cold_err!(val.store_at(&mut mem.inner, addr))?; - Ok(()) + crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; + cold_err!(val.store_at(&mut *mem, addr))?; + Ok(()) + }) } #[inline(always)] @@ -1543,10 +1548,11 @@ impl<'store> Executor<'store> { offset: u64, val: U, ) -> Result<(), Trap> { - let mem = self.store.state.get_mem_mut(memory_addr); - let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = cold_err!(mem.effective_addr::(base, offset))?; - cold_err!(val.store_at(&mut mem.inner, addr)) + crate::store::with_memory!(self.store.state, memory_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::(base, offset))?; + cold_err!(val.store_at(&mut *mem, addr)) + }) } fn exec_table_get(&mut self, table_index: u32) -> Result<(), Trap> { diff --git a/crates/tinywasm/src/reference/store.rs b/crates/tinywasm/src/reference/store.rs index 11c7ae58..b0771efa 100644 --- a/crates/tinywasm/src/reference/store.rs +++ b/crates/tinywasm/src/reference/store.rs @@ -377,7 +377,14 @@ impl Memory { /// Create a new memory in the given store. pub fn try_new(store: &mut Store, ty: MemoryType) -> Result { - let addr = store.state.memories.len() as MemAddr; + if ty.shared() { + return Err(Error::UnsupportedFeature("use MemoryShared for shared memory")); + } + let addr = MemAddr::try_from(store.state.memories.len()) + .map_err(|_| Error::UnsupportedFeature("too many memories"))?; + if addr >= crate::store::SHARED_MEM_BIT { + return Err(Error::UnsupportedFeature("too many memories")); + } let limiter = store.engine.config().resource_limiter.clone(); store.state.memories.push(MemoryInstance::new(ty, limiter.as_deref())?); Ok(Self(StoreItem::new(store.id(), addr))) diff --git a/crates/tinywasm/src/store/memory/instance.rs b/crates/tinywasm/src/store/memory/instance.rs index 9da327b1..64d70d2a 100644 --- a/crates/tinywasm/src/store/memory/instance.rs +++ b/crates/tinywasm/src/store/memory/instance.rs @@ -24,7 +24,7 @@ impl core::fmt::Debug for MemoryInstance { impl MemoryInstance { #[inline] - fn host_size(kind: MemoryType, pages: u64) -> Option { + pub(super) fn host_size(kind: MemoryType, pages: u64) -> Option { pages.checked_mul(kind.page_size()).and_then(|size| usize::try_from(size).ok()) } @@ -34,32 +34,21 @@ impl MemoryInstance { } #[inline] - fn page_count_max(kind: MemoryType) -> u64 { + pub(super) fn page_count_max(kind: MemoryType) -> u64 { match kind.arch() { MemoryArch::I32 => kind.page_count_max(), MemoryArch::I64 => kind.page_count_max().min(MEMORY64_MAX_BYTES / kind.page_size()), } } - #[cfg(target_pointer_width = "64")] - #[inline(always)] - pub(crate) fn effective_addr(&self, base: usize, offset: u64) -> Result { - match base.checked_add(offset as usize) { - Some(addr) => Ok(addr), - None => cold!(Err(memory_oob(base, N, self.inner.len()))), + pub(crate) fn new(kind: MemoryType, limiter: Option<&dyn ResourceLimiter>) -> Result { + if kind.shared() && kind.page_count_max_declared().is_none() { + return Err(Error::UnsupportedFeature("shared memory requires a maximum")); } - } - - #[cfg(not(target_pointer_width = "64"))] - #[inline(always)] - pub(crate) fn effective_addr(&self, base: usize, offset: u64) -> Result { - match usize::try_from(offset).ok().and_then(|offset| base.checked_add(offset)) { - Some(addr) => Ok(addr), - None => cold!(Err(memory_oob(base, N, self.inner.len()))), + #[cfg(not(feature = "std"))] + if kind.shared() { + return Err(Error::UnsupportedFeature("shared memory requires std")); } - } - - pub(crate) fn new(kind: MemoryType, limiter: Option<&dyn ResourceLimiter>) -> Result { if kind.page_size() == 0 { return Err(Error::UnsupportedFeature("zero-byte memory pages")); } @@ -90,10 +79,6 @@ impl MemoryInstance { Ok(Self { kind, inner: storage, page_count: kind.page_count_initial() as usize }) } - pub(crate) const fn is_64bit(&self) -> bool { - matches!(self.kind.arch(), MemoryArch::I64) - } - pub(crate) fn copy_from_memory( &mut self, dst: usize, @@ -113,12 +98,22 @@ impl MemoryInstance { pages_delta: i64, limiter: Option<&dyn ResourceLimiter>, ) -> Result, Trap> { - let current_pages = self.page_count; + Self::grow_storage(self.kind, &mut self.inner, &mut self.page_count, pages_delta, limiter) + } + + pub(super) fn grow_storage( + kind: MemoryType, + inner: &mut MemoryStorage, + page_count: &mut usize, + pages_delta: i64, + limiter: Option<&dyn ResourceLimiter>, + ) -> Result, Trap> { + let current_pages = *page_count; let Some(new_pages) = usize::try_from(pages_delta).ok().and_then(|delta| current_pages.checked_add(delta)) else { return cold!(Ok(None)); }; - let max_pages = Self::page_count_max(self.kind).try_into().unwrap_or(usize::MAX); + let max_pages = Self::page_count_max(kind).try_into().unwrap_or(usize::MAX); if new_pages > max_pages { return cold!({ @@ -127,23 +122,23 @@ impl MemoryInstance { }); } - let Some(new_size) = Self::host_size(self.kind, new_pages as u64) else { + let Some(new_size) = Self::host_size(kind, new_pages as u64) else { return cold!(Ok(None)); }; - if new_size == self.inner.len() { + if new_size == inner.len() { return Ok(i64::try_from(current_pages).ok()); } if let Some(limiter) = limiter - && !limiter.memory_growing(self.inner.len(), new_size, Self::maximum_size(self.kind))? + && !limiter.memory_growing(inner.len(), new_size, Self::maximum_size(kind))? { return cold!(Ok(None)); } - if self.inner.grow_to(new_size).is_err() { + if inner.grow_to(new_size).is_err() { return cold!(Ok(None)); } - self.page_count = new_pages; + *page_count = new_pages; Ok(i64::try_from(current_pages).ok()) } } diff --git a/crates/tinywasm/src/store/memory/mod.rs b/crates/tinywasm/src/store/memory/mod.rs index 9f059ac1..f7dfcd99 100644 --- a/crates/tinywasm/src/store/memory/mod.rs +++ b/crates/tinywasm/src/store/memory/mod.rs @@ -1,9 +1,13 @@ use crate::interpreter::Value128; mod instance; +#[cfg(feature = "std")] +mod shared; mod vec; pub(crate) use instance::MemoryInstance; +#[cfg(feature = "std")] +pub use shared::{MemoryShared, MemorySharedGuard}; pub(crate) use vec::VecMemory; /// Internal storage for a linear memory. @@ -70,6 +74,6 @@ impl MemValue<16> for Value128 { } } -const fn memory_oob(offset: usize, len: usize, max: usize) -> crate::Trap { +pub(super) const fn memory_oob(offset: usize, len: usize, max: usize) -> crate::Trap { crate::Trap::MemoryOutOfBounds { offset, len, max } } diff --git a/crates/tinywasm/src/store/memory/shared.rs b/crates/tinywasm/src/store/memory/shared.rs new file mode 100644 index 00000000..c92ab045 --- /dev/null +++ b/crates/tinywasm/src/store/memory/shared.rs @@ -0,0 +1,194 @@ +use crate::std::sync::{Mutex, MutexGuard}; +use alloc::{sync::Arc, vec::Vec}; +use core::sync::atomic::{AtomicUsize, Ordering}; +use tinywasm_types::MemoryType; + +use super::{MemoryInstance, MemoryStorage, memory_oob}; +use crate::{ResourceLimiter, Result, Trap}; + +/// A cloneable handle to one shared WebAssembly memory, importable into multiple stores. +#[derive(Clone)] +#[cfg_attr(feature = "debug", derive(Debug))] +pub struct MemoryShared(Arc); + +struct MemorySharedInstance { + kind: MemoryType, + pages: AtomicUsize, + bytes: Mutex, +} + +#[cfg(feature = "debug")] +impl core::fmt::Debug for MemorySharedInstance { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MemorySharedInstance").field("kind", &self.kind).field("pages", &self.pages).finish() + } +} + +/// Holds the memory lock while accessing its bytes. +pub struct MemorySharedGuard<'a> { + pub(crate) kind: MemoryType, + pub(crate) inner: MutexGuard<'a, MemoryStorage>, +} + +impl MemoryShared { + pub(crate) fn from_instance(instance: MemoryInstance) -> Self { + let kind = instance.kind; + let pages = AtomicUsize::new(instance.page_count); + let bytes = Mutex::new(instance.inner); + Self(Arc::new(MemorySharedInstance { kind, pages, bytes })) + } + + /// Whether two handles reference the same instance, even across store slots. + pub(crate) fn same_instance(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + /// Creates shared memory with a declared maximum size. + /// + /// The memory type is marked shared automatically. + pub fn try_new(ty: MemoryType) -> Result { + Ok(Self::from_instance(MemoryInstance::new(ty.with_shared(true), None)?)) + } + + /// Locks the memory for scoped access to its bytes and page count. + pub fn lock(&self) -> MemorySharedGuard<'_> { + MemorySharedGuard { kind: self.ty(), inner: self.0.bytes.lock().unwrap_or_else(|poison| poison.into_inner()) } + } + + /// Reads the published size without locking the byte storage. + pub fn page_count(&self) -> usize { + self.0.pages.load(Ordering::Acquire) + } + + /// Reads the immutable memory type without locking. + pub fn ty(&self) -> MemoryType { + self.0.kind + } + + /// Returns the current byte length without locking. + pub fn len(&self) -> usize { + MemoryInstance::host_size(self.ty(), self.page_count() as u64).expect("published memory size fits on this host") + } + + /// Returns whether the memory has no allocated bytes. + pub fn is_empty(&self) -> bool { + self.page_count() == 0 + } + + /// Reads up to `dst.len()` bytes, returning the count read. + pub fn read(&self, offset: usize, dst: &mut [u8]) -> Result { + Ok(self.lock().inner.read(offset, dst)) + } + + /// Writes up to `src.len()` bytes, returning the count written. + pub fn write(&self, offset: usize, src: &[u8]) -> Result { + Ok(self.lock().inner.write(offset, src)) + } + + /// Reads an exact range or returns an out-of-bounds trap. + pub fn read_exact(&self, offset: usize, dst: &mut [u8]) -> Result<()> { + let guard = self.lock(); + guard.inner.read_exact(offset, dst).ok_or_else(|| memory_oob(offset, dst.len(), guard.inner.len()).into()) + } + + /// Copies an exact range out of the memory. + pub fn read_vec(&self, offset: usize, len: usize) -> Result> { + let guard = self.lock(); + guard.inner.read_vec(offset, len).ok_or_else(|| memory_oob(offset, len, guard.inner.len()).into()) + } + + /// Copies an entire slice into the memory. + pub fn copy_from_slice(&self, offset: usize, bytes: &[u8]) -> Result<()> { + let mut guard = self.lock(); + guard.inner.write_all(offset, bytes).ok_or_else(|| memory_oob(offset, bytes.len(), guard.inner.len()).into()) + } + + /// Copies a range within the same memory. + pub fn copy_within(&self, src: usize, dst: usize, len: usize) -> Result<()> { + let mut guard = self.lock(); + guard.inner.copy_within(dst, src, len).ok_or_else(|| memory_oob(dst, len, guard.inner.len()).into()) + } + + /// Fills a range with a byte value. + pub fn fill(&self, offset: usize, len: usize, val: u8) -> Result<()> { + let mut guard = self.lock(); + guard.inner.fill(offset, len, val).ok_or_else(|| memory_oob(offset, len, guard.inner.len()).into()) + } + + /// Grows the memory by the given number of pages, returning its previous size. + pub fn grow(&self, pages: i64) -> Result> { + Ok(self.grow_with_limiter(pages, None)?) + } + + /// Calls host limiters outside the memory lock, then rechecks concurrent growth. + pub(crate) fn grow_with_limiter( + &self, + pages: i64, + limiter: Option<&dyn ResourceLimiter>, + ) -> core::result::Result, Trap> { + let Some(limiter) = limiter else { + let mut guard = self.lock(); + return self.grow_locked(pages, &mut guard.inner); + }; + loop { + let guard = self.lock(); + let kind = self.ty(); + let current_pages = self.page_count(); + let Some(new_pages) = usize::try_from(pages).ok().and_then(|delta| current_pages.checked_add(delta)) else { + return Ok(None); + }; + if new_pages as u64 > MemoryInstance::page_count_max(kind) { + return Ok(None); + } + let Some(desired) = MemoryInstance::host_size(kind, new_pages as u64) else { + return Ok(None); + }; + let current = guard.inner.len(); + let maximum = + kind.page_count_max_declared().map(|max| MemoryInstance::host_size(kind, max).unwrap_or(usize::MAX)); + drop(guard); + + let allowed = desired == current || limiter.memory_growing(current, desired, maximum)?; + + let mut guard = self.lock(); + if self.page_count() != current_pages { + continue; + } + if !allowed { + return Ok(None); + } + return self.grow_locked(pages, &mut guard.inner); + } + } + + fn grow_locked(&self, pages: i64, bytes: &mut MemoryStorage) -> core::result::Result, Trap> { + let mut current_pages = self.page_count(); + let result = MemoryInstance::grow_storage(self.ty(), bytes, &mut current_pages, pages, None); + if result.as_ref().is_ok_and(Option::is_some) { + self.0.pages.store(current_pages, Ordering::Release); + } + result + } +} + +impl MemorySharedGuard<'_> { + /// Returns the current memory type. + pub fn ty(&self) -> MemoryType { + self.kind + } + + /// Returns the current page count. + pub fn page_count(&self) -> usize { + (self.inner.len() as u64 / self.kind.page_size()) as usize + } + + /// Borrows the bytes while this guard holds the lock. + pub fn data(&self) -> &[u8] { + self.inner.data() + } + + /// Mutably borrows the bytes while this guard holds the lock. + pub fn data_mut(&mut self) -> &mut [u8] { + self.inner.data_mut() + } +} diff --git a/crates/tinywasm/src/store/memory/vec.rs b/crates/tinywasm/src/store/memory/vec.rs index b14f54dd..29cae7a2 100644 --- a/crates/tinywasm/src/store/memory/vec.rs +++ b/crates/tinywasm/src/store/memory/vec.rs @@ -7,26 +7,24 @@ use super::memory_oob; /// A contiguous `Vec`-backed linear memory storage. /// /// This is the default internal storage for [`super::MemoryInstance`]. -pub(crate) struct VecMemory { - data: Vec, -} +pub(crate) struct VecMemory(Vec); impl VecMemory { /// Returns a raw mutable pointer to the memory's backing allocation. /// Growth may invalidate the pointer. The caller must keep the store alive /// and uphold Rust's aliasing rules when dereferencing it. pub(crate) fn data_ptr(&mut self) -> *mut u8 { - self.data.as_mut_ptr() + self.0.as_mut_ptr() } /// Borrows the backing bytes. pub(crate) fn data(&self) -> &[u8] { - &self.data + &self.0 } /// Borrows the backing bytes exclusively. pub(crate) fn data_mut(&mut self) -> &mut [u8] { - &mut self.data + &mut self.0 } /// Tries to create a new memory with `len` zero-initialized bytes. @@ -34,77 +32,87 @@ impl VecMemory { let mut data = Vec::new(); cold_err!(data.try_reserve(len)).map_err(|_| crate::Trap::OutOfMemory)?; data.resize(len, 0); - Ok(Self { data }) + Ok(Self(data)) } #[inline(always)] pub(crate) fn len(&self) -> usize { - self.data.len() + self.0.len() + } + + /// Resolves the memory offset while retaining the byte length for bounds errors. + #[inline(always)] + pub(crate) fn effective_addr(&self, base: usize, offset: u64) -> Result { + #[cfg(target_pointer_width = "64")] + let address = base.checked_add(offset as usize); + #[cfg(not(target_pointer_width = "64"))] + let address = usize::try_from(offset).ok().and_then(|offset| base.checked_add(offset)); + address.ok_or_else(|| memory_oob(base, N, self.len())) } #[inline(always)] - pub(super) fn checked_range(&self, addr: usize, len: usize) -> Option> { + pub(crate) fn checked_range(&self, addr: usize, len: usize) -> Option> { let end = addr.checked_add(len)?; - (end <= self.data.len()).then_some(addr..end) + (end <= self.0.len()).then_some(addr..end) } /// Grows the backing allocation to `new_len`. Only called after the Wasm limits and any user /// limiter have accepted the grow. #[inline(always)] pub(crate) fn grow_to(&mut self, new_len: usize) -> Result<(), crate::Trap> { - debug_assert!(new_len >= self.data.len(), "memory only grows"); - cold_err!(self.data.try_reserve(new_len - self.data.len())).map_err(|_| crate::Trap::OutOfMemory)?; - self.data.resize(new_len, 0); + debug_assert!(new_len >= self.0.len(), "memory only grows"); + cold_err!(self.0.try_reserve(new_len - self.0.len())).map_err(|_| crate::Trap::OutOfMemory)?; + self.0.resize(new_len, 0); Ok(()) } /// Reads exactly `N` bytes at `addr` into a fixed-size array. #[inline(always)] pub(crate) fn read_fixed(&self, addr: usize) -> Result<[u8; N], crate::Trap> { - if N > self.data.len() || addr > self.data.len() - N { - return cold!(Err(memory_oob(addr, N, self.data.len()))); + if N > self.0.len() || addr > self.0.len() - N { + return cold!(Err(memory_oob(addr, N, self.0.len()))); } let mut bytes = [0u8; N]; - bytes.copy_from_slice(&self.data[addr..addr + N]); + bytes.copy_from_slice(&self.0[addr..addr + N]); Ok(bytes) } /// Writes exactly `N` bytes from `bytes` at `addr`. #[inline(always)] pub(crate) fn write_fixed(&mut self, addr: usize, bytes: &[u8; N]) -> Result<(), crate::Trap> { - if N > self.data.len() || addr > self.data.len() - N { - return cold!(Err(memory_oob(addr, N, self.data.len()))); + if N > self.0.len() || addr > self.0.len() - N { + return cold!(Err(memory_oob(addr, N, self.0.len()))); } - self.data[addr..addr + N].copy_from_slice(bytes); + self.0[addr..addr + N].copy_from_slice(bytes); Ok(()) } /// Reads up to `dst.len()` bytes starting at `addr` and returns the number of bytes read. #[inline(always)] pub(crate) fn read(&self, addr: usize, dst: &mut [u8]) -> usize { - if addr >= self.data.len() { + if addr >= self.0.len() { return 0; } - let read_len = dst.len().min(self.data.len() - addr); - dst[..read_len].copy_from_slice(&self.data[addr..addr + read_len]); + let read_len = dst.len().min(self.0.len() - addr); + dst[..read_len].copy_from_slice(&self.0[addr..addr + read_len]); read_len } /// Writes up to `src.len()` bytes starting at `addr` and returns the number of bytes written. #[inline(always)] pub(crate) fn write(&mut self, addr: usize, src: &[u8]) -> usize { - if addr >= self.data.len() { + if addr >= self.0.len() { return 0; } - let write_len = src.len().min(self.data.len() - addr); - self.data[addr..addr + write_len].copy_from_slice(&src[..write_len]); + let write_len = src.len().min(self.0.len() - addr); + self.0[addr..addr + write_len].copy_from_slice(&src[..write_len]); write_len } /// Reads exactly `dst.len()` bytes starting at `addr`, returning `None` for an invalid range. #[inline(always)] pub(crate) fn read_exact(&self, addr: usize, dst: &mut [u8]) -> Option<()> { - dst.copy_from_slice(&self.data[self.checked_range(addr, dst.len())?]); + dst.copy_from_slice(&self.0[self.checked_range(addr, dst.len())?]); Some(()) } @@ -112,14 +120,14 @@ impl VecMemory { /// invalid range. #[inline(always)] pub(crate) fn read_vec(&self, addr: usize, len: usize) -> Option> { - Some(self.data[self.checked_range(addr, len)?].to_vec()) + Some(self.0[self.checked_range(addr, len)?].to_vec()) } /// Writes all of `src` at `addr`, returning `None` for an invalid range. #[inline(always)] pub(crate) fn write_all(&mut self, addr: usize, src: &[u8]) -> Option<()> { let range = self.checked_range(addr, src.len())?; - self.data[range].copy_from_slice(src); + self.0[range].copy_from_slice(src); Some(()) } @@ -127,7 +135,7 @@ impl VecMemory { #[inline(always)] pub(crate) fn fill(&mut self, addr: usize, len: usize, val: u8) -> Option<()> { let range = self.checked_range(addr, len)?; - self.data[range].fill(val); + self.0[range].fill(val); Some(()) } @@ -137,7 +145,7 @@ impl VecMemory { pub(crate) fn copy_within(&mut self, dst: usize, src: usize, len: usize) -> Option<()> { let src = self.checked_range(src, len)?; self.checked_range(dst, len)?; - self.data.copy_within(src, dst); + self.0.copy_within(src, dst); Some(()) } @@ -153,7 +161,7 @@ impl VecMemory { let src_range = cold_err!(src_memory.checked_range(src, len).ok_or_else(|| memory_oob(src, len, src_memory.len())))?; let dst_range = cold_err!(self.checked_range(dst, len).ok_or_else(|| memory_oob(dst, len, self.len())))?; - self.data[dst_range].copy_from_slice(&src_memory.data[src_range]); + self.0[dst_range].copy_from_slice(&src_memory.0[src_range]); Ok(()) } } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index b065f2a5..e89650a2 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -24,10 +24,16 @@ mod table; mod tag; mod types; +// The high bit selects shared storage. MemAddr::MAX is reserved for modules without a first memory. +pub(crate) const SHARED_MEM_BIT: MemAddr = 1 << 31; + use const_expr::eval_const; pub(crate) use gc::{GcObjectKind, data_range, decode_data, default_value, pop_value, push_value}; pub(crate) use memory::{MemValue, MemoryInstance}; +#[cfg(feature = "std")] +pub use memory::{MemoryShared, MemorySharedGuard}; pub(crate) use state::State; +pub(crate) use state::with_memory; pub(crate) use types::{canonicalize_ref_type, canonicalize_value_type}; pub(crate) use {data::*, element::*, function::*, global::*, table::*, tag::*}; @@ -626,12 +632,33 @@ impl Store { memories: &[MemoryType], init: impl Fn(MemoryType) -> Result, ) -> Result> { - let start = self.state.memories.len() as MemAddr; - self.state.memories.reserve_exact(memories.len()); - for mem in memories { - self.state.memories.push(cold_err!(init(*mem))?); + let mut addresses = Vec::with_capacity(memories.len()); + for &ty in memories { + let instance = cold_err!(init(ty))?; + if ty.shared() { + #[cfg(feature = "std")] + { + let index = MemAddr::try_from(self.state.shared_memories.len()) + .map_err(|_| Error::UnsupportedFeature("too many shared memories"))?; + if index >= SHARED_MEM_BIT - 1 { + return Err(Error::UnsupportedFeature("too many shared memories")); + } + self.state.shared_memories.push(MemoryShared::from_instance(instance)); + addresses.push(index | SHARED_MEM_BIT); + } + #[cfg(not(feature = "std"))] + unreachable!("shared memory instantiation requires std"); + } else { + let index = MemAddr::try_from(self.state.memories.len()) + .map_err(|_| Error::UnsupportedFeature("too many memories"))?; + if index >= SHARED_MEM_BIT { + return Err(Error::UnsupportedFeature("too many memories")); + } + self.state.memories.push(instance); + addresses.push(index); + } } - Ok(start..start + memories.len() as MemAddr) + Ok(addresses.into_iter()) } /// Add globals to the store, returning their addresses in the store @@ -793,19 +820,12 @@ impl Store { RuntimeValue::Value64(value) => value, other => return Err(Error::Other(format!("expected i32 or i64, got {other:?}"))), }; - let Some(mem) = self.state.memories.get_mut(*mem_addr as usize) else { - return Err(Error::Other(format!("memory {mem_addr} not found for data segment {i}"))); - }; - let offset = usize::try_from(offset).unwrap_or(usize::MAX); - match mem.inner.write_all(offset, &data.data) { - Some(()) => self.state.data[data_addrs[i] as usize].drop(), - None => { - return Err( - Trap::MemoryOutOfBounds { offset, len: data.data.len(), max: mem.inner.len() }.into() - ); - } - } + with_memory!(self.state, *mem_addr, |mem, kind| { + mem.write_all(offset, &data.data) + .ok_or_else(|| memory::memory_oob(offset, data.data.len(), mem.len()))?; + }); + self.state.data[data_addrs[i] as usize].drop(); } tinywasm_types::DataKind::Passive => {} }; diff --git a/crates/tinywasm/src/store/state.rs b/crates/tinywasm/src/store/state.rs index 0a72fc0d..3d2e682e 100644 --- a/crates/tinywasm/src/store/state.rs +++ b/crates/tinywasm/src/store/state.rs @@ -3,6 +3,7 @@ use alloc::vec::Vec; use super::*; use crate::engine::Config; use crate::interpreter::{InternalValue, Value32, Value64, Value128}; +use crate::store::memory::memory_oob; /// Global state that can be manipulated by WebAssembly programs /// @@ -15,6 +16,8 @@ pub(crate) struct State { pub(crate) funcs: Functions, pub(crate) tables: Vec, pub(crate) memories: Vec, + #[cfg(feature = "std")] + pub(crate) shared_memories: Vec, pub(crate) globals: Globals, pub(crate) tags: Vec, pub(crate) elements: Vec, @@ -23,7 +26,117 @@ pub(crate) struct State { pub(crate) roots: gc::Roots, } +// Select ordinary bytes or a locked shared backing, then run the operation once. +macro_rules! with_memory { + ($state:expr, $addr:expr, |$memory:ident, $kind:ident| $body:block) => {{ + let state = &mut $state; + let addr = $addr; + #[cfg(feature = "std")] + let mut guard; + #[cfg(feature = "std")] + let (kind, bytes) = if addr & $crate::store::SHARED_MEM_BIT != 0 { + core::hint::cold_path(); + guard = state.shared_memories[(addr & !$crate::store::SHARED_MEM_BIT) as usize].lock(); + (guard.kind, &mut *guard.inner) + } else { + let ordinary = &mut state.memories[addr as usize]; + (ordinary.kind, &mut ordinary.inner) + }; + #[cfg(not(feature = "std"))] + let (kind, bytes) = { + let ordinary = state.get_mem_mut(addr); + (ordinary.kind, &mut ordinary.inner) + }; + #[allow(unused_variables)] + let $kind = kind; + let $memory = bytes; + $body + }}; +} +pub(crate) use with_memory; + impl State { + /// Returns the immutable memory type without taking a shared-memory lock. + pub(crate) fn memory_type(&self, addr: MemAddr) -> MemoryType { + #[cfg(feature = "std")] + if addr & SHARED_MEM_BIT != 0 { + return self.shared_memories[(addr & !SHARED_MEM_BIT) as usize].ty(); + } + self.get_mem(addr).kind + } + + /// Returns the memory architecture and published page count without locking. + pub(crate) fn memory_size(&self, addr: MemAddr) -> (MemoryArch, usize) { + #[cfg(feature = "std")] + if addr & SHARED_MEM_BIT != 0 { + let memory = &self.shared_memories[(addr & !SHARED_MEM_BIT) as usize]; + return (memory.ty().arch(), memory.page_count()); + } + let memory = self.get_mem(addr); + (memory.kind.arch(), memory.page_count) + } + + /// Grows ordinary or shared memory without holding a shared lock during limiter callbacks. + pub(crate) fn grow_mem( + &mut self, + addr: MemAddr, + pages: i64, + limiter: Option<&dyn ResourceLimiter>, + ) -> core::result::Result, Trap> { + #[cfg(feature = "std")] + if addr & SHARED_MEM_BIT != 0 { + return self.shared_memories[(addr & !SHARED_MEM_BIT) as usize].grow_with_limiter(pages, limiter); + } + self.get_mem_mut(addr).grow(pages, limiter) + } + + /// Copies between memories, locking at most one shared backing at a time. + pub(crate) fn copy_memories( + &mut self, + dst_addr: MemAddr, + dst: usize, + src_addr: MemAddr, + src: usize, + size: usize, + ) -> core::result::Result<(), Trap> { + if dst_addr != src_addr && (dst_addr | src_addr) & SHARED_MEM_BIT == 0 { + let (destination, source) = self.get_mems_mut(dst_addr, src_addr); + return destination.copy_from_memory(dst, source, src, size); + } + + let same_memory = dst_addr == src_addr; + #[cfg(feature = "std")] + let same_memory = same_memory + || (dst_addr & src_addr & SHARED_MEM_BIT != 0 + && self.shared_memories[(dst_addr & !SHARED_MEM_BIT) as usize] + .same_instance(&self.shared_memories[(src_addr & !SHARED_MEM_BIT) as usize])); + + if same_memory { + return with_memory!(*self, dst_addr, |memory, kind| { + memory.copy_within(dst, src, size).ok_or_else(|| memory_oob(dst, size, memory.len())) + }); + } + + // Growth cannot invalidate these ranges. Check both before changing the destination. + with_memory!(*self, src_addr, |source, kind| { + source.checked_range(src, size).ok_or_else(|| memory_oob(src, size, source.len()))?; + }); + with_memory!(*self, dst_addr, |destination, kind| { + destination.checked_range(dst, size).ok_or_else(|| memory_oob(dst, size, destination.len()))?; + }); + let mut bytes = [0u8; 4096]; + for offset in (0..size).step_by(bytes.len()) { + let chunk = bytes.len().min(size - offset); + with_memory!(*self, src_addr, |source, kind| { + source.read_exact(src + offset, &mut bytes[..chunk]).expect("source range checked before copy"); + }); + with_memory!(*self, dst_addr, |destination, kind| { + destination.write_all(dst + offset, &bytes[..chunk]).expect("destination range checked before copy"); + }); + } + Ok(()) + } + pub(crate) fn new(config: &Config) -> Self { Self { canonical_types: Vec::new(), @@ -31,6 +144,8 @@ impl State { funcs: Functions::default(), tables: Vec::new(), memories: Vec::new(), + #[cfg(feature = "std")] + shared_memories: Vec::new(), globals: Globals::default(), tags: Vec::new(), elements: Vec::new(), @@ -297,18 +412,21 @@ impl State { /// Get the memory at the actual index in the store #[inline] pub(crate) fn get_mem(&self, addr: MemAddr) -> &MemoryInstance { + debug_assert_eq!(addr & SHARED_MEM_BIT, 0); &self.memories[addr as usize] } /// Get the memory at the actual index in the store #[inline] pub(crate) fn get_mem_mut(&mut self, addr: MemAddr) -> &mut MemoryInstance { + debug_assert_eq!(addr & SHARED_MEM_BIT, 0); &mut self.memories[addr as usize] } /// Get the memory at the actual index in the store #[inline] pub(crate) fn get_mems_mut(&mut self, addr: MemAddr, addr2: MemAddr) -> (&mut MemoryInstance, &mut MemoryInstance) { + debug_assert_eq!((addr | addr2) & SHARED_MEM_BIT, 0); Self::get_disjoint_mut(&mut self.memories, addr, addr2, "memory") } diff --git a/crates/tinywasm/tests/generated/wasm-threads.csv b/crates/tinywasm/tests/generated/wasm-threads.csv index 454014e7..290307af 100644 --- a/crates/tinywasm/tests/generated/wasm-threads.csv +++ b/crates/tinywasm/tests/generated/wasm-threads.csv @@ -1,3 +1,3 @@ 0.9.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] 0.10.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] -0.11.0,609,10,[{"name":"atomic.wast","passed":292,"failed":5},{"name":"exports.wast","passed":88,"failed":0},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":82,"failed":0}] +0.11.0,614,5,[{"name":"atomic.wast","passed":292,"failed":5},{"name":"exports.wast","passed":88,"failed":0},{"name":"imports.wast","passed":152,"failed":0},{"name":"memory.wast","passed":82,"failed":0}] diff --git a/crates/tinywasm/tests/shared_memory.rs b/crates/tinywasm/tests/shared_memory.rs new file mode 100644 index 00000000..36ff10f4 --- /dev/null +++ b/crates/tinywasm/tests/shared_memory.rs @@ -0,0 +1,303 @@ +#![cfg(feature = "std")] + +use tinywasm::engine::Config; +use tinywasm::types::{MemoryArch, MemoryType, ModuleInner}; +use tinywasm::{Engine, Imports, MemoryShared, ModuleInstance, ResourceLimiter, Store, Trap}; + +type TestResult = Result<(), Box>; + +#[test] +fn shared_memory_is_visible_across_stores() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (import "host" "memory" (memory 1 2 shared)) + (export "memory" (memory 0)) + (func (export "add") (result i32) + i32.const 0 + i32.const 1 + i32.atomic.rmw.add) + (func (export "size") (result i32) memory.size) + (func (export "grow") (result i32) i32.const 1 memory.grow)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(2), None))?; + let mut imports = Imports::new(); + imports.define("host", "memory", memory.clone()); + + let mut first_store = Store::default(); + let first = ModuleInstance::instantiate(&mut first_store, &module, Some(&imports))?; + let mut second_store = Store::default(); + let second = ModuleInstance::instantiate(&mut second_store, &module, Some(&imports))?; + + memory.lock().data_mut()[..4].copy_from_slice(&41u32.to_le_bytes()); + assert_eq!(first.func::<(), i32>(&first_store, "add")?.call(&mut first_store, ())?, 41); + assert_eq!(second.func::<(), i32>(&second_store, "add")?.call(&mut second_store, ())?, 42); + assert_eq!(memory.read_vec(0, 4)?, 43u32.to_le_bytes()); + assert_eq!(first.memory_shared("memory")?.lock().data()[..4], 43u32.to_le_bytes()); + + assert_eq!(first.func::<(), i32>(&first_store, "grow")?.call(&mut first_store, ())?, 1); + assert_eq!(second.func::<(), i32>(&second_store, "size")?.call(&mut second_store, ())?, 2); + assert_eq!(memory.lock().page_count(), 2); + Ok(()) +} + +#[test] +fn concurrent_growth_publishes_page_count() -> TestResult { + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(3), None))?; + let barrier = std::sync::Barrier::new(3); + let mut sizes = std::thread::scope(|scope| { + let workers: Vec<_> = (0..2) + .map(|_| { + let memory = memory.clone(); + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + memory.grow(1).unwrap().unwrap() + }) + }) + .collect(); + barrier.wait(); + workers.into_iter().map(|worker| worker.join().unwrap()).collect::>() + }); + sizes.sort(); + assert_eq!(sizes, [1, 2]); + assert_eq!(memory.page_count(), 3); + assert_eq!(memory.len(), 3 * 65536); + Ok(()) +} + +#[test] +fn shared_import_type_must_match() -> TestResult { + let wasm = wat::parse_str("(module (import \"host\" \"memory\" (memory 1 2 shared)))")?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let ordinary = tinywasm::Memory::try_new(&mut store, MemoryType::new(MemoryArch::I32, 1, Some(2), None))?; + let mut imports = Imports::new(); + imports.define("host", "memory", ordinary); + assert!(ModuleInstance::instantiate(&mut store, &module, Some(&imports)).is_err()); + Ok(()) +} + +#[test] +fn shared_memory_requires_declared_maximum_even_when_unused() { + let ty = MemoryType::new(MemoryArch::I32, 1, None, None).with_shared(true); + let module = tinywasm::Module::from(ModuleInner { + memory_types: vec![ty].into_boxed_slice(), + skip_local_memory_allocation: true, + ..ModuleInner::default() + }); + assert!(ModuleInstance::instantiate(&mut Store::default(), &module, None).is_err()); +} + +#[test] +fn atomic_rmw_is_indivisible_across_stores() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (import "host" "memory" (memory 1 1 shared)) + (func (export "add") + i32.const 0 + i32.const 1 + i32.atomic.rmw.add + drop)) + "#, + )?; + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(1), None))?; + std::thread::scope(|scope| -> TestResult { + let mut workers = Vec::new(); + for _ in 0..4 { + let memory = memory.clone(); + let wasm = &wasm; + workers.push(scope.spawn(move || -> tinywasm::Result<()> { + let module = tinywasm::parse_bytes(wasm)?; + let mut imports = Imports::new(); + imports.define("host", "memory", memory); + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, Some(&imports))?; + let add = instance.func::<(), ()>(&store, "add")?; + for _ in 0..1000 { + add.call(&mut store, ())?; + } + Ok(()) + })); + } + for worker in workers { + worker.join().unwrap()?; + } + Ok(()) + })?; + assert_eq!(memory.read_vec(0, 4)?, 4000u32.to_le_bytes()); + Ok(()) +} + +#[test] +fn defined_shared_memory_supports_data_and_bulk_access() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (memory (export "memory") 1 2 shared) + (data (i32.const 0) "abc") + (func (export "copy") + i32.const 4 + i32.const 0 + i32.const 3 + memory.copy) + (func (export "read") (result i32) + i32.const 4 + i32.load8_u)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert!(instance.memory("memory").is_err()); + let memory = instance.memory_shared("memory")?; + assert!(matches!( + instance.exports().find(|(name, _)| *name == "memory"), + Some((_, tinywasm::ExternItem::MemoryShared(_))) + )); + assert_eq!(memory.read_vec(0, 3)?, b"abc"); + instance.func::<(), ()>(&store, "copy")?.call(&mut store, ())?; + assert_eq!(memory.read_vec(4, 3)?, b"abc"); + memory.lock().data_mut()[4] = b'z'; + assert_eq!(instance.func::<(), i32>(&store, "read")?.call(&mut store, ())?, i32::from(b'z')); + + let consumer = tinywasm::parse_bytes(&wat::parse_str( + r#" + (module + (import "producer" "memory" (memory 1 2 shared)) + (func (export "read") (result i32) + i32.const 4 + i32.load8_u)) + "#, + )?)?; + let mut imports = Imports::new(); + imports.link_module("producer", instance)?; + let linked = ModuleInstance::instantiate(&mut store, &consumer, Some(&imports))?; + assert_eq!(linked.func::<(), i32>(&store, "read")?.call(&mut store, ())?, i32::from(b'z')); + Ok(()) +} + +#[cfg(feature = "archive")] +#[test] +fn shared_memory_flag_survives_serialization() -> TestResult { + let wasm = wat::parse_str("(module (memory 1 2 shared))")?; + let module = tinywasm::parse_bytes(&wasm)?; + let roundtrip = tinywasm::Module::try_from_twasm(&module.serialize_twasm()?)?; + assert!(roundtrip.memory_types[0].shared()); + Ok(()) +} + +#[test] +fn copy_between_shared_memories_checks_ranges() -> TestResult { + let wasm = wat::parse_str( + r#" + (module + (memory (export "source") 1 1 shared) + (memory (export "destination") 1 1 shared) + (func (export "copy") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + memory.copy 1 0)) + "#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let source = instance.memory_shared("source")?; + let destination = instance.memory_shared("destination")?; + let bytes = vec![0xa5; 8192]; + source.copy_from_slice(0, &bytes)?; + let copy = instance.func::<(i32, i32, i32), ()>(&store, "copy")?; + copy.call(&mut store, (8, 0, bytes.len() as i32))?; + assert_eq!(destination.read_vec(8, bytes.len())?, bytes); + assert!(copy.call(&mut store, (65535, 0, 2)).is_err()); + assert_eq!(destination.read_vec(8, bytes.len())?, bytes); + Ok(()) +} + +#[test] +fn copy_between_ordinary_and_shared_memory() -> TestResult { + let module = tinywasm::parse_bytes(&wat::parse_str( + r#" + (module + (memory (export "ordinary") 1) + (memory (export "shared") 1 1 shared) + (func (export "to_shared") + i32.const 4 + i32.const 0 + i32.const 3 + memory.copy 1 0) + (func (export "to_ordinary") + i32.const 8 + i32.const 4 + i32.const 3 + memory.copy 0 1)) + "#, + )?)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let ordinary = instance.memory("ordinary")?; + let shared = instance.memory_shared("shared")?; + + ordinary.copy_from_slice(&mut store, 0, b"abc")?; + instance.func::<(), ()>(&store, "to_shared")?.call(&mut store, ())?; + assert_eq!(shared.read_vec(4, 3)?, b"abc"); + + shared.copy_from_slice(4, b"xyz")?; + instance.func::<(), ()>(&store, "to_ordinary")?.call(&mut store, ())?; + assert_eq!(ordinary.read_vec(&store, 8, 3)?, b"xyz"); + Ok(()) +} + +#[test] +fn copy_between_aliases_of_one_shared_memory() -> TestResult { + let module = tinywasm::parse_bytes(&wat::parse_str( + r#" + (module + (import "host" "source" (memory 1 1 shared)) + (import "host" "destination" (memory 1 1 shared)) + (func (export "copy") + i32.const 1 + i32.const 0 + i32.const 3 + memory.copy 1 0)) + "#, + )?)?; + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(1), None))?; + memory.copy_from_slice(0, b"abc")?; + let mut imports = Imports::new(); + imports.define("host", "source", memory.clone()).define("host", "destination", memory.clone()); + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, Some(&imports))?; + instance.func::<(), ()>(&store, "copy")?.call(&mut store, ())?; + assert_eq!(memory.read_vec(0, 4)?, b"aabc"); + Ok(()) +} + +struct DenyGrowth; + +impl ResourceLimiter for DenyGrowth { + fn memory_growing(&self, current: usize, _desired: usize, _maximum: Option) -> Result { + Ok(current == 0) + } +} + +#[test] +fn shared_memory_growth_respects_store_limiter() -> TestResult { + let module = tinywasm::parse_bytes(&wat::parse_str( + r#" + (module + (memory (export "memory") 1 2 shared) + (func (export "grow") (result i32) i32.const 1 memory.grow)) + "#, + )?)?; + let mut store = Store::new(Engine::new(Config::new().with_resource_limiter(DenyGrowth))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert_eq!(instance.func::<(), i32>(&store, "grow")?.call(&mut store, ())?, -1); + assert_eq!(instance.memory_shared("memory")?.lock().page_count(), 1); + Ok(()) +} diff --git a/crates/types/src/archive.rs b/crates/types/src/archive.rs index 6a036056..bdce379d 100644 --- a/crates/types/src/archive.rs +++ b/crates/types/src/archive.rs @@ -7,7 +7,7 @@ use crate::Module; #[rustfmt::skip] const TWASM_MAGIC: [u8; 16] = [ TWASM_MAGIC_PREFIX[0], TWASM_MAGIC_PREFIX[1], TWASM_MAGIC_PREFIX[2], TWASM_MAGIC_PREFIX[3], TWASM_VERSION[0], TWASM_VERSION[1], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; const TWASM_MAGIC_PREFIX: &[u8; 4] = b"TWAS"; -const TWASM_VERSION: &[u8; 2] = b"05"; +const TWASM_VERSION: &[u8; 2] = b"06"; fn validate_magic(wasm: &[u8]) -> Result { if wasm.len() < TWASM_MAGIC.len() || &wasm[..TWASM_MAGIC_PREFIX.len()] != TWASM_MAGIC_PREFIX { @@ -97,7 +97,7 @@ mod tests { let module = Module::from(ModuleInner { funcs: Box::new([Shared::new(function)]), ..ModuleInner::default() }); let archive = module.serialize_twasm().expect("serialize archive"); - assert_eq!(&archive[..6], b"TWAS05"); + assert_eq!(&archive[..6], b"TWAS06"); let decoded = Module::try_from_twasm(&archive).expect("deserialize archive"); let function = &decoded.funcs[0]; diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index a3f26f23..b3474237 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -617,6 +617,7 @@ pub struct MemoryType { page_count_initial: u64, page_count_max: Option, page_size: Option, + shared: bool, } impl MemoryType { @@ -627,7 +628,18 @@ impl MemoryType { page_count_max: Option, page_size: Option, ) -> Self { - Self { arch, page_count_initial, page_count_max, page_size } + Self { arch, page_count_initial, page_count_max, page_size, shared: false } + } + + /// Whether this memory is shared. + pub const fn shared(&self) -> bool { + self.shared + } + + /// Sets whether this memory is shared. + pub const fn with_shared(mut self, shared: bool) -> Self { + self.shared = shared; + self } #[inline] diff --git a/examples/threads.rs b/examples/threads.rs new file mode 100644 index 00000000..37ac7d6d --- /dev/null +++ b/examples/threads.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use tinywasm::types::{MemoryArch, MemoryType}; +use tinywasm::{Imports, MemoryShared, ModuleInstance, Store}; + +const WASM: &str = r#" +(module + (import "host" "memory" (memory 1 2 shared)) + (func (export "increment") + i32.const 0 + i32.const 1 + i32.atomic.rmw.add + drop)) +"#; + +fn main() -> Result<()> { + let wasm = wat::parse_str(WASM)?; + let module = tinywasm::parse_bytes(&wasm)?; + + // Prepare one shared memory and import set for all workers. + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(2), None))?; + let mut imports = Imports::new(); + imports.define("host", "memory", memory.clone()); + + std::thread::scope(|scope| -> Result<()> { + let mut workers = Vec::new(); + for _ in 0..4 { + let module = &module; + let imports = &imports; + workers.push(scope.spawn(move || -> Result<()> { + // Each worker has its own store and module instance, but imports the same bytes. + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, module, Some(imports))?; + let increment = instance.func::<(), ()>(&store, "increment")?; + for _ in 0..1000 { + increment.call(&mut store, ())?; + } + Ok(()) + })); + } + for worker in workers { + worker.join().expect("worker thread panicked")?; + } + Ok(()) + })?; + + // Atomic read-modify-write operations serialize updates to the shared memory. + assert_eq!(memory.read_vec(0, 4)?, 4000u32.to_le_bytes()); + assert_eq!(memory.grow(1)?, Some(1)); + assert_eq!(memory.page_count(), 2); + Ok(()) +} From 455344baba0b047f9d2cd6a27f797d1acc0d4ed8 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 18:04:26 +0200 Subject: [PATCH 03/10] feat: add support for atomic waiters / notify Signed-off-by: Henry --- CHANGELOG.md | 1 + README.md | 4 +- crates/parser/src/visit.rs | 33 ++++-- crates/tinywasm/Cargo.toml | 1 - .../src/interpreter/executor/instructions.rs | 1 + .../tinywasm/src/interpreter/executor/mod.rs | 111 ++++++++++++------ crates/tinywasm/src/store/memory/shared.rs | 88 +++++++++++++- crates/tinywasm/tests/atomic_wait.rs | 31 +++++ .../tinywasm/tests/generated/wasm-threads.csv | 2 +- crates/tinywasm/tests/shared_memory.rs | 84 +++++++++++++ crates/types/src/instructions.rs | 11 ++ 11 files changed, 312 insertions(+), 55 deletions(-) create mode 100644 crates/tinywasm/tests/atomic_wait.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 253f5a9b..1dab0c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added WebAssembly threads support, including atomic wait/notify and cross-store shared memory with `MemoryShared`. - Added new `Memory::data` and `Memory::data_mut` methods for direct access to linear memory. - Added several internal methods to support a future c-api crate diff --git a/README.md b/README.md index 8e847019..5fb864c8 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ See the [examples](./examples) directory and [documentation](https://docs.rs/tin ## Cargo Features - **`full`:** Enables `archive`, `debug`, `parallel-parser`, `parser`, `state`, and `validate`. Enabled by default. -- **`std`:** Enables `std` and parsing from files and streams. Enabled by default. +- **`std`:** Enables parsing from files and streams, shared memory handles, and blocking atomic wait/notify across stores. Enabled by default. - **`parser`:** Enables `tinywasm-parser` and top-level parse helpers. Enabled by default. - **`validate`:** Enables WebAssembly validation while parsing. Enabled by default and configurable through [`ParserOptions`](https://docs.rs/tinywasm/latest/tinywasm/parser/struct.ParserOptions.html). - **`parallel-parser`:** Parallelizes function parsing when `std` is enabled. Enabled by default. @@ -104,8 +104,8 @@ TinyWasm generally implements non-JavaScript core proposals at [phase 4](https:/ | [**Garbage Collection**](https://github.com/WebAssembly/gc) | 🟢 | 0.11.0 | | [**Exception Handling**](https://github.com/WebAssembly/exception-handling) | 🟢 | 0.11.0 | | [**Compact Import Section**](https://github.com/WebAssembly/compact-import-section) | 🟢 | 0.11.0 | +| [**Threads**](https://github.com/WebAssembly/threads) | 🚧 | next | | [**Stack Switching**](https://github.com/WebAssembly/stack-switching) | 🌑 | - | -| [**Threads**](https://github.com/WebAssembly/threads) | 🌑 | - | **Legend**\ 🌑 -- not available\ diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index 844eda1c..d5cb9f02 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -11,9 +11,9 @@ use alloc::{ vec::Vec, }; use tinywasm_types::{ - AtomicArg, AtomicOp, AtomicWidth, BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, ImportKind, - Instruction, MemoryType, Operand64, Operand64Idx, Operand128, Operand128Idx, StorageType, TableDefinition, TagType, - TypeSection, ValueCounts, ValueLane, WasmFunctionData, + AtomicArg, AtomicOp, AtomicWaitOp, AtomicWidth, BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, + ImportKind, Instruction, MemoryType, Operand64, Operand64Idx, Operand128, Operand128Idx, StorageType, + TableDefinition, TagType, TypeSection, ValueCounts, ValueLane, WasmFunctionData, }; use wasmparser::{FunctionBody, OperatorsReader, OperatorsReaderAllocations, VisitSimdOperator}; @@ -231,6 +231,21 @@ impl<'a> FunctionBuilder<'a> { self.uses_local_memory |= memory >= self.metadata.imported_memories; } + fn atomic_wait(&mut self, memarg: wasmparser::MemArg, width: u8, op: AtomicWaitOp) -> Result<()> { + if memarg.align != width.trailing_zeros() as u8 { + return Err(crate::ParseError::Other("invalid atomic alignment".into())); + } + let address = self.metadata.memory_size(memarg.memory)?; + self.mark_memory(memarg.memory); + let memory = self.push128(Operand128::::new(memarg.offset, memarg.memory))?; + let inputs: &[ValueLane] = match op { + AtomicWaitOp::Notify => &[address, ValueLane::S32], + AtomicWaitOp::Wait32 => &[address, ValueLane::S32, ValueLane::S64], + AtomicWaitOp::Wait64 => &[address, ValueLane::S64, ValueLane::S64], + }; + self.emit(inputs, &[ValueLane::S32], Instruction::AtomicWait(memory, op)) + } + fn visit_struct_get_impl( &mut self, type_index: u32, @@ -539,14 +554,14 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { self.emit(&[], &[], Instruction::AtomicFence) } - fn visit_memory_atomic_notify(&mut self, _: wasmparser::MemArg) -> Self::Output { - Err(crate::ParseError::UnsupportedOperator("memory.atomic.notify".into())) + fn visit_memory_atomic_notify(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.atomic_wait(memarg, 4, AtomicWaitOp::Notify) } - fn visit_memory_atomic_wait32(&mut self, _: wasmparser::MemArg) -> Self::Output { - Err(crate::ParseError::UnsupportedOperator("memory.atomic.wait32".into())) + fn visit_memory_atomic_wait32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.atomic_wait(memarg, 4, AtomicWaitOp::Wait32) } - fn visit_memory_atomic_wait64(&mut self, _: wasmparser::MemArg) -> Self::Output { - Err(crate::ParseError::UnsupportedOperator("memory.atomic.wait64".into())) + fn visit_memory_atomic_wait64(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + self.atomic_wait(memarg, 8, AtomicWaitOp::Wait64) } atomic_visitors! { diff --git a/crates/tinywasm/Cargo.toml b/crates/tinywasm/Cargo.toml index 4f354666..7788770f 100644 --- a/crates/tinywasm/Cargo.toml +++ b/crates/tinywasm/Cargo.toml @@ -42,7 +42,6 @@ name = "test-wasm-multi-memory" [[test]] harness = false name = "test-wasm-threads" -test = false [[test]] harness = false diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index c139af39..57eef102 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -324,6 +324,7 @@ macro_rules! instruction_handlers { MemorySize(addr) => executor.exec_memory_size(*addr)?, MemoryGrow(addr) => executor.exec_memory_grow(*addr)?, Atomic(arg) => executor.exec_atomic(*arg)?, + AtomicWait(memory, op) => executor.exec_atomic_wait(*memory, *op)?, AtomicFence => crate::shared::fence(crate::shared::Ordering::SeqCst), // Bulk memory operations diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 8a562164..963df859 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -561,10 +561,7 @@ impl<'store> Executor<'store> { fn throw_exception(&mut self, exception: ValueRef, protected_ip: usize) -> ExecResult { match self.dispatch_exception(exception, protected_ip)? { Some(landing_pad) => Ok(ExecFlow::next(landing_pad)), - None => match self.store.root_exception(exception) { - Ok(exception) => Err(Error::Exception(exception).into()), - Err(error) => Err(error.into()), - }, + None => Err(Error::Exception(self.store.root_exception(exception)?).into()), } } @@ -734,6 +731,7 @@ impl<'store> Executor<'store> { let type_addr = operand.a(); let table_addr = operand.b(); self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + // verify that the table is of the right type, this should be validated by the parser already let table_addr = self.module.resolve_table_addr(table_addr); let table_idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; @@ -1100,14 +1098,13 @@ impl<'store> Executor<'store> { let type_index = operand.a(); let len = operand.b(); let type_addr = self.module.resolve_type_addr(type_index); - let storage = - self.store.state.get_type(type_addr).as_array().expect("validated array.new_fixed type").field.storage; + let arr_type = self.store.state.get_type(type_addr).as_array().expect("validated array.new_fixed type"); let len = len as usize; self.store.state.gc.check_allocation(len, self.store.state.gc_type_has_references(type_addr))?; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; for _ in 0..len { - values.push(pop_value(&mut self.store.value_stack, storage)); + values.push(pop_value(&mut self.store.value_stack, arr_type.field.storage)); } values.reverse(); self.push_gc_object(type_addr, values) @@ -1257,8 +1254,8 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object_len = self.store.state.gc.get_handle(object).expect("validated array").values.len(); dst.checked_add(len).filter(|end| *end <= object_len).ok_or(Trap::ArrayOutOfBounds)?; - let items = - self.store.state.elements[self.module.resolve_elem_addr(elem_index) as usize].items_range(src, len)?; + let elem_addr = self.module.resolve_elem_addr(elem_index); + let items = self.store.state.elements[elem_addr as usize].items_range(src, len)?; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; values.extend(items.iter().copied().map(RuntimeValue::ValueRef)); @@ -1267,11 +1264,9 @@ impl<'store> Executor<'store> { } fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { - let mem_addr = self.mem_addr(addr); - let (arch, pages) = self.store.state.memory_size(mem_addr); - match arch { - MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, pages as i64), - MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, pages as i32), + match self.store.state.memory_size(self.mem_addr(addr)) { + (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), + (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), } } @@ -1441,24 +1436,65 @@ impl<'store> Executor<'store> { } } + fn exec_atomic_wait(&mut self, arg: Operand128Idx, op: AtomicWaitOp) -> Result<(), Trap> { + let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.store.value_stack) }; + let value = match op { + AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.store.value_stack), + _ => u32::stack_pop(&mut self.store.value_stack) as u64, + }; + let memory = arg.resolve(&self.func.data); + let mem_addr = self.mem_addr(memory.memory()); + let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; + let addr = crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; + if addr % width != 0 { + return cold!(Err(Trap::UnalignedAtomic)); + } + + if op != AtomicWaitOp::Notify && !kind.shared() { + return cold!(Err(Trap::Other("atomic wait requires shared memory"))); + } + + // Check the entire access even for notify and even when no waiter exists. + if mem.checked_range(addr, width).is_none() { + return cold!(Err(Trap::MemoryOutOfBounds { offset: addr, len: width, max: mem.len() })); + } + + Ok(addr) + })?; + + #[cfg(not(feature = "std"))] + let _ = (timeout, value, addr); + + #[cfg(feature = "std")] + if mem_addr & crate::store::SHARED_MEM_BIT != 0 { + let shared = &self.store.state.shared_memories[(mem_addr & !crate::store::SHARED_MEM_BIT) as usize]; + let result = match op { + AtomicWaitOp::Notify => shared.notify(addr, value as u32), + AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, + AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, + }; + return u32::stack_push(&mut self.store.value_stack, result); + } + + u32::stack_push(&mut self.store.value_stack, 0) + } + fn exec_atomic_width(&mut self, arg: AtomicArg) -> Result<(), Trap> { let op = arg.op(); - let value = if op == AtomicOp::Load { - 0 - } else if arg.is_64() { - u64::stack_pop(&mut self.store.value_stack) - } else { - u32::stack_pop(&mut self.store.value_stack) as u64 + let value = match op { + AtomicOp::Load => 0, + _ if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), + _ => u32::stack_pop(&mut self.store.value_stack) as u64, }; - let expected = if op == AtomicOp::Cmpxchg { - if arg.is_64() { - u64::stack_pop(&mut self.store.value_stack) - } else { - u32::stack_pop(&mut self.store.value_stack) as u64 - } - } else { - 0 + + let expected = match op { + AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), + AtomicOp::Cmpxchg => u32::stack_pop(&mut self.store.value_stack) as u64, + _ => 0, }; + let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { @@ -1493,10 +1529,10 @@ impl<'store> Executor<'store> { bytes.copy_from_slice(&next.to_le_bytes()[..N]); cold_err!(mem.write_fixed::(addr, &bytes))?; } - if arg.is_64() { - u64::stack_push(&mut self.store.value_stack, old)?; - } else { - u32::stack_push(&mut self.store.value_stack, old as u32)?; + + match arg.is_64() { + true => u64::stack_push(&mut self.store.value_stack, old)?, + false => u32::stack_push(&mut self.store.value_stack, old as u32)?, } Ok(()) }) @@ -1580,19 +1616,16 @@ impl<'store> Executor<'store> { fn exec_table_init(&mut self, index: Operand64Idx<(u32, u32)>) -> Result<(), Trap> { let operand = index.resolve(&self.func.data); - let elem_index = operand.a(); - let table_index = operand.b(); let size = self.pop_table_operand(MemoryArch::I32)?; // n let offset = self.pop_table_operand(MemoryArch::I32)?; // s - let table_addr = self.module.resolve_table_addr(table_index); + let table_addr = self.module.resolve_table_addr(operand.b()); let dst = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; // d - let elem_addr = self.module.resolve_elem_addr(elem_index) as usize; + let elem_addr = self.module.resolve_elem_addr(operand.a()) as usize; let elem = self.store.state.elements.get(elem_addr).ok_or_else(|| Trap::Other("element not found"))?; let items = elem.items_range(offset, size)?; - let table = - self.store.state.tables.get_mut(table_addr as usize).ok_or_else(|| Trap::Other("table not found"))?; - table.init(dst, items) + let table = self.store.state.tables.get_mut(table_addr as usize); + table.ok_or_else(|| Trap::Other("table not found"))?.init(dst, items) } fn exec_table_grow(&mut self, table_index: u32) -> Result<(), Trap> { diff --git a/crates/tinywasm/src/store/memory/shared.rs b/crates/tinywasm/src/store/memory/shared.rs index c92ab045..d8c6635b 100644 --- a/crates/tinywasm/src/store/memory/shared.rs +++ b/crates/tinywasm/src/store/memory/shared.rs @@ -1,5 +1,10 @@ -use crate::std::sync::{Mutex, MutexGuard}; -use alloc::{sync::Arc, vec::Vec}; +use crate::std::sync::{Condvar, Mutex, MutexGuard}; +use crate::std::time::Duration; +use alloc::{ + collections::{BTreeMap, VecDeque}, + sync::Arc, + vec::Vec, +}; use core::sync::atomic::{AtomicUsize, Ordering}; use tinywasm_types::MemoryType; @@ -15,6 +20,13 @@ struct MemorySharedInstance { kind: MemoryType, pages: AtomicUsize, bytes: Mutex, + // Lock order: bytes, waiters, then an individual waiter's notified flag. + waiters: Mutex>>>, +} + +struct Waiter { + notified: Mutex, + wake: Condvar, } #[cfg(feature = "debug")] @@ -35,7 +47,7 @@ impl MemoryShared { let kind = instance.kind; let pages = AtomicUsize::new(instance.page_count); let bytes = Mutex::new(instance.inner); - Self(Arc::new(MemorySharedInstance { kind, pages, bytes })) + Self(Arc::new(MemorySharedInstance { kind, pages, bytes, waiters: Mutex::new(BTreeMap::new()) })) } /// Whether two handles reference the same instance, even across store slots. @@ -115,6 +127,76 @@ impl MemoryShared { guard.inner.fill(offset, len, val).ok_or_else(|| memory_oob(offset, len, guard.inner.len()).into()) } + /// Waits until notified or the signed nanosecond timeout expires. + pub(crate) fn wait( + &self, + addr: usize, + expected: u64, + timeout: i64, + ) -> core::result::Result { + let guard = self.lock(); + let bytes = guard.inner.read_fixed::(addr)?; + let mut value = [0u8; 8]; + value[..N].copy_from_slice(&bytes); + if u64::from_le_bytes(value) != expected { + return Ok(1); + } + if timeout == 0 { + return Ok(2); + } + let waiter = Arc::new(Waiter { notified: Mutex::new(false), wake: Condvar::new() }); + self.0 + .waiters + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .entry(addr) + .or_default() + .push_back(waiter.clone()); + drop(guard); + + let notified = waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()); + if timeout < 0 { + drop(waiter.wake.wait_while(notified, |ready| !*ready).unwrap_or_else(|poison| poison.into_inner())); + } else { + drop( + waiter + .wake + .wait_timeout_while(notified, Duration::from_nanos(timeout as u64), |ready| !*ready) + .unwrap_or_else(|poison| poison.into_inner()) + .0, + ); + } + // The condvar guard was dropped before taking waiters, preserving lock order. + let mut waiters = self.0.waiters.lock().unwrap_or_else(|poison| poison.into_inner()); + let notified = *waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()); + if let Some(queue) = waiters.get_mut(&addr) { + queue.retain(|entry| !Arc::ptr_eq(entry, &waiter)); + if queue.is_empty() { + waiters.remove(&addr); + } + } + Ok(if notified { 0 } else { 2 }) + } + + /// Wakes at most `count` waiters at the given byte address. + pub(crate) fn notify(&self, addr: usize, count: u32) -> u32 { + let _guard = self.lock(); + let mut waiters = self.0.waiters.lock().unwrap_or_else(|poison| poison.into_inner()); + let mut woken = 0; + if let Some(queue) = waiters.get_mut(&addr) { + while woken < count { + let Some(waiter) = queue.pop_front() else { break }; + *waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()) = true; + waiter.wake.notify_one(); + woken += 1; + } + if queue.is_empty() { + waiters.remove(&addr); + } + } + woken + } + /// Grows the memory by the given number of pages, returning its previous size. pub fn grow(&self, pages: i64) -> Result> { Ok(self.grow_with_limiter(pages, None)?) diff --git a/crates/tinywasm/tests/atomic_wait.rs b/crates/tinywasm/tests/atomic_wait.rs new file mode 100644 index 00000000..22f8a972 --- /dev/null +++ b/crates/tinywasm/tests/atomic_wait.rs @@ -0,0 +1,31 @@ +#![cfg(feature = "parser")] + +use tinywasm::{Error, ModuleInstance, Store, Trap}; + +#[test] +fn ordinary_memory_atomics_and_wait() -> Result<(), Box> { + let wasm = wat::parse_str( + r#"(module + (memory 1) + (func (export "add") (result i32) + i32.const 0 i32.const 1 i32.atomic.rmw.add) + (func (export "notify") (result i32) + i32.const 0 i32.const 1 memory.atomic.notify) + (func (export "wait") (param i32) (result i32) + local.get 0 i32.const 0 i64.const 0 memory.atomic.wait32))"#, + )?; + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert_eq!(instance.func::<(), i32>(&store, "add")?.call(&mut store, ())?, 0); + assert_eq!(instance.func::<(), i32>(&store, "add")?.call(&mut store, ())?, 1); + assert_eq!(instance.func::<(), i32>(&store, "notify")?.call(&mut store, ())?, 0); + let wait = instance.func::(&store, "wait")?; + assert!(matches!(wait.call(&mut store, 0), Err(Error::Trap(Trap::Other("atomic wait requires shared memory"))))); + assert!(matches!( + wait.call(&mut store, 65536), + Err(Error::Trap(Trap::Other("atomic wait requires shared memory"))) + )); + assert!(matches!(wait.call(&mut store, 1), Err(Error::Trap(Trap::UnalignedAtomic)))); + Ok(()) +} diff --git a/crates/tinywasm/tests/generated/wasm-threads.csv b/crates/tinywasm/tests/generated/wasm-threads.csv index 290307af..f7513747 100644 --- a/crates/tinywasm/tests/generated/wasm-threads.csv +++ b/crates/tinywasm/tests/generated/wasm-threads.csv @@ -1,3 +1,3 @@ 0.9.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] 0.10.0,357,262,[{"name":"atomic.wast","passed":48,"failed":249},{"name":"exports.wast","passed":82,"failed":6},{"name":"imports.wast","passed":147,"failed":5},{"name":"memory.wast","passed":80,"failed":2}] -0.11.0,614,5,[{"name":"atomic.wast","passed":292,"failed":5},{"name":"exports.wast","passed":88,"failed":0},{"name":"imports.wast","passed":152,"failed":0},{"name":"memory.wast","passed":82,"failed":0}] +0.11.0,619,0,[{"name":"atomic.wast","passed":297,"failed":0},{"name":"exports.wast","passed":88,"failed":0},{"name":"imports.wast","passed":152,"failed":0},{"name":"memory.wast","passed":82,"failed":0}] diff --git a/crates/tinywasm/tests/shared_memory.rs b/crates/tinywasm/tests/shared_memory.rs index 36ff10f4..5a5e3d74 100644 --- a/crates/tinywasm/tests/shared_memory.rs +++ b/crates/tinywasm/tests/shared_memory.rs @@ -132,6 +132,90 @@ fn atomic_rmw_is_indivisible_across_stores() -> TestResult { Ok(()) } +#[test] +fn wait_notify_across_stores() -> TestResult { + let wasm = wat::parse_str( + r#"(module + (import "host" "memory" (memory 1 1 shared)) + (func (export "wait32") (param i32 i32 i64) (result i32) + local.get 0 local.get 1 local.get 2 memory.atomic.wait32) + (func (export "wait64") (param i32 i64 i64) (result i32) + local.get 0 local.get 1 local.get 2 memory.atomic.wait64) + (func (export "notify") (param i32 i32) (result i32) + local.get 0 local.get 1 memory.atomic.notify))"#, + )?; + let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(1), None))?; + std::thread::scope(|scope| -> TestResult { + let mut workers = Vec::new(); + for (address, width) in [(0, 4), (0, 4), (8, 8)] { + let worker_memory = memory.clone(); + let wasm = &wasm; + workers.push(scope.spawn(move || -> tinywasm::Result { + let module = tinywasm::parse_bytes(wasm)?; + let mut imports = Imports::new(); + imports.define("host", "memory", worker_memory); + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, Some(&imports))?; + if width == 4 { + instance + .func::<(i32, i32, i64), i32>(&store, "wait32")? + .call(&mut store, (address, 0, 10_000_000_000)) + } else { + instance + .func::<(i32, i64, i64), i32>(&store, "wait64")? + .call(&mut store, (address, 0, 10_000_000_000)) + } + })); + } + let module = tinywasm::parse_bytes(&wasm)?; + let mut imports = Imports::new(); + imports.define("host", "memory", memory.clone()); + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, Some(&imports))?; + let notify = instance.func::<(i32, i32), i32>(&store, "notify")?; + for address in [8, 0, 0] { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + if notify.call(&mut store, (address, 1))? == 1 { + break; + } + assert!(std::time::Instant::now() < deadline, "waiter did not register"); + std::thread::yield_now(); + } + } + for worker in workers { + assert_eq!(worker.join().unwrap()?, 0); + } + assert_eq!(notify.call(&mut store, (0, 1))?, 0); + assert_eq!(notify.call(&mut store, (8, 1))?, 0); + Ok(()) + }) +} + +#[test] +fn wait_checks_values_timeouts_and_addresses() -> TestResult { + let module = tinywasm::parse_bytes(&wat::parse_str( + r#"(module + (memory 1 1 shared) + (func (export "wait") (param i32 i64 i64) (result i32) + local.get 0 local.get 1 local.get 2 memory.atomic.wait64) + (func (export "notify") (param i32 i32) (result i32) + local.get 0 local.get 1 memory.atomic.notify))"#, + )?)?; + let mut store = Store::default(); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let wait = instance.func::<(i32, i64, i64), i32>(&store, "wait")?; + assert_eq!(wait.call(&mut store, (0, 1, -1))?, 1); + assert_eq!(wait.call(&mut store, (0, 0, 0))?, 2); + assert_eq!(wait.call(&mut store, (0, 0, 1_000_000))?, 2); + assert!(wait.call(&mut store, (1, 0, 0)).is_err()); + assert!(wait.call(&mut store, (65536, 0, 0)).is_err()); + let notify = instance.func::<(i32, i32), i32>(&store, "notify")?; + assert_eq!(notify.call(&mut store, (0, 1))?, 0); + assert!(notify.call(&mut store, (65536, 1)).is_err()); + Ok(()) +} + #[test] fn defined_shared_memory_supports_data_and_bulk_access() -> TestResult { let wasm = wat::parse_str( diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index e45a5aba..3d1f9dbf 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -61,6 +61,16 @@ pub struct AtomicArg { code: u8, } +/// A wait or notify instruction with its module-local memory operand. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +pub enum AtomicWaitOp { + Notify, + Wait32, + Wait64, +} + impl AtomicArg { /// Packs the operation, access width, and result type into one byte. pub const fn new(memory: Operand128Idx, width: AtomicWidth, is_64: bool, op: AtomicOp) -> Self { @@ -736,6 +746,7 @@ define_instructions! { MemorySize(MemAddr), MemoryGrow(MemAddr), Atomic(AtomicArg), + AtomicWait(Operand128Idx, AtomicWaitOp), AtomicFence, // > Constants From 17c5dc6a92aac339880c7345745ff8c88f6d194b Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 18:36:31 +0200 Subject: [PATCH 04/10] chore: cleanup, tweak api Signed-off-by: Henry --- crates/parser/src/visit.rs | 4 +- crates/tinywasm/src/imports.rs | 25 +- crates/tinywasm/src/instance.rs | 48 ++-- .../tinywasm/src/interpreter/executor/mod.rs | 63 ++--- crates/tinywasm/src/store/memory/instance.rs | 6 +- crates/tinywasm/src/store/memory/shared.rs | 248 ++++++++++-------- crates/tinywasm/src/store/state.rs | 7 +- crates/tinywasm/tests/import_linking.rs | 10 + crates/tinywasm/tests/shared_memory.rs | 54 ++-- crates/types/src/instructions.rs | 4 +- examples/threads.rs | 7 +- 11 files changed, 247 insertions(+), 229 deletions(-) diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index d5cb9f02..20c220dd 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -527,10 +527,12 @@ macro_rules! atomic_visitors { let address = self.metadata.memory_size(memarg.memory)?; self.mark_memory(memarg.memory); let memory = self.push128(Operand128::::new(memarg.offset, memarg.memory))?; + let is_64 = atomic_visitors!(@is64 [$($input),*] [$($output),*]); + let arg = AtomicArg::new(memory, AtomicWidth::from_bytes($width), is_64, AtomicOp::$op); self.emit( &[$(atomic_visitors!(@size $input, address)),*], &[$(atomic_visitors!(@size $output, address)),*], - Instruction::Atomic(AtomicArg::new(memory, AtomicWidth::from_bytes($width), atomic_visitors!(@is64 [$($input),*] [$($output),*]), AtomicOp::$op)), + Instruction::Atomic(arg), ) } }; diff --git a/crates/tinywasm/src/imports.rs b/crates/tinywasm/src/imports.rs index 870d76bb..a89e47d5 100644 --- a/crates/tinywasm/src/imports.rs +++ b/crates/tinywasm/src/imports.rs @@ -237,15 +237,14 @@ impl ResolvedImports { fn compare_memory_types( import: &Import, - expected: &MemoryType, actual: &MemoryType, + expected: &MemoryType, real_size: usize, ) -> Result<()> { Self::compare_types(import, &expected.arch(), &actual.arch())?; Self::compare_types(import, &expected.shared(), &actual.shared())?; - if actual.page_count_initial() > expected.page_count_initial() && actual.page_count_initial() > real_size as u64 - { + if (real_size as u64) < expected.page_count_initial() { return Err(LinkingError::incompatible_import_type(import).into()); } @@ -253,7 +252,9 @@ impl ResolvedImports { return Err(LinkingError::incompatible_import_type(import).into()); } - if expected.page_count_max() > actual.page_count_max() { + if let Some(max) = expected.page_count_max_declared() + && actual.page_count_max_declared().is_none_or(|actual_max| actual_max > max) + { return Err(LinkingError::incompatible_import_type(import).into()); } @@ -364,20 +365,8 @@ impl ResolvedImports { imports.tables.push(table_addr); } (ExternVal::Memory(memory_addr), ImportKind::Memory(ty)) => { - #[cfg(feature = "std")] - let (kind, pages) = if memory_addr & crate::store::SHARED_MEM_BIT != 0 { - let memory = - &store.state.shared_memories[(memory_addr & !crate::store::SHARED_MEM_BIT) as usize]; - (memory.ty(), memory.page_count()) - } else { - let memory = store.state.get_mem(memory_addr); - (memory.kind, memory.page_count) - }; - #[cfg(not(feature = "std"))] - let (kind, pages) = { - let memory = store.state.get_mem(memory_addr); - (memory.kind, memory.page_count) - }; + let kind = store.state.memory_type(memory_addr); + let (_, pages) = store.state.memory_size(memory_addr); Self::compare_memory_types(import, &kind, ty, pages)?; imports.memories.push(memory_addr); } diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index 9cce7b88..f92765dd 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -215,9 +215,9 @@ impl ModuleInstance { addrs.tags.extend(store.init_tags(&module.tags, &type_addrs)); let limiter = store.engine.config().resource_limiter.clone(); if !module.skip_local_memory_allocation { - addrs - .memories - .extend(store.init_memories(&module.memory_types, |ty| MemoryInstance::new(ty, limiter.as_deref()))?); + let memories = + store.init_memories(&module.memory_types, |ty| MemoryInstance::new(ty, limiter.as_deref()))?; + addrs.memories.extend(memories); } store.init_globals(&mut addrs.globals, &module.globals, &addrs.funcs, &type_addrs)?; @@ -316,35 +316,15 @@ impl ModuleInstance { /// ``` pub fn exports(&self) -> impl Iterator + '_ { self.0.exports.iter().map(move |export| { - let item = match export.kind { - ExternalKind::Func => ExternItem::Func(Function { - item: StoreItem::new(self.0.store_id, self.resolve_func_addr(export.index)), - module_id: self.id(), - }), - ExternalKind::Table => { - ExternItem::Table(Table(StoreItem::new(self.0.store_id, self.resolve_table_addr(export.index)))) - } - ExternalKind::Memory => { - let addr = self.resolve_mem_addr(export.index); - #[cfg(feature = "std")] - if addr & SHARED_MEM_BIT != 0 { - ExternItem::MemoryShared( - self.shared_backing(addr).expect("exported shared memory has a backing"), - ) - } else { - ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr))) - } - #[cfg(not(feature = "std"))] - ExternItem::Memory(Memory(StoreItem::new(self.0.store_id, addr))) - } - ExternalKind::Global => { - ExternItem::Global(Global(StoreItem::new(self.0.store_id, self.resolve_global_addr(export.index)))) - } - ExternalKind::Tag => { - ExternItem::Tag(Tag(StoreItem::new(self.0.store_id, self.resolve_tag_addr(export.index)))) - } + let addr = match export.kind { + ExternalKind::Func => self.resolve_func_addr(export.index), + ExternalKind::Table => self.resolve_table_addr(export.index), + ExternalKind::Memory => self.resolve_mem_addr(export.index), + ExternalKind::Global => self.resolve_global_addr(export.index), + ExternalKind::Tag => self.resolve_tag_addr(export.index), }; - + let value = ExternVal::new(export.kind, addr); + let item = self.resolve_extern(value).expect("exported extern has a backing"); (export.name.as_ref(), item) }) } @@ -388,7 +368,11 @@ impl ModuleInstance { /// # } /// ``` pub fn extern_item(&self, name: &str) -> Result { - match self.require_export(name)? { + self.resolve_extern(self.require_export(name)?) + } + + fn resolve_extern(&self, value: ExternVal) -> Result { + match value { ExternVal::Func(addr) => { Ok(ExternItem::Func(Function { item: StoreItem::new(self.0.store_id, addr), module_id: self.id() })) } diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 963df859..83111569 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -866,18 +866,17 @@ impl<'store> Executor<'store> { let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let addr = if kind.arch() == MemoryArch::I64 { + let base = if kind.arch() == MemoryArch::I64 { let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { + cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), - }))?; - cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? + }))? } else { - let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? + u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize }; + let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; value.store_at(&mut *mem, addr) }) } @@ -891,18 +890,17 @@ impl<'store> Executor<'store> { let memarg = index.resolve(&self.func.data); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let addr = if kind.arch() == MemoryArch::I64 { + let base = if kind.arch() == MemoryArch::I64 { let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; - let base = cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { + cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), - }))?; - cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))? + }))? } else { - let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); - cold_err!(mem.effective_addr::(base as usize, u64::from(memarg.offset())))? + u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize }; + let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; let value = cold_err!(T::load_at(&*mem, addr))?; increment(value).store_at(&mut *mem, addr) @@ -1343,28 +1341,19 @@ impl<'store> Executor<'store> { let dst = self.store.value_stack.pop_memory_operand(arch)?; let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); + // Dropped segments behave like empty segments, including valid zero-length copies. + let data = data.as_deref().unwrap_or_default(); + let end = offset.checked_add(size).filter(|&end| end <= data.len()).ok_or(Trap::MemoryOutOfBounds { + offset, + len: size, + max: data.len(), + })?; crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let data_len = data.as_ref().map_or(0, |d| d.len()); - let mem_len = mem.len(); - if offset.checked_add(size).is_none_or(|end| end > data_len) { - return cold!(Err(Trap::MemoryOutOfBounds { offset, len: size, max: data_len })); - } - if dst.checked_add(size).is_none_or(|end| end > mem_len) { - return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); - } - - if size == 0 { - return Ok(()); - } - - let Some(data) = &data else { - return cold!(Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 })); - }; - - if mem.write_all(dst, &data[offset..offset + size]).is_none() { - return cold!(Err(Trap::MemoryOutOfBounds { offset: dst, len: size, max: mem_len })); - } - Ok(()) + cold_err!(mem.write_all(dst, &data[offset..end]).ok_or_else(|| Trap::MemoryOutOfBounds { + offset: dst, + len: size, + max: mem.len(), + })) }) } @@ -1531,10 +1520,9 @@ impl<'store> Executor<'store> { } match arg.is_64() { - true => u64::stack_push(&mut self.store.value_stack, old)?, - false => u32::stack_push(&mut self.store.value_stack, old as u32)?, + true => u64::stack_push(&mut self.store.value_stack, old), + false => u32::stack_push(&mut self.store.value_stack, old as u32), } - Ok(()) }) } @@ -1550,8 +1538,7 @@ impl<'store> Executor<'store> { crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = self.store.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; - cold_err!(val.store_at(&mut *mem, addr))?; - Ok(()) + cold_err!(val.store_at(&mut *mem, addr)) }) } diff --git a/crates/tinywasm/src/store/memory/instance.rs b/crates/tinywasm/src/store/memory/instance.rs index 64d70d2a..0222cc04 100644 --- a/crates/tinywasm/src/store/memory/instance.rs +++ b/crates/tinywasm/src/store/memory/instance.rs @@ -23,16 +23,19 @@ impl core::fmt::Debug for MemoryInstance { } impl MemoryInstance { + /// Converts a page count to a byte length that fits the host address space. #[inline] pub(super) fn host_size(kind: MemoryType, pages: u64) -> Option { pages.checked_mul(kind.page_size()).and_then(|size| usize::try_from(size).ok()) } + /// Returns the declared byte limit, saturating when it exceeds the host address space. #[inline] - fn maximum_size(kind: MemoryType) -> Option { + pub(super) fn maximum_size(kind: MemoryType) -> Option { kind.page_count_max_declared().map(|pages| Self::host_size(kind, pages).unwrap_or(usize::MAX)) } + /// Applies the runtime's memory64 allocation cap to the declared page limit. #[inline] pub(super) fn page_count_max(kind: MemoryType) -> u64 { match kind.arch() { @@ -101,6 +104,7 @@ impl MemoryInstance { Self::grow_storage(self.kind, &mut self.inner, &mut self.page_count, pages_delta, limiter) } + /// Grows exclusively borrowed storage after checking limits and the host limiter. pub(super) fn grow_storage( kind: MemoryType, inner: &mut MemoryStorage, diff --git a/crates/tinywasm/src/store/memory/shared.rs b/crates/tinywasm/src/store/memory/shared.rs index d8c6635b..fa68a472 100644 --- a/crates/tinywasm/src/store/memory/shared.rs +++ b/crates/tinywasm/src/store/memory/shared.rs @@ -12,6 +12,19 @@ use super::{MemoryInstance, MemoryStorage, memory_oob}; use crate::{ResourceLimiter, Result, Trap}; /// A cloneable handle to one shared WebAssembly memory, importable into multiple stores. +/// +/// # Synchronization +/// +/// A single mutex protects the memory's bytes. Both host and Wasm code take this +/// lock to access the bytes or grow the memory. This keeps concurrent accesses safe +/// under Rust's memory model and lets host code borrow slices through [`Self::lock`]. +/// +/// Wasm atomics use the same mutex. An atomic read-modify-write holds the lock for +/// the whole operation, while separate loads and stores can interleave with other +/// threads. Atomic wait releases the lock while sleeping. +/// +/// A lock-free implementation would require a different storage and host-access design, +/// with particular care for overlapping accesses and WebAssembly's memory model. #[derive(Clone)] #[cfg_attr(feature = "debug", derive(Debug))] pub struct MemoryShared(Arc); @@ -37,12 +50,16 @@ impl core::fmt::Debug for MemorySharedInstance { } /// Holds the memory lock while accessing its bytes. +/// +/// Until this guard is dropped, other threads cannot access the memory's bytes or grow it. pub struct MemorySharedGuard<'a> { pub(crate) kind: MemoryType, pub(crate) inner: MutexGuard<'a, MemoryStorage>, + pages: &'a AtomicUsize, } impl MemoryShared { + /// Moves a newly allocated memory into a shared backing. pub(crate) fn from_instance(instance: MemoryInstance) -> Self { let kind = instance.kind; let pages = AtomicUsize::new(instance.page_count); @@ -62,111 +79,60 @@ impl MemoryShared { Ok(Self::from_instance(MemoryInstance::new(ty.with_shared(true), None)?)) } - /// Locks the memory for scoped access to its bytes and page count. + /// Locks the memory for scoped byte access, size queries, and growth. + /// + /// Drop the guard before calling other locking methods or running Wasm that accesses + /// this memory, including through another handle. The lock is not reentrant. pub fn lock(&self) -> MemorySharedGuard<'_> { - MemorySharedGuard { kind: self.ty(), inner: self.0.bytes.lock().unwrap_or_else(|poison| poison.into_inner()) } + // A host panic cannot invalidate the byte storage, so poisoned locks remain usable. + let inner = self.0.bytes.lock().unwrap_or_else(|poison| poison.into_inner()); + MemorySharedGuard { kind: self.ty(), inner, pages: &self.0.pages } } /// Reads the published size without locking the byte storage. - pub fn page_count(&self) -> usize { + pub(crate) fn page_count(&self) -> usize { self.0.pages.load(Ordering::Acquire) } /// Reads the immutable memory type without locking. - pub fn ty(&self) -> MemoryType { + pub(crate) fn ty(&self) -> MemoryType { self.0.kind } - /// Returns the current byte length without locking. - pub fn len(&self) -> usize { - MemoryInstance::host_size(self.ty(), self.page_count() as u64).expect("published memory size fits on this host") - } - - /// Returns whether the memory has no allocated bytes. - pub fn is_empty(&self) -> bool { - self.page_count() == 0 - } - - /// Reads up to `dst.len()` bytes, returning the count read. - pub fn read(&self, offset: usize, dst: &mut [u8]) -> Result { - Ok(self.lock().inner.read(offset, dst)) - } - - /// Writes up to `src.len()` bytes, returning the count written. - pub fn write(&self, offset: usize, src: &[u8]) -> Result { - Ok(self.lock().inner.write(offset, src)) - } - - /// Reads an exact range or returns an out-of-bounds trap. - pub fn read_exact(&self, offset: usize, dst: &mut [u8]) -> Result<()> { - let guard = self.lock(); - guard.inner.read_exact(offset, dst).ok_or_else(|| memory_oob(offset, dst.len(), guard.inner.len()).into()) - } - - /// Copies an exact range out of the memory. - pub fn read_vec(&self, offset: usize, len: usize) -> Result> { - let guard = self.lock(); - guard.inner.read_vec(offset, len).ok_or_else(|| memory_oob(offset, len, guard.inner.len()).into()) - } - - /// Copies an entire slice into the memory. - pub fn copy_from_slice(&self, offset: usize, bytes: &[u8]) -> Result<()> { - let mut guard = self.lock(); - guard.inner.write_all(offset, bytes).ok_or_else(|| memory_oob(offset, bytes.len(), guard.inner.len()).into()) - } - - /// Copies a range within the same memory. - pub fn copy_within(&self, src: usize, dst: usize, len: usize) -> Result<()> { - let mut guard = self.lock(); - guard.inner.copy_within(dst, src, len).ok_or_else(|| memory_oob(dst, len, guard.inner.len()).into()) - } - - /// Fills a range with a byte value. - pub fn fill(&self, offset: usize, len: usize, val: u8) -> Result<()> { - let mut guard = self.lock(); - guard.inner.fill(offset, len, val).ok_or_else(|| memory_oob(offset, len, guard.inner.len()).into()) - } - /// Waits until notified or the signed nanosecond timeout expires. - pub(crate) fn wait( - &self, - addr: usize, - expected: u64, - timeout: i64, - ) -> core::result::Result { - let guard = self.lock(); - let bytes = guard.inner.read_fixed::(addr)?; - let mut value = [0u8; 8]; - value[..N].copy_from_slice(&bytes); - if u64::from_le_bytes(value) != expected { - return Ok(1); - } - if timeout == 0 { - return Ok(2); - } - let waiter = Arc::new(Waiter { notified: Mutex::new(false), wake: Condvar::new() }); - self.0 - .waiters - .lock() - .unwrap_or_else(|poison| poison.into_inner()) - .entry(addr) - .or_default() - .push_back(waiter.clone()); - drop(guard); - - let notified = waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()); - if timeout < 0 { - drop(waiter.wake.wait_while(notified, |ready| !*ready).unwrap_or_else(|poison| poison.into_inner())); - } else { - drop( - waiter - .wake - .wait_timeout_while(notified, Duration::from_nanos(timeout as u64), |ready| !*ready) - .unwrap_or_else(|poison| poison.into_inner()) - .0, - ); + pub(crate) fn wait(&self, addr: usize, expected: u64, timeout: i64) -> Result { + let waiter = { + let guard = self.lock(); + let bytes = guard.inner.read_fixed::(addr)?; + let mut value = [0u8; 8]; + value[..N].copy_from_slice(&bytes); + if u64::from_le_bytes(value) != expected { + return Ok(1); + } + if timeout == 0 { + return Ok(2); + } + let waiter = Arc::new(Waiter { notified: Mutex::new(false), wake: Condvar::new() }); + // Keep the value check and registration under the byte lock so a store followed + // by notify cannot slip between them and leave this waiter asleep. + let mut waiters = self.0.waiters.lock().unwrap_or_else(|poison| poison.into_inner()); + waiters.entry(addr).or_default().push_back(waiter.clone()); + waiter + }; + + { + let notified = waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()); + let _notified = if timeout < 0 { + let result = waiter.wake.wait_while(notified, |ready| !*ready); + result.unwrap_or_else(|poison| poison.into_inner()) + } else { + let timeout = Duration::from_nanos(timeout as u64); + let result = waiter.wake.wait_timeout_while(notified, timeout, |ready| !*ready); + result.unwrap_or_else(|poison| poison.into_inner()).0 + }; } - // The condvar guard was dropped before taking waiters, preserving lock order. + + // Release the condvar guard before taking waiters, preserving lock order. let mut waiters = self.0.waiters.lock().unwrap_or_else(|poison| poison.into_inner()); let notified = *waiter.notified.lock().unwrap_or_else(|poison| poison.into_inner()); if let Some(queue) = waiters.get_mut(&addr) { @@ -197,25 +163,22 @@ impl MemoryShared { woken } - /// Grows the memory by the given number of pages, returning its previous size. - pub fn grow(&self, pages: i64) -> Result> { - Ok(self.grow_with_limiter(pages, None)?) - } - /// Calls host limiters outside the memory lock, then rechecks concurrent growth. pub(crate) fn grow_with_limiter( &self, pages: i64, limiter: Option<&dyn ResourceLimiter>, - ) -> core::result::Result, Trap> { + ) -> Result, Trap> { let Some(limiter) = limiter else { - let mut guard = self.lock(); - return self.grow_locked(pages, &mut guard.inner); + return self.lock().grow_inner(pages); }; + let kind = self.ty(); + let maximum = MemoryInstance::maximum_size(kind); loop { - let guard = self.lock(); - let kind = self.ty(); - let current_pages = self.page_count(); + let (current_pages, current) = { + let guard = self.lock(); + (self.page_count(), guard.inner.len()) + }; let Some(new_pages) = usize::try_from(pages).ok().and_then(|delta| current_pages.checked_add(delta)) else { return Ok(None); }; @@ -225,35 +188,88 @@ impl MemoryShared { let Some(desired) = MemoryInstance::host_size(kind, new_pages as u64) else { return Ok(None); }; - let current = guard.inner.len(); - let maximum = - kind.page_count_max_declared().map(|max| MemoryInstance::host_size(kind, max).unwrap_or(usize::MAX)); - drop(guard); - let allowed = desired == current || limiter.memory_growing(current, desired, maximum)?; let mut guard = self.lock(); if self.page_count() != current_pages { + // The limiter approved a different size transition. Ask again with the new size. continue; } if !allowed { return Ok(None); } - return self.grow_locked(pages, &mut guard.inner); + return guard.grow_inner(pages); } } +} + +impl MemorySharedGuard<'_> { + /// Returns the memory byte length. + pub fn len(&self) -> usize { + self.inner.len() + } + + /// Returns whether the memory has no allocated bytes. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Reads up to `dst.len()` bytes, returning the count read. + /// + /// Use [`Self::read_exact`] or [`Self::read_vec`] when you need a full range. + pub fn read(&self, offset: usize, dst: &mut [u8]) -> Result { + Ok(self.inner.read(offset, dst)) + } + + /// Writes up to `src.len()` bytes, returning the count written. + /// + /// Use [`Self::copy_from_slice`] when you need the full slice written. + pub fn write(&mut self, offset: usize, src: &[u8]) -> Result { + Ok(self.inner.write(offset, src)) + } + + /// Reads exactly `dst.len()` bytes or returns an out-of-bounds trap. + pub fn read_exact(&self, offset: usize, dst: &mut [u8]) -> Result<()> { + self.inner.read_exact(offset, dst).ok_or_else(|| memory_oob(offset, dst.len(), self.len()).into()) + } - fn grow_locked(&self, pages: i64, bytes: &mut MemoryStorage) -> core::result::Result, Trap> { + /// Reads `len` bytes into a newly allocated buffer. + pub fn read_vec(&self, offset: usize, len: usize) -> Result> { + self.inner.read_vec(offset, len).ok_or_else(|| memory_oob(offset, len, self.len()).into()) + } + + /// Copies a full slice into memory. + pub fn copy_from_slice(&mut self, offset: usize, data: &[u8]) -> Result<()> { + self.inner.write_all(offset, data).ok_or_else(|| memory_oob(offset, data.len(), self.len()).into()) + } + + /// Copies a range within the same memory. + pub fn copy_within(&mut self, src: usize, dst: usize, len: usize) -> Result<()> { + self.inner.copy_within(dst, src, len).ok_or_else(|| memory_oob(dst, len, self.len()).into()) + } + + /// Fills a range with a byte value. + pub fn fill(&mut self, offset: usize, len: usize, val: u8) -> Result<()> { + self.inner.fill(offset, len, val).ok_or_else(|| memory_oob(offset, len, self.len()).into()) + } + + /// Grows the memory by the given number of pages, returning its previous page count. + /// + /// Returns `None` if the memory cannot grow. New bytes are zero-initialized. + /// Host growth does not consult a store's resource limiter. + pub fn grow(&mut self, pages: i64) -> Result> { + Ok(self.grow_inner(pages)?) + } + + fn grow_inner(&mut self, pages: i64) -> Result, Trap> { let mut current_pages = self.page_count(); - let result = MemoryInstance::grow_storage(self.ty(), bytes, &mut current_pages, pages, None); - if result.as_ref().is_ok_and(Option::is_some) { - self.0.pages.store(current_pages, Ordering::Release); + let result = MemoryInstance::grow_storage(self.kind, &mut self.inner, &mut current_pages, pages, None)?; + if result.is_some() { + self.pages.store(current_pages, Ordering::Release); } - result + Ok(result) } -} -impl MemorySharedGuard<'_> { /// Returns the current memory type. pub fn ty(&self) -> MemoryType { self.kind diff --git a/crates/tinywasm/src/store/state.rs b/crates/tinywasm/src/store/state.rs index 3d2e682e..50262014 100644 --- a/crates/tinywasm/src/store/state.rs +++ b/crates/tinywasm/src/store/state.rs @@ -26,7 +26,7 @@ pub(crate) struct State { pub(crate) roots: gc::Roots, } -// Select ordinary bytes or a locked shared backing, then run the operation once. +// Dispatch once per operation, keeping ordinary memory on the direct-access path macro_rules! with_memory { ($state:expr, $addr:expr, |$memory:ident, $kind:ident| $body:block) => {{ let state = &mut $state; @@ -82,7 +82,7 @@ impl State { addr: MemAddr, pages: i64, limiter: Option<&dyn ResourceLimiter>, - ) -> core::result::Result, Trap> { + ) -> Result, Trap> { #[cfg(feature = "std")] if addr & SHARED_MEM_BIT != 0 { return self.shared_memories[(addr & !SHARED_MEM_BIT) as usize].grow_with_limiter(pages, limiter); @@ -98,7 +98,7 @@ impl State { src_addr: MemAddr, src: usize, size: usize, - ) -> core::result::Result<(), Trap> { + ) -> Result<(), Trap> { if dst_addr != src_addr && (dst_addr | src_addr) & SHARED_MEM_BIT == 0 { let (destination, source) = self.get_mems_mut(dst_addr, src_addr); return destination.copy_from_memory(dst, source, src, size); @@ -117,6 +117,7 @@ impl State { }); } + // Never hold two backing locks, since another store may copy in the opposite direction. // Growth cannot invalidate these ranges. Check both before changing the destination. with_memory!(*self, src_addr, |source, kind| { source.checked_range(src, size).ok_or_else(|| memory_oob(src, size, source.len()))?; diff --git a/crates/tinywasm/tests/import_linking.rs b/crates/tinywasm/tests/import_linking.rs index a65c8a43..f5ce8f4e 100644 --- a/crates/tinywasm/tests/import_linking.rs +++ b/crates/tinywasm/tests/import_linking.rs @@ -39,6 +39,16 @@ fn link_module_links_same_store_instance() -> Result<(), Box Result<(), Box> { + let module = tinywasm::parse_bytes(&wat::parse_str(r#"(module (import "host" "memory" (memory 0 65536)))"#)?)?; + let mut store = Store::default(); + let mut imports = Imports::new(); + imports.define("host", "memory", tinywasm::Memory::try_new(&mut store, tinywasm::types::MemoryType::default())?); + assert!(matches!(ModuleInstance::instantiate(&mut store, &module, Some(&imports)), Err(Error::Linker(_)))); + Ok(()) +} + #[test] fn link_module_rejects_cross_store_instance() -> Result<(), Box> { let (add_module, import_module) = parse_modules()?; diff --git a/crates/tinywasm/tests/shared_memory.rs b/crates/tinywasm/tests/shared_memory.rs index 5a5e3d74..80641e97 100644 --- a/crates/tinywasm/tests/shared_memory.rs +++ b/crates/tinywasm/tests/shared_memory.rs @@ -34,10 +34,18 @@ fn shared_memory_is_visible_across_stores() -> TestResult { memory.lock().data_mut()[..4].copy_from_slice(&41u32.to_le_bytes()); assert_eq!(first.func::<(), i32>(&first_store, "add")?.call(&mut first_store, ())?, 41); assert_eq!(second.func::<(), i32>(&second_store, "add")?.call(&mut second_store, ())?, 42); - assert_eq!(memory.read_vec(0, 4)?, 43u32.to_le_bytes()); + assert_eq!(memory.lock().read_vec(0, 4)?, 43u32.to_le_bytes()); assert_eq!(first.memory_shared("memory")?.lock().data()[..4], 43u32.to_le_bytes()); - assert_eq!(first.func::<(), i32>(&first_store, "grow")?.call(&mut first_store, ())?, 1); + { + let mut guard = memory.lock(); + assert_eq!(guard.grow(1)?, Some(1)); + assert_eq!(guard.page_count(), 2); + assert!(guard.data()[65536..].iter().all(|&byte| byte == 0)); + assert_eq!(guard.grow(1)?, None); + } + assert_eq!(first.func::<(), i32>(&first_store, "size")?.call(&mut first_store, ())?, 2); + assert_eq!(first.func::<(), i32>(&first_store, "grow")?.call(&mut first_store, ())?, -1); assert_eq!(second.func::<(), i32>(&second_store, "size")?.call(&mut second_store, ())?, 2); assert_eq!(memory.lock().page_count(), 2); Ok(()) @@ -54,7 +62,7 @@ fn concurrent_growth_publishes_page_count() -> TestResult { let barrier = &barrier; scope.spawn(move || { barrier.wait(); - memory.grow(1).unwrap().unwrap() + memory.lock().grow(1).unwrap().unwrap() }) }) .collect(); @@ -63,8 +71,9 @@ fn concurrent_growth_publishes_page_count() -> TestResult { }); sizes.sort(); assert_eq!(sizes, [1, 2]); - assert_eq!(memory.page_count(), 3); - assert_eq!(memory.len(), 3 * 65536); + let guard = memory.lock(); + assert_eq!(guard.page_count(), 3); + assert_eq!(guard.data().len(), 3 * 65536); Ok(()) } @@ -128,7 +137,7 @@ fn atomic_rmw_is_indivisible_across_stores() -> TestResult { } Ok(()) })?; - assert_eq!(memory.read_vec(0, 4)?, 4000u32.to_le_bytes()); + assert_eq!(memory.lock().read_vec(0, 4)?, 4000u32.to_le_bytes()); Ok(()) } @@ -223,6 +232,10 @@ fn defined_shared_memory_supports_data_and_bulk_access() -> TestResult { (module (memory (export "memory") 1 2 shared) (data (i32.const 0) "abc") + (data $passive "xyz") + (func (export "init") (param i32 i32 i32) + local.get 0 local.get 1 local.get 2 memory.init $passive) + (func (export "drop") data.drop $passive) (func (export "copy") i32.const 4 i32.const 0 @@ -242,12 +255,23 @@ fn defined_shared_memory_supports_data_and_bulk_access() -> TestResult { instance.exports().find(|(name, _)| *name == "memory"), Some((_, tinywasm::ExternItem::MemoryShared(_))) )); - assert_eq!(memory.read_vec(0, 3)?, b"abc"); + assert_eq!(memory.lock().read_vec(0, 3)?, b"abc"); instance.func::<(), ()>(&store, "copy")?.call(&mut store, ())?; - assert_eq!(memory.read_vec(4, 3)?, b"abc"); + assert_eq!(memory.lock().read_vec(4, 3)?, b"abc"); memory.lock().data_mut()[4] = b'z'; assert_eq!(instance.func::<(), i32>(&store, "read")?.call(&mut store, ())?, i32::from(b'z')); + let init = instance.func::<(i32, i32, i32), ()>(&store, "init")?; + init.call(&mut store, (8, 0, 3))?; + assert_eq!(memory.lock().read_vec(8, 3)?, b"xyz"); + assert!(init.call(&mut store, (8, 2, 2)).is_err()); + assert_eq!(memory.lock().read_vec(8, 3)?, b"xyz"); + instance.func::<(), ()>(&store, "drop")?.call(&mut store, ())?; + init.call(&mut store, (65536, 0, 0))?; + assert!(init.call(&mut store, (65537, 0, 0)).is_err()); + assert!(init.call(&mut store, (0, 1, 0)).is_err()); + assert!(init.call(&mut store, (0, 0, 1)).is_err()); + let consumer = tinywasm::parse_bytes(&wat::parse_str( r#" (module @@ -294,12 +318,12 @@ fn copy_between_shared_memories_checks_ranges() -> TestResult { let source = instance.memory_shared("source")?; let destination = instance.memory_shared("destination")?; let bytes = vec![0xa5; 8192]; - source.copy_from_slice(0, &bytes)?; + source.lock().copy_from_slice(0, &bytes)?; let copy = instance.func::<(i32, i32, i32), ()>(&store, "copy")?; copy.call(&mut store, (8, 0, bytes.len() as i32))?; - assert_eq!(destination.read_vec(8, bytes.len())?, bytes); + assert_eq!(destination.lock().read_vec(8, bytes.len())?, bytes); assert!(copy.call(&mut store, (65535, 0, 2)).is_err()); - assert_eq!(destination.read_vec(8, bytes.len())?, bytes); + assert_eq!(destination.lock().read_vec(8, bytes.len())?, bytes); Ok(()) } @@ -329,9 +353,9 @@ fn copy_between_ordinary_and_shared_memory() -> TestResult { ordinary.copy_from_slice(&mut store, 0, b"abc")?; instance.func::<(), ()>(&store, "to_shared")?.call(&mut store, ())?; - assert_eq!(shared.read_vec(4, 3)?, b"abc"); + assert_eq!(shared.lock().read_vec(4, 3)?, b"abc"); - shared.copy_from_slice(4, b"xyz")?; + shared.lock().copy_from_slice(4, b"xyz")?; instance.func::<(), ()>(&store, "to_ordinary")?.call(&mut store, ())?; assert_eq!(ordinary.read_vec(&store, 8, 3)?, b"xyz"); Ok(()) @@ -352,13 +376,13 @@ fn copy_between_aliases_of_one_shared_memory() -> TestResult { "#, )?)?; let memory = MemoryShared::try_new(MemoryType::new(MemoryArch::I32, 1, Some(1), None))?; - memory.copy_from_slice(0, b"abc")?; + memory.lock().copy_from_slice(0, b"abc")?; let mut imports = Imports::new(); imports.define("host", "source", memory.clone()).define("host", "destination", memory.clone()); let mut store = Store::default(); let instance = ModuleInstance::instantiate(&mut store, &module, Some(&imports))?; instance.func::<(), ()>(&store, "copy")?.call(&mut store, ())?; - assert_eq!(memory.read_vec(0, 4)?, b"aabc"); + assert_eq!(memory.lock().read_vec(0, 4)?, b"aabc"); Ok(()) } diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index 3d1f9dbf..fb2960b1 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -61,7 +61,7 @@ pub struct AtomicArg { code: u8, } -/// A wait or notify instruction with its module-local memory operand. +/// Operation performed by an atomic wait or notify instruction. #[derive(Clone, Copy, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] @@ -72,7 +72,7 @@ pub enum AtomicWaitOp { } impl AtomicArg { - /// Packs the operation, access width, and result type into one byte. + /// Packs the operation, access width, and value type into one byte. pub const fn new(memory: Operand128Idx, width: AtomicWidth, is_64: bool, op: AtomicOp) -> Self { Self { memory, code: (op as u8) | ((width as u8).trailing_zeros() as u8) << 4 | (is_64 as u8) << 6 } } diff --git a/examples/threads.rs b/examples/threads.rs index 37ac7d6d..9bb68610 100644 --- a/examples/threads.rs +++ b/examples/threads.rs @@ -44,8 +44,9 @@ fn main() -> Result<()> { })?; // Atomic read-modify-write operations serialize updates to the shared memory. - assert_eq!(memory.read_vec(0, 4)?, 4000u32.to_le_bytes()); - assert_eq!(memory.grow(1)?, Some(1)); - assert_eq!(memory.page_count(), 2); + let mut guard = memory.lock(); + assert_eq!(guard.data()[..4], 4000u32.to_le_bytes()); + assert_eq!(guard.grow(1)?, Some(1)); + assert_eq!(guard.page_count(), 2); Ok(()) } From 7ea65234eb6d568dc8943919e0b2c3a6fb6c144d Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 18:40:57 +0200 Subject: [PATCH 05/10] chore: simplify visitor macros Signed-off-by: Henry --- crates/parser/src/macros.rs | 30 +++++++++ crates/parser/src/visit.rs | 122 ++++++++++++++++++++---------------- 2 files changed, 97 insertions(+), 55 deletions(-) diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index 3779d5e0..c49d5850 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -45,6 +45,12 @@ pub(crate) mod visit { // and fallback opcode. Instructions without a family use plain emission. macro_rules! lowering_ops { () => {}; + (atomic $op:ident $inputs:tt => $outputs:tt { + $($visit:ident => $width:literal),* $(,)? + } $($rest:tt)*) => { + $(lowering_ops!(@atomic $op $inputs => $outputs $visit $width);)* + lowering_ops!($($rest)*); + }; ($kind:ident $inputs:tt => $outputs:tt { $($visit:ident $(($($arg:ident: $ty:ty),+))? => $instr:ident $([$family:ident($($rule_arg:expr),*)])?),* $(,)? } $($rest:tt)*) => { @@ -104,6 +110,30 @@ pub(crate) mod visit { (@memory_arg $memory_arg_idx:ident, $lane:ident) => { tinywasm_types::MemoryLaneArg { memory_arg_idx: $memory_arg_idx, lane: $lane } }; + (@atomic $op:ident [$($input:ident),*] => [$($output:ident),*] $visit:ident $width:literal) => { + fn $visit(&mut self, memarg: wasmparser::MemArg) -> Self::Output { + if memarg.align != ($width as u32).trailing_zeros() as u8 { + return Err(crate::ParseError::Other("invalid atomic alignment".into())); + } + let address = self.metadata.memory_size(memarg.memory)?; + self.mark_memory(memarg.memory); + let memory = self.push128(tinywasm_types::Operand128::::new( + memarg.offset, memarg.memory, + ))?; + let is_64 = lowering_ops!(@atomic_is64 [$($input),*] [$($output),*]); + let arg = tinywasm_types::AtomicArg::new( + memory, tinywasm_types::AtomicWidth::from_bytes($width), is_64, tinywasm_types::AtomicOp::$op, + ); + self.emit( + &[$(lowering_ops!(@size $input, address)),*], + &[$(lowering_ops!(@size $output, address)),*], + tinywasm_types::Instruction::Atomic(arg), + ) + } + }; + (@atomic_is64 [Addr, S64 $(, S64)*] $outputs:tt) => { true }; + (@atomic_is64 [Addr] [S64]) => { true }; + (@atomic_is64 $inputs:tt $outputs:tt) => { false }; (@global $inputs:tt => $outputs:tt $($operator:tt)*) => { lowering_ops!(@resolved global_size $inputs => $outputs $($operator)*); }; diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index 20c220dd..b9550c1d 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -11,9 +11,9 @@ use alloc::{ vec::Vec, }; use tinywasm_types::{ - AtomicArg, AtomicOp, AtomicWaitOp, AtomicWidth, BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, - ImportKind, Instruction, MemoryType, Operand64, Operand64Idx, Operand128, Operand128Idx, StorageType, - TableDefinition, TagType, TypeSection, ValueCounts, ValueLane, WasmFunctionData, + AtomicWaitOp, BinOp, BinOp128, CmpOp, ExceptionHandler, Global, Import, ImportKind, Instruction, MemoryType, + Operand64, Operand64Idx, Operand128, Operand128Idx, StorageType, TableDefinition, TagType, TypeSection, + ValueCounts, ValueLane, WasmFunctionData, }; use wasmparser::{FunctionBody, OperatorsReader, OperatorsReaderAllocations, VisitSimdOperator}; @@ -512,37 +512,6 @@ pub(crate) fn process_operators_and_validate( Ok((instructions, builder.data, builder.uses_local_memory, validator.into_allocations(), reader.into_allocations())) } -macro_rules! atomic_visitors { - ($($op:ident, $inputs:tt => $outputs:tt: $(($visit:ident, $width:literal)),+;)*) => { - $(atomic_visitors!(@group $op, $inputs => $outputs: $(($visit, $width)),+);)* - }; - (@group $op:ident, $inputs:tt => $outputs:tt: $(($visit:ident, $width:literal)),+) => { - $(atomic_visitors!(@one $op, $inputs => $outputs: $visit, $width);)+ - }; - (@one $op:ident, [$($input:ident),*] => [$($output:ident),*]: $visit:ident, $width:literal) => { - fn $visit(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - if memarg.align != ($width as u32).trailing_zeros() as u8 { - return Err(crate::ParseError::Other("invalid atomic alignment".into())); - } - let address = self.metadata.memory_size(memarg.memory)?; - self.mark_memory(memarg.memory); - let memory = self.push128(Operand128::::new(memarg.offset, memarg.memory))?; - let is_64 = atomic_visitors!(@is64 [$($input),*] [$($output),*]); - let arg = AtomicArg::new(memory, AtomicWidth::from_bytes($width), is_64, AtomicOp::$op); - self.emit( - &[$(atomic_visitors!(@size $input, address)),*], - &[$(atomic_visitors!(@size $output, address)),*], - Instruction::Atomic(arg), - ) - } - }; - (@size Addr, $address:ident) => { $address }; - (@size $lane:ident, $address:ident) => { ValueLane::$lane }; - (@is64 [Addr, S64 $(, S64)*] $outputs:tt) => { true }; - (@is64 [Addr] [S64]) => { true }; - (@is64 $inputs:tt $outputs:tt) => { false }; -} - impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { type Output = Result<()>; @@ -566,28 +535,71 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { self.atomic_wait(memarg, 8, AtomicWaitOp::Wait64) } - atomic_visitors! { - Load, [Addr] => [S32]: (visit_i32_atomic_load, 4), (visit_i32_atomic_load8_u, 1), (visit_i32_atomic_load16_u, 2); - Load, [Addr] => [S64]: (visit_i64_atomic_load, 8), (visit_i64_atomic_load8_u, 1), (visit_i64_atomic_load16_u, 2), (visit_i64_atomic_load32_u, 4); - Store, [Addr, S32] => []: (visit_i32_atomic_store, 4), (visit_i32_atomic_store8, 1), (visit_i32_atomic_store16, 2); - Store, [Addr, S64] => []: (visit_i64_atomic_store, 8), (visit_i64_atomic_store8, 1), (visit_i64_atomic_store16, 2), (visit_i64_atomic_store32, 4); - Add, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_add, 4), (visit_i32_atomic_rmw8_add_u, 1), (visit_i32_atomic_rmw16_add_u, 2); - Add, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_add, 8), (visit_i64_atomic_rmw8_add_u, 1), (visit_i64_atomic_rmw16_add_u, 2), (visit_i64_atomic_rmw32_add_u, 4); - Sub, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_sub, 4), (visit_i32_atomic_rmw8_sub_u, 1), (visit_i32_atomic_rmw16_sub_u, 2); - Sub, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_sub, 8), (visit_i64_atomic_rmw8_sub_u, 1), (visit_i64_atomic_rmw16_sub_u, 2), (visit_i64_atomic_rmw32_sub_u, 4); - And, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_and, 4), (visit_i32_atomic_rmw8_and_u, 1), (visit_i32_atomic_rmw16_and_u, 2); - And, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_and, 8), (visit_i64_atomic_rmw8_and_u, 1), (visit_i64_atomic_rmw16_and_u, 2), (visit_i64_atomic_rmw32_and_u, 4); - Or, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_or, 4), (visit_i32_atomic_rmw8_or_u, 1), (visit_i32_atomic_rmw16_or_u, 2); - Or, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_or, 8), (visit_i64_atomic_rmw8_or_u, 1), (visit_i64_atomic_rmw16_or_u, 2), (visit_i64_atomic_rmw32_or_u, 4); - Xor, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_xor, 4), (visit_i32_atomic_rmw8_xor_u, 1), (visit_i32_atomic_rmw16_xor_u, 2); - Xor, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_xor, 8), (visit_i64_atomic_rmw8_xor_u, 1), (visit_i64_atomic_rmw16_xor_u, 2), (visit_i64_atomic_rmw32_xor_u, 4); - Xchg, [Addr, S32] => [S32]: (visit_i32_atomic_rmw_xchg, 4), (visit_i32_atomic_rmw8_xchg_u, 1), (visit_i32_atomic_rmw16_xchg_u, 2); - Xchg, [Addr, S64] => [S64]: (visit_i64_atomic_rmw_xchg, 8), (visit_i64_atomic_rmw8_xchg_u, 1), (visit_i64_atomic_rmw16_xchg_u, 2), (visit_i64_atomic_rmw32_xchg_u, 4); - Cmpxchg, [Addr, S32, S32] => [S32]: (visit_i32_atomic_rmw_cmpxchg, 4), (visit_i32_atomic_rmw8_cmpxchg_u, 1), (visit_i32_atomic_rmw16_cmpxchg_u, 2); - Cmpxchg, [Addr, S64, S64] => [S64]: (visit_i64_atomic_rmw_cmpxchg, 8), (visit_i64_atomic_rmw8_cmpxchg_u, 1), (visit_i64_atomic_rmw16_cmpxchg_u, 2), (visit_i64_atomic_rmw32_cmpxchg_u, 4); - } - lowering_ops! { + atomic Load [Addr] => [S32] { + visit_i32_atomic_load => 4, visit_i32_atomic_load8_u => 1, visit_i32_atomic_load16_u => 2, + } + atomic Load [Addr] => [S64] { + visit_i64_atomic_load => 8, visit_i64_atomic_load8_u => 1, + visit_i64_atomic_load16_u => 2, visit_i64_atomic_load32_u => 4, + } + atomic Store [Addr, S32] => [] { + visit_i32_atomic_store => 4, visit_i32_atomic_store8 => 1, visit_i32_atomic_store16 => 2, + } + atomic Store [Addr, S64] => [] { + visit_i64_atomic_store => 8, visit_i64_atomic_store8 => 1, + visit_i64_atomic_store16 => 2, visit_i64_atomic_store32 => 4, + } + atomic Add [Addr, S32] => [S32] { + visit_i32_atomic_rmw_add => 4, visit_i32_atomic_rmw8_add_u => 1, visit_i32_atomic_rmw16_add_u => 2, + } + atomic Add [Addr, S64] => [S64] { + visit_i64_atomic_rmw_add => 8, visit_i64_atomic_rmw8_add_u => 1, + visit_i64_atomic_rmw16_add_u => 2, visit_i64_atomic_rmw32_add_u => 4, + } + atomic Sub [Addr, S32] => [S32] { + visit_i32_atomic_rmw_sub => 4, visit_i32_atomic_rmw8_sub_u => 1, visit_i32_atomic_rmw16_sub_u => 2, + } + atomic Sub [Addr, S64] => [S64] { + visit_i64_atomic_rmw_sub => 8, visit_i64_atomic_rmw8_sub_u => 1, + visit_i64_atomic_rmw16_sub_u => 2, visit_i64_atomic_rmw32_sub_u => 4, + } + atomic And [Addr, S32] => [S32] { + visit_i32_atomic_rmw_and => 4, visit_i32_atomic_rmw8_and_u => 1, visit_i32_atomic_rmw16_and_u => 2, + } + atomic And [Addr, S64] => [S64] { + visit_i64_atomic_rmw_and => 8, visit_i64_atomic_rmw8_and_u => 1, + visit_i64_atomic_rmw16_and_u => 2, visit_i64_atomic_rmw32_and_u => 4, + } + atomic Or [Addr, S32] => [S32] { + visit_i32_atomic_rmw_or => 4, visit_i32_atomic_rmw8_or_u => 1, visit_i32_atomic_rmw16_or_u => 2, + } + atomic Or [Addr, S64] => [S64] { + visit_i64_atomic_rmw_or => 8, visit_i64_atomic_rmw8_or_u => 1, + visit_i64_atomic_rmw16_or_u => 2, visit_i64_atomic_rmw32_or_u => 4, + } + atomic Xor [Addr, S32] => [S32] { + visit_i32_atomic_rmw_xor => 4, visit_i32_atomic_rmw8_xor_u => 1, visit_i32_atomic_rmw16_xor_u => 2, + } + atomic Xor [Addr, S64] => [S64] { + visit_i64_atomic_rmw_xor => 8, visit_i64_atomic_rmw8_xor_u => 1, + visit_i64_atomic_rmw16_xor_u => 2, visit_i64_atomic_rmw32_xor_u => 4, + } + atomic Xchg [Addr, S32] => [S32] { + visit_i32_atomic_rmw_xchg => 4, visit_i32_atomic_rmw8_xchg_u => 1, visit_i32_atomic_rmw16_xchg_u => 2, + } + atomic Xchg [Addr, S64] => [S64] { + visit_i64_atomic_rmw_xchg => 8, visit_i64_atomic_rmw8_xchg_u => 1, + visit_i64_atomic_rmw16_xchg_u => 2, visit_i64_atomic_rmw32_xchg_u => 4, + } + atomic Cmpxchg [Addr, S32, S32] => [S32] { + visit_i32_atomic_rmw_cmpxchg => 4, visit_i32_atomic_rmw8_cmpxchg_u => 1, + visit_i32_atomic_rmw16_cmpxchg_u => 2, + } + atomic Cmpxchg [Addr, S64, S64] => [S64] { + visit_i64_atomic_rmw_cmpxchg => 8, visit_i64_atomic_rmw8_cmpxchg_u => 1, + visit_i64_atomic_rmw16_cmpxchg_u => 2, visit_i64_atomic_rmw32_cmpxchg_u => 4, + } memory [Addr] => [S32] { visit_i32_load => I32Load [load(Instruction::LoadLocal32)], visit_f32_load => F32Load [load(Instruction::LoadLocal32)], From 72a3baae301853b9d6f9d0e3e1a9f4f0dffa5998 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Thu, 24 Sep 2026 11:16:14 -0700 Subject: [PATCH 06/10] perf: grow the value stack out of line (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of a three-PR stack on `next` (`785be0e`), each on top of the previous one with individual wins that combine nicely: 1. #57 **perf: grow the value stack out of line** (this PR) 2. #58 perf: inline the fused binop and compare helpers 3. #59 perf: reserve each function's operand stack on entry `Stack::push` now checks `len == capacity` itself and hands the full case to a `#[cold] #[inline(never)] push_grow`, which keeps the existing overflow check and growth. After that check `Vec::push` can no longer reach its own growth path, so the allocator call leaves the instruction handlers. It was inlined into every handler that pushes, and the call made those handlers save and restore four pairs of callee-saved registers on every execution, even though the default config's stacks are fixed and never grow. `LocalGet32`'s hot path goes from 57 to 37 instructions on aarch64. No behavior change: fixed stacks still trap at capacity, and dynamic stacks still grow up to `max_size`. **iPhone 12 efficiency cores** (A14 Icestorm, `.utility` QoS). XCode 27 PMU template for low-level CPU stats. 16 benchmark rows, 5 launches per build, interleaved with the other builds of the stack; cycles and instructions per call, geomean against `next` as of `b45a98a`: | | cycles | instructions | rows faster | |---|---:|---:|---:| | this PR vs `next` | **−4.1 %** | −3.5 % | 16/16 | **Checks** on this branch: tinywasm's CI matrix run on my local M4 Mac: `cargo test --workspace` and `cargo test --workspace --examples` with `--features tinywasm/nightly-tail-calls`, each with and without default features --- .../src/interpreter/stack/value_stack.rs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 814195ac..a582ae5d 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -46,10 +46,10 @@ impl Stack { #[inline(always)] pub(crate) fn push(&mut self, value: T) -> Result<(), Trap> { - // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may - // intentionally overshoot max_size. Revisit when Vec::push_within_capacity is stable. - if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { - return cold!(Err(Trap::ValueStackOverflow)); + // At capacity, grow (or trap) out of line. After this check `Vec::push` cannot reach its + // own growth path, so the allocator call stays out of the instruction handlers. + if self.data.len() == self.data.capacity() { + return self.push_grow(value); } self.data.push(value); Ok(()) @@ -57,11 +57,18 @@ impl Stack { #[inline(always)] pub(crate) fn push_copy(&mut self, index: usize) -> Result<(), Trap> { - // Keep the same capacity-based limit check as push, including intentional overshoot. - if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { - return cold!(Err(Trap::ValueStackOverflow)); - } let value = self.data[index]; + self.push(value) + } + + #[cold] + #[inline(never)] + fn push_grow(&mut self, value: T) -> Result<(), Trap> { + // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may + // intentionally overshoot max_size. + if !self.dynamic || self.data.len() >= self.max_size { + return Err(Trap::ValueStackOverflow); + } self.data.push(value); Ok(()) } From 6b4dc3ff76e60942aaa6b8b6f56a6a55c00b611e Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Thu, 24 Sep 2026 11:16:21 -0700 Subject: [PATCH 07/10] perf: inline the fused binop and compare helpers (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second of a three-PR stack on `next` (`785be0e`): 1. #57 perf: grow the value stack out of line 2. #58 **perf: inline the fused binop and compare helpers** (this PR) 3. #59 perf: reserve each function's operand stack on entry This branch includes #57 below it; the last commit is new here. `#[inline(always)]` on `exec_binop_32/64` and `exec_cmp_32/64`. LLVM kept them out of line, so every fused `BinOp*` / compare handler called out for the operator `match` and spilled registers around the call. Inlined, the `match` runs inside the handler. The binary grows by about 16 KB, and I didn't see a notable rise in L1 cache misses on Apple A14 efficiency cores in my benchmarks. No behavior change. **iPhone 12 E-cores**, same setup as the first PR; this PR against the first: | | cycles | instructions | rows faster | |---|---:|---:|---:| | this PR vs the first | **−1.3 %** | −2.0 % | 12/16 | | first two PRs vs `next` | −5.3 % | −5.5 % | 16/16 | The four rows that didn't get faster moved by 0.5 % or less. **Checks** on this branch: tinywasm's CI matrix run on my local M4 Mac: `cargo test --workspace` and `cargo test --workspace --examples` on rust 1.98 and with `--features tinywasm/nightly-tail-calls`, each with and without default features. --- crates/tinywasm/src/interpreter/values.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/tinywasm/src/interpreter/values.rs b/crates/tinywasm/src/interpreter/values.rs index 31a507fe..ec69fe97 100644 --- a/crates/tinywasm/src/interpreter/values.rs +++ b/crates/tinywasm/src/interpreter/values.rs @@ -273,6 +273,7 @@ macro_rules! impl_value_ops { } } + #[inline(always)] fn $binop(op: BinOp, lhs: $unsigned, rhs: $unsigned) -> $unsigned { match op { BinOp::IAdd => lhs.wrapping_add(rhs), @@ -296,6 +297,7 @@ macro_rules! impl_value_ops { } } + #[inline(always)] fn $cmp(op: CmpOp, lhs: $signed, rhs: $signed) -> bool { match op { CmpOp::Eq => lhs == rhs, From fbc35315ac90fad52087ace8c1bfb4d9fcb2be9b Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Thu, 24 Sep 2026 11:21:43 -0700 Subject: [PATCH 08/10] perf: reserve each function's operand stack on entry (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third of a three-PR stack on `next` (`785be0e`): 1. #57 perf: grow the value stack out of line 2. #58 perf: inline the fused binop and compare helpers 3. #59 **perf: reserve each function's operand stack on entry** (this PR) This branch includes #58 and #57 below it; the last three commits are new here. The first PR moved stack growth out of the instruction handlers but kept a cold call to it, and that call still costs the handlers their register saves. This PR removes growth from `push` entirely: - The parser already tracks the operand stack per lane (`lane_counts` in `visit.rs`). It now also records the highest point each function body reaches, as `WasmFunction::max_stack`. - `enter_locals` reserves `locals + max_stack` in each lane when a function is entered, and grows a dynamic stack there if needed. It already grew for the locals at that point. - `Stack::push`, used by the instruction handlers, then traps at capacity instead of calling a growth path: a full stack inside a body means the limit. Typed guest operand handlers now use reserved capacity; host value marshaling can still grow. - Host argument and result marshaling through `push_dyn` retains `push_or_grow`, including host calls made by a guest. - Catch landing reservations now include tag payload lanes and optional exception references, including function-label catches and cross-frame throws. Self-calls preserve the actual value-stack overflow trap; the WAST exhaustion assertion accepts either internal stack reaching its limit first. GC field reads and exception payload injection use reserved no-growth pushes. Changes to know about: - **Public API**: `tinywasm_types::WasmFunction` gains a `pub max_stack: ValueCounts` field. - **Archive format**: the layout of `WasmFunction` changed, so `TWASM_VERSION` goes to `06` (the first format change since 0.11.0 shipped with `05`). - **Stack limits**: overflow is detected when a function is entered, not at the push that crosses the limit. A function whose deepest operand stack does not fit traps on entry, even if a particular call would not have reached that depth. Deep recursion runs out slightly earlier, by at most one function's operand stack. - `tests/value_stack_reservation.rs` runs a function with a deep operand stack in all three lanes, recursively, on dynamic stacks that start empty or tiny and on a fixed stack. It also checks that a limit smaller than the body's needs still traps with `ValueStackOverflow`. **iPhone 12 E-cores**, same setup as the first PR, with XCode 27 PMU template for low-level CPU stats. This PR against the second, which ittself is against `next`, and against an upper bound: the second PR plus a `push` with no growth path and no reservation, which behaves the same on fixed stacks (the default) but would trap on dynamic ones. | | cycles | instructions | rows faster | |---|---:|---:|---:| | this PR vs the second | **−2.8 %** | −1.9 % | 15/16 | | the whole stack vs `next` | **−8.0 %** | −7.2 % | 16/16 | | upper bound vs the second | −2.7 % | −2.4 % | 16/16 | | this PR vs the upper bound | −0.1 % | +0.5 % | 8/16 | The reservation keeps the whole upper-bound gain on the geomean. Its extra work on function entry shows up on the most call-heavy rows. The EH parser (exnref) is the one row that is slower than with the first two PRs: +1.3 %, consistently across reps. Against the upper bound, it is +2.3 %, and the tail-call FSM +1.2 %. A few rows come out about 3 % faster than the upper bound (sieve, xmrsplayer), which is most likely code layout rather than this change. Cycles per call on `next`, and every build as a ratio to it (median of 5 launches): | row | `next` Mcycles | first PR | first two | whole stack | upper bound | |---|---:|---:|---:|---:|---:| | fib(30) | 303.8 | 0.964 | 0.931 | 0.913 | 0.905 | | matmul relaxed-simd FMA | 8.17 | 0.960 | 0.951 | 0.889 | 0.879 | | audio DSP | 3089 | 0.960 | 0.926 | 0.881 | 0.888 | | call_indirect | 63.36 | 0.970 | 0.975 | 0.952 | 0.953 | | xmrsplayer | 45.52 | 0.943 | 0.946 | 0.909 | 0.938 | | vtable_poly4 | 128.5 | 0.984 | 0.961 | 0.937 | 0.934 | | graphql-validation (AssemblyScript) | 36.61 | 0.975 | 0.945 | 0.936 | 0.926 | | graphql-validation (Porffor) | 31.03 | 0.970 | 0.958 | 0.957 | 0.952 | | sieve (scalar) | 2.04 | 0.909 | 0.908 | 0.857 | 0.889 | | crc32 (scalar) | 14.75 | 0.915 | 0.904 | 0.866 | 0.869 | | convolution (scalar) | 40.93 | 0.935 | 0.933 | 0.882 | 0.887 | | bulk_memory (scalar) | 30.59 | 0.936 | 0.907 | 0.869 | 0.877 | | tail-call FSM | 14.59 | 0.978 | 0.967 | 0.953 | 0.942 | | EH parser (exnref) | 42.01 | 0.980 | 0.981 | 0.994 | 0.972 | | GC binary trees | 284.4 | 0.989 | 0.990 | 0.984 | 0.984 | | call_ref twin (call_indirect) | 55.03 | 0.983 | 0.970 | 0.962 | 0.954 | | **geomean** | | **0.959** | **0.947** | **0.920** | **0.921** | **Checks** on this branch: tinywasm's CI matrix , run on my local M4 MacBook: `cargo test --workspace` and `cargo test --workspace --examples` . I went the distance and made sure the whole suite passes with the default value stacks switched to dynamic stacks that start empty and grow to exactly each function's reservation (a local stress configuration, not in this PR), where a function whose recorded maximum undercounted its pushes would trap. Last, I did a coverage-driven fuzzer pass overnight while I slept, and didn't get any anomalous exceptions or state tainting. --------- Co-authored-by: Henry Gressmann --- crates/cli/src/wast_runner.rs | 7 +- crates/parser/src/conversion.rs | 35 ++-- crates/parser/src/module.rs | 2 + crates/parser/src/visit.rs | 72 ++++++-- crates/tinywasm/src/func/mod.rs | 14 +- crates/tinywasm/src/func/resume.rs | 12 +- .../tinywasm/src/interpreter/executor/mod.rs | 18 +- .../src/interpreter/stack/value_stack.rs | 68 +++++-- crates/tinywasm/src/store/gc/mod.rs | 2 +- .../tinywasm/tests/value_stack_reservation.rs | 169 ++++++++++++++++++ crates/types/src/archive.rs | 4 +- crates/types/src/lib.rs | 2 + 12 files changed, 330 insertions(+), 75 deletions(-) create mode 100644 crates/tinywasm/tests/value_stack_reservation.rs diff --git a/crates/cli/src/wast_runner.rs b/crates/cli/src/wast_runner.rs index a3f17e59..50d1df53 100644 --- a/crates/cli/src/wast_runner.rs +++ b/crates/cli/src/wast_runner.rs @@ -349,11 +349,14 @@ impl WastRunner { ); continue; }; - if !message.starts_with(trap.message()) && !trap.message().starts_with(message) { + // The core test suite's "call stack exhausted" text denotes stack exhaustion, + // not which internal stack reaches its configured limit first. Function-entry + // operand reservation can exhaust a value lane before the call-frame stack. + if !matches!(trap, tinywasm::Trap::CallStackOverflow | tinywasm::Trap::ValueStackOverflow) { test_group.add_result( &format!("AssertExhaustion({i})"), span.linecol_in(wast_raw), - Err(anyhow!("expected trap: {}, got: {}", message, trap.message())), + Err(anyhow!("expected stack exhaustion ({message}), got: {}", trap.message())), ); continue; } diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 25743229..5482d4d8 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -198,34 +198,35 @@ pub(crate) fn convert_module_code( } #[cfg(feature = "validate")] - let (body, data, uses_local_memory, validator_allocs, reader_allocs) = match validator { + let (body, data, uses_local_memory, max_stack, validator_allocs, reader_allocs) = match validator { Some(validator) => { - let (body, data, uses_local_memory, validator_allocs, reader_allocs) = process_operators_and_validate( - validator, - func, - (local_types, local_addr_map), - metadata, - context, - reader_allocs, - options, - )?; - (body, data, uses_local_memory, Some(validator_allocs), reader_allocs) + let (body, data, uses_local_memory, max_stack, validator_allocs, reader_allocs) = + process_operators_and_validate( + validator, + func, + (local_types, local_addr_map), + metadata, + context, + reader_allocs, + options, + )?; + (body, data, uses_local_memory, max_stack, Some(validator_allocs), reader_allocs) } None => { - let (body, data, uses_local_memory, reader_allocs) = + let (body, data, uses_local_memory, max_stack, reader_allocs) = process_operators(func, (local_types, local_addr_map), metadata, context, reader_allocs, options)?; - (body, data, uses_local_memory, None, reader_allocs) + (body, data, uses_local_memory, max_stack, None, reader_allocs) } }; #[cfg(not(feature = "validate"))] - let (body, data, uses_local_memory, validator_allocs, reader_allocs) = { + let (body, data, uses_local_memory, max_stack, validator_allocs, reader_allocs) = { let _ = validator; - let (body, data, uses_local_memory, reader_allocs) = + let (body, data, uses_local_memory, max_stack, reader_allocs) = process_operators(func, (local_types, local_addr_map), metadata, context, reader_allocs, options)?; - (body, data, uses_local_memory, None, reader_allocs) + (body, data, uses_local_memory, max_stack, None, reader_allocs) }; Ok(( - FunctionCode { instructions: body, data: data.finish(), locals: local_counts, uses_local_memory }, + FunctionCode { instructions: body, data: data.finish(), locals: local_counts, max_stack, uses_local_memory }, validator_allocs, reader_allocs, )) diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 03c725c9..fcaba401 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -14,6 +14,7 @@ pub(crate) struct FunctionCode { pub instructions: Vec, pub data: WasmFunctionData, pub locals: ValueCounts, + pub max_stack: ValueCounts, pub uses_local_memory: bool, } @@ -553,6 +554,7 @@ impl<'a> ModuleReader<'a> { locals: code.locals, params, results, + max_stack: code.max_stack, })) }) .collect::>()?; diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index b9550c1d..7ca82e3b 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -185,6 +185,7 @@ pub(crate) struct FunctionBuilder<'a> { control_stack: Vec>, operand_stack: Vec, lane_counts: ValueCounts, + max_lane_counts: ValueCounts, metadata: &'a ModuleMetadata, local_types: Vec, local_addr_map: Vec, @@ -223,6 +224,7 @@ impl<'a> FunctionBuilder<'a> { }], operand_stack: Vec::new(), lane_counts: ValueCounts::default(), + max_lane_counts: ValueCounts::default(), uses_local_memory: false, } } @@ -451,7 +453,7 @@ pub(crate) fn process_operators( context: FunctionLoweringContext, allocs: OperatorsReaderAllocations, options: &ParserOptions, -) -> Result<(Vec, FunctionDataBuilder, bool, OperatorsReaderAllocations)> { +) -> Result<(Vec, FunctionDataBuilder, bool, ValueCounts, OperatorsReaderAllocations)> { let (local_types, local_addr_map) = locals; let body_size = body.as_bytes().len(); let reader = body.get_binary_reader_for_operators()?; @@ -474,7 +476,7 @@ pub(crate) fn process_operators( reader.finish()?; let instructions = builder.emitter.finish(&mut builder.data)?; - Ok((instructions, builder.data, builder.uses_local_memory, reader.into_allocations())) + Ok((instructions, builder.data, builder.uses_local_memory, builder.max_lane_counts, reader.into_allocations())) } #[cfg(feature = "validate")] @@ -486,7 +488,14 @@ pub(crate) fn process_operators_and_validate( context: FunctionLoweringContext, allocs: OperatorsReaderAllocations, options: &ParserOptions, -) -> Result<(Vec, FunctionDataBuilder, bool, FuncValidatorAllocations, OperatorsReaderAllocations)> { +) -> Result<( + Vec, + FunctionDataBuilder, + bool, + ValueCounts, + FuncValidatorAllocations, + OperatorsReaderAllocations, +)> { let (local_types, local_addr_map) = locals; let body_size = body.as_bytes().len(); let reader = body.get_binary_reader_for_operators()?; @@ -509,7 +518,14 @@ pub(crate) fn process_operators_and_validate( reader.finish()?; let instructions = builder.emitter.finish(&mut builder.data)?; - Ok((instructions, builder.data, builder.uses_local_memory, validator.into_allocations(), reader.into_allocations())) + Ok(( + instructions, + builder.data, + builder.uses_local_memory, + builder.max_lane_counts, + validator.into_allocations(), + reader.into_allocations(), + )) } impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { @@ -952,11 +968,23 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { wasmparser::Catch::All { label } => (None, label, false), wasmparser::Catch::AllRef { label } => (None, label, true), }; - if let Some(tag) = tag { - self.metadata.tag_signature(tag)?; - } let target_idx = self.get_ctx_idx(depth)?; let target_base = self.control_stack[target_idx].base; + // The runtime truncates to the target base, then injects the exception payload and + // optional reference before executing this landing pad. These pushes do not pass + // through the ordinary logical operand stack, but still need function-entry capacity. + let mut landing_counts = target_base; + if let Some(tag) = tag { + for &lane in &self.metadata.tag_signature(tag)?.params { + Self::increment_lane(&mut landing_counts, lane)?; + } + } + if with_ref { + Self::increment_lane(&mut landing_counts, ValueLane::S32)?; + } + self.max_lane_counts.c32 = self.max_lane_counts.c32.max(landing_counts.c32); + self.max_lane_counts.c64 = self.max_lane_counts.c64.max(landing_counts.c64); + self.max_lane_counts.c128 = self.max_lane_counts.c128.max(landing_counts.c128); let landing_label = self.emitter.new_label(); self.emitter.bind(landing_label)?; self.emit_branch_jump_or_return(depth)?; @@ -1525,17 +1553,29 @@ impl<'a> FunctionBuilder<'a> { Ok((size, addr)) } - /// Pushes logical operands while maintaining the lane counts used by `DropKeep`. + /// Increments a physical lane count, rejecting functions too large for the encoded count. + fn increment_lane(counts: &mut ValueCounts, lane: ValueLane) -> Result<()> { + let count = match lane { + ValueLane::S32 => &mut counts.c32, + ValueLane::S64 => &mut counts.c64, + ValueLane::S128 => &mut counts.c128, + }; + *count = count + .checked_add(1) + .ok_or_else(|| crate::ParseError::Other("logical operand lane count is too large".into()))?; + Ok(()) + } + + /// Pushes logical operands while maintaining the lane counts used by `DropKeep` and their + /// maximum, which the runtime reserves when it enters the function. fn push_sizes(&mut self, sizes: &[ValueLane]) -> Result<()> { for &size in sizes { - let count = match size { - ValueLane::S32 => &mut self.lane_counts.c32, - ValueLane::S64 => &mut self.lane_counts.c64, - ValueLane::S128 => &mut self.lane_counts.c128, - }; - *count = count - .checked_add(1) - .ok_or_else(|| crate::ParseError::Other("logical operand lane count is too large".into()))?; + Self::increment_lane(&mut self.lane_counts, size)?; + match size { + ValueLane::S32 => self.max_lane_counts.c32 = self.max_lane_counts.c32.max(self.lane_counts.c32), + ValueLane::S64 => self.max_lane_counts.c64 = self.max_lane_counts.c64.max(self.lane_counts.c64), + ValueLane::S128 => self.max_lane_counts.c128 = self.max_lane_counts.c128.max(self.lane_counts.c128), + } self.operand_stack.push(size); } Ok(()) diff --git a/crates/tinywasm/src/func/mod.rs b/crates/tinywasm/src/func/mod.rs index cf3c8550..44a2c812 100644 --- a/crates/tinywasm/src/func/mod.rs +++ b/crates/tinywasm/src/func/mod.rs @@ -157,16 +157,14 @@ impl Function { let host = store.state.funcs.host(self.addr()).func.clone(); host.call_values(store, self.module_id, type_addr, params, results) } else { - let (wasm_params, wasm_locals) = { + let (wasm_params, wasm_locals, wasm_max_stack) = { let wasm = store.state.funcs.wasm(self.addr()); - let wasm_params = wasm.func.params; - let wasm_locals = wasm.func.locals; - (wasm_params, wasm_locals) + (wasm.func.params, wasm.func.locals, wasm.func.max_stack) }; store.push_wasm_values(params).inspect_err(|_| store.value_stack.truncate_to_base(value_stack_base))?; let locals_base = store .value_stack - .enter_locals(&wasm_params, &wasm_locals) + .enter_locals(&wasm_params, &wasm_locals, &wasm_max_stack) .inspect_err(|_| store.value_stack.truncate_to_base(value_stack_base))?; let callframe = CallFrame::new(self.addr(), locals_base, wasm_locals); InterpreterRuntime::exec(store, callframe, call_stack_base).inspect_err(|_| { @@ -213,14 +211,14 @@ impl Function { }; } - let (type_addr, wasm_params, wasm_locals) = { + let (type_addr, wasm_params, wasm_locals, wasm_max_stack) = { let wasm = store.state.funcs.wasm(self.addr()); - (wasm.type_addr, wasm.func.params, wasm.func.locals) + (wasm.type_addr, wasm.func.params, wasm.func.locals, wasm.func.max_stack) }; store.push_typed_values::(type_addr, params, value_stack_base)?; let locals_base = store .value_stack - .enter_locals(&wasm_params, &wasm_locals) + .enter_locals(&wasm_params, &wasm_locals, &wasm_max_stack) .inspect_err(|_| store.value_stack.truncate_to_base(value_stack_base))?; let callframe = CallFrame::new(self.addr(), locals_base, wasm_locals); InterpreterRuntime::exec(store, callframe, call_stack_base).inspect_err(|_| { diff --git a/crates/tinywasm/src/func/resume.rs b/crates/tinywasm/src/func/resume.rs index a035ffc7..106aea64 100644 --- a/crates/tinywasm/src/func/resume.rs +++ b/crates/tinywasm/src/func/resume.rs @@ -79,15 +79,15 @@ impl Function { return Ok(ExecState::Completed(Some(CallResult::Written))); } - let (wasm_params, wasm_locals) = { + let (wasm_params, wasm_locals, wasm_max_stack) = { let wasm = store.state.funcs.wasm(self.addr()); - (wasm.func.params, wasm.func.locals) + (wasm.func.params, wasm.func.locals, wasm.func.max_stack) }; store.call_stack.clear(); store.value_stack.clear(); store.push_wasm_values(params)?; - let locals_base = store.value_stack.enter_locals(&wasm_params, &wasm_locals)?; + let locals_base = store.value_stack.enter_locals(&wasm_params, &wasm_locals, &wasm_max_stack)?; let callframe = CallFrame::new(self.addr(), locals_base, wasm_locals); Ok(ExecState::Running { callframe, root_func_addr: self.addr() }) @@ -224,9 +224,9 @@ impl FunctionTyped { let execution = ExecutionInner { store, state: ExecState::Completed(None) }; return Ok(FuncExecutionTyped { execution, result: Some(result) }); } - let (type_addr, wasm_params, wasm_locals) = { + let (type_addr, wasm_params, wasm_locals, wasm_max_stack) = { let wasm = store.state.funcs.wasm(self.func.addr()); - (wasm.type_addr, wasm.func.params, wasm.func.locals) + (wasm.type_addr, wasm.func.params, wasm.func.locals, wasm.func.max_stack) }; store.enter_execution()?; @@ -236,7 +236,7 @@ impl FunctionTyped { store.push_typed_values::(type_addr, params.into_wasm_values(), StackBase::default())?; let locals_base = store .value_stack - .enter_locals(&wasm_params, &wasm_locals) + .enter_locals(&wasm_params, &wasm_locals, &wasm_max_stack) .inspect_err(|_| store.value_stack.clear())?; let callframe = CallFrame::new(self.func.addr(), locals_base, wasm_locals); Ok(ExecState::Running { callframe, root_func_addr: self.func.addr() }) diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 83111569..4f2c27fa 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -617,7 +617,7 @@ impl<'store> Executor<'store> { let Store { state, value_stack, .. } = self.store; let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_dyn(value)?; + value_stack.push_reserved(value)?; } } if with_ref { @@ -703,9 +703,8 @@ impl<'store> Executor<'store> { fn exec_call_self(&mut self, return_instr_ptr: usize) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let Ok(locals_base) = self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) else { - return cold!(Err(Trap::CallStackOverflow.into())); - }; + let locals_base = + self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; let new = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); self.store.call_stack.push(core::mem::replace(&mut self.cf, new), return_instr_ptr)?; Ok(()) @@ -715,9 +714,8 @@ impl<'store> Executor<'store> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let Ok(locals_base) = self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) else { - return cold!(Err(Trap::CallStackOverflow.into())); - }; + let locals_base = + self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -778,14 +776,14 @@ impl<'store> Executor<'store> { return_instr_ptr: usize, ) -> ExecResult { let wasm_func = self.store.state.funcs.wasm(func_addr); - let (params, locals, owner, next_func) = { + let (params, locals, max_stack, owner, next_func) = { let next_func = (!Shared::ptr_eq(&self.func, &wasm_func.func)).then(|| wasm_func.func.clone()); - (wasm_func.func.params, wasm_func.func.locals, wasm_func.owner, next_func) + (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) }; if TAIL { self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals)?; + let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; if TAIL { self.cf = CallFrame::new(func_addr, locals_base, locals); } else { diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index a582ae5d..b3ac91a0 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -44,10 +44,22 @@ impl Stack { self.data.len() } + /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand + /// stack, so a full stack here is the limit and there is nothing to grow. After this check + /// `Vec::push` cannot reach its own growth path, so the instruction handlers make no calls. #[inline(always)] pub(crate) fn push(&mut self, value: T) -> Result<(), Trap> { - // At capacity, grow (or trap) out of line. After this check `Vec::push` cannot reach its - // own growth path, so the allocator call stays out of the instruction handlers. + if self.data.len() == self.data.capacity() { + return cold!(Err(Trap::ValueStackOverflow)); + } + self.data.push(value); + Ok(()) + } + + /// Pushes a value outside a function body (host arguments and results), which no reservation + /// covers, so a dynamic stack grows here if needed. + #[inline(always)] + pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> { if self.data.len() == self.data.capacity() { return self.push_grow(value); } @@ -131,22 +143,31 @@ impl Stack { self.data.push(last); } + /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves + /// room for its operand stack (`max_stack` values above the locals), so [`Self::push`] never + /// has to grow the stack while the function runs. #[inline] - pub(crate) fn enter_locals(&mut self, param_count: usize, local_count: usize) -> Result { + pub(crate) fn enter_locals( + &mut self, + param_count: usize, + local_count: usize, + max_stack: usize, + ) -> Result { debug_assert!(param_count <= local_count); debug_assert!(param_count <= self.data.len()); let len = self.data.len(); let start = len - param_count; let end = start + local_count; + let reserve = end + max_stack; - if end > self.data.capacity() { + if reserve > self.data.capacity() { core::hint::cold_path(); - if end > self.max_size || !self.dynamic { + if reserve > self.max_size || !self.dynamic { return Err(Trap::ValueStackOverflow); } let cap = self.data.capacity(); - let target = end.max(cap.max(1).saturating_mul(2)).min(self.max_size); + let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size); if self.data.try_reserve(target - len).is_err() { return Err(Trap::ValueStackOverflow); } @@ -247,10 +268,18 @@ impl ValueStack { } #[inline(always)] - pub(crate) fn enter_locals(&mut self, params: &ValueCounts, locals: &ValueCounts) -> Result { - let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize)?; - let locals_base64 = self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize)?; - let locals_base128 = self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize)?; + pub(crate) fn enter_locals( + &mut self, + params: &ValueCounts, + locals: &ValueCounts, + max_stack: &ValueCounts, + ) -> Result { + let locals_base32 = + self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize, max_stack.c32 as usize)?; + let locals_base64 = + self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)?; + let locals_base128 = + self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)?; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } @@ -268,12 +297,23 @@ impl ValueStack { self.stack_128.truncate_to(base.s128 as usize); } + /// Pushes a dynamically typed value inside a function body using its entry reservation. + pub(crate) fn push_reserved(&mut self, value: RuntimeValue) -> Result<(), Trap> { + match value { + RuntimeValue::Value32(value) => self.stack_32.push(value), + RuntimeValue::Value64(value) => self.stack_64.push(value), + RuntimeValue::Value128(value) => self.stack_128.push(value), + RuntimeValue::ValueRef(value) => self.stack_32.push(value.raw()), + } + } + + /// Pushes a value from outside a function body's reservation; see [`Stack::push_or_grow`]. pub(crate) fn push_dyn(&mut self, value: RuntimeValue) -> Result<(), Trap> { match value { - RuntimeValue::Value32(value) => Value32::stack_push(self, value), - RuntimeValue::Value64(value) => Value64::stack_push(self, value), - RuntimeValue::Value128(value) => Value128::stack_push(self, value), - RuntimeValue::ValueRef(value) => ValueRef::stack_push(self, value), + RuntimeValue::Value32(value) => self.stack_32.push_or_grow(value), + RuntimeValue::Value64(value) => self.stack_64.push_or_grow(value), + RuntimeValue::Value128(value) => self.stack_128.push_or_grow(value), + RuntimeValue::ValueRef(value) => self.stack_32.push_or_grow(value.raw()), } } } diff --git a/crates/tinywasm/src/store/gc/mod.rs b/crates/tinywasm/src/store/gc/mod.rs index 152d6515..598907d9 100644 --- a/crates/tinywasm/src/store/gc/mod.rs +++ b/crates/tinywasm/src/store/gc/mod.rs @@ -55,7 +55,7 @@ pub(crate) fn push_value( (value, _, None) => value, _ => unreachable!("validated packed field access"), }; - stack.push_dyn(value) + stack.push_reserved(value) } /// Decodes numeric array elements from a data segment. diff --git a/crates/tinywasm/tests/value_stack_reservation.rs b/crates/tinywasm/tests/value_stack_reservation.rs new file mode 100644 index 00000000..ae22e6fd --- /dev/null +++ b/crates/tinywasm/tests/value_stack_reservation.rs @@ -0,0 +1,169 @@ +use tinywasm::engine::{Config, StackConfig}; +use tinywasm::types::{RefValue, WasmValue}; +use tinywasm::{Engine, Error, ModuleInstance, Store, Trap}; + +const DEPTH: usize = 64; + +/// `deep(n)` pushes `DEPTH` operands in each value lane before folding them, so its operand stack +/// is far deeper than its locals, and recurses `n` times to stack those frames on each other. +/// It returns `DEPTH * n * (n + 2)`. +fn deep_module() -> Vec { + let i32_lane = "local.get 0\n".repeat(DEPTH) + &"i32.add\n".repeat(DEPTH - 1); + let i64_lane = "local.get 1\n".repeat(DEPTH) + &"i64.add\n".repeat(DEPTH - 1); + let v128_lane = "v128.const i64x2 1 1\n".repeat(DEPTH) + &"i64x2.add\n".repeat(DEPTH - 1); + wat::parse_str(format!( + r#"(module + (func $deep (export "deep") (param i32) (result i32) + (local i64) + (if (i32.eqz (local.get 0)) (then (return (i32.const 0)))) + (local.set 1 (i64.extend_i32_u (local.get 0))) + {i32_lane} + {i64_lane} + i32.wrap_i64 + i32.add + {v128_lane} + i64x2.extract_lane 0 + i32.wrap_i64 + i32.add + (call $deep (i32.sub (local.get 0) (i32.const 1))) + i32.add))"# + )) + .unwrap() +} + +fn call_deep(stack: StackConfig, n: i32) -> tinywasm::Result { + let module = tinywasm::parse_bytes(&deep_module())?; + let mut store = Store::new(Engine::new(Config::new().with_value_stack(stack))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + instance.func::(&store, "deep")?.call(&mut store, n) +} + +#[test] +fn dynamic_value_stacks_grow_when_a_function_is_entered() -> tinywasm::Result<()> { + // Stacks that start empty or tiny must grow at each function entry to cover the body's whole + // operand stack, since pushes inside the body no longer grow them. + for stack in [StackConfig::dynamic(0, 4096), StackConfig::dynamic(1, 4096), StackConfig::fixed(4096)] { + assert_eq!(call_deep(stack, 10)?, (DEPTH * 10 * 12) as i32); + } + Ok(()) +} + +#[test] +fn value_stack_limit_still_traps() { + // The body needs more than 32 slots in each lane, so entering it exceeds the limit. + for stack in [StackConfig::dynamic(0, 32), StackConfig::fixed(32)] { + let result = call_deep(stack, 1); + assert!(matches!(result, Err(Error::Trap(Trap::ValueStackOverflow))), "{result:?}"); + } +} + +#[test] +fn exception_references_reserve_their_landing_stack() -> tinywasm::Result<()> { + for catch in ["catch_ref $tag 0", "catch_all_ref 0"] { + let wasm = wat::parse_str(format!( + r#"(module + (tag $tag) + (func (export "catch") (result exnref) + (try_table ({catch}) + throw $tag) + unreachable))"# + )) + .unwrap(); + let module = tinywasm::parse_bytes(&wasm)?; + assert_eq!(module.funcs[0].max_stack.c32, 1); + + for stack in [StackConfig::dynamic(0, 1), StackConfig::fixed(1)] { + let mut store = Store::new(Engine::new(Config::new().with_value_stack(stack))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let function = instance.func_untyped(&store, "catch")?; + let mut result = [WasmValue::Ref(RefValue::Null)]; + function.call(&mut store, &[], &mut result)?; + assert!(matches!(result[0], WasmValue::Ref(RefValue::Exn(_)))); + } + } + Ok(()) +} + +#[test] +fn exception_payloads_reserve_every_landing_lane() -> tinywasm::Result<()> { + let wasm = wat::parse_str( + r#"(module + (tag $tag (param i32 i64 v128)) + (func $throw + i32.const 7 + i64.const 9 + v128.const i64x2 1 2 + throw $tag) + (func (export "catch") (result i32 i64 v128 exnref) + (try_table (catch_ref $tag 0) + call $throw) + unreachable))"#, + ) + .unwrap(); + let module = tinywasm::parse_bytes(&wasm)?; + let max = module.funcs[1].max_stack; + assert_eq!((max.c32, max.c64, max.c128), (2, 1, 1)); + + let mut store = Store::new(Engine::new(Config::new().with_value_stack(StackConfig::dynamic(0, 2)))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let function = instance.func_untyped(&store, "catch")?; + let mut result = [WasmValue::I32(0), WasmValue::I64(0), WasmValue::V128([0; 16]), WasmValue::Ref(RefValue::Null)]; + function.call(&mut store, &[], &mut result)?; + assert_eq!(result[0], WasmValue::I32(7)); + assert_eq!(result[1], WasmValue::I64(9)); + assert_eq!(result[2], WasmValue::V128([1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0])); + assert!(matches!(result[3], WasmValue::Ref(RefValue::Exn(_)))); + Ok(()) +} + +#[test] +fn gc_field_reads_use_the_function_reservation() -> tinywasm::Result<()> { + let wasm = wat::parse_str( + r#"(module + (type $struct (struct (field i32))) + (type $array (array (mut i32))) + (func (export "struct") (result i32) + i32.const 17 + struct.new $struct + struct.get $struct 0) + (func (export "array") (result i32) + i32.const 23 + i32.const 1 + array.new $array + i32.const 0 + array.get $array))"#, + ) + .unwrap(); + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::new(Engine::new(Config::new().with_value_stack(StackConfig::dynamic(0, 4)))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + assert_eq!(instance.func::<(), i32>(&store, "struct")?.call(&mut store, ())?, 17); + assert_eq!(instance.func::<(), i32>(&store, "array")?.call(&mut store, ())?, 23); + Ok(()) +} + +#[test] +fn recursive_self_call_reports_value_stack_exhaustion() -> tinywasm::Result<()> { + let wasm = wat::parse_str( + r#"(module + (func $f (export "f") (param i32) (result i32) + local.get 0 + if (result i32) + local.get 0 + i32.const 1 + i32.sub + call $f + else + i32.const 0 + end))"#, + ) + .unwrap(); + let module = tinywasm::parse_bytes(&wasm)?; + let mut store = Store::new(Engine::new(Config::new().with_value_stack(StackConfig::fixed(3)))); + let instance = ModuleInstance::instantiate(&mut store, &module, None)?; + let function = instance.func::(&store, "f")?; + assert_eq!(function.call(&mut store, 0)?, 0); + assert!(matches!(function.call(&mut store, 1), Err(Error::Trap(Trap::ValueStackOverflow)))); + assert_eq!(function.call(&mut store, 0)?, 0); + Ok(()) +} diff --git a/crates/types/src/archive.rs b/crates/types/src/archive.rs index bdce379d..f24dfd25 100644 --- a/crates/types/src/archive.rs +++ b/crates/types/src/archive.rs @@ -67,7 +67,7 @@ mod tests { use crate::Operand128Idx; use crate::{ AbstractHeapType, ConstInstruction, Global, GlobalType, Instruction, ModuleFuncIdx, ModuleInner, Operand128, - RefType, Shared, WasmFunction, WasmType, + RefType, Shared, ValueCounts, WasmFunction, WasmType, }; use alloc::boxed::Box; @@ -91,6 +91,7 @@ mod tests { fn v128_operands_round_trip_archive() { let bytes = [0x00, 0x01, 0x02, 0x03, 0x7f, 0x80, 0xfe, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x90]; let mut function = WasmFunction::default(); + function.max_stack = ValueCounts { c32: 2, c64: 3, c128: 4 }; let constant = Operand128Idx::new(0); function.data.operands128 = Box::new([Operand128::<[u8; 16]>::new(bytes).cast()]); function.instructions = Box::new([Instruction::Const128(constant), Instruction::I8x16Shuffle(constant)]); @@ -100,6 +101,7 @@ mod tests { assert_eq!(&archive[..6], b"TWAS06"); let decoded = Module::try_from_twasm(&archive).expect("deserialize archive"); let function = &decoded.funcs[0]; + assert!(function.max_stack == ValueCounts { c32: 2, c64: 3, c128: 4 }); for instruction in function.instructions.iter() { let index = match instruction { diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b3474237..a6ba7c30 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -495,6 +495,8 @@ pub struct WasmFunction { pub locals: ValueCounts, pub params: ValueCounts, pub results: ValueCounts, + /// The highest operand stack the body reaches in each lane, on top of its locals. + pub max_stack: ValueCounts, } #[derive(Clone, PartialEq, Eq, Default)] From 8299745f5945bf2b6faeaf835cc90dc0225ca0d1 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 20:28:09 +0200 Subject: [PATCH 09/10] ci: add windows runner, update runner images Signed-off-by: Henry --- .github/workflows/audit.yaml | 4 ++-- .github/workflows/release.yaml | 2 +- .github/workflows/test.yaml | 41 ++++++++++++++++++++++------------ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml index fa785ca1..c88dff72 100644 --- a/.github/workflows/audit.yaml +++ b/.github/workflows/audit.yaml @@ -29,7 +29,7 @@ jobs: security-events: write uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc # v2.6.0 with: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 upload-sarif: false scan-args: --lockfile=./Cargo.lock @@ -41,6 +41,6 @@ jobs: security-events: write uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc # v2.6.0 with: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 fail-on-vuln: false scan-args: --lockfile=./Cargo.lock diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ae4bb2d1..757f31c8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,7 +5,7 @@ on: jobs: publish: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 environment: release permissions: id-token: write diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1b75a831..5e9274b3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,36 +16,38 @@ jobs: strategy: matrix: include: - - os: ubuntu-latest + - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable)" - - os: ubuntu-latest + - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly)" - - os: ubuntu-latest + - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly tail calls)" args: "--features tinywasm/nightly-tail-calls" - - os: ubuntu-latest + - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" args: "--no-default-features" - - os: ubuntu-latest + - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly, no default features)" args: "--no-default-features" - - os: ubuntu-latest + - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly tail calls, no default features)" args: "--no-default-features --features tinywasm/nightly-tail-calls" - - os: macos-14 + - os: macos-26 rust: stable name: "macOS arm64 (Apple M1)" - - os: ubuntu-latest + - os: windows-2025 + rust: stable + name: "Windows x86_64" + - os: ubuntu-26.04-arm rust: stable name: "Linux arm64" - target: aarch64-unknown-linux-gnu - - os: ubuntu-latest + - os: ubuntu-26.04-arm rust: stable name: "Linux armv7" target: armv7-unknown-linux-gnueabihf @@ -75,10 +77,21 @@ jobs: run: cargo clippy --workspace ${{ matrix.args }} if: matrix.target == '' - - name: Run tests (${{ matrix.target }}) - uses: houseabsolute/actions-rust-cross@21b0f18dc621b25bfae556ff2791fca4173121e8 # v1.0.8 + - name: Install armv7 tools + run: sudo apt-get update && sudo apt-get install -y gcc-arm-linux-gnueabihf qemu-user + if: matrix.target != '' + + - name: Install Rust toolchain (${{ matrix.target }}) + uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: - command: test - target: ${{ matrix.target }} toolchain: ${{ matrix.rust }} + target: ${{ matrix.target }} + rustflags: "" + if: matrix.target != '' + + - name: Run tests (${{ matrix.target }}) + run: cargo test --workspace --target ${{ matrix.target }} + env: + CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc + CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER: qemu-arm -L /usr/arm-linux-gnueabihf if: matrix.target != '' From 4ee43596a8f67b1b78c0e0fb965b5fdde17e44e1 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 24 Sep 2026 20:37:14 +0200 Subject: [PATCH 10/10] chore: fix wasi build on windows Signed-off-by: Henry --- .github/workflows/test.yaml | 43 +++++++++++----------- crates/cli/src/testsuite.rs | 6 ++- crates/wasi/src/p1/abi.rs | 3 ++ crates/wasi/src/p1/imports/fd.rs | 8 ++++ crates/wasi/src/p1/imports/mod.rs | 4 ++ crates/wasi/src/p1/imports/poll.rs | 40 ++++++++++++++++++-- crates/wasi/src/p1/imports/sock_windows.rs | 21 +++++++++++ crates/wasi/tests/p1.rs | 2 + 8 files changed, 102 insertions(+), 25 deletions(-) create mode 100644 crates/wasi/src/p1/imports/sock_windows.rs diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5e9274b3..53feb08e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,34 +19,43 @@ jobs: - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable)" + target: x86_64-unknown-linux-gnu - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly)" + target: x86_64-unknown-linux-gnu - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly tail calls)" + target: x86_64-unknown-linux-gnu args: "--features tinywasm/nightly-tail-calls" - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" + target: x86_64-unknown-linux-gnu args: "--no-default-features" - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly, no default features)" + target: x86_64-unknown-linux-gnu args: "--no-default-features" - os: ubuntu-26.04 rust: nightly name: "Linux x86 (nightly tail calls, no default features)" + target: x86_64-unknown-linux-gnu args: "--no-default-features --features tinywasm/nightly-tail-calls" - os: macos-26 rust: stable name: "macOS arm64 (Apple M1)" + target: aarch64-apple-darwin - os: windows-2025 rust: stable name: "Windows x86_64" + target: x86_64-pc-windows-msvc - os: ubuntu-26.04-arm rust: stable name: "Linux arm64" + target: aarch64-unknown-linux-gnu - os: ubuntu-26.04-arm rust: stable name: "Linux armv7" @@ -65,33 +74,25 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: toolchain: ${{ matrix.rust }} + target: ${{ matrix.target }} rustflags: "" + build-warnings: warn components: rustfmt, clippy - if: matrix.target == '' - - - name: Run tests - run: cargo test --workspace ${{ matrix.args }} && cargo test --workspace ${{ matrix.args }} --examples - if: matrix.target == '' - - - name: Run clippy - run: cargo clippy --workspace ${{ matrix.args }} - if: matrix.target == '' - name: Install armv7 tools run: sudo apt-get update && sudo apt-get install -y gcc-arm-linux-gnueabihf qemu-user - if: matrix.target != '' + if: matrix.target == 'armv7-unknown-linux-gnueabihf' - - name: Install Rust toolchain (${{ matrix.target }}) - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 - with: - toolchain: ${{ matrix.rust }} - target: ${{ matrix.target }} - rustflags: "" - if: matrix.target != '' - - - name: Run tests (${{ matrix.target }}) - run: cargo test --workspace --target ${{ matrix.target }} + - name: Run tests + run: cargo test --workspace --target ${{ matrix.target }} ${{ matrix.args }} env: CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER: qemu-arm -L /usr/arm-linux-gnueabihf - if: matrix.target != '' + + - name: Run examples + run: cargo test --workspace --target ${{ matrix.target }} ${{ matrix.args }} --examples + if: matrix.target != 'armv7-unknown-linux-gnueabihf' + + - name: Run clippy + run: cargo clippy --workspace --target ${{ matrix.target }} ${{ matrix.args }} + if: matrix.target != 'armv7-unknown-linux-gnueabihf' diff --git a/crates/cli/src/testsuite.rs b/crates/cli/src/testsuite.rs index b23b2c89..c376203c 100644 --- a/crates/cli/src/testsuite.rs +++ b/crates/cli/src/testsuite.rs @@ -103,7 +103,11 @@ impl TestSuite { use std::fs::OpenOptions; use std::io::Write; - let mut file = OpenOptions::new().create(true).append(true).read(true).open(path)?; + if std::env::var("CI").is_ok_and(|value| value == "true") { + return Ok(()); + } + + let mut file = OpenOptions::new().create(true).read(true).write(true).open(path)?; let last_line = BufReader::new(&file).lines().last().transpose()?; if let Some(last) = last_line diff --git a/crates/wasi/src/p1/abi.rs b/crates/wasi/src/p1/abi.rs index c331a6b8..cb662e6f 100644 --- a/crates/wasi/src/p1/abi.rs +++ b/crates/wasi/src/p1/abi.rs @@ -31,6 +31,7 @@ pub(super) const INTR: Errno = Errno(27); pub(super) const INVAL: Errno = Errno(28); pub(super) const IO: Errno = Errno(29); pub(super) const ISDIR: Errno = Errno(31); +#[cfg(any(unix, target_os = "android"))] pub(super) const LOOP: Errno = Errno(32); pub(super) const MFILE: Errno = Errno(33); pub(super) const MLINK: Errno = Errno(34); @@ -41,6 +42,7 @@ pub(super) const NOSPC: Errno = Errno(51); pub(super) const NOTCONN: Errno = Errno(53); pub(super) const NOTDIR: Errno = Errno(54); pub(super) const NOTEMPTY: Errno = Errno(55); +#[cfg(unix)] pub(super) const NOTSOCK: Errno = Errno(57); pub(super) const NOTSUP: Errno = Errno(58); pub(super) const OVERFLOW: Errno = Errno(61); @@ -187,6 +189,7 @@ fn platform_io_errno(_error: &io::Error) -> Errno { IO } +#[cfg(unix)] impl From for Errno { fn from(error: rustix::io::Errno) -> Self { use rustix::io::Errno as E; diff --git a/crates/wasi/src/p1/imports/fd.rs b/crates/wasi/src/p1/imports/fd.rs index 18eb1223..965eaba1 100644 --- a/crates/wasi/src/p1/imports/fd.rs +++ b/crates/wasi/src/p1/imports/fd.rs @@ -1,6 +1,7 @@ use std::io::{Read, Seek, SeekFrom, Write}; use cap_fs_ext::{DirEntryExt, DirExt, MetadataExt}; +#[cfg(unix)] use cap_std::fs::FileExt; use tinywasm::FuncContext; @@ -185,9 +186,12 @@ fn read_from_fd( buffer.resize(iovecs.byte_len, 0); let read = match &mut descriptor.resource { Resource::Stdin => std::io::stdin().lock().read(&mut buffer), + #[cfg(unix)] Resource::File(file) if let Some(offset) = offset => { file.read_at(&mut buffer, checked_offset(offset, iovecs.byte_len)?) } + #[cfg(not(unix))] + Resource::File(_) if offset.is_some() => return Ok(NOTSUP), Resource::File(file) => file.read(&mut buffer), Resource::TcpStream(stream) => stream.read(&mut buffer), Resource::UdpSocket(socket) => socket.recv(&mut buffer), @@ -224,7 +228,10 @@ fn write_to_fd( let written = match &mut descriptor.resource { Resource::Stdout => std::io::stdout().lock().write(&data), Resource::Stderr => std::io::stderr().lock().write(&data), + #[cfg(unix)] Resource::File(file) if let Some(offset) = offset => file.write_at(&data, checked_offset(offset, data.len())?), + #[cfg(not(unix))] + Resource::File(_) if offset.is_some() => return Ok(NOTSUP), Resource::File(file) => file.write(&data), Resource::TcpStream(stream) => stream.write(&data), Resource::UdpSocket(socket) => socket.send(&data), @@ -237,6 +244,7 @@ fn write_to_fd( .map_or_else(|errno| errno, |()| SUCCESS)) } +#[cfg(unix)] fn checked_offset(offset: u64, len: usize) -> Result { offset.checked_add(len as u64).map(|_| offset).ok_or(OVERFLOW) } diff --git a/crates/wasi/src/p1/imports/mod.rs b/crates/wasi/src/p1/imports/mod.rs index eff8d0fb..d7a8af9b 100644 --- a/crates/wasi/src/p1/imports/mod.rs +++ b/crates/wasi/src/p1/imports/mod.rs @@ -5,6 +5,10 @@ mod path; mod poll; mod process; mod random; +#[cfg(unix)] +mod sock; +#[cfg(not(unix))] +#[path = "sock_windows.rs"] mod sock; use tinywasm::{HostFunction, Imports}; diff --git a/crates/wasi/src/p1/imports/poll.rs b/crates/wasi/src/p1/imports/poll.rs index 8098b74e..a4192db4 100644 --- a/crates/wasi/src/p1/imports/poll.rs +++ b/crates/wasi/src/p1/imports/poll.rs @@ -1,5 +1,6 @@ use std::io::Seek; +#[cfg(unix)] use rustix::event::{PollFd, PollFlags, Timespec}; use tinywasm::FuncContext; @@ -12,6 +13,7 @@ use super::WasiResult; enum Readiness { Ready, Clock(u64), + #[cfg(unix)] Poll, } @@ -20,12 +22,16 @@ struct PendingEvent { readiness: Readiness, } +#[cfg(unix)] struct PollTarget { event_index: usize, fd: i32, event_type: u8, } +#[cfg(not(unix))] +struct PollTarget; + impl PendingEvent { fn new(userdata: u64, error: Errno, event_type: u8, available: u64, readiness: Readiness) -> Self { let mut bytes = [0; 32]; @@ -134,11 +140,21 @@ fn subscribe_fd( pending_events.push(PendingEvent::new(userdata, SUCCESS, event_type, available, Readiness::Ready)); return Ok(()); } - poll_targets.push(PollTarget { event_index: pending_events.len(), fd, event_type }); - pending_events.push(PendingEvent::new(userdata, SUCCESS, event_type, 0, Readiness::Poll)); - Ok(()) + #[cfg(not(unix))] + { + let _ = poll_targets; + pending_events.push(PendingEvent::new(userdata, NOTSUP, event_type, 0, Readiness::Ready)); + Ok(()) + } + #[cfg(unix)] + { + poll_targets.push(PollTarget { event_index: pending_events.len(), fd, event_type }); + pending_events.push(PendingEvent::new(userdata, SUCCESS, event_type, 0, Readiness::Poll)); + Ok(()) + } } +#[cfg(unix)] fn wait_for_events( ctx: &FuncContext<'_>, poll_targets: &[PollTarget], @@ -217,3 +233,21 @@ fn wait_for_events( } Ok(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)) } + +#[cfg(not(unix))] +fn wait_for_events( + _ctx: &FuncContext<'_>, + poll_targets: &[PollTarget], + _pending_events: &mut [PendingEvent], + timeout_ns: Option, +) -> WasiResult { + if !poll_targets.is_empty() { + return Err(NOTSUP.into()); + } + if let Some(nanoseconds) = timeout_ns { + std::thread::sleep(std::time::Duration::from_nanos(nanoseconds)); + Ok(nanoseconds) + } else { + Ok(0) + } +} diff --git a/crates/wasi/src/p1/imports/sock_windows.rs b/crates/wasi/src/p1/imports/sock_windows.rs new file mode 100644 index 00000000..4d952b04 --- /dev/null +++ b/crates/wasi/src/p1/imports/sock_windows.rs @@ -0,0 +1,21 @@ +use tinywasm::FuncContext; + +use crate::p1::abi::{Errno, NOTSUP}; + +use super::WasiResult; + +pub(super) fn sock_accept(_ctx: FuncContext<'_>, _args: (i32, i32, i32)) -> WasiResult { + Ok(NOTSUP) +} + +pub(super) fn sock_recv(_ctx: FuncContext<'_>, _args: (i32, i32, i32, i32, i32, i32)) -> WasiResult { + Ok(NOTSUP) +} + +pub(super) fn sock_send(_ctx: FuncContext<'_>, _args: (i32, i32, i32, i32, i32)) -> WasiResult { + Ok(NOTSUP) +} + +pub(super) fn sock_shutdown(_ctx: FuncContext<'_>, _args: (i32, i32)) -> WasiResult { + Ok(NOTSUP) +} diff --git a/crates/wasi/tests/p1.rs b/crates/wasi/tests/p1.rs index e41ca390..e17ca476 100644 --- a/crates/wasi/tests/p1.rs +++ b/crates/wasi/tests/p1.rs @@ -236,6 +236,7 @@ fn provides_clocks_randomness_and_timer_polling() -> tinywasm::Result<()> { Ok(()) } +#[cfg(unix)] #[test] fn polls_accepts_and_uses_tcp_streams() -> tinywasm::Result<()> { use std::io::{Read, Write}; @@ -282,6 +283,7 @@ fn polls_accepts_and_uses_tcp_streams() -> tinywasm::Result<()> { Ok(()) } +#[cfg(unix)] #[test] fn sends_and_receives_udp_datagrams() -> tinywasm::Result<()> { let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();