perf(go): own a Vec<u8>'s storage while only the translation reaches it

go/Cargo.toml

@@ -29,6 +29,7 @@ toml = "1.1.8"
 codec = { path = "fixtures/codec" }
 enums = { path = "fixtures/enums" }
 fmt = { path = "fixtures/fmt" }
+hand = { path = "fixtures/hand" }
 ints = { path = "fixtures/ints" }
 items = { path = "fixtures/items" }
 order = { path = "fixtures/order" }

go/README.md

@@ -53,6 +53,23 @@ std translates from its own MIR, down to bindings for what Go does itself: a `Ve
 growth, a slice's iterator, UTF-8 validation, and the formatting machinery, which
 `runtime/rt_fmt.go` ports from core so `{:?}` and `{:#?}` print as Rust's do.
 
+A `Vec<u8>`'s storage is the runtime's own while only the translation reaches it: it
+grows to twice its capacity without zeroing, through the runtime's `growslice`, and an
+outgrown buffer is recycled for the next grow. Own storage is told apart by its shape, a
+capacity that is a power of two of at least 8 and never filled. A root function, which
+Go calls, takes Go's storage it is passed off that shape, a byte less capacity or a copy
+when full, so it is never recycled, and clips own storage it hands back to Go, in its
+result or behind an argument's `&mut`, to its length, so Go never sees past it and it is
+never recycled either; Go's `append` to such a slice copies. It works on a copy of each
+container an argument owns, a `Box`'s pointee or a `Vec`'s elements, so Go code still
+holding one never sees what the translation moved or grew out of it: a `Box` Go passes
+by value is left as it was, as Rust's move would leave it unread. Only roots are
+exported. A root that reaches byte storage refuses a function pointer or trait object in
+its signature, since Go code could fill it with a function that keeps what it is handed
+or reads a place the translation is still writing; a raw pointer, which could see what
+handing in copied; and a reference anywhere in an argument but its outermost type,
+which the translation could move out of what handing back walks.
+
 A panic is a Go panic of `RustPanic` carrying native Rust's message. As with
 `panic = "abort"`, no cleanup runs on the way out, so a caller that recovers one must
 drop what the translation held.
@@ -63,7 +80,9 @@ drop what the translation held.
 `Drop` impl, a trait object of any trait but `Display` and `Debug`, a union, a type
 generic over a constant, such as an array's by-value iterator, a `pub` item that is
 generic, a `static mut`, a static with interior mutability, such as an atomic, or of another
-language, and a reference into part of a static.
+language, a reference into part of a static, and, in the signature of a root that
+reaches byte storage, a function pointer, trait object, raw pointer, or reference
+inside an argument.
 
 ## Tests
 
@@ -73,8 +92,9 @@ Rust on the same inputs, failing on any difference in output or panic message:
 integer width's arithmetic, overflow, division by zero, and shifts, `fixtures/order`
 calls, reads, and overflow checks a folded temporary could reorder or skip, and
 `fixtures/enums` enums held as their parts, whose pointers a callee could write through
-or see changed. `fixtures/items` also checks its exported constants and statics as
-`tests/exports` reads them from Go.
+or see changed, and `fixtures/hand` byte storage made, grown, and shrunk each way, which
+`tests/hand/hand_test.go` then holds onto from Go as Go code may. `fixtures/items` also
+checks its exported constants and statics as `tests/exports` reads them from Go.
 
 `just go bench` times the codec's workloads translated against the same crate exported by
 rust2-ffi and called through cgo: per-call varints and frame headers, and `stream/`, a

go/bench/ffi.toml

@@ -1,4 +1,3 @@
-# The codec fixture's C ABI: only its entry points, as the rest is no API.
+# The codec fixture's C ABI: only its entry points, the rest being crate-private.
 package = "codec"
 prefix = "codec"
-exclude = ["SessionError", "Timescale", "Version", "stats"]

go/fixtures/hand/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "hand"
+version = "0.0.0"
+edition = "2024"
+publish = false
+
+[lints]
+workspace = true

go/fixtures/hand/src/lib.rs

@@ -0,0 +1,190 @@
+//! Byte storage handed between Go code and the translation every way a root function's
+//! signature can pass it, which `tests/hand/hand_test.go` holds onto from Go, and `run`,
+//! each way the translation grows, copies, and shrinks a `Vec<u8>`.
+
+/// Bytes Go code holds only through methods.
+pub struct Buf {
+	data: Vec<u8>,
+}
+
+impl Buf {
+	/// An empty buffer.
+	pub fn new() -> Self {
+		Self { data: Vec::new() }
+	}
+
+	/// Append `n` bytes of `n`.
+	pub fn fill(&mut self, n: u8) {
+		append(&mut self.data, n);
+	}
+
+	/// What it holds.
+	pub fn bytes(&self) -> &[u8] {
+		&self.data
+	}
+}
+
+impl Default for Buf {
+	fn default() -> Self {
+		Self::new()
+	}
+}
+
+/// Bytes Go code reads directly.
+pub struct Open {
+	pub data: Vec<u8>,
+}
+
+/// One of two holders of bytes.
+pub enum Either {
+	Left(Vec<u8>),
+	Right(u8, Vec<u8>),
+	Neither,
+}
+
+/// Append `n` bytes of `n`.
+pub fn append(v: &mut Vec<u8>, n: u8) {
+	for _ in 0..n {
+		v.push(n);
+	}
+}
+
+/// Append `n` bytes of `n`, then panic if `n` is odd.
+pub fn append_or_panic(v: &mut Vec<u8>, n: u8) {
+	append(v, n);
+	assert!(n.is_multiple_of(2), "odd");
+}
+
+/// `n` bytes of `n`, with the room of `n` more.
+pub fn fill(n: u8) -> Vec<u8> {
+	let mut v = Vec::with_capacity(2 * n as usize);
+	append(&mut v, n);
+	v
+}
+
+/// `v` with `n` bytes of `n` appended.
+pub fn grow(mut v: Vec<u8>, n: u8) -> Vec<u8> {
+	append(&mut v, n);
+	v
+}
+
+/// `b` with `n` bytes of `n` appended to what it holds.
+pub fn boxed(mut b: Box<Open>, n: u8) -> Box<Open> {
+	append(&mut b.data, n);
+	b
+}
+
+/// Append `n` bytes of `n` to what `b` holds.
+pub fn reboxed(b: &mut Box<Open>, n: u8) {
+	append(&mut b.data, n);
+}
+
+/// Clear `v`, then append `n` bytes of `n`.
+pub fn refill(v: &mut Vec<u8>, n: u8) {
+	v.clear();
+	append(v, n);
+}
+
+/// Replace `v`'s first buffer with `n` bytes of `n`, outgrow `v`, then that buffer: its
+/// length.
+pub fn regrow(v: &mut Vec<Vec<u8>>, n: u8) -> usize {
+	v[0] = fill(n);
+	for _ in 0..64 {
+		v.push(Vec::new());
+	}
+	append(&mut v[0], 200);
+	v[0].len()
+}
+
+/// `n` buffers, the `i`th of `i` bytes.
+pub fn nested(n: u8) -> Vec<Vec<u8>> {
+	let mut out = Vec::new();
+	for i in 0..n {
+		out.push(fill(i));
+	}
+	out
+}
+
+/// `n` bytes of `n`, if `n` is not zero.
+pub fn maybe(n: u8) -> Option<Vec<u8>> {
+	match n {
+		0 => None,
+		n => Some(fill(n)),
+	}
+}
+
+/// Append `n` bytes of `n` to `v`, giving what it then holds.
+pub fn view(v: &mut Vec<u8>, n: u8) -> &[u8] {
+	append(v, n);
+	v
+}
+
+/// An iterator over `xs`.
+pub fn iter(xs: &[u8]) -> std::slice::Iter<'_, u8> {
+	xs.iter()
+}
+
+/// An iterator over `v`'s buffers.
+pub fn iter_nested(v: &[Vec<u8>]) -> std::slice::Iter<'_, Vec<u8>> {
+	v.iter()
+}
+
+/// Push `n` bytes of `n` onto `v`, then pop and outgrow them, leaving the slot vacated.
+pub fn vacate(v: &mut Vec<Vec<u8>>, n: u8) -> usize {
+	v.push(fill(n));
+	let mut last = v.pop().unwrap_or_default();
+	append(&mut last, 200);
+	last.len()
+}
+
+/// Switch `e` to the other side, its bytes with `n` bytes of `n` appended.
+pub fn flip(e: &mut Either, n: u8) {
+	*e = match std::mem::replace(e, Either::Neither) {
+		Either::Left(v) => Either::Right(n, grow(v, n)),
+		Either::Right(_, v) => Either::Left(grow(v, n)),
+		Either::Neither => Either::Left(fill(n)),
+	};
+}
+
+/// Each way the translation makes, grows, copies, and shrinks a `Vec<u8>`, its sizes
+/// from the input's bytes: what each ends up holding.
+pub fn run(input: &[u8]) -> Vec<u8> {
+	let mut out = Vec::new();
+	let mut held: Vec<Vec<u8>> = Vec::new();
+	for (i, &b) in input.iter().enumerate() {
+		let n = b as usize * (i + 1);
+		let mut v = match b % 4 {
+			0 => Vec::new(),
+			1 => Vec::with_capacity(n),
+			2 => input.to_vec(),
+			_ => match held.last() {
+				Some(last) => last.clone(),
+				None => Vec::new(),
+			},
+		};
+		for k in 0..n {
+			v.push(k as u8);
+		}
+		v.extend_from_slice(input);
+		match b % 3 {
+			0 => v.truncate(n / 2),
+			1 => v.clear(),
+			_ => {
+				v.pop();
+			}
+		}
+		v.extend_from_slice(&out);
+		out.extend_from_slice(&(v.len() as u32).to_be_bytes());
+		let mut sum = 0u8;
+		for &x in &v {
+			sum = sum.wrapping_mul(31).wrapping_add(x);
+		}
+		out.push(sum);
+		held.push(v);
+		out.truncate(out.len().min(4096));
+	}
+	for v in &held {
+		out.extend_from_slice(&v[..v.len().min(3)]);
+	}
+	out
+}

go/fixtures/reject/expected.txt

@@ -4,12 +4,18 @@ a generic `pub` function, which translates only as its callers instantiate it, i
 `READY`, a static of `std::sync::atomic::Atomic<bool>`, which has interior mutability Go callers would share unsynchronized, is not supported at fixtures/reject/src/lib.rs:14:1
 `ENVIRON`, a static another language defines, is not supported at fixtures/reject/src/lib.rs:18:2
 a `pub` constant of a generic type, which translates only as its uses instantiate it, is not supported at fixtures/reject/src/lib.rs:25:2
+`*const [u8]`, a raw pointer in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:97:1
 `Callback`, an enum held as its parts that a function pointer in it takes or returns, is not supported at fixtures/reject/src/lib.rs:119:7
 `std::ptr::NonNull`, a raw pointer's wrapper is not supported at fixtures/reject/src/lib.rs:145:19
 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:151:2
 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:152:2
 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:153:2
 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:154:2
+`for<'a> fn(&'a mut std::vec::Vec<u8>)`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:165:1
+`&mut std::vec::Vec<u8>`, a reference inside an argument of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:165:1
+`fn()`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:173:1
+`&mut std::vec::Vec<u8>`, a reference inside an argument of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:181:1
+`*const u8`, a raw pointer in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:188:1
 a type with `pub` methods reached only through a specialized alias is not supported at fixtures/reject/src/lib.rs:142:1
 a reference into part of a static is not supported at fixtures/reject/src/lib.rs:34:3
 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:38:1

go/fixtures/reject/src/lib.rs

@@ -159,3 +159,32 @@ pub struct Fields {
 pub struct _1 {
 	pub count: u8,
 }
+
+/// A function pointer over byte storage, which Go code could fill with a function that
+/// keeps what it is handed past the call.
+pub fn handed(f: fn(&mut Vec<u8>)) -> usize {
+	let mut v = Vec::new();
+	f(&mut v);
+	v.len()
+}
+
+/// A function pointer beside byte storage, which Go code could fill with a closure that
+/// reads `v` while it holds storage of the translation's own.
+pub fn beside(v: &mut Vec<u8>, f: fn()) {
+	v.push(1);
+	f();
+	v.push(2);
+}
+
+/// A reference inside an argument beside byte storage, which the translation could move
+/// out of what handing back walks.
+pub fn popped(mut refs: Vec<&mut Vec<u8>>) {
+	if let Some(v) = refs.pop() {
+		v.push(1);
+	}
+}
+
+/// A raw pointer beside byte storage, which could see that handing in copied it.
+pub fn same(v: Vec<u8>, p: *const u8) -> bool {
+	std::ptr::eq(&v[0], p)
+}