From 63c4a239aaf0106042fdbd38612b2a814700df96 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:18:08 -0800 Subject: [PATCH 01/10] chore(quest): claim quest/m4/go/speed.md Quest-Claim: c686a62dbe6127f6 From bf18c2e8e886039e2f0a0638bc0902a8569f1e81 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 04:29:29 -0800 Subject: [PATCH 02/10] perf(go): own a Vec's storage while only the translation reaches it The runtime grows a byte buffer it owns without zeroing it, through growslice, and recycles an outgrown one through power-of-two pools. Own storage is told by its shape: a power-of-two capacity of at least 8, never filled. A root function trims Go's storage of that shape on the way in and clips its own to its length on the way out, in its result and behind its arguments' &mut and Box, so storage Go can reach is never recycled and shows nothing past its length. A function pointer over byte storage in a root's signature is refused. The bench's ffi.toml drops excludes of crate-private items, which rust2-ffi refuses since #607. Co-Authored-By: Claude Opus 5.5 --- go/Cargo.lock | 5 + go/Cargo.toml | 1 + go/README.md | 17 +- go/bench/ffi.toml | 3 +- go/fixtures/hand/Cargo.toml | 8 + go/fixtures/hand/src/lib.rs | 149 ++++++++++++++ go/fixtures/reject/expected.txt | 1 + go/fixtures/reject/src/lib.rs | 8 + go/runtime/rt.go | 133 +++++++++++++ go/runtime/rt_fmt.go | 2 +- go/src/lower/hand.rs | 330 ++++++++++++++++++++++++++++++++ go/src/lower/library.rs | 7 + go/src/lower/mod.rs | 42 +++- go/tests/fixtures.rs | 10 + go/tests/hand/hand_test.go | 147 ++++++++++++++ 15 files changed, 852 insertions(+), 11 deletions(-) create mode 100644 go/fixtures/hand/Cargo.toml create mode 100644 go/fixtures/hand/src/lib.rs create mode 100644 go/src/lower/hand.rs create mode 100644 go/tests/hand/hand_test.go diff --git a/go/Cargo.lock b/go/Cargo.lock index fec1f2b20..3abbfab79 100644 --- a/go/Cargo.lock +++ b/go/Cargo.lock @@ -31,6 +31,10 @@ checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" name = "fmt" version = "0.0.0" +[[package]] +name = "hand" +version = "0.0.0" + [[package]] name = "hashbrown" version = "0.17.1" @@ -127,6 +131,7 @@ dependencies = [ "codec", "enums", "fmt", + "hand", "ints", "items", "order", diff --git a/go/Cargo.toml b/go/Cargo.toml index a40dbbc43..61512499f 100644 --- a/go/Cargo.toml +++ b/go/Cargo.toml @@ -29,6 +29,7 @@ toml = "1.1.8" codec = { path = "fixtures/codec" } enums = { path = "fixtures/enums" } fmt = { path = "fixtures/fmt" } +hand = { path = "fixtures/hand" } ints = { path = "fixtures/ints" } items = { path = "fixtures/items" } order = { path = "fixtures/order" } diff --git a/go/README.md b/go/README.md index bcfe3b047..9c596fcbe 100644 --- a/go/README.md +++ b/go/README.md @@ -45,6 +45,17 @@ std translates from its own MIR, down to bindings for what Go does itself: a `Ve growth, a slice's iterator, UTF-8 validation, and the formatting machinery, which `runtime/rt_fmt.go` ports from core so `{:?}` and `{:#?}` print as Rust's do. +A `Vec`'s storage is the runtime's own while only the translation reaches it: it +grows to twice its capacity without zeroing, through the runtime's `growslice`, and an +outgrown buffer is recycled for the next grow. Own storage is told apart by its shape, a +capacity that is a power of two of at least 8 and never filled. A root function, which +Go calls, trims Go's storage it is passed off that shape, a byte less capacity, so it is +never recycled, and clips own storage it hands back to Go, in its result or behind an +argument's `&mut` or `Box`, to its length, so Go never sees past it and it is never +recycled either; Go's `append` to such a slice copies. A function pointer over byte +storage in a root's signature is refused, since Go code could fill it with a function +that keeps what it is handed. + A panic is a Go panic of `RustPanic` carrying native Rust's message. As with `panic = "abort"`, no cleanup runs on the way out, so a caller that recovers one must drop what the translation held. @@ -54,7 +65,8 @@ drop what the translation held. `fixtures/reject/expected.txt` lists them with their spans: 128-bit integers, floats, a `Drop` impl, a trait object of any trait but `Display` and `Debug`, a union, a type generic over a constant, such as an array's by-value iterator, and a `pub` item that is -generic, a constant, or a static. +generic, a constant, or a static, or whose signature holds a function pointer over byte +storage. ## Tests @@ -64,7 +76,8 @@ Rust on the same inputs, failing on any difference in output or panic message: integer width's arithmetic, overflow, division by zero, and shifts, `fixtures/order` calls, reads, and overflow checks a folded temporary could reorder or skip, and `fixtures/enums` enums held as their parts, whose pointers a callee could write through -or see changed. +or see changed, and `fixtures/hand` byte storage made, grown, and shrunk each way, which +`tests/hand/hand_test.go` then holds onto from Go as Go code may. `just go bench` times the codec's workloads translated against the same crate exported by rust2-ffi and called through cgo: per-call varints and frame headers, and `stream/`, a diff --git a/go/bench/ffi.toml b/go/bench/ffi.toml index eff5b1f9c..3dce410e7 100644 --- a/go/bench/ffi.toml +++ b/go/bench/ffi.toml @@ -1,4 +1,3 @@ -# The codec fixture's C ABI: only its entry points, as the rest is no API. +# The codec fixture's C ABI: only its entry points, the rest being crate-private. package = "codec" prefix = "codec" -exclude = ["SessionError", "Timescale", "Version", "stats"] diff --git a/go/fixtures/hand/Cargo.toml b/go/fixtures/hand/Cargo.toml new file mode 100644 index 000000000..16fe78c02 --- /dev/null +++ b/go/fixtures/hand/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hand" +version = "0.0.0" +edition = "2024" +publish = false + +[lints] +workspace = true diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs new file mode 100644 index 000000000..a5ab61a06 --- /dev/null +++ b/go/fixtures/hand/src/lib.rs @@ -0,0 +1,149 @@ +//! Byte storage handed between Go code and the translation every way a root function's +//! signature can pass it, which `tests/hand/hand_test.go` holds onto from Go, and `run`, +//! each way the translation grows, copies, and shrinks a `Vec`. + +/// Bytes Go code holds only through methods. +pub struct Buf { + data: Vec, +} + +impl Buf { + /// An empty buffer. + pub fn new() -> Self { + Self { data: Vec::new() } + } + + /// Append `n` bytes of `n`. + pub fn fill(&mut self, n: u8) { + append(&mut self.data, n); + } + + /// What it holds. + pub fn bytes(&self) -> &[u8] { + &self.data + } +} + +impl Default for Buf { + fn default() -> Self { + Self::new() + } +} + +/// Bytes Go code reads directly. +pub struct Open { + pub data: Vec, +} + +/// One of two holders of bytes. +pub enum Either { + Left(Vec), + Right(u8, Vec), + Neither, +} + +/// Append `n` bytes of `n`. +pub fn append(v: &mut Vec, n: u8) { + for _ in 0..n { + v.push(n); + } +} + +/// Append `n` bytes of `n`, then panic if `n` is odd. +pub fn append_or_panic(v: &mut Vec, n: u8) { + append(v, n); + assert!(n % 2 == 0, "odd"); +} + +/// `n` bytes of `n`, with the room of `n` more. +pub fn fill(n: u8) -> Vec { + let mut v = Vec::with_capacity(2 * n as usize); + append(&mut v, n); + v +} + +/// `v` with `n` bytes of `n` appended. +pub fn grow(mut v: Vec, n: u8) -> Vec { + append(&mut v, n); + v +} + +/// Append `n` bytes of `n` to what `b` holds. +pub fn boxed(mut b: Box, n: u8) { + append(&mut b.data, n); +} + +/// `n` buffers, the `i`th of `i` bytes. +pub fn nested(n: u8) -> Vec> { + let mut out = Vec::new(); + for i in 0..n { + out.push(fill(i)); + } + out +} + +/// `n` bytes of `n`, if `n` is not zero. +pub fn maybe(n: u8) -> Option> { + match n { + 0 => None, + n => Some(fill(n)), + } +} + +/// Append `n` bytes of `n` to `v`, giving what it then holds. +pub fn view(v: &mut Vec, n: u8) -> &[u8] { + append(v, n); + v +} + +/// Switch `e` to the other side, its bytes with `n` bytes of `n` appended. +pub fn flip(e: &mut Either, n: u8) { + *e = match std::mem::replace(e, Either::Neither) { + Either::Left(v) => Either::Right(n, grow(v, n)), + Either::Right(_, v) => Either::Left(grow(v, n)), + Either::Neither => Either::Left(fill(n)), + }; +} + +/// Each way the translation makes, grows, copies, and shrinks a `Vec`, its sizes +/// from the input's bytes: what each ends up holding. +pub fn run(input: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut held: Vec> = Vec::new(); + for (i, &b) in input.iter().enumerate() { + let n = b as usize * (i + 1); + let mut v = match b % 4 { + 0 => Vec::new(), + 1 => Vec::with_capacity(n), + 2 => input.to_vec(), + _ => match held.last() { + Some(last) => last.clone(), + None => Vec::new(), + }, + }; + for k in 0..n { + v.push(k as u8); + } + v.extend_from_slice(input); + match b % 3 { + 0 => v.truncate(n / 2), + 1 => v.clear(), + _ => { + v.pop(); + } + } + v.extend_from_slice(&out); + out.extend_from_slice(&(v.len() as u32).to_be_bytes()); + let mut sum = 0u8; + for &x in &v { + sum = sum.wrapping_mul(31).wrapping_add(x); + } + out.push(sum); + held.push(v); + out.truncate(out.len().min(4096)); + } + for v in &held { + out.extend_from_slice(&v[..v.len().min(3)]); + } + out +} diff --git a/go/fixtures/reject/expected.txt b/go/fixtures/reject/expected.txt index 44378d118..ab6fdb031 100644 --- a/go/fixtures/reject/expected.txt +++ b/go/fixtures/reject/expected.txt @@ -7,6 +7,7 @@ a `pub` field not named in ASCII with a leading lowercase letter, which Go canno a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:128:2 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:129:2 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:130:2 +`for<'a> fn(&'a mut std::vec::Vec)`, a function pointer over byte storage in a root's signature, is not supported at fixtures/reject/src/lib.rs:141:1 a type with `pub` methods reached only through a specialized alias is not supported at fixtures/reject/src/lib.rs:118:1 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:14:1 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:15:2 diff --git a/go/fixtures/reject/src/lib.rs b/go/fixtures/reject/src/lib.rs index 923f702fc..bcdfec613 100644 --- a/go/fixtures/reject/src/lib.rs +++ b/go/fixtures/reject/src/lib.rs @@ -135,3 +135,11 @@ pub struct Fields { pub struct _1 { pub count: u8, } + +/// A function pointer over byte storage, which Go code could fill with a function that +/// keeps what it is handed past the call. +pub fn handed(f: fn(&mut Vec)) -> usize { + let mut v = Vec::new(); + f(&mut v); + v.len() +} diff --git a/go/runtime/rt.go b/go/runtime/rt.go index 25256763b..f16f2fab6 100644 --- a/go/runtime/rt.go +++ b/go/runtime/rt.go @@ -9,6 +9,8 @@ import ( "fmt" "math/bits" "slices" + "sync" + "unsafe" ) type tuple1[A any] struct{ F0 A } @@ -319,6 +321,131 @@ func vecGrow[T any](v *[]T, n int) { *v = slices.Grow(*v, max(2*cap(*v), len(*v)+n)-len(*v)) } +// A `Vec`'s storage is the translation's own, to grow without zeroing and to recycle +// once outgrown, until Go code outside the translation can reach it. Own storage is told +// apart by its shape: a capacity that is a power of two of at least 8, never filled, since +// own growth always leaves a byte spare. A root function hands Go storage in through +// bytesEnter, which trims a Go slice of that shape by a byte, and own storage out through +// bytesEscape, which clips it to its length, so storage Go can reach never has the shape +// and is never recycled. Empty storage, such as `Vec::new`'s, grows into own storage. + +// bytePools[k] holds own buffers of capacity 1<= 8 && c&(c-1) == 0 && len(s) < c +} + +// bytesTake is own storage of capacity 1<::with_capacity`, as vecWithCapacity is for others. +func bytesWithCapacity(n uint) []uint8 { + if n > 1<<63-1 { + panic(RustPanic{"capacity overflow"}) + } + if n == 0 { + return nil + } + if n >= 1<<62 { + // More than any allocation, which fails where Rust's would abort. + return make([]uint8, 0, n) + } + return bytesTake(max(bits.Len(n), 3)) +} + +// bytesPush is `Vec::::push`. +func bytesPush(v *[]uint8, x uint8) { + if len(*v)+1 >= cap(*v) { + bytesGrow(v, 1) + } + *v = append(*v, x) +} + +// bytesExtend is `Vec::::extend_from_slice`. +func bytesExtend(v *[]uint8, xs []uint8) { + if len(*v)+len(xs) >= cap(*v) { + bytesGrow(v, len(xs)) + } + *v = append(*v, xs...) +} + +// bytesClone is a copy of xs in own storage: `Vec::::clone` and `<[u8]>::to_vec`. +func bytesClone(xs []uint8) []uint8 { + var v []uint8 + bytesExtend(&v, xs) + return v +} + +// bytesGrow makes room for n more bytes and a spare one, as vecGrow does. Own storage +// grows into own storage and is recycled; Go's grows as `append` would, zeroed, and is +// trimmed off the shape of own storage. +func bytesGrow(v *[]uint8, n int) { + s := *v + want := max(2*cap(s), len(s)+n+1) + if cap(s) == 0 || bytesOwn(s) { + next := bytesTake(max(bits.Len(uint(want-1)), 3))[:len(s)] + copy(next, s) + if cap(s) != 0 { + bytePools[bits.Len(uint(cap(s)))-1].Put(unsafe.Pointer(unsafe.SliceData(s))) + } + *v = next + return + } + s = slices.Grow(s, want-len(s)) + if c := cap(s); c&(c-1) == 0 { + s = s[: len(s) : c-1] + } + *v = s +} + +// bytesEnter takes Go's storage into a root function, trimmed off the shape of own +// storage so it is never recycled. +func bytesEnter(v *[]uint8) { + if bytesOwn(*v) { + *v = (*v)[: len(*v) : cap(*v)-1] + } +} + +// bytesEscape hands own storage out of a root function to Go, clipped to its length, +// past which lie another buffer's bytes, and so never recycled. +func bytesEscape(v *[]uint8) { + if bytesOwn(*v) { + clip(v) + } +} + +// clip hides a slice's capacity past its length, which may hold another buffer's bytes. +func clip[T any](v *[]T) { + *v = (*v)[:len(*v):len(*v)] +} + +type rawSlice struct { + ptr unsafe.Pointer + len, cap int +} + +// growslice is the runtime's, which keeps its signature for the packages that link it: +// go.dev/issue/67401. +// +//go:linkname growslice runtime.growslice +func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et unsafe.Pointer) rawSlice + func vecTruncate[T any](v *[]T, n uint) { if n < uint(len(*v)) { *v = (*v)[:n] @@ -430,3 +557,9 @@ func fromBe64(b [8]uint8) uint64 { return binary.BigEndian.Uint64(b[:]) } func fromLe16(b [2]uint8) uint16 { return binary.LittleEndian.Uint16(b[:]) } func fromLe32(b [4]uint8) uint32 { return binary.LittleEndian.Uint32(b[:]) } func fromLe64(b [8]uint8) uint64 { return binary.LittleEndian.Uint64(b[:]) } + +// zero sets *p to its zero value. +func zero[T any](p *T) { + var z T + *p = z +} diff --git a/go/runtime/rt_fmt.go b/go/runtime/rt_fmt.go index 3e6bbbd58..853a0beec 100644 --- a/go/runtime/rt_fmt.go +++ b/go/runtime/rt_fmt.go @@ -38,7 +38,7 @@ type sink interface{ writeStr(s []uint8) fmtResult } type byteSink struct{ buf *[]uint8 } func (b byteSink) writeStr(s []uint8) fmtResult { - vecExtend(b.buf, s) + bytesExtend(b.buf, s) return fmtOk } diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs new file mode 100644 index 000000000..d8c5356a6 --- /dev/null +++ b/go/src/lower/hand.rs @@ -0,0 +1,330 @@ +//! Handing a `Vec`'s storage between Go code and the translation. The runtime grows +//! storage it owns without zeroing it and recycles it once outgrown, telling it apart by +//! its shape (`bytesOwn` in `runtime/rt.go`). Go code reaches storage only through a root +//! function, so the root hands Go's storage in trimmed off that shape, and its own out +//! clipped to its length: in, through each argument it is passed by value or through +//! `&mut` or `Box`; out, through its result and whatever its arguments reach behind a +//! `&mut` or `Box`, which Go code keeps. + +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::ty::util::IntTypeExt; +use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt, TypingEnv, Unnormalized}; +use rustc_span::Span; + +use super::body::int_bits; +use super::library::{self, Mapped}; +use super::ty::variant_field; +use super::{Translator, names}; + +/// Which way a walk hands storage, and so what it does with each buffer it reaches. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum Hand { + /// Into a root: each `Vec` of Go's is trimmed off the shape of own storage. + In, + /// Out of a root, in its result: each `Vec` of own storage, and each byte slice, + /// is clipped to its length. + Out, + /// Out of a root, in an argument: as `Out`, but only behind a pointer, where Go's + /// memory is, since the argument's own slots are the root's copies. + Back, +} + +impl Hand { + /// The way of what lies behind a pointer, or in a slice's elements. + fn behind(self) -> Self { + match self { + Hand::Back => Hand::Out, + hand => hand, + } + } +} + +fn is_u8(ty: Ty<'_>) -> bool { + matches!(ty.kind(), ty::Uint(ty::UintTy::U8)) +} + +impl<'tcx> Translator<'tcx> { + /// Whether a walk handing a value of `ty` this way has anything to do. + pub fn hands(&self, ty: Ty<'tcx>, hand: Hand) -> bool { + self.hands_in(ty, hand, &mut Vec::new()) + } + + /// `hands` of `ty` inside the walks in `seen`: one met again, as a recursive type's + /// is, adds nothing to its first. + fn hands_in(&self, ty: Ty<'tcx>, hand: Hand, seen: &mut Vec<(Ty<'tcx>, Hand)>) -> bool { + if seen.contains(&(ty, hand)) { + return false; + } + seen.push((ty, hand)); + let tcx = self.tcx; + let found = match *ty.kind() { + ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { + Some(Mapped::Vec) => match is_u8(args.type_at(0)) { + true => hand != Hand::Back, + false => self.hands_in(args.type_at(0), hand.behind(), seen), + }, + Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, hand, seen), + Some(Mapped::SliceIter) => hand == Hand::Out && is_u8(args.type_at(0)), + Some(Mapped::Transparent) => self.hands_in(args.type_at(0), hand, seen), + Some(Mapped::Option | Mapped::Result) | None => adt + .all_fields() + .any(|field| self.hands_in(self.mono_field_ty(field, args), hand, seen)), + Some(Mapped::Runtime(_) | Mapped::Refused(_)) => false, + }, + ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => self.hands_behind(inner, m.is_mut(), hand, seen), + ty::Tuple(tys) => tys.iter().any(|t| self.hands_in(t, hand, seen)), + ty::Array(elem, _) => self.hands_in(elem, hand, seen), + ty::Closure(_, args) => args + .as_closure() + .upvar_tys() + .iter() + .any(|t| self.hands_in(t, hand, seen)), + _ => false, + }; + seen.pop(); + found + } + + /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`. + fn hands_behind(&self, inner: Ty<'tcx>, mutable: bool, hand: Hand, seen: &mut Vec<(Ty<'tcx>, Hand)>) -> bool { + match *inner.kind() { + ty::Str => hand == Hand::Out, + ty::Slice(elem) => { + (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, hand.behind(), seen)) + } + ty::Dynamic(..) => false, + _ => mutable && self.hands_in(inner, hand.behind(), seen), + } + } + + /// A field's type at `args`. + fn mono_field_ty(&self, field: &ty::FieldDef, args: ty::GenericArgsRef<'tcx>) -> Ty<'tcx> { + let fty = field.ty(self.tcx, args).skip_norm_wip(); + self.tcx + .normalize_erasing_regions(TypingEnv::fully_monomorphized(), Unnormalized::new_wip(fty)) + } + + /// The Go function that walks a value of `ty` behind a pointer, handing its storage + /// this way, declared if new. Only for a `ty` that `hands`. + pub fn hand(&mut self, ty: Ty<'tcx>, hand: Hand, span: Span) -> String { + if let Some(name) = self.hand_fns.get(&(ty, hand)) { + return name.clone(); + } + let tcx = self.tcx; + let way = match hand { + Hand::In => "In", + Hand::Out => "Out", + Hand::Back => "Back", + }; + let name = self.fresh(&format!("hand{way}_{}", names::ty_short(tcx, ty))); + self.hand_fns.insert((ty, hand), name.clone()); + let go = self.ty(ty, span); + let mut body = Vec::new(); + match *ty.kind() { + ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { + Some(Mapped::Vec) => { + let elem = args.type_at(0); + if is_u8(elem) { + body.push(match hand { + Hand::In => "bytesEnter(p)".into(), + _ => "bytesEscape(p)".into(), + }); + } else { + let f = self.hand(elem, hand.behind(), span); + if hand != Hand::In { + // A slot past the length holds what was moved out of it, which + // Go code could still reach. + body.push("clear((*p)[len(*p):cap(*p)])".into()); + } + body.push(format!("for i := range *p {{ {f}(&(*p)[i]) }}")); + } + } + Some(Mapped::Box) => self.hand_behind(args.type_at(0), true, hand, "p", &mut body, span), + Some(Mapped::SliceIter) => body.push("clip(&p.s)".into()), + Some(Mapped::Transparent) => { + let f = self.hand(args.type_at(0), hand, span); + body.push(format!("{f}(p)")); + } + _ => self.hand_adt(adt, args, hand, &mut body, span), + }, + ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => { + self.hand_behind(inner, m.is_mut(), hand, "p", &mut body, span) + } + ty::Tuple(tys) => { + for (i, t) in tys.iter().enumerate() { + if self.hands(t, hand) { + let f = self.hand(t, hand, span); + body.push(format!("{f}(&p.F{i})")); + } + } + } + ty::Array(elem, _) => { + let f = self.hand(elem, hand, span); + body.push(format!("for i := range p {{ {f}(&p[i]) }}")); + } + ty::Closure(_, args) => { + for (i, t) in args.as_closure().upvar_tys().iter().enumerate() { + if self.hands(t, hand) { + let f = self.hand(t, hand, span); + body.push(format!("{f}(&p.F{i})")); + } + } + } + _ => unreachable!("a walk of `{ty}`, which hands nothing"), + } + let what = match hand { + Hand::In => "into a root function", + Hand::Out => "out of a root function", + Hand::Back => "back to Go in a root function's argument", + }; + let text = format!( + "// {name} hands the storage of a `{ty}` {what}.\nfunc {name}(p *{go}) {{\n\t{}\n}}\n", + body.join("\n\t") + ); + self.out + .entry(LOCAL_CRATE) + .or_default() + .funcs + .push((name.clone(), text)); + name + } + + /// The walk of a pointer to `inner`, at `p`, a pointer to the pointer, or to the + /// slice of an unsized `inner`. + fn hand_behind(&mut self, inner: Ty<'tcx>, mutable: bool, hand: Hand, p: &str, body: &mut Vec, span: Span) { + match *inner.kind() { + ty::Str => body.push(format!("clip({p})")), + ty::Slice(elem) => { + if hand == Hand::Out && is_u8(elem) { + body.push(format!("clip({p})")); + } + if mutable && self.hands(elem, hand.behind()) { + let f = self.hand(elem, hand.behind(), span); + body.push(format!("for i := range *{p} {{ {f}(&(*{p})[i]) }}")); + } + } + _ => { + let f = self.hand(inner, hand.behind(), span); + body.push(format!("if *{p} != nil {{ {f}(*{p}) }}")); + } + } + } + + /// The walk of a struct's fields, or of an enum's variant it holds. Handing out also + /// zeroes the variants it does not hold, whose fields may be what was moved out. + fn hand_adt( + &mut self, + adt: AdtDef<'tcx>, + args: ty::GenericArgsRef<'tcx>, + hand: Hand, + body: &mut Vec, + span: Span, + ) { + let tcx = self.tcx; + for (v, variant) in adt.variants().iter_enumerated() { + let mut walks = Vec::new(); + for (i, field) in variant.fields.iter().enumerate() { + let fty = self.mono_field_ty(field, args); + if !self.hands(fty, hand) { + continue; + } + let f = self.hand(fty, hand, span); + let field = names::field(field, i); + walks.push(match adt.is_enum() { + true => format!("{f}(&p.{}.{field})", variant_field(variant.name.as_str())), + false => format!("{f}(&p.{field})"), + }); + } + if walks.is_empty() { + continue; + } + if !adt.is_enum() { + body.extend(walks); + continue; + } + let discr_ty = adt.repr().discr_type().to_ty(tcx); + let bits = adt.discriminant_for_variant(tcx, v).val; + let size = int_bits(tcx, discr_ty); + let value = match discr_ty.kind() { + ty::Int(_) => (((bits << (128 - size)) as i128) >> (128 - size)).to_string(), + _ => (bits & (u128::MAX >> (128 - size))).to_string(), + }; + let tag = format!("{}({value})", self.ty(discr_ty, span)); + let mut text = format!("if p.tag == {tag} {{ {} }}", walks.join("; ")); + if hand != Hand::In { + text.push_str(&format!( + " else {{ zero(&p.{}) }}", + variant_field(variant.name.as_str()) + )); + } + body.push(text); + } + } +} + +impl<'tcx> Translator<'tcx> { + /// Refuse a function pointer a root's signature reaches whose own signature reaches + /// byte storage: Go code could pass a function of its own, which the translation would + /// hand its own storage to, or take Go's from, past the root's walks. + pub fn refuse_callback(&mut self, ty: Ty<'tcx>, span: Span) { + let mut seen = Vec::new(); + let found = self.find_part(ty, &mut seen, &|t, this| { + matches!(t.kind(), ty::FnPtr(..)) + && this + .find_part(t, &mut Vec::new(), &|t, this| bytes(this.tcx, t)) + .is_some() + }); + if let Some(f) = found { + self.err( + span, + format!("`{f}`, a function pointer over byte storage in a root's signature, is not supported"), + ); + } + } + + /// The first type `pick` takes among `ty` and every type it is made of: fields, generic + /// arguments, pointees, elements, captures, and a function pointer's parameters and result. + fn find_part( + &self, + ty: Ty<'tcx>, + seen: &mut Vec>, + pick: &dyn Fn(Ty<'tcx>, &Self) -> bool, + ) -> Option> { + if seen.contains(&ty) { + return None; + } + seen.push(ty); + if pick(ty, self) { + return Some(ty); + } + let tcx = self.tcx; + let parts: Vec> = match *ty.kind() { + ty::Adt(adt, args) => { + let mut parts: Vec> = args.types().collect(); + if library::mapped(tcx, adt.did()).is_none() { + parts.extend(adt.all_fields().map(|f| self.mono_field_ty(f, args))); + } + parts + } + ty::Ref(_, inner, _) | ty::RawPtr(inner, _) | ty::Slice(inner) | ty::Array(inner, _) => vec![inner], + ty::Tuple(tys) => tys.to_vec(), + ty::Closure(_, args) => args.as_closure().upvar_tys().to_vec(), + ty::FnPtr(..) => { + let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx)); + sig.inputs_and_output.to_vec() + } + _ => Vec::new(), + }; + parts.into_iter().find_map(|t| self.find_part(t, seen, pick)) + } +} + +/// Whether `ty` is byte storage: `str`, `[u8]`, or `Vec`. +fn bytes<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + match ty.kind() { + ty::Str => true, + ty::Slice(elem) => is_u8(*elem), + ty::Adt(adt, args) => library::mapped(tcx, adt.did()) == Some(Mapped::Vec) && is_u8(args.type_at(0)), + _ => false, + } +} diff --git a/go/src/lower/library.rs b/go/src/lower/library.rs index f0d7ec6d2..4e1a9dd90 100644 --- a/go/src/lower/library.rs +++ b/go/src/lower/library.rs @@ -224,6 +224,8 @@ pub fn call_with<'tcx>( } let key = key(tcx, did); let targ = |i: usize| instance.args.types().nth(i).expect("a binding's type argument"); + // A `Vec`, whose storage the runtime owns while the translation holds it. + let bytes = || matches!(targ(0).kind(), ty::Uint(ty::UintTy::U8)); let dest = |fx: &mut Fx<'_, 'tcx>| fx.ty(dest_ty, span); let expr = Bound::Expr; let stmt = Bound::Stmt; @@ -235,10 +237,12 @@ pub fn call_with<'tcx>( "::new" => expr(format!("ptrTo({})", a[0])), "alloc::boxed::box_assume_init_into_vec_unsafe" => expr(format!("(*{})[:]", a[0])), "::new" => expr(format!("{}(nil)", dest(fx))), + "::with_capacity" if bytes() => expr(format!("bytesWithCapacity({})", a[0])), "::with_capacity" => { let size = tcx.layout_of(fx.t_env().as_query_input(targ(0))).ok()?.size.bytes(); expr(format!("vecWithCapacity[{}]({}, {size})", fx.ty(targ(0), span), a[0])) } + "::push" if bytes() => stmt(format!("bytesPush({}, {})", a[0], a[1])), "::push" => stmt(format!("vecPush({}, {})", a[0], a[1])), "::pop" => expr(format!("vecPop({})", a[0])), "::len" => expr(format!("uint(len(*{}))", a[0])), @@ -254,6 +258,7 @@ pub fn call_with<'tcx>( fx_drop_check(fx, targ(0), span); stmt(format!("*{0} = (*{0})[:0]", a[0])) } + "::extend_from_slice" if bytes() => stmt(format!("bytesExtend({}, {})", a[0], a[1])), "::extend_from_slice" => { copy_check(fx, targ(0), span); stmt(format!("vecExtend({}, {})", a[0], a[1])) @@ -262,6 +267,7 @@ pub fn call_with<'tcx>( | "::as_mut_slice" | "::deref" | "::deref_mut" => expr(format!("*{}", a[0])), + "::clone" if bytes() => expr(format!("bytesClone(*{})", a[0])), "::clone" => { copy_check(fx, targ(0), span); expr(format!("append({}(nil), (*{})...)", dest(fx), a[0])) @@ -305,6 +311,7 @@ pub fn call_with<'tcx>( "::next" => { expr(format!("sliceIterNext({})", a[0])) } + "<[T]>::to_vec" if bytes() => expr(format!("bytesClone({})", a[0])), "<[T]>::to_vec" => { copy_check(fx, targ(0), span); expr(format!("append({}(nil), {}...)", dest(fx), a[0])) diff --git a/go/src/lower/mod.rs b/go/src/lower/mod.rs index 1bcb6e881..9074a170f 100644 --- a/go/src/lower/mod.rs +++ b/go/src/lower/mod.rs @@ -7,6 +7,7 @@ mod body; mod consts; mod fold; +mod hand; mod library; mod names; mod split; @@ -21,6 +22,8 @@ use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc_middle::ty::{Instance, Ty, TyCtxt, TyKind, TypingEnv, Unnormalized}; use rustc_span::Span; +use hand::Hand; + /// The runtime's files, copied into the package beside the translation. const RUNTIME: &[(&str, &str)] = &[ ("rt.go", include_str!("../../runtime/rt.go")), @@ -177,6 +180,8 @@ pub struct Translator<'tcx> { splits: HashMap, Option>>>, /// The types whose parts are being found. splitting: Vec>, + /// The Go function walking each type to hand its storage each way. + hand_fns: HashMap<(Ty<'tcx>, hand::Hand), String>, /// Each refusal reported, so a construct met again at the same place is reported once. reported: HashSet<(Span, String)>, failed: bool, @@ -212,6 +217,7 @@ impl<'tcx> Translator<'tcx> { allocs: HashMap::new(), splits: HashMap::new(), splitting: Vec::new(), + hand_fns: HashMap::new(), reported: HashSet::new(), failed: false, } @@ -336,19 +342,26 @@ impl<'tcx> Translator<'tcx> { let inputs: Vec> = sig.inputs().iter().map(|t| normal(*t)).collect(); let output = normal(sig.output()); let splits = inputs.iter().chain([&output]).any(|t| self.split(*t, span).is_some()); - if !splits { + let hands = inputs + .iter() + .any(|t| self.hands(*t, Hand::In) || self.hands(*t, Hand::Back)) + || self.hands(output, Hand::Out); + for t in inputs.iter().chain([&output]) { + self.refuse_callback(*t, span); + } + if !splits && !hands { self.funcs.insert(instance, name); self.queue.push_back((instance, span)); return; } - // Its callers' Go holds its enums whole. + // Its callers' Go holds its enums whole and its storage handed over. let inner = self.func(instance, span); let text = self.wrapper(&name, path, &inner, &inputs, output, span); self.out.entry(def_id.krate).or_default().funcs.push((name, text)); } /// The root `name` of the function at `path`, calling `inner`, the function as - /// translated code calls it, with its enums whole. + /// translated code calls it, with its enums whole and its storage handed over. fn wrapper( &mut self, name: &str, @@ -360,9 +373,17 @@ impl<'tcx> Translator<'tcx> { ) -> String { let mut params = Vec::new(); let mut args = Vec::new(); + let mut enter = String::new(); + let mut back = Vec::new(); for (i, input) in inputs.iter().enumerate() { let param = format!("_{}", i + 1); params.push(format!("{param} {}", self.ty(*input, span))); + if self.hands(*input, Hand::In) { + enter.push_str(&format!("\t{}(&{param})\n", self.hand(*input, Hand::In, span))); + } + if self.hands(*input, Hand::Back) { + back.push(format!("{}(&{param})", self.hand(*input, Hand::Back, span))); + } match self.split(*input, span) { Some(split) => args.extend(split.parts.iter().map(|p| match p.big { true => format!("&{param}{}", p.path), @@ -375,6 +396,14 @@ impl<'tcx> Translator<'tcx> { true => String::new(), false => format!(" {}", self.ty(output, span)), }; + if !back.is_empty() { + // Even when a panic unwinds out, what Go keeps is handed back. + enter.push_str(&format!("\tdefer func() {{ {} }}()\n", back.join("; "))); + } + let out = match self.hands(output, Hand::Out) { + true => format!("\t{}(&_0)\n", self.hand(output, Hand::Out, span)), + false => String::new(), + }; let body = match self.split(output, span) { Some(split) => { let mut lhs = Vec::new(); @@ -385,16 +414,17 @@ impl<'tcx> Translator<'tcx> { } } format!( - "\tvar _0{ret}\n\t{} = {inner}({})\n\treturn _0\n", + "\tvar _0{ret}\n\t{} = {inner}({})\n{out}\treturn _0\n", lhs.join(", "), args.join(", ") ) } None if ret.is_empty() => format!("\t{inner}({})\n", args.join(", ")), - None => format!("\treturn {inner}({})\n", args.join(", ")), + None if out.is_empty() => format!("\treturn {inner}({})\n", args.join(", ")), + None => format!("\t_0 := {inner}({})\n{out}\treturn _0\n", args.join(", ")), }; format!( - "// {name} is `{path}`.\nfunc {name}({}){ret} {{\n{body}}}\n", + "// {name} is `{path}`.\nfunc {name}({}){ret} {{\n{enter}{body}}}\n", params.join(", ") ) } diff --git a/go/tests/fixtures.rs b/go/tests/fixtures.rs index b787166b1..f68c16935 100644 --- a/go/tests/fixtures.rs +++ b/go/tests/fixtures.rs @@ -297,6 +297,16 @@ fn vecs() { differential("vecs", vecs::run, "Run", cases); } +/// Byte storage the runtime owns, made, grown, copied, and shrunk each way, and handed +/// to Go code, which `tests/hand/hand_test.go` holds onto as Go code may. +#[test] +fn hand() { + differential("hand", hand::run, "Run", cases(0, 0, 500)); + let out = scratch().join("hand/out"); + std::fs::copy(manifest_dir().join("tests/hand/hand_test.go"), out.join("hand_test.go")).unwrap(); + go(&out, &["test", "-count=1", "."]); +} + /// Fields and variants whose escaped names could meet, constant memory, which each use /// shares, and the `pub` API, exported by the paths the crate exports it at. #[test] diff --git a/go/tests/hand/hand_test.go b/go/tests/hand/hand_test.go new file mode 100644 index 000000000..bee918fdf --- /dev/null +++ b/go/tests/hand/hand_test.go @@ -0,0 +1,147 @@ +// Holds onto byte storage the translated `fixtures/hand` hands Go code, as Go code may: +// what it is handed has no capacity past its length, and nothing it keeps is recycled. +package hand_test + +import ( + "bytes" + "testing" + "unsafe" + + fixture "fixture" +) + +// churn grows and outgrows buffers of every size, which would overwrite any buffer Go +// code keeps that the translation recycled. +func churn() { + for i := 0; i < 64; i++ { + var v []uint8 + fixture.Append(&v, uint8(255-i)) + fixture.Run(bytes.Repeat([]uint8{uint8(i)}, 40)) + } +} + +func clipped(t *testing.T, what string, s []uint8) { + t.Helper() + if len(s) != cap(s) { + t.Errorf("%s: length %d, capacity %d", what, len(s), cap(s)) + } +} + +// clean checks that s shows nothing past its length but zeros, as Go storage the +// translation grew does, which `append` would have grown the same. +func clean(t *testing.T, what string, s []uint8) { + t.Helper() + if !bytes.Equal(s[len(s):cap(s)], make([]uint8, cap(s)-len(s))) { + t.Errorf("%s: bytes past its length", what) + } +} + +// kept checks that keep still holds what it did, all the way to its capacity, after +// the translation outgrew it. +func kept(t *testing.T, what string, keep []uint8) { + t.Helper() + want := bytes.Clone(keep[:cap(keep)]) + churn() + if got := keep[:cap(keep)]; !bytes.Equal(got, want) { + t.Errorf("%s changed after the translation outgrew it", what) + } +} + +func TestResultsClipped(t *testing.T) { + for _, n := range []uint8{1, 7, 8, 100, 255} { + clipped(t, "Fill", fixture.Fill(n)) + clipped(t, "Grow", fixture.Grow(nil, n)) + clipped(t, "Maybe", fixture.Maybe(n).Some.F0) + for i, v := range fixture.Nested(n) { + clipped(t, "Nested", v) + if len(v) != i { + t.Errorf("Nested: buffer %d holds %d bytes", i, len(v)) + } + } + var v []uint8 + clipped(t, "View", fixture.View(&v, n)) + clipped(t, "View's argument", v) + b := fixture.BufNew() + fixture.BufFill(&b, n) + clipped(t, "BufBytes", fixture.BufBytes(&b)) + } +} + +// Go's storage of the shape of the translation's own is trimmed, not recycled. +func TestGoStorageKept(t *testing.T) { + s := make([]uint8, 5, 64) + for i := range s[:cap(s)] { + s[:cap(s)][i] = 0xaa + } + keep := s + fixture.Append(&s, 200) + if len(s) != 205 { + t.Fatalf("Append: %d bytes", len(s)) + } + kept(t, "Go's slice passed by pointer", keep) + g := make([]uint8, 3, 128) + keepG := g + r := fixture.Grow(g, 200) + if len(r) != 203 { + t.Fatalf("Grow: %d bytes", len(r)) + } + kept(t, "Go's slice passed by value", keepG) +} + +// What the translation hands out, handed back and outgrown, is not recycled. +func TestHandedKept(t *testing.T) { + r := fixture.Fill(100) + keep := r + fixture.Append(&r, 200) + kept(t, "a result passed back", keep) + + o := &fixture.Open{} + fixture.Boxed(o, 100) + clipped(t, "a boxed field", o.Data) + keep = o.Data + fixture.Boxed(o, 200) + clean(t, "a boxed field grown again", o.Data) + kept(t, "a boxed field", keep) + + var e fixture.Either + fixture.Flip(&e, 50) + clipped(t, "Right", e.Right.F1) + keep = e.Right.F1 + fixture.Flip(&e, 200) + clean(t, "Left", e.Left.F0) + if e.Right.F1 != nil { + t.Errorf("Right still holds %d bytes after switching to Left", len(e.Right.F1)) + } + kept(t, "a variant switched away from", keep) +} + +// A panic unwinding out of the translation still hands back what Go keeps. +func TestPanicHandsBack(t *testing.T) { + var v []uint8 + func() { + defer func() { recover() }() + fixture.AppendOrPanic(&v, 101) + }() + if len(v) != 101 { + t.Fatalf("AppendOrPanic: %d bytes", len(v)) + } + clipped(t, "AppendOrPanic's argument", v) +} + +// Go's storage the translation grows keeps growing as `append` would: each call to +// append one byte copies the buffer only as often as doubling it does. +func TestGoStorageAmortized(t *testing.T) { + var v []uint8 + moves := 0 + var last *uint8 + for range 10_000 { + fixture.Append(&v, 1) + if p := unsafe.SliceData(v); p != last { + moves++ + last = p + } + } + if moves > 20 { + t.Errorf("10000 one-byte appends moved the buffer %d times", moves) + } +} From 0844b9581a5682c791be7ec69dcc827fff52075e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 04:43:58 -0800 Subject: [PATCH 03/10] fix(go): zero the slots a Vec vacates instead of a root's walk clearing past its length A root's walk cleared a Go-held Vec's slots past its length, which may be Go's own elements of a larger array. Pop, truncate, and clear zero what they vacate instead, as slices.Delete does; a Vec's have nothing to zero. Co-Authored-By: Claude Opus 5.5 --- go/fixtures/hand/src/lib.rs | 8 ++++++++ go/runtime/rt.go | 11 +++++++++++ go/src/lower/hand.rs | 5 ----- go/src/lower/library.rs | 4 +++- go/tests/hand/hand_test.go | 11 +++++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs index a5ab61a06..aef75ce62 100644 --- a/go/fixtures/hand/src/lib.rs +++ b/go/fixtures/hand/src/lib.rs @@ -96,6 +96,14 @@ pub fn view(v: &mut Vec, n: u8) -> &[u8] { v } +/// Push `n` bytes of `n` onto `v`, then pop and outgrow them, leaving the slot vacated. +pub fn vacate(v: &mut Vec>, n: u8) -> usize { + v.push(fill(n)); + let mut last = v.pop().unwrap_or_default(); + append(&mut last, 200); + last.len() +} + /// Switch `e` to the other side, its bytes with `n` bytes of `n` appended. pub fn flip(e: &mut Either, n: u8) { *e = match std::mem::replace(e, Either::Neither) { diff --git a/go/runtime/rt.go b/go/runtime/rt.go index f16f2fab6..337a531d9 100644 --- a/go/runtime/rt.go +++ b/go/runtime/rt.go @@ -280,6 +280,7 @@ func vecPop[T any](v *[]T) Option[T] { return Option[T]{} } x := (*v)[n-1] + zero(&(*v)[n-1]) *v = (*v)[:n-1] return Option[T]{tag: 1, Some: struct{ F0 T }{x}} } @@ -446,7 +447,17 @@ type rawSlice struct { //go:linkname growslice runtime.growslice func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et unsafe.Pointer) rawSlice +// vecTruncate is `Vec::truncate`, which zeroes the slots it vacates, as `slices.Delete` +// does, so Go code reading past the length sees nothing moved out of them. func vecTruncate[T any](v *[]T, n uint) { + if n < uint(len(*v)) { + clear((*v)[n:]) + *v = (*v)[:n] + } +} + +// bytesTruncate is `Vec::::truncate`, which has nothing to zero. +func bytesTruncate(v *[]uint8, n uint) { if n < uint(len(*v)) { *v = (*v)[:n] } diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index d8c5356a6..706ac0785 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -131,11 +131,6 @@ impl<'tcx> Translator<'tcx> { }); } else { let f = self.hand(elem, hand.behind(), span); - if hand != Hand::In { - // A slot past the length holds what was moved out of it, which - // Go code could still reach. - body.push("clear((*p)[len(*p):cap(*p)])".into()); - } body.push(format!("for i := range *p {{ {f}(&(*p)[i]) }}")); } } diff --git a/go/src/lower/library.rs b/go/src/lower/library.rs index 4e1a9dd90..fd6cc0684 100644 --- a/go/src/lower/library.rs +++ b/go/src/lower/library.rs @@ -250,13 +250,15 @@ pub fn call_with<'tcx>( expr(fx.err(span, "`Vec::capacity`, which Go's growth would make differ from Rust's")) } "::is_empty" => expr(format!("len(*{}) == 0", a[0])), + "::truncate" if bytes() => stmt(format!("bytesTruncate({}, {})", a[0], a[1])), "::truncate" => { fx_drop_check(fx, targ(0), span); stmt(format!("vecTruncate({}, {})", a[0], a[1])) } + "::clear" if bytes() => stmt(format!("*{0} = (*{0})[:0]", a[0])), "::clear" => { fx_drop_check(fx, targ(0), span); - stmt(format!("*{0} = (*{0})[:0]", a[0])) + stmt(format!("vecTruncate({}, 0)", a[0])) } "::extend_from_slice" if bytes() => stmt(format!("bytesExtend({}, {})", a[0], a[1])), "::extend_from_slice" => { diff --git a/go/tests/hand/hand_test.go b/go/tests/hand/hand_test.go index bee918fdf..19ab4e407 100644 --- a/go/tests/hand/hand_test.go +++ b/go/tests/hand/hand_test.go @@ -115,6 +115,17 @@ func TestHandedKept(t *testing.T) { kept(t, "a variant switched away from", keep) } +// A slot the translation vacates past a length shows nothing it held. +func TestVacatedZeroed(t *testing.T) { + v := make([][]uint8, 0, 4) + if n := fixture.Vacate(&v, 100); n != 300 { + t.Fatalf("Vacate: %d bytes", n) + } + if s := v[:cap(v)][0]; s != nil { + t.Errorf("a vacated slot holds %d bytes", len(s)) + } +} + // A panic unwinding out of the translation still hands back what Go keeps. func TestPanicHandsBack(t *testing.T) { var v []uint8 From dd8ee2f04f8708cfced9fbd15a381f81b881fd82 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 04:56:03 -0800 Subject: [PATCH 04/10] chore(go): record the stream ratios with own storage, and the choice of how it escapes Co-Authored-By: Claude Opus 5.5 --- go/fixtures/hand/src/lib.rs | 3 ++- quest/m4/go/speed.md | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs index aef75ce62..c53be330f 100644 --- a/go/fixtures/hand/src/lib.rs +++ b/go/fixtures/hand/src/lib.rs @@ -52,7 +52,7 @@ pub fn append(v: &mut Vec, n: u8) { /// Append `n` bytes of `n`, then panic if `n` is odd. pub fn append_or_panic(v: &mut Vec, n: u8) { append(v, n); - assert!(n % 2 == 0, "odd"); + assert!(n.is_multiple_of(2), "odd"); } /// `n` bytes of `n`, with the room of `n` more. @@ -69,6 +69,7 @@ pub fn grow(mut v: Vec, n: u8) -> Vec { } /// Append `n` bytes of `n` to what `b` holds. +#[expect(clippy::boxed_local, reason = "a Box that Go code passes as its own pointer")] pub fn boxed(mut b: Box, n: u8) { append(&mut b.data, n); } diff --git a/quest/m4/go/speed.md b/quest/m4/go/speed.md index c5149fdb6..513fc7c5a 100644 --- a/quest/m4/go/speed.md +++ b/quest/m4/go/speed.md @@ -83,8 +83,29 @@ Decided with the maintainer on 2026-11-07, settling - ✅ **The bar is unchanged:** every `stream/` workload within 2x of cgo, measured with the ownership bookkeeping in place, else the epic's drop rule. -Open: how a grow tells storage the translation owns from storage Go code has seen. That -bookkeeping eats into the 1.6 to 1.9x margin above, so measure it on the streams. +The runtime tells own storage by its shape, so a grow costs no lookup: a `Vec`'s +capacity a power of two of at least 8 and never filled, since own growth always leaves a +byte spare. A root function trims Go's storage of that shape by a byte on the way in and +clips its own on the way out, through its result and its arguments' `&mut` and `Box`; +`fixtures/hand` holds onto both from Go. Ratios to cgo, CPU time then wall time, medians +of ten runs alternating the variants, on a host at load 11 to 30 of 32 cores: + +| Workload | main | Clip at escape | Copy at escape | +| --- | --- | --- | --- | +| `stream/video/lite-06/encode` | 2.8x, 3.0x | 1.9x, 2.0x | 1.2x, 1.5x | +| `stream/video/lite-06/decode` | 2.6x, 2.7x | 1.5x, 1.7x | 1.3x, 1.8x | +| `stream/audio/lite-06/encode` | 3.1x, 3.0x | 2.0x, 2.0x | 1.4x, 1.7x | +| `stream/audio/lite-06/decode` | 2.8x, 2.8x | 1.4x, 1.7x | 1.5x, 1.9x | +| `stream/video/lite-07/encode` | 2.7x, 2.9x | 1.4x, 1.5x | 1.1x, 1.4x | +| `stream/video/lite-07/decode` | 2.6x, 2.9x | 1.4x, 1.5x | 1.2x, 1.4x | +| `stream/audio/lite-07/encode` | 3.5x, 3.4x | 2.1x, 2.0x | 1.3x, 1.6x | +| `stream/audio/lite-07/decode` | 2.9x, 2.9x | 1.4x, 1.7x | 1.6x, 2.0x | + +Clipping, as decided, leaves audio encoding just past 2x: the result Go keeps is own +storage, up to twice its length, which each run allocates anew, so the collector runs +about twice as often as cgo's, whose result is exact. Copying own storage out at escape +into an exact Go buffer, and recycling it, costs a copy as `C.GoBytes` does and brings +every stream within 2x; it changes the 2026-11-07 decision, so it waits on the maintainer. A video group is 60 frames, 577 KB; an audio group is 50 frames of 160 bytes. Measure each change on these, comparing `cpu-ns/item` and `faults/op` as well as wall time on a @@ -94,3 +115,14 @@ loaded host, since CPU time counts the collector's workers. The per-call diagnos Public API: none beyond clipped capacity: a slice the translation returns has no spare capacity, so a Go caller's `append` to it copies. + +## Waiting + +- Maintainer: how own storage escapes to Go code, now that clipping, as decided on + 2026-11-07, leaves `stream/audio/lite-07/encode` at 2.07x CPU and 2.03x wall time, + and `stream/video/lite-06/encode` at 2.01x wall time. + Options: copy it out at escape into an exact Go buffer and recycle it, measured 1.1 to + 1.6x CPU and 1.4 to 2.0x wall time on every stream (recommended: Go code then never + holds own storage, and the copy is the one cgo's result makes); keep clipping and take + the miss as noise; or drop the translator, as the epic's rule would. Check that this + Plan records the choice. From 91cb8c90b0321889d8fe3e7fbd1760a26af5db38 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:39:28 -0800 Subject: [PATCH 05/10] quest: finish m4/go/speed, keeping clipping as the bar's result The maintainer decided on 2026-11-07 that clipping's 1 to 4% miss of the 2x bar on two stream workloads is noise, so speed meets its bar and coverage splits. Co-Authored-By: Claude Opus 5.5 --- quest/m4/go/README.md | 9 ++- quest/m4/go/coverage.md | 16 +++-- quest/m4/go/speed.md | 128 ---------------------------------------- 3 files changed, 14 insertions(+), 139 deletions(-) delete mode 100644 quest/m4/go/speed.md diff --git a/quest/m4/go/README.md b/quest/m4/go/README.md index 25001d26a..93aad8c2f 100644 --- a/quest/m4/go/README.md +++ b/quest/m4/go/README.md @@ -15,7 +15,7 @@ exporter stays moq's Go route until a translation matches it. Decided with the maintainer on 2026-11-01, settling [#410](https://github.com/moq-dev/rust2/pull/410): -- **Continue as research,** [speed](/quest/m4/go/speed.md) first, then +- **Continue as research,** speed first, then [coverage](/quest/m4/go/coverage.md). If speed cannot bring the translated crate within 2x of cgo on a realistic whole-crate workload (a group's stream end to end, decided on 2026-11-02 in [#428](https://github.com/moq-dev/rust2/pull/428)), delete @@ -25,6 +25,12 @@ Decided with the maintainer on 2026-11-01, settling of a Go use. About a quarter of `ts/src` would serve Go, nearly all of it inside files whose lowering returns TypeScript's AST. +Decided with the maintainer on 2026-11-07, settling +[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643): speed met the bar. With +the translation owning a `Vec`'s storage and clipping it at escape, every `stream/` +workload of `just go bench` runs within 2x of cgo, the worst at 2.07x CPU time, which +counts as noise. + The spike ([moq-dev/rust2#410](https://github.com/moq-dev/rust2/pull/410)) translated moq's codec held to rust2-ts's bar: 35,273 codec cases and 124,193 integer cases match native Rust, panics included. It runs 2 to 113 times slower than cgo on batched loops, @@ -33,7 +39,6 @@ adds 324 KB to a Go binary where cgo adds 1.15 MB and a libc dependency. ## Required -- [Match cgo on the crate](/quest/m4/go/speed.md) - the translated crate within 2x of cgo on a group's stream end to end - [moq-net beyond the codec](/quest/m4/go/coverage.md) - `Drop`, floats, 128-bit integers, trait objects, and collections, so moq-net translates whole - [pub constants and statics](/quest/m4/go/consts.md) - exported as Go declarations instead of refused, so a crate with any translates - [Exported names off the runtime's](/quest/m4/go/runtime-names.md) - a crate's own `Result` or `Formatter` exports beside the runtime's diff --git a/quest/m4/go/coverage.md b/quest/m4/go/coverage.md index 37af9cadb..2581c46a0 100644 --- a/quest/m4/go/coverage.md +++ b/quest/m4/go/coverage.md @@ -26,16 +26,14 @@ The spike refuses each of these with its span (`go/fixtures/reject/expected.txt` The translation targets the host, 64-bit `usize`; a 32-bit Go target needs its own translation. -Decided with the maintainer on 2026-11-06: coverage waits on [speed](/quest/m4/go/speed.md), -as the epic orders them, and splits into one quest per construct above only once speed -reaches its bar. A failed speed result deletes `go/`, so planning the constructs earlier -would be wasted. The last coverage child reruns `just go bench` against speed's bar, since -constructs such as `Vec::capacity` change what the stream workload runs. +Decided with the maintainer on 2026-11-06: coverage waits on speed, as the epic orders +them, and splits into one quest per construct above only once speed reaches its bar. A +failed speed result deletes `go/`, so planning the constructs earlier would be wasted. +The last coverage child reruns `just go bench` against speed's bar, every `stream/` +workload within 2x of cgo, since constructs such as `Vec::capacity` change what the +stream workload runs. Speed reached the bar on 2026-11-07 in +[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643), so coverage now splits. - ✅ Gate on speed, split later. - Split now, each child requiring speed. - Split now, ungated. - -## Required - -- [Match cgo on the crate](/quest/m4/go/speed.md) - the translated crate within 2x of cgo on a group's stream end to end diff --git a/quest/m4/go/speed.md b/quest/m4/go/speed.md deleted file mode 100644 index 513fc7c5a..000000000 --- a/quest/m4/go/speed.md +++ /dev/null @@ -1,128 +0,0 @@ -# [L] Match cgo on the crate - -## Goal - -The translated crate, on `just go bench`'s whole-crate workloads (`stream/`: a group's -stream encoded and decoded end to end, header, frames, and payloads, as moq's consumer -runs it), runs within 2x of the same crate called through cgo, and the differential -tests still pass. If it cannot, the translator is dropped, as -[the epic](/quest/m4/go/README.md) records. The per-call workloads, varints and frame -headers, are diagnostics, not the bar. - -## Plan - -Decided with the maintainer on 2026-11-02, settling -[moq-dev/rust2#428](https://github.com/moq-dev/rust2/pull/428): the 2x bar is for the -whole crate on a realistic workload, not an encode or decode micro benchmark. Even -hand-written Go, a loop parsing as quic-go's `quicvarint.Parse` does, decodes varints -3 to 8 times slower than cgo, so the old per-call bar measured Go against Rust rather -than the translation. - -Folding each temporary assigned once and read once into its reader, as rust2-ts does, -cut decoding's time 35 to 55%. Growing a `Vec` as Rust's `RawVec` does, to at least -twice its capacity where Go's `append` grows a slice past 256 elements by about 1.25x, -cut a video stream's time 37 to 56%. Structuring control flow as rust2-ts does, through the `cfg` both now share -(`driver/src/cfg.rs`), moved none of these past noise. Splitting enums into their parts -cut decoding's time by more than half: an enum of over four words, which Go keeps in -memory, is its tag and its variants' fields, each a Go variable, parameter, and result, -and a field of over four words goes behind a pointer, which a function returning it -writes only for that variant, so `?` hands an error on in its caller's storage. Each -variable is declared where it is used, so Go zeroes it only on a path that reaches it. - -What remains is the **allocator**. The translated codec is now a small part of each -stream's profile; most of the rest is growing the output: allocating each larger -buffer, zeroing and copying into it, and the collector reclaiming the ones it leaves -behind, along with the page faults of memory the scavenger returned to the kernel. -Rust's `realloc` reuses what the old buffer frees and leaves spare capacity -uninitialized. Two levers do the same in Go: - -- **No zeroing:** growing through the runtime's `growslice` (linked as - [go.dev/issue/67401](https://go.dev/issue/67401) keeps open), which leaves a - pointer-free buffer's new storage unzeroed until appends write it. -- **Recycling:** a free list of power-of-two byte buffers, each grow taking its storage - from it and giving back the buffer it outgrew. - -Neither is sound once a `Vec`'s storage reaches Go code outside the translation, where -Rust's borrow checker no longer holds. Go code may read a slice's spare capacity -through `s[:cap(s)]`, which would show stale heap contents; and Go keeps an array an -`append` moved off untouched, so a caller may still read a grown-out buffer through -another slice, such as a struct's field read before a method that grows it. Ratios of -wall time to cgo's, medians of six runs alternating the variants, on a host at load 17 -to 50 of 32 cores, with both levers applied everywhere (unsound, measured in -[moq-dev/rust2#609](https://github.com/moq-dev/rust2/pull/609)): - -| Workload | Go | No zeroing | Both | cgo | -| --- | --- | --- | --- | --- | -| `stream/video/lite-06/encode` | 3.0x | 2.1x | 1.6x | 111..139 µs | -| `stream/video/lite-06/decode` | 2.9x | 2.2x | 1.9x | 100..130 µs | -| `stream/audio/lite-06/encode` | 3.0x | 3.3x | 1.9x | 2.4..3.1 µs | -| `stream/audio/lite-06/decode` | 2.6x | 2.7x | 1.7x | 2.7..2.8 µs | -| `stream/video/lite-07/encode` | 2.7x | 1.9x | 1.6x | 121..158 µs | -| `stream/video/lite-07/decode` | 3.6x | 2.4x | 1.6x | 117..135 µs | -| `stream/audio/lite-07/encode` | 2.9x | 2.8x | 1.8x | 2.7..3.5 µs | -| `stream/audio/lite-07/decode` | 3.4x | 2.5x | 1.8x | 2.6..3.7 µs | - -Recycling alone, zeroing each fresh buffer, measured 1.6 to 3.0x. Growing by 4x cut -fewer bytes than recycling while overshooting memory, and `GODEBUG=madvdontneed=0` -removed the page faults but is the main program's setting, not a library's; both are -dropped. - -Decided with the maintainer on 2026-11-07, settling -[moq-dev/rust2#609](https://github.com/moq-dev/rust2/pull/609): - -- ✅ **The translation owns a `Vec`'s storage** while it stays inside the translation: - the runtime grows it without zeroing, through `growslice`, and recycles outgrown - power-of-two buffers. Where storage reaches Go code outside the translation, as a - public function's argument or result or a field of one, it is handed over safely and - never recycled again. Rejected: skipping zeroing and recycling everywhere, with - documented rules for Go callers, since breaking a rule leaks or corrupts data - silently; and dropping the translator now. -- ✅ **Escape hides spare capacity by clipping:** the translation hands Go code - `s[:len(s):len(s)]`. Rejected: zeroing `[len:cap]` lazily at escape, measuring both, - and a wrapper buffer type in the public Go API in place of `[]byte`. -- ✅ **The bar is unchanged:** every `stream/` workload within 2x of cgo, measured with - the ownership bookkeeping in place, else the epic's drop rule. - -The runtime tells own storage by its shape, so a grow costs no lookup: a `Vec`'s -capacity a power of two of at least 8 and never filled, since own growth always leaves a -byte spare. A root function trims Go's storage of that shape by a byte on the way in and -clips its own on the way out, through its result and its arguments' `&mut` and `Box`; -`fixtures/hand` holds onto both from Go. Ratios to cgo, CPU time then wall time, medians -of ten runs alternating the variants, on a host at load 11 to 30 of 32 cores: - -| Workload | main | Clip at escape | Copy at escape | -| --- | --- | --- | --- | -| `stream/video/lite-06/encode` | 2.8x, 3.0x | 1.9x, 2.0x | 1.2x, 1.5x | -| `stream/video/lite-06/decode` | 2.6x, 2.7x | 1.5x, 1.7x | 1.3x, 1.8x | -| `stream/audio/lite-06/encode` | 3.1x, 3.0x | 2.0x, 2.0x | 1.4x, 1.7x | -| `stream/audio/lite-06/decode` | 2.8x, 2.8x | 1.4x, 1.7x | 1.5x, 1.9x | -| `stream/video/lite-07/encode` | 2.7x, 2.9x | 1.4x, 1.5x | 1.1x, 1.4x | -| `stream/video/lite-07/decode` | 2.6x, 2.9x | 1.4x, 1.5x | 1.2x, 1.4x | -| `stream/audio/lite-07/encode` | 3.5x, 3.4x | 2.1x, 2.0x | 1.3x, 1.6x | -| `stream/audio/lite-07/decode` | 2.9x, 2.9x | 1.4x, 1.7x | 1.6x, 2.0x | - -Clipping, as decided, leaves audio encoding just past 2x: the result Go keeps is own -storage, up to twice its length, which each run allocates anew, so the collector runs -about twice as often as cgo's, whose result is exact. Copying own storage out at escape -into an exact Go buffer, and recycling it, costs a copy as `C.GoBytes` does and brings -every stream within 2x; it changes the 2026-11-07 decision, so it waits on the maintainer. - -A video group is 60 frames, 577 KB; an audio group is 50 frames of 160 bytes. Measure -each change on these, comparing `cpu-ns/item` and `faults/op` as well as wall time on a -loaded host, since CPU time counts the collector's workers. The per-call diagnostics: a -2-byte QUIC varint decodes in 8.2 ns, where cgo takes 1.0, and an audio frame header in -25 ns, where cgo takes 8.6. - -Public API: none beyond clipped capacity: a slice the translation returns has no spare -capacity, so a Go caller's `append` to it copies. - -## Waiting - -- Maintainer: how own storage escapes to Go code, now that clipping, as decided on - 2026-11-07, leaves `stream/audio/lite-07/encode` at 2.07x CPU and 2.03x wall time, - and `stream/video/lite-06/encode` at 2.01x wall time. - Options: copy it out at escape into an exact Go buffer and recycle it, measured 1.1 to - 1.6x CPU and 1.4 to 2.0x wall time on every stream (recommended: Go code then never - holds own storage, and the copy is the one cgo's result makes); keep clipping and take - the miss as noise; or drop the translator, as the epic's rule would. Check that this - Plan records the choice. From 2d29bed8884f94f03134977611ae781d72a0d3d2 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 07:01:55 -0800 Subject: [PATCH 06/10] fix(go): copy what an argument owns, keep full storage Go's, export only roots Round 5 of the review found ways own storage reaches memory Go holds: - A full Go slice of own-shaped capacity, shortened inside, took the shape; bytesEnter now copies it. - Internal functions on a pub type exported as `Type_method`, callable past the handover; they are now unexported. - A Go callback beside `&mut Vec` could read or keep storage mid-call; any function pointer or trait object in a root whose signature reaches byte storage is now refused. - Go aliases of a container the translation writes, then abandons (an outgrown array, a Box), kept own storage; a root now copies each Box pointee and Vec element array an argument owns, so a by-value Box is the translation's copy. Co-Authored-By: Claude Opus 5.5 --- go/README.md | 20 +++--- go/fixtures/hand/src/lib.rs | 26 +++++++- go/fixtures/reject/expected.txt | 3 +- go/fixtures/reject/src/lib.rs | 8 +++ go/runtime/rt.go | 20 ++++-- go/src/lower/hand.rs | 114 ++++++++++++++++++++------------ go/src/lower/mod.rs | 10 +-- go/tests/fixtures.rs | 9 +++ go/tests/hand/hand_test.go | 50 ++++++++++++-- 9 files changed, 193 insertions(+), 67 deletions(-) diff --git a/go/README.md b/go/README.md index 9c596fcbe..c234a3a85 100644 --- a/go/README.md +++ b/go/README.md @@ -49,12 +49,16 @@ A `Vec`'s storage is the runtime's own while only the translation reaches it grows to twice its capacity without zeroing, through the runtime's `growslice`, and an outgrown buffer is recycled for the next grow. Own storage is told apart by its shape, a capacity that is a power of two of at least 8 and never filled. A root function, which -Go calls, trims Go's storage it is passed off that shape, a byte less capacity, so it is -never recycled, and clips own storage it hands back to Go, in its result or behind an -argument's `&mut` or `Box`, to its length, so Go never sees past it and it is never -recycled either; Go's `append` to such a slice copies. A function pointer over byte -storage in a root's signature is refused, since Go code could fill it with a function -that keeps what it is handed. +Go calls, takes Go's storage it is passed off that shape, a byte less capacity or a copy +when full, so it is never recycled, and clips own storage it hands back to Go, in its +result or behind an argument's `&mut`, to its length, so Go never sees past it and it is +never recycled either; Go's `append` to such a slice copies. It works on a copy of each +container an argument owns, a `Box`'s pointee or a `Vec`'s elements, so Go code still +holding one never sees what the translation moved or grew out of it: a `Box` Go passes +by value is left as it was, as Rust's move would leave it unread. Only roots are +exported. A function pointer or trait object in the signature of a root that reaches +byte storage is refused, since Go code could fill it with a function that keeps what it +is handed or reads a place the translation is still writing. A panic is a Go panic of `RustPanic` carrying native Rust's message. As with `panic = "abort"`, no cleanup runs on the way out, so a caller that recovers one must @@ -65,8 +69,8 @@ drop what the translation held. `fixtures/reject/expected.txt` lists them with their spans: 128-bit integers, floats, a `Drop` impl, a trait object of any trait but `Display` and `Debug`, a union, a type generic over a constant, such as an array's by-value iterator, and a `pub` item that is -generic, a constant, or a static, or whose signature holds a function pointer over byte -storage. +generic, a constant, or a static, or whose signature holds a function pointer or trait +object beside byte storage. ## Tests diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs index c53be330f..b71c99667 100644 --- a/go/fixtures/hand/src/lib.rs +++ b/go/fixtures/hand/src/lib.rs @@ -68,12 +68,34 @@ pub fn grow(mut v: Vec, n: u8) -> Vec { v } +/// `b` with `n` bytes of `n` appended to what it holds. +pub fn boxed(mut b: Box, n: u8) -> Box { + append(&mut b.data, n); + b +} + /// Append `n` bytes of `n` to what `b` holds. -#[expect(clippy::boxed_local, reason = "a Box that Go code passes as its own pointer")] -pub fn boxed(mut b: Box, n: u8) { +pub fn reboxed(b: &mut Box, n: u8) { append(&mut b.data, n); } +/// Clear `v`, then append `n` bytes of `n`. +pub fn refill(v: &mut Vec, n: u8) { + v.clear(); + append(v, n); +} + +/// Replace `v`'s first buffer with `n` bytes of `n`, outgrow `v`, then that buffer: its +/// length. +pub fn regrow(v: &mut Vec>, n: u8) -> usize { + v[0] = fill(n); + for _ in 0..64 { + v.push(Vec::new()); + } + append(&mut v[0], 200); + v[0].len() +} + /// `n` buffers, the `i`th of `i` bytes. pub fn nested(n: u8) -> Vec> { let mut out = Vec::new(); diff --git a/go/fixtures/reject/expected.txt b/go/fixtures/reject/expected.txt index ab6fdb031..0709209ca 100644 --- a/go/fixtures/reject/expected.txt +++ b/go/fixtures/reject/expected.txt @@ -7,7 +7,8 @@ a `pub` field not named in ASCII with a leading lowercase letter, which Go canno a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:128:2 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:129:2 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:130:2 -`for<'a> fn(&'a mut std::vec::Vec)`, a function pointer over byte storage in a root's signature, is not supported at fixtures/reject/src/lib.rs:141:1 +`for<'a> fn(&'a mut std::vec::Vec)`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:141:1 +`fn()`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:149:1 a type with `pub` methods reached only through a specialized alias is not supported at fixtures/reject/src/lib.rs:118:1 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:14:1 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:15:2 diff --git a/go/fixtures/reject/src/lib.rs b/go/fixtures/reject/src/lib.rs index bcdfec613..c9f0c4dc1 100644 --- a/go/fixtures/reject/src/lib.rs +++ b/go/fixtures/reject/src/lib.rs @@ -143,3 +143,11 @@ pub fn handed(f: fn(&mut Vec)) -> usize { f(&mut v); v.len() } + +/// A function pointer beside byte storage, which Go code could fill with a closure that +/// reads `v` while it holds storage of the translation's own. +pub fn beside(v: &mut Vec, f: fn()) { + v.push(1); + f(); + v.push(2); +} diff --git a/go/runtime/rt.go b/go/runtime/rt.go index 337a531d9..88ad28d56 100644 --- a/go/runtime/rt.go +++ b/go/runtime/rt.go @@ -326,7 +326,7 @@ func vecGrow[T any](v *[]T, n int) { // once outgrown, until Go code outside the translation can reach it. Own storage is told // apart by its shape: a capacity that is a power of two of at least 8, never filled, since // own growth always leaves a byte spare. A root function hands Go storage in through -// bytesEnter, which trims a Go slice of that shape by a byte, and own storage out through +// bytesEnter, which takes a Go slice of that capacity off it, and own storage out through // bytesEscape, which clips it to its length, so storage Go can reach never has the shape // and is never recycled. Empty storage, such as `Vec::new`'s, grows into own storage. @@ -415,12 +415,22 @@ func bytesGrow(v *[]uint8, n int) { *v = s } -// bytesEnter takes Go's storage into a root function, trimmed off the shape of own -// storage so it is never recycled. +// bytesEnter takes Go's storage into a root function off the shape of own storage, so it +// is never recycled: trimmed by a byte, or copied when full, since shortening it would +// give it the shape. func bytesEnter(v *[]uint8) { - if bytesOwn(*v) { - *v = (*v)[: len(*v) : cap(*v)-1] + s := *v + c := cap(s) + if c < 8 || c&(c-1) != 0 { + return + } + if len(s) < c { + *v = s[: len(s) : c-1] + return } + next := make([]uint8, len(s), len(s)+1) + copy(next, s) + *v = next } // bytesEscape hands own storage out of a root function to Go, clipped to its length, diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index 706ac0785..f645bc1ae 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -1,10 +1,12 @@ //! Handing a `Vec`'s storage between Go code and the translation. The runtime grows //! storage it owns without zeroing it and recycles it once outgrown, telling it apart by //! its shape (`bytesOwn` in `runtime/rt.go`). Go code reaches storage only through a root -//! function, so the root hands Go's storage in trimmed off that shape, and its own out -//! clipped to its length: in, through each argument it is passed by value or through -//! `&mut` or `Box`; out, through its result and whatever its arguments reach behind a -//! `&mut` or `Box`, which Go code keeps. +//! function, so the root hands Go's storage in off that shape, and its own out clipped to +//! its length. In, it also copies each container an argument owns, a `Box`'s pointee or a +//! `Vec`'s elements, by value or behind `&mut`, so the translation never writes storage of +//! its own where Go code may still hold the container, which a move, a grow, or a +//! replaced `Box` would leave behind. Out, it clips its result and what its arguments +//! reach behind `&mut`, the places Go code keeps. use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::ty::util::IntTypeExt; @@ -19,21 +21,22 @@ use super::{Translator, names}; /// Which way a walk hands storage, and so what it does with each buffer it reaches. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum Hand { - /// Into a root: each `Vec` of Go's is trimmed off the shape of own storage. + /// Into a root: each `Vec` of Go's is taken off the shape of own storage, and each + /// container holding one is copied. In, /// Out of a root, in its result: each `Vec` of own storage, and each byte slice, /// is clipped to its length. Out, - /// Out of a root, in an argument: as `Out`, but only behind a pointer, where Go's - /// memory is, since the argument's own slots are the root's copies. + /// Out of a root, in an argument: as `Out`, but only behind a reference, where Go's + /// places are, since what the argument owns is the root's copy. Back, } impl Hand { - /// The way of what lies behind a pointer, or in a slice's elements. - fn behind(self) -> Self { + /// The way of what lies behind a pointer, which `owns` it if a `Box`. + fn behind(self, owns: bool) -> Self { match self { - Hand::Back => Hand::Out, + Hand::Back if !owns => Hand::Out, hand => hand, } } @@ -61,9 +64,9 @@ impl<'tcx> Translator<'tcx> { ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { Some(Mapped::Vec) => match is_u8(args.type_at(0)) { true => hand != Hand::Back, - false => self.hands_in(args.type_at(0), hand.behind(), seen), + false => self.hands_in(args.type_at(0), hand, seen), }, - Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, hand, seen), + Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, true, hand, seen), Some(Mapped::SliceIter) => hand == Hand::Out && is_u8(args.type_at(0)), Some(Mapped::Transparent) => self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Option | Mapped::Result) | None => adt @@ -71,7 +74,7 @@ impl<'tcx> Translator<'tcx> { .any(|field| self.hands_in(self.mono_field_ty(field, args), hand, seen)), Some(Mapped::Runtime(_) | Mapped::Refused(_)) => false, }, - ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => self.hands_behind(inner, m.is_mut(), hand, seen), + ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => self.hands_behind(inner, m.is_mut(), false, hand, seen), ty::Tuple(tys) => tys.iter().any(|t| self.hands_in(t, hand, seen)), ty::Array(elem, _) => self.hands_in(elem, hand, seen), ty::Closure(_, args) => args @@ -85,15 +88,23 @@ impl<'tcx> Translator<'tcx> { found } - /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`. - fn hands_behind(&self, inner: Ty<'tcx>, mutable: bool, hand: Hand, seen: &mut Vec<(Ty<'tcx>, Hand)>) -> bool { + /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`, + /// and which `owns` it if a `Box`. + fn hands_behind( + &self, + inner: Ty<'tcx>, + mutable: bool, + owns: bool, + hand: Hand, + seen: &mut Vec<(Ty<'tcx>, Hand)>, + ) -> bool { + // The pointer is where `hand` is; what it points to is behind it. + let next = hand.behind(owns); match *inner.kind() { ty::Str => hand == Hand::Out, - ty::Slice(elem) => { - (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, hand.behind(), seen)) - } + ty::Slice(elem) => (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, next, seen)), ty::Dynamic(..) => false, - _ => mutable && self.hands_in(inner, hand.behind(), seen), + _ => mutable && self.hands_in(inner, next, seen), } } @@ -130,11 +141,14 @@ impl<'tcx> Translator<'tcx> { _ => "bytesEscape(p)".into(), }); } else { - let f = self.hand(elem, hand.behind(), span); + if hand == Hand::In { + body.push("*p = append((*p)[:0:0], *p...)".into()); + } + let f = self.hand(elem, hand, span); body.push(format!("for i := range *p {{ {f}(&(*p)[i]) }}")); } } - Some(Mapped::Box) => self.hand_behind(args.type_at(0), true, hand, "p", &mut body, span), + Some(Mapped::Box) => self.hand_behind(args.type_at(0), true, true, hand, "p", &mut body, span), Some(Mapped::SliceIter) => body.push("clip(&p.s)".into()), Some(Mapped::Transparent) => { let f = self.hand(args.type_at(0), hand, span); @@ -143,7 +157,7 @@ impl<'tcx> Translator<'tcx> { _ => self.hand_adt(adt, args, hand, &mut body, span), }, ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => { - self.hand_behind(inner, m.is_mut(), hand, "p", &mut body, span) + self.hand_behind(inner, m.is_mut(), false, hand, "p", &mut body, span) } ty::Tuple(tys) => { for (i, t) in tys.iter().enumerate() { @@ -185,22 +199,40 @@ impl<'tcx> Translator<'tcx> { } /// The walk of a pointer to `inner`, at `p`, a pointer to the pointer, or to the - /// slice of an unsized `inner`. - fn hand_behind(&mut self, inner: Ty<'tcx>, mutable: bool, hand: Hand, p: &str, body: &mut Vec, span: Span) { + /// slice of an unsized `inner`, which a `Box` `owns` and so copies in. + #[expect(clippy::too_many_arguments, reason = "the walk's position, as hands_behind takes it")] + fn hand_behind( + &mut self, + inner: Ty<'tcx>, + mutable: bool, + owns: bool, + hand: Hand, + p: &str, + body: &mut Vec, + span: Span, + ) { + let copy = owns && hand == Hand::In; + let next = hand.behind(owns); match *inner.kind() { ty::Str => body.push(format!("clip({p})")), ty::Slice(elem) => { if hand == Hand::Out && is_u8(elem) { body.push(format!("clip({p})")); } - if mutable && self.hands(elem, hand.behind()) { - let f = self.hand(elem, hand.behind(), span); + if mutable && self.hands(elem, next) { + if copy { + body.push(format!("*{p} = append((*{p})[:0:0], *{p}...)")); + } + let f = self.hand(elem, next, span); body.push(format!("for i := range *{p} {{ {f}(&(*{p})[i]) }}")); } } _ => { - let f = self.hand(inner, hand.behind(), span); - body.push(format!("if *{p} != nil {{ {f}(*{p}) }}")); + let f = self.hand(inner, next, span); + match copy { + true => body.push(format!("if *{p} != nil {{ c := **{p}; *{p} = &c; {f}(&c) }}")), + false => body.push(format!("if *{p} != nil {{ {f}(*{p}) }}")), + } } } } @@ -246,7 +278,7 @@ impl<'tcx> Translator<'tcx> { }; let tag = format!("{}({value})", self.ty(discr_ty, span)); let mut text = format!("if p.tag == {tag} {{ {} }}", walks.join("; ")); - if hand != Hand::In { + if hand == Hand::Out { text.push_str(&format!( " else {{ zero(&p.{}) }}", variant_field(variant.name.as_str()) @@ -258,21 +290,21 @@ impl<'tcx> Translator<'tcx> { } impl<'tcx> Translator<'tcx> { - /// Refuse a function pointer a root's signature reaches whose own signature reaches + /// Refuse a function pointer or trait object in the signature of a root that reaches /// byte storage: Go code could pass a function of its own, which the translation would - /// hand its own storage to, or take Go's from, past the root's walks. - pub fn refuse_callback(&mut self, ty: Ty<'tcx>, span: Span) { - let mut seen = Vec::new(); - let found = self.find_part(ty, &mut seen, &|t, this| { - matches!(t.kind(), ty::FnPtr(..)) - && this - .find_part(t, &mut Vec::new(), &|t, this| bytes(this.tcx, t)) - .is_some() - }); - if let Some(f) = found { + /// hand its storage to, or run while a place Go holds keeps storage of its own. + pub fn refuse_callback(&mut self, sig: &[Ty<'tcx>], span: Span) { + let find = |pick: &dyn Fn(Ty<'tcx>, &Self) -> bool| { + let mut seen = Vec::new(); + sig.iter().find_map(|t| self.find_part(*t, &mut seen, pick)) + }; + if find(&|t, this| bytes(this.tcx, t)).is_none() { + return; + } + if let Some(f) = find(&|t, _| matches!(t.kind(), ty::FnPtr(..) | ty::Dynamic(..))) { self.err( span, - format!("`{f}`, a function pointer over byte storage in a root's signature, is not supported"), + format!("`{f}`, a callback in the signature of a root that reaches byte storage, is not supported"), ); } } diff --git a/go/src/lower/mod.rs b/go/src/lower/mod.rs index 9074a170f..4e13413cf 100644 --- a/go/src/lower/mod.rs +++ b/go/src/lower/mod.rs @@ -346,9 +346,8 @@ impl<'tcx> Translator<'tcx> { .iter() .any(|t| self.hands(*t, Hand::In) || self.hands(*t, Hand::Back)) || self.hands(output, Hand::Out); - for t in inputs.iter().chain([&output]) { - self.refuse_callback(*t, span); - } + let sig: Vec> = inputs.iter().copied().chain([output]).collect(); + self.refuse_callback(&sig, span); if !splits && !hands { self.funcs.insert(instance, name); self.queue.push_back((instance, span)); @@ -464,7 +463,10 @@ impl<'tcx> Translator<'tcx> { if let Some(name) = self.funcs.get(&instance) { return name.clone(); } - let base = names::func_base(self.tcx, instance); + // Unexported, so Go callers reach the translation only through its roots, which + // hand storage over. + let mut base = names::ident(&names::func_base(self.tcx, instance)); + base[..1].make_ascii_lowercase(); let name = self.fresh(&base); self.funcs.insert(instance, name.clone()); self.queue.push_back((instance, span)); diff --git a/go/tests/fixtures.rs b/go/tests/fixtures.rs index f68c16935..f97097920 100644 --- a/go/tests/fixtures.rs +++ b/go/tests/fixtures.rs @@ -303,6 +303,15 @@ fn vecs() { fn hand() { differential("hand", hand::run, "Run", cases(0, 0, 500)); let out = scratch().join("hand/out"); + // Go code reaches the translation only through its roots, named in camel case, which + // hand storage over. + let text = std::fs::read_to_string(out.join("hand.go")).unwrap(); + for name in text.lines().filter_map(|l| l.strip_prefix("func ")?.split('(').next()) { + assert!( + !name.starts_with(|c: char| c.is_ascii_uppercase()) || !name.contains('_'), + "hand: `{name}` is exported past the roots" + ); + } std::fs::copy(manifest_dir().join("tests/hand/hand_test.go"), out.join("hand_test.go")).unwrap(); go(&out, &["test", "-count=1", "."]); } diff --git a/go/tests/hand/hand_test.go b/go/tests/hand/hand_test.go index 19ab4e407..9016c92e0 100644 --- a/go/tests/hand/hand_test.go +++ b/go/tests/hand/hand_test.go @@ -95,12 +95,11 @@ func TestHandedKept(t *testing.T) { fixture.Append(&r, 200) kept(t, "a result passed back", keep) - o := &fixture.Open{} - fixture.Boxed(o, 100) - clipped(t, "a boxed field", o.Data) - keep = o.Data - fixture.Boxed(o, 200) - clean(t, "a boxed field grown again", o.Data) + b := fixture.Boxed(&fixture.Open{}, 100) + clipped(t, "a boxed field", b.Data) + keep = b.Data + fixture.Reboxed(&b, 200) + clean(t, "a boxed field grown again", b.Data) kept(t, "a boxed field", keep) var e fixture.Either @@ -115,6 +114,45 @@ func TestHandedKept(t *testing.T) { kept(t, "a variant switched away from", keep) } +// Go's full storage of the capacity of the translation's own, shortened and grown, is +// not recycled, whether Go made it or was handed it. +func TestFullKept(t *testing.T) { + s := bytes.Repeat([]uint8{0xaa}, 64) + keep := s + fixture.Refill(&s, 200) + kept(t, "Go's full slice", keep) + r := fixture.Fill(8) + keep = r + fixture.Refill(&r, 200) + kept(t, "a full result passed back", keep) +} + +// What Go passes, the translation works on a copy of wherever it could leave storage of +// its own behind: Go's Box and an array it outgrows keep what Go put there. +func TestContainersCopied(t *testing.T) { + g := make([]uint8, 3, 16) + o := &fixture.Open{Data: g} + b := fixture.Boxed(o, 100) + if b == o || unsafe.SliceData(o.Data) != unsafe.SliceData(g) || len(o.Data) != 3 { + t.Errorf("Boxed wrote into the Box Go passed") + } + old := b + fixture.Reboxed(&b, 100) + if b == old || len(old.Data) != 103 { + t.Errorf("Reboxed wrote into the Box Go passed behind a pointer") + } + + v := make([][]uint8, 1) + alias := v + if n := fixture.Regrow(&v, 100); n != 300 { + t.Fatalf("Regrow: %d bytes", n) + } + if alias[0] != nil { + t.Errorf("an array the translation outgrew holds %d bytes Go did not put there", len(alias[0])) + } + clipped(t, "Regrow's argument", v[0]) +} + // A slot the translation vacates past a length shows nothing it held. func TestVacatedZeroed(t *testing.T) { v := make([][]uint8, 0, 4) From 91da239b86f85955f64f6b0ea6c5b0ad7eca02f2 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:00:47 -0800 Subject: [PATCH 07/10] fix(go): narrow the handover; park the Go epic A root that reaches byte storage refuses a raw pointer in its signature and a reference anywhere in an argument but its outermost type, so the exit walk only follows top-level &mut arguments and no argument sees an entry copy's address. The Back walk and raw-pointer arms go. The Go epic and its children wait until a consumer needs pure Go. Co-Authored-By: Claude Opus 5.5 --- go/README.md | 13 ++-- go/fixtures/reject/expected.txt | 4 ++ go/fixtures/reject/src/lib.rs | 13 ++++ go/src/lower/hand.rs | 122 +++++++++++++++++--------------- go/src/lower/mod.rs | 11 ++- quest/m4/go/README.md | 16 +++-- quest/m4/go/alias-methods.md | 5 ++ quest/m4/go/coverage.md | 8 ++- quest/m4/go/enums.md | 5 ++ quest/m4/go/runtime-names.md | 5 ++ quest/m4/go/trait-impls.md | 5 ++ 11 files changed, 132 insertions(+), 75 deletions(-) diff --git a/go/README.md b/go/README.md index 36bae1104..8c52629e1 100644 --- a/go/README.md +++ b/go/README.md @@ -64,9 +64,11 @@ never recycled either; Go's `append` to such a slice copies. It works on a copy container an argument owns, a `Box`'s pointee or a `Vec`'s elements, so Go code still holding one never sees what the translation moved or grew out of it: a `Box` Go passes by value is left as it was, as Rust's move would leave it unread. Only roots are -exported. A function pointer or trait object in the signature of a root that reaches -byte storage is refused, since Go code could fill it with a function that keeps what it -is handed or reads a place the translation is still writing. +exported. A root that reaches byte storage refuses a function pointer or trait object in +its signature, since Go code could fill it with a function that keeps what it is handed +or reads a place the translation is still writing; a raw pointer, which could see what +handing in copied; and a reference anywhere in an argument but its outermost type, +which the translation could move out of what handing back walks. A panic is a Go panic of `RustPanic` carrying native Rust's message. As with `panic = "abort"`, no cleanup runs on the way out, so a caller that recovers one must @@ -78,8 +80,9 @@ drop what the translation held. `Drop` impl, a trait object of any trait but `Display` and `Debug`, a union, a type generic over a constant, such as an array's by-value iterator, a `pub` item that is generic, a `static mut`, a static with interior mutability, such as an atomic, or of another -language, a reference into part of a static, and a function pointer or trait object in -the signature of a root that reaches byte storage. +language, a reference into part of a static, and, in the signature of a root that +reaches byte storage, a function pointer, trait object, raw pointer, or reference +inside an argument. ## Tests diff --git a/go/fixtures/reject/expected.txt b/go/fixtures/reject/expected.txt index 5c9cbdbe4..7fc4e4fca 100644 --- a/go/fixtures/reject/expected.txt +++ b/go/fixtures/reject/expected.txt @@ -4,6 +4,7 @@ a generic `pub` function, which translates only as its callers instantiate it, i `READY`, a static of `std::sync::atomic::Atomic`, which has interior mutability Go callers would share unsynchronized, is not supported at fixtures/reject/src/lib.rs:14:1 `ENVIRON`, a static another language defines, is not supported at fixtures/reject/src/lib.rs:18:2 a `pub` constant of a generic type, which translates only as its uses instantiate it, is not supported at fixtures/reject/src/lib.rs:25:2 +`*const [u8]`, a raw pointer in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:97:1 `Callback`, an enum held as its parts that a function pointer in it takes or returns, is not supported at fixtures/reject/src/lib.rs:119:7 `std::ptr::NonNull`, a raw pointer's wrapper is not supported at fixtures/reject/src/lib.rs:145:19 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:151:2 @@ -11,7 +12,10 @@ a `pub` field not named in ASCII with a leading lowercase letter, which Go canno a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:153:2 a `pub` field not named in ASCII with a leading lowercase letter, which Go cannot export one to one, is not supported at fixtures/reject/src/lib.rs:154:2 `for<'a> fn(&'a mut std::vec::Vec)`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:165:1 +`&mut std::vec::Vec`, a reference inside an argument of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:165:1 `fn()`, a callback in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:173:1 +`&mut std::vec::Vec`, a reference inside an argument of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:181:1 +`*const u8`, a raw pointer in the signature of a root that reaches byte storage, is not supported at fixtures/reject/src/lib.rs:188:1 a type with `pub` methods reached only through a specialized alias is not supported at fixtures/reject/src/lib.rs:142:1 a reference into part of a static is not supported at fixtures/reject/src/lib.rs:34:3 `u128`, which Go has no integer for is not supported at fixtures/reject/src/lib.rs:38:1 diff --git a/go/fixtures/reject/src/lib.rs b/go/fixtures/reject/src/lib.rs index 8ff8cad0a..e223c8688 100644 --- a/go/fixtures/reject/src/lib.rs +++ b/go/fixtures/reject/src/lib.rs @@ -175,3 +175,16 @@ pub fn beside(v: &mut Vec, f: fn()) { f(); v.push(2); } + +/// A reference inside an argument beside byte storage, which the translation could move +/// out of what handing back walks. +pub fn popped(mut refs: Vec<&mut Vec>) { + if let Some(v) = refs.pop() { + v.push(1); + } +} + +/// A raw pointer beside byte storage, which could see that handing in copied it. +pub fn same(v: Vec, p: *const u8) -> bool { + std::ptr::eq(&v[0], p) +} diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index f645bc1ae..069502e32 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -5,8 +5,9 @@ //! its length. In, it also copies each container an argument owns, a `Box`'s pointee or a //! `Vec`'s elements, by value or behind `&mut`, so the translation never writes storage of //! its own where Go code may still hold the container, which a move, a grow, or a -//! replaced `Box` would leave behind. Out, it clips its result and what its arguments -//! reach behind `&mut`, the places Go code keeps. +//! replaced `Box` would leave behind. Out, it clips its result and what its `&mut` +//! arguments reach, the places Go code keeps. A root refuses what would let a place Go +//! holds leave those walks, or see what handing in copied (`refuse_unhandable`). use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::ty::util::IntTypeExt; @@ -24,22 +25,9 @@ pub enum Hand { /// Into a root: each `Vec` of Go's is taken off the shape of own storage, and each /// container holding one is copied. In, - /// Out of a root, in its result: each `Vec` of own storage, and each byte slice, - /// is clipped to its length. + /// Out of a root, in its result or behind a `&mut` argument: each `Vec` of own + /// storage, and each byte slice, is clipped to its length. Out, - /// Out of a root, in an argument: as `Out`, but only behind a reference, where Go's - /// places are, since what the argument owns is the root's copy. - Back, -} - -impl Hand { - /// The way of what lies behind a pointer, which `owns` it if a `Box`. - fn behind(self, owns: bool) -> Self { - match self { - Hand::Back if !owns => Hand::Out, - hand => hand, - } - } } fn is_u8(ty: Ty<'_>) -> bool { @@ -62,11 +50,8 @@ impl<'tcx> Translator<'tcx> { let tcx = self.tcx; let found = match *ty.kind() { ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { - Some(Mapped::Vec) => match is_u8(args.type_at(0)) { - true => hand != Hand::Back, - false => self.hands_in(args.type_at(0), hand, seen), - }, - Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, true, hand, seen), + Some(Mapped::Vec) => is_u8(args.type_at(0)) || self.hands_in(args.type_at(0), hand, seen), + Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, hand, seen), Some(Mapped::SliceIter) => hand == Hand::Out && is_u8(args.type_at(0)), Some(Mapped::Transparent) => self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Option | Mapped::Result) | None => adt @@ -74,7 +59,7 @@ impl<'tcx> Translator<'tcx> { .any(|field| self.hands_in(self.mono_field_ty(field, args), hand, seen)), Some(Mapped::Runtime(_) | Mapped::Refused(_)) => false, }, - ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => self.hands_behind(inner, m.is_mut(), false, hand, seen), + ty::Ref(_, inner, m) => self.hands_behind(inner, m.is_mut(), hand, seen), ty::Tuple(tys) => tys.iter().any(|t| self.hands_in(t, hand, seen)), ty::Array(elem, _) => self.hands_in(elem, hand, seen), ty::Closure(_, args) => args @@ -88,23 +73,28 @@ impl<'tcx> Translator<'tcx> { found } - /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`, - /// and which `owns` it if a `Box`. - fn hands_behind( - &self, - inner: Ty<'tcx>, - mutable: bool, - owns: bool, - hand: Hand, - seen: &mut Vec<(Ty<'tcx>, Hand)>, - ) -> bool { - // The pointer is where `hand` is; what it points to is behind it. - let next = hand.behind(owns); + /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`. + fn hands_behind(&self, inner: Ty<'tcx>, mutable: bool, hand: Hand, seen: &mut Vec<(Ty<'tcx>, Hand)>) -> bool { match *inner.kind() { ty::Str => hand == Hand::Out, - ty::Slice(elem) => (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, next, seen)), + ty::Slice(elem) => (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, hand, seen)), ty::Dynamic(..) => false, - _ => mutable && self.hands_in(inner, next, seen), + _ => mutable && self.hands_in(inner, hand, seen), + } + } + + /// Whether a root hands back to Go what its argument of `ty` reaches: what lies behind + /// a `&mut`, the only place of Go's the translation can write, since a reference + /// anywhere else in an argument is refused. A slice or `str` it points to is the + /// root's own copy of the slice, so only its elements are Go's. + pub fn hands_back(&self, ty: Ty<'tcx>) -> bool { + let ty::Ref(_, inner, ty::Mutability::Mut) = *ty.kind() else { + return false; + }; + match *inner.kind() { + ty::Str => false, + ty::Slice(elem) => self.hands(elem, Hand::Out), + _ => self.hands(inner, Hand::Out), } } @@ -125,7 +115,6 @@ impl<'tcx> Translator<'tcx> { let way = match hand { Hand::In => "In", Hand::Out => "Out", - Hand::Back => "Back", }; let name = self.fresh(&format!("hand{way}_{}", names::ty_short(tcx, ty))); self.hand_fns.insert((ty, hand), name.clone()); @@ -156,9 +145,7 @@ impl<'tcx> Translator<'tcx> { } _ => self.hand_adt(adt, args, hand, &mut body, span), }, - ty::Ref(_, inner, m) | ty::RawPtr(inner, m) => { - self.hand_behind(inner, m.is_mut(), false, hand, "p", &mut body, span) - } + ty::Ref(_, inner, m) => self.hand_behind(inner, m.is_mut(), false, hand, "p", &mut body, span), ty::Tuple(tys) => { for (i, t) in tys.iter().enumerate() { if self.hands(t, hand) { @@ -184,7 +171,6 @@ impl<'tcx> Translator<'tcx> { let what = match hand { Hand::In => "into a root function", Hand::Out => "out of a root function", - Hand::Back => "back to Go in a root function's argument", }; let text = format!( "// {name} hands the storage of a `{ty}` {what}.\nfunc {name}(p *{go}) {{\n\t{}\n}}\n", @@ -212,23 +198,22 @@ impl<'tcx> Translator<'tcx> { span: Span, ) { let copy = owns && hand == Hand::In; - let next = hand.behind(owns); match *inner.kind() { ty::Str => body.push(format!("clip({p})")), ty::Slice(elem) => { if hand == Hand::Out && is_u8(elem) { body.push(format!("clip({p})")); } - if mutable && self.hands(elem, next) { + if mutable && self.hands(elem, hand) { if copy { body.push(format!("*{p} = append((*{p})[:0:0], *{p}...)")); } - let f = self.hand(elem, next, span); + let f = self.hand(elem, hand, span); body.push(format!("for i := range *{p} {{ {f}(&(*{p})[i]) }}")); } } _ => { - let f = self.hand(inner, next, span); + let f = self.hand(inner, hand, span); match copy { true => body.push(format!("if *{p} != nil {{ c := **{p}; *{p} = &c; {f}(&c) }}")), false => body.push(format!("if *{p} != nil {{ {f}(*{p}) }}")), @@ -290,22 +275,43 @@ impl<'tcx> Translator<'tcx> { } impl<'tcx> Translator<'tcx> { - /// Refuse a function pointer or trait object in the signature of a root that reaches - /// byte storage: Go code could pass a function of its own, which the translation would - /// hand its storage to, or run while a place Go holds keeps storage of its own. - pub fn refuse_callback(&mut self, sig: &[Ty<'tcx>], span: Span) { - let find = |pick: &dyn Fn(Ty<'tcx>, &Self) -> bool| { + /// Refuse what a root that reaches byte storage cannot hand over: a function pointer or + /// trait object, which Go code could fill with a function that keeps what the + /// translation hands it, or reads a place Go holds while it keeps storage of its own; a + /// raw pointer, which could see that handing in copied what it points into; and a + /// reference anywhere in an argument but its outermost type, which the translation + /// could move out of what handing back walks, leaving the place of Go's it points to + /// with storage of its own. + pub fn refuse_unhandable(&mut self, inputs: &[Ty<'tcx>], output: Ty<'tcx>, span: Span) { + let find = |tys: &[Ty<'tcx>], pick: &dyn Fn(Ty<'tcx>, &Self) -> bool| { let mut seen = Vec::new(); - sig.iter().find_map(|t| self.find_part(*t, &mut seen, pick)) + tys.iter().find_map(|t| self.find_part(*t, &mut seen, pick)) }; - if find(&|t, this| bytes(this.tcx, t)).is_none() { + let sig: Vec> = inputs.iter().copied().chain([output]).collect(); + if find(&sig, &|t, this| bytes(this.tcx, t)).is_none() { return; } - if let Some(f) = find(&|t, _| matches!(t.kind(), ty::FnPtr(..) | ty::Dynamic(..))) { - self.err( - span, - format!("`{f}`, a callback in the signature of a root that reaches byte storage, is not supported"), - ); + let mut refused = Vec::new(); + if let Some(f) = find(&sig, &|t, _| matches!(t.kind(), ty::FnPtr(..) | ty::Dynamic(..))) { + refused.push(format!("`{f}`, a callback in the signature of a root that reaches byte storage,")); + } + if let Some(p) = find(&sig, &|t, _| t.is_raw_ptr()) { + refused.push(format!("`{p}`, a raw pointer in the signature of a root that reaches byte storage,")); + } + let below: Vec> = inputs + .iter() + .map(|t| match *t.kind() { + ty::Ref(_, inner, _) => inner, + _ => *t, + }) + .collect(); + if let Some(r) = find(&below, &|t, _| t.is_ref()) { + refused.push(format!( + "`{r}`, a reference inside an argument of a root that reaches byte storage," + )); + } + for msg in refused { + self.err(span, format!("{msg} is not supported")); } } diff --git a/go/src/lower/mod.rs b/go/src/lower/mod.rs index 21d02fe51..d4b4bc65a 100644 --- a/go/src/lower/mod.rs +++ b/go/src/lower/mod.rs @@ -418,12 +418,9 @@ impl<'tcx> Translator<'tcx> { let inputs: Vec> = sig.inputs().iter().map(|t| normal(*t)).collect(); let output = normal(sig.output()); let splits = inputs.iter().chain([&output]).any(|t| self.split(*t, span).is_some()); - let hands = inputs - .iter() - .any(|t| self.hands(*t, Hand::In) || self.hands(*t, Hand::Back)) + let hands = inputs.iter().any(|t| self.hands(*t, Hand::In) || self.hands_back(*t)) || self.hands(output, Hand::Out); - let sig: Vec> = inputs.iter().copied().chain([output]).collect(); - self.refuse_callback(&sig, span); + self.refuse_unhandable(&inputs, output, span); if !splits && !hands { self.funcs.insert(instance, name); self.queue.push_back((instance, span)); @@ -456,8 +453,8 @@ impl<'tcx> Translator<'tcx> { if self.hands(*input, Hand::In) { enter.push_str(&format!("\t{}(&{param})\n", self.hand(*input, Hand::In, span))); } - if self.hands(*input, Hand::Back) { - back.push(format!("{}(&{param})", self.hand(*input, Hand::Back, span))); + if self.hands_back(*input) { + back.push(format!("{}(&{param})", self.hand(*input, Hand::Out, span))); } match self.split(*input, span) { Some(split) => args.extend(split.parts.iter().map(|p| match p.big { diff --git a/quest/m4/go/README.md b/quest/m4/go/README.md index c11085bbf..84a82dd13 100644 --- a/quest/m4/go/README.md +++ b/quest/m4/go/README.md @@ -26,10 +26,18 @@ Decided with the maintainer on 2026-11-01, settling whose lowering returns TypeScript's AST. Decided with the maintainer on 2026-11-07, settling -[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643): speed met the bar. With -the translation owning a `Vec`'s storage and clipping it at escape, every `stream/` -workload of `just go bench` runs within 2x of cgo, the worst at 2.07x CPU time, which -counts as noise. +[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643): + +- **Speed met the bar.** With the translation owning a `Vec`'s storage and clipping + it at escape, every `stream/` workload of `just go bench` runs within 2x of cgo, the + worst at 2.07x CPU time, which counts as noise. +- ✅ **Parked until a consumer needs pure Go.** None has asked, and the cgo exporter + stays moq's Go route. This epic and its children don't start until one does; each + child's `Waiting` item holds it. When one does, first weigh running moq compiled to + WASM inside Go on a pure-Go runtime such as [wazero](https://wazero.io) (no cgo, a + plain `go get`; unmeasured for moq-net's I/O) against finishing this translation. + Rejected: splitting [coverage](/quest/m4/go/coverage.md) into a quest per construct + now; deleting `go/`, whose translator and measurements are kept. The spike ([moq-dev/rust2#410](https://github.com/moq-dev/rust2/pull/410)) translated moq's codec held to rust2-ts's bar: 35,273 codec cases and 124,193 integer cases match diff --git a/quest/m4/go/alias-methods.md b/quest/m4/go/alias-methods.md index c2390a4eb..97ba41363 100644 --- a/quest/m4/go/alias-methods.md +++ b/quest/m4/go/alias-methods.md @@ -29,3 +29,8 @@ with impls that apply to each, to neither, and under a bound, checked by differe and by exported names; reject loses its specialized-alias case. Public API: what a translation exports. + +## Waiting + +- Go consumers: one needs moq in pure Go, which [the epic](/quest/m4/go/README.md) is + parked until; check with the maintainer before starting. diff --git a/quest/m4/go/coverage.md b/quest/m4/go/coverage.md index 2581c46a0..7457a0e76 100644 --- a/quest/m4/go/coverage.md +++ b/quest/m4/go/coverage.md @@ -32,8 +32,14 @@ failed speed result deletes `go/`, so planning the constructs earlier would be w The last coverage child reruns `just go bench` against speed's bar, every `stream/` workload within 2x of cgo, since constructs such as `Vec::capacity` change what the stream workload runs. Speed reached the bar on 2026-11-07 in -[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643), so coverage now splits. +[moq-dev/rust2#643](https://github.com/moq-dev/rust2/pull/643), but the epic is parked +until a consumer needs pure Go, so the split waits for that too. - ✅ Gate on speed, split later. - Split now, each child requiring speed. - Split now, ungated. + +## Waiting + +- Go consumers: one needs moq in pure Go, which [the epic](/quest/m4/go/README.md) is + parked until; check with the maintainer before starting. diff --git a/quest/m4/go/enums.md b/quest/m4/go/enums.md index 7121f9d0c..85d5c0e36 100644 --- a/quest/m4/go/enums.md +++ b/quest/m4/go/enums.md @@ -38,3 +38,8 @@ with the other fixture tests. Public API: what a translation exports for an enum (breaking for Go code reading the lowercase `tag`, which only the package itself could). + +## Waiting + +- Go consumers: one needs moq in pure Go, which [the epic](/quest/m4/go/README.md) is + parked until; check with the maintainer before starting. diff --git a/quest/m4/go/runtime-names.md b/quest/m4/go/runtime-names.md index 253702a2b..911efa427 100644 --- a/quest/m4/go/runtime-names.md +++ b/quest/m4/go/runtime-names.md @@ -37,3 +37,8 @@ named as another runtime export, and its Go callers use both. Public API: what a translation exports, and the runtime's package (breaking for Go code naming `Option` or `Result` from the translated package). + +## Waiting + +- Go consumers: one needs moq in pure Go, which [the epic](/quest/m4/go/README.md) is + parked until; check with the maintainer before starting. diff --git a/quest/m4/go/trait-impls.md b/quest/m4/go/trait-impls.md index 5e40e56ee..d395e1cdc 100644 --- a/quest/m4/go/trait-impls.md +++ b/quest/m4/go/trait-impls.md @@ -36,3 +36,8 @@ Fixtures: `go/fixtures/items` gives an exported type `Display`, `Debug`, `Partia two `From` impls, called from Go and checked by differential. Public API: what a translation exports. + +## Waiting + +- Go consumers: one needs moq in pure Go, which [the epic](/quest/m4/go/README.md) is + parked until; check with the maintainer before starting. From 8c0d1b6dbe80d46b1f0dc8f9efd1b2809e8e80cc Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:01:11 -0800 Subject: [PATCH 08/10] style(go): format Co-Authored-By: Claude Opus 5.5 --- go/src/lower/hand.rs | 8 ++++++-- go/src/lower/mod.rs | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index 069502e32..797b334e4 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -293,10 +293,14 @@ impl<'tcx> Translator<'tcx> { } let mut refused = Vec::new(); if let Some(f) = find(&sig, &|t, _| matches!(t.kind(), ty::FnPtr(..) | ty::Dynamic(..))) { - refused.push(format!("`{f}`, a callback in the signature of a root that reaches byte storage,")); + refused.push(format!( + "`{f}`, a callback in the signature of a root that reaches byte storage," + )); } if let Some(p) = find(&sig, &|t, _| t.is_raw_ptr()) { - refused.push(format!("`{p}`, a raw pointer in the signature of a root that reaches byte storage,")); + refused.push(format!( + "`{p}`, a raw pointer in the signature of a root that reaches byte storage," + )); } let below: Vec> = inputs .iter() diff --git a/go/src/lower/mod.rs b/go/src/lower/mod.rs index d4b4bc65a..080546a2c 100644 --- a/go/src/lower/mod.rs +++ b/go/src/lower/mod.rs @@ -418,8 +418,8 @@ impl<'tcx> Translator<'tcx> { let inputs: Vec> = sig.inputs().iter().map(|t| normal(*t)).collect(); let output = normal(sig.output()); let splits = inputs.iter().chain([&output]).any(|t| self.split(*t, span).is_some()); - let hands = inputs.iter().any(|t| self.hands(*t, Hand::In) || self.hands_back(*t)) - || self.hands(output, Hand::Out); + let hands = + inputs.iter().any(|t| self.hands(*t, Hand::In) || self.hands_back(*t)) || self.hands(output, Hand::Out); self.refuse_unhandable(&inputs, output, span); if !splits && !hands { self.funcs.insert(instance, name); From 6242af2e3be9112d7b2dd77aa79004e895e580c0 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:25:15 -0800 Subject: [PATCH 09/10] fix(go): walk shared references out of a result; skip slice::Iter's lifetime Co-Authored-By: Claude Opus 5.5 --- go/fixtures/hand/src/lib.rs | 5 +++++ go/src/lower/hand.rs | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs index b71c99667..b61b09cb2 100644 --- a/go/fixtures/hand/src/lib.rs +++ b/go/fixtures/hand/src/lib.rs @@ -119,6 +119,11 @@ pub fn view(v: &mut Vec, n: u8) -> &[u8] { v } +/// An iterator over `xs`. +pub fn iter(xs: &[u8]) -> std::slice::Iter<'_, u8> { + xs.iter() +} + /// Push `n` bytes of `n` onto `v`, then pop and outgrow them, leaving the slot vacated. pub fn vacate(v: &mut Vec>, n: u8) -> usize { v.push(fill(n)); diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index 797b334e4..cf10b21aa 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -34,6 +34,14 @@ fn is_u8(ty: Ty<'_>) -> bool { matches!(ty.kind(), ty::Uint(ty::UintTy::U8)) } +/// Whether a walk this way goes behind a pointer, which is `mutable` if the translation +/// may write through it: handing in, only where the translation may grow or replace what +/// Go passed; handing out, behind any, since Go code gets what the result reaches once +/// the call is done, shared or not. +fn reaches(mutable: bool, hand: Hand) -> bool { + mutable || hand == Hand::Out +} + impl<'tcx> Translator<'tcx> { /// Whether a walk handing a value of `ty` this way has anything to do. pub fn hands(&self, ty: Ty<'tcx>, hand: Hand) -> bool { @@ -52,7 +60,8 @@ impl<'tcx> Translator<'tcx> { ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { Some(Mapped::Vec) => is_u8(args.type_at(0)) || self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, hand, seen), - Some(Mapped::SliceIter) => hand == Hand::Out && is_u8(args.type_at(0)), + // Its first argument is the slice's lifetime. + Some(Mapped::SliceIter) => hand == Hand::Out && args.types().next().is_some_and(is_u8), Some(Mapped::Transparent) => self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Option | Mapped::Result) | None => adt .all_fields() @@ -75,11 +84,12 @@ impl<'tcx> Translator<'tcx> { /// `hands` of a pointer to `inner`, through which the translation may write if `mutable`. fn hands_behind(&self, inner: Ty<'tcx>, mutable: bool, hand: Hand, seen: &mut Vec<(Ty<'tcx>, Hand)>) -> bool { + let reach = reaches(mutable, hand); match *inner.kind() { ty::Str => hand == Hand::Out, - ty::Slice(elem) => (hand == Hand::Out && is_u8(elem)) || (mutable && self.hands_in(elem, hand, seen)), + ty::Slice(elem) => (hand == Hand::Out && is_u8(elem)) || (reach && self.hands_in(elem, hand, seen)), ty::Dynamic(..) => false, - _ => mutable && self.hands_in(inner, hand, seen), + _ => reach && self.hands_in(inner, hand, seen), } } @@ -204,7 +214,7 @@ impl<'tcx> Translator<'tcx> { if hand == Hand::Out && is_u8(elem) { body.push(format!("clip({p})")); } - if mutable && self.hands(elem, hand) { + if reaches(mutable, hand) && self.hands(elem, hand) { if copy { body.push(format!("*{p} = append((*{p})[:0:0], *{p}...)")); } From 36a27d738f03e0c323c50ade0a1039da29d7b407 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:33:04 -0800 Subject: [PATCH 10/10] fix(go): hand out what a slice iterator result holds Co-Authored-By: Claude Opus 5.5 --- go/fixtures/hand/src/lib.rs | 5 +++++ go/src/lower/hand.rs | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/go/fixtures/hand/src/lib.rs b/go/fixtures/hand/src/lib.rs index b61b09cb2..cca3141d0 100644 --- a/go/fixtures/hand/src/lib.rs +++ b/go/fixtures/hand/src/lib.rs @@ -124,6 +124,11 @@ pub fn iter(xs: &[u8]) -> std::slice::Iter<'_, u8> { xs.iter() } +/// An iterator over `v`'s buffers. +pub fn iter_nested(v: &[Vec]) -> std::slice::Iter<'_, Vec> { + v.iter() +} + /// Push `n` bytes of `n` onto `v`, then pop and outgrow them, leaving the slot vacated. pub fn vacate(v: &mut Vec>, n: u8) -> usize { v.push(fill(n)); diff --git a/go/src/lower/hand.rs b/go/src/lower/hand.rs index cf10b21aa..e326199dd 100644 --- a/go/src/lower/hand.rs +++ b/go/src/lower/hand.rs @@ -60,8 +60,10 @@ impl<'tcx> Translator<'tcx> { ty::Adt(adt, args) => match library::mapped(tcx, adt.did()) { Some(Mapped::Vec) => is_u8(args.type_at(0)) || self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Box) => self.hands_behind(args.type_at(0), true, hand, seen), - // Its first argument is the slice's lifetime. - Some(Mapped::SliceIter) => hand == Hand::Out && args.types().next().is_some_and(is_u8), + // What it holds is shared, so only a result's is handed. + Some(Mapped::SliceIter) => { + hand == Hand::Out && self.hands_behind(self.iter_slice(args), false, hand, seen) + } Some(Mapped::Transparent) => self.hands_in(args.type_at(0), hand, seen), Some(Mapped::Option | Mapped::Result) | None => adt .all_fields() @@ -108,6 +110,12 @@ impl<'tcx> Translator<'tcx> { } } + /// The slice a `slice::Iter` with `args` walks, whose first is the slice's lifetime. + fn iter_slice(&self, args: ty::GenericArgsRef<'tcx>) -> Ty<'tcx> { + let elem = args.types().next().expect("a slice iterator's element type"); + Ty::new_slice(self.tcx, elem) + } + /// A field's type at `args`. fn mono_field_ty(&self, field: &ty::FieldDef, args: ty::GenericArgsRef<'tcx>) -> Ty<'tcx> { let fty = field.ty(self.tcx, args).skip_norm_wip(); @@ -148,7 +156,10 @@ impl<'tcx> Translator<'tcx> { } } Some(Mapped::Box) => self.hand_behind(args.type_at(0), true, true, hand, "p", &mut body, span), - Some(Mapped::SliceIter) => body.push("clip(&p.s)".into()), + Some(Mapped::SliceIter) => { + let slice = self.iter_slice(args); + self.hand_behind(slice, false, false, hand, "&p.s", &mut body, span) + } Some(Mapped::Transparent) => { let f = self.hand(args.type_at(0), hand, span); body.push(format!("{f}(p)"));