From b670c1c890693b49153f0b1dbb931fead59bb5b9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:26:50 -0800 Subject: [PATCH 01/13] chore(quest): claim quest/m1/ffi-export/python.md Quest-Claim: 21142239fd409ac6 From 0c103add1908070c01a8f28a88e4f2025f2cdf1f Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:02:17 -0800 Subject: [PATCH 02/13] wip(ffi): Python generator, runtime, and the api script Co-Authored-By: Claude Opus 5.5 --- ffi/runtime/python.rs | 1705 +++++++++++++++++++++++++++++++++++++++++ ffi/runtime/rt.py | 104 +++ ffi/src/glue.rs | 14 +- ffi/src/kotlin.rs | 8 +- ffi/src/main.rs | 1 + ffi/src/python.rs | 1358 ++++++++++++++++++++++++++++++++ ffi/tests/api.py | 650 ++++++++++++++++ 7 files changed, 3835 insertions(+), 5 deletions(-) create mode 100644 ffi/runtime/python.rs create mode 100644 ffi/runtime/rt.py create mode 100644 ffi/src/python.rs create mode 100644 ffi/tests/api.py diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs new file mode 100644 index 000000000..8c4962970 --- /dev/null +++ b/ffi/runtime/python.rs @@ -0,0 +1,1705 @@ +//! The runtime the Python glue's extension functions share: the conversions between +//! Python's values and the C ABI's, the handles objects own, and the queue a future's +//! and a waker's wakes reach the event loop through. +//! +//! The glue holds the GIL only while it touches Python: it releases it around each call +//! into the C ABI, so Rust may call Python back from any thread, as a lent closure or a +//! trait method, while the caller waits. A wake never takes the GIL: it queues a token +//! and writes a byte to a socket the event loop reads, and the loop does the rest. + +use std::cell::UnsafeCell; +use std::ffi::{CStr, c_char, c_int, c_void}; +use std::io::{Read, Write}; +use std::ptr::{NonNull, null, null_mut}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicPtr, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; + +pub use pyo3_ffi::*; + +use crate::rt; + +/// A Python exception is set, for the caller to return null with. +pub struct Raised; + +pub type R = Result; + +/// A value only the thread holding the GIL touches. +pub struct Gil(UnsafeCell); + +// SAFETY: every access holds the GIL, which this abi3 extension's interpreters have. +unsafe impl Sync for Gil {} + +impl Gil { + pub const fn new(value: T) -> Self { + Self(UnsafeCell::new(value)) + } + + /// The value. + /// + /// # Safety + /// The caller holds the GIL, and no other reference to the value is live. + #[allow(clippy::mut_from_ref, reason = "the GIL makes the access exclusive")] + pub unsafe fn get(&self) -> &mut T { + // SAFETY: the caller's contract. + unsafe { &mut *self.0.get() } + } +} + +/// A strong reference, released when dropped. +pub struct Obj(NonNull); + +impl Obj { + /// A new reference a call returned, or the exception it raised. + /// + /// # Safety + /// `ptr` is null with an exception set, or a new reference, and the GIL is held. + pub unsafe fn new(ptr: *mut PyObject) -> R { + NonNull::new(ptr).map(Self).ok_or(Raised) + } + + /// A new reference to `ptr`, which the caller borrows. + /// + /// # Safety + /// `ptr` is a live object, and the GIL is held. + pub unsafe fn borrowed(ptr: *mut PyObject) -> Self { + // SAFETY: the caller's contract. + unsafe { Py_IncRef(ptr) }; + Self(NonNull::new(ptr).expect("a live object")) + } + + pub fn ptr(&self) -> *mut PyObject { + self.0.as_ptr() + } + + /// The reference, for a caller that steals it. + pub fn into_ptr(self) -> *mut PyObject { + let ptr = self.ptr(); + std::mem::forget(self); + ptr + } +} + +impl Drop for Obj { + fn drop(&mut self) { + // SAFETY: a strong reference, dropped with the GIL held, as every `Obj` is. + unsafe { Py_DecRef(self.ptr()) }; + } +} + +/// Raise `exc` with `msg`. +pub fn raise(exc: *mut PyObject, msg: &str) -> R { + // SAFETY: the GIL is held, and `exc` is an exception type. + unsafe { + let msg = PyUnicode_FromStringAndSize(msg.as_ptr().cast(), msg.len() as Py_ssize_t); + if !msg.is_null() { + PyErr_SetObject(exc, msg); + Py_DecRef(msg); + } + } + Err(Raised) +} + +pub fn type_error(msg: &str) -> R { + // SAFETY: a static exception type. + raise(unsafe { PyExc_TypeError }, msg) +} + +pub fn value_error(msg: &str) -> R { + // SAFETY: a static exception type. + raise(unsafe { PyExc_ValueError }, msg) +} + +fn overflow(msg: &str) -> R { + // SAFETY: a static exception type. + raise(unsafe { PyExc_OverflowError }, msg) +} + +/// Run an extension function's body, returning null once it raised. +pub fn wrap(f: impl FnOnce() -> R) -> *mut PyObject { + match f() { + Ok(obj) => obj.into_ptr(), + Err(Raised) => null_mut(), + } +} + +/// The `n` arguments of a fast call of `name`. +/// +/// # Safety +/// `args` points at `nargs` live objects. +pub unsafe fn args<'a>(args: *mut *mut PyObject, nargs: Py_ssize_t, n: usize, name: &str) -> R<&'a [*mut PyObject]> { + if nargs as usize != n { + return type_error(&format!("{name}() takes {n} arguments ({nargs} given)")); + } + match n { + 0 => Ok(&[]), + // SAFETY: the caller's contract. + n => Ok(unsafe { std::slice::from_raw_parts(args, n) }), + } +} + +/// Call into the C ABI with the GIL released, so Rust may call Python back from any +/// thread while this one waits. +pub fn call(f: impl FnOnce() -> T) -> T { + // SAFETY: this thread holds the GIL, and touches no Python object until it retakes it. + let state = unsafe { PyEval_SaveThread() }; + let result = f(); + // SAFETY: released just above. + unsafe { PyEval_RestoreThread(state) }; + result +} + +/// Run `f` calling into Python, from whatever thread Rust calls on, holding the GIL; an +/// exception it raises aborts with what it was, since Rust has nowhere to unwind it to. +pub fn upcall(what: &str, f: impl FnOnce() -> R) -> T { + // SAFETY: any thread may take the GIL. + let gil = unsafe { PyGILState_Ensure() }; + match f() { + Ok(value) => { + // SAFETY: taken above. + unsafe { PyGILState_Release(gil) }; + value + } + Err(Raised) => { + // SAFETY: the GIL is held, and an exception is set. + unsafe { PyErr_Print() }; + rt::fail(what, "a call into Python raised") + } + } +} + +/// A Python string interned once, such as a field's name. +pub struct Key(&'static CStr, AtomicPtr); + +impl Key { + pub const fn new(name: &'static CStr) -> Self { + Self(name, AtomicPtr::new(null_mut())) + } + + /// The string, which lives as long as the interpreter. + pub fn get(&self) -> R<*mut PyObject> { + let ptr = self.1.load(Ordering::Relaxed); + if !ptr.is_null() { + return Ok(ptr); + } + // SAFETY: the GIL is held. + let ptr = unsafe { PyUnicode_InternFromString(self.0.as_ptr()) }; + if ptr.is_null() { + return Err(Raised); + } + self.1.store(ptr, Ordering::Relaxed); + Ok(ptr) + } +} + +/// What a call borrows from Python until it returns: the objects whose memory it reads, +/// the buffers it views, the handles it holds and those it moves out of, and what it +/// gives Rust, taken back if the call never happens. +#[derive(Default)] +pub struct Scope { + keep: Vec, + views: Vec>, + arrays: Vec>, + held: Vec, + moved: Vec<*mut Handle>, + /// The objects Rust lends a callback, closed once it returns. + lent: Vec, + /// Python implementations and waker nodes given to Rust, taken back unless the call + /// happens. + given: Vec, + nodes: Vec, +} + +impl Scope { + /// Keep `obj` until the call returns, returning it borrowed. + pub fn keep(&mut self, obj: Obj) -> *mut PyObject { + let ptr = obj.ptr(); + self.keep.push(obj); + ptr + } + + /// Note that the call happened: what it moved out of is closed, and what it was given + /// is Rust's. + pub fn done(&mut self) { + for handle in self.moved.drain(..) { + // SAFETY: a handle this scope holds, so live. + unsafe { close(handle) }; + } + for given in self.given.drain(..) { + given.into_ptr(); + } + self.nodes.clear(); + } + + /// The handles held, which a future keeps until it ends. + pub fn take_held(&mut self) -> Vec { + std::mem::take(&mut self.held) + } +} + +impl Drop for Scope { + fn drop(&mut self) { + for waker in self.nodes.drain(..) { + // SAFETY: a reference the call never took. + unsafe { node_drop(waker.data) }; + } + for lent in self.lent.drain(..) { + // SAFETY: an object [`lent`] made. + unsafe { close(lent.ptr().cast()) }; + } + for view in &mut self.views { + // SAFETY: a view `PyObject_GetBuffer` filled, released once. + unsafe { PyBuffer_Release(&mut **view) }; + } + for held in self.held.drain(..) { + // SAFETY: a handle this scope holds. + unsafe { release(held.ptr().cast()) }; + } + } +} + +/// The classes the generated Python registers, by the index the glue names each with: +/// the runtime's, then the API's. +static TYPES: Gil> = Gil::new(Vec::new()); + +/// The plain enums' members, by class, with their discriminants. +static MEMBERS: Gil>>> = Gil::new(Vec::new()); + +/// The runtime's classes, at the start of the registry: the awaitable a pending future +/// is, and `datetime.timedelta`. +pub const T_FUTURE: usize = 0; +const T_TIMEDELTA: usize = 1; + +/// The class registered at `index`. +pub fn ty(index: usize) -> *mut PyObject { + // SAFETY: the GIL is held, and the module registered its classes when it loaded. + (unsafe { TYPES.get() })[index].ptr() +} + +/// `_register(*classes)`: note the classes the glue makes values of. +unsafe extern "C" fn register(_: *mut PyObject, args: *mut *mut PyObject, nargs: Py_ssize_t) -> *mut PyObject { + // SAFETY: Python's call. + let args = unsafe { std::slice::from_raw_parts(args, nargs as usize) }; + // SAFETY: the GIL is held, and each argument a live class. + unsafe { + *TYPES.get() = args.iter().map(|&class| Obj::borrowed(class)).collect(); + *MEMBERS.get() = (0..args.len()).map(|_| None).collect(); + } + none().map_or(null_mut(), Obj::into_ptr) +} + +pub fn none() -> R { + // SAFETY: `None` is immortal; a reference to it is still counted on older versions. + Ok(unsafe { Obj::borrowed(Py_None()) }) +} + +pub fn is_none(obj: *mut PyObject) -> bool { + // SAFETY: comparing addresses. + obj == unsafe { Py_None() } +} + +/// Whether `obj` is an instance of the class registered at `index`. +pub fn is(obj: *mut PyObject, index: usize) -> bool { + // SAFETY: the GIL is held, and `obj` is live. + unsafe { PyObject_TypeCheck(obj, ty(index).cast()) != 0 } +} + +/// A `TypeError` naming the class registered at `index` and what came instead. +fn expected(obj: *mut PyObject, index: usize) -> R { + expected_named(obj, &type_name(ty(index))) +} + +fn expected_named(obj: *mut PyObject, want: &str) -> R { + // SAFETY: `obj` is live. + let got = type_name(unsafe { Py_TYPE(obj) }.cast()); + type_error(&format!("expected {want}, got {got}")) +} + +/// A class's name, for an error. +fn type_name(class: *mut PyObject) -> String { + // SAFETY: the GIL is held, and `class` is a type. + unsafe { + let Ok(name) = Obj::new(PyType_GetName(class.cast())) else { + PyErr_Clear(); + return "?".to_string(); + }; + let mut len = 0; + let ptr = PyUnicode_AsUTF8AndSize(name.ptr(), &mut len); + if ptr.is_null() { + PyErr_Clear(); + return "?".to_string(); + } + String::from_utf8_lossy(std::slice::from_raw_parts(ptr.cast(), len as usize)).into_owned() + } +} + +/// The attribute `key` of `obj`, kept for the call. +pub fn attr(obj: *mut PyObject, key: &Key, s: &mut Scope) -> R<*mut PyObject> { + // SAFETY: the GIL is held, and `obj` is live. + let value = unsafe { Obj::new(PyObject_GetAttr(obj, key.get()?)) }?; + Ok(s.keep(value)) +} + +pub fn bool_in(obj: *mut PyObject) -> R { + // SAFETY: comparing addresses. + unsafe { + if obj == Py_True() { + Ok(true) + } else if obj == Py_False() { + Ok(false) + } else { + expected_named(obj, "bool") + } + } +} + +pub fn bool_out(value: bool) -> R { + // SAFETY: the GIL is held. + unsafe { Obj::new(PyBool_FromLong(value as _)) } +} + +/// An integer of Rust's type `T`, from any `int` or object with `__index__` in its range. +pub fn int_in>(obj: *mut PyObject) -> R { + // SAFETY: the GIL is held, and `obj` is live. + let value = unsafe { + let index = Obj::new(PyNumber_Index(obj))?; + let mut overflow = 0; + let v = PyLong_AsLongLongAndOverflow(index.ptr(), &mut overflow); + match overflow { + 0 if v == -1 && !PyErr_Occurred().is_null() => return Err(Raised), + 0 => v as i128, + // Past `i64`: the `u64`s above it, or out of every range. + _ => { + let v = PyLong_AsUnsignedLongLong(index.ptr()); + if v == u64::MAX && !PyErr_Occurred().is_null() { + PyErr_Clear(); + return overflow_int::(); + } + v as i128 + } + } + }; + T::try_from(value).or_else(|_| overflow_int::()) +} + +fn overflow_int() -> R { + let name = std::any::type_name::(); + overflow(&format!("int out of range for Rust's {name}")) +} + +pub fn int_out(value: i64) -> R { + // SAFETY: the GIL is held. + unsafe { Obj::new(PyLong_FromLongLong(value)) } +} + +pub fn uint_out(value: u64) -> R { + // SAFETY: the GIL is held. + unsafe { Obj::new(PyLong_FromUnsignedLongLong(value)) } +} + +pub fn float_in(obj: *mut PyObject) -> R { + // SAFETY: the GIL is held, and `obj` is live. + let value = unsafe { PyFloat_AsDouble(obj) }; + // SAFETY: as above. + if value == -1.0 && !unsafe { PyErr_Occurred() }.is_null() { + return Err(Raised); + } + Ok(value) +} + +pub fn float_out(value: f64) -> R { + // SAFETY: the GIL is held. + unsafe { Obj::new(PyFloat_FromDouble(value)) } +} + +/// A `char`, from a `str` of one code point that is a Unicode scalar value. +pub fn char_in(obj: *mut PyObject) -> R { + // SAFETY: the GIL is held, and `obj` is live. + unsafe { + if PyUnicode_Check(obj) == 0 || PyUnicode_GetLength(obj) != 1 { + return expected_named(obj, "a str of one character"); + } + let code = PyUnicode_ReadChar(obj, 0); + match char::from_u32(code) { + Some(_) => Ok(code), + None => value_error("a surrogate is no Rust char"), + } + } +} + +pub fn char_out(code: u32) -> R { + // SAFETY: the GIL is held. + unsafe { Obj::new(PyUnicode_FromOrdinal(code as c_int)) } +} + +/// A view of a `str`'s UTF-8, which Python caches in the string, so it lives as long as +/// `obj`: the caller keeps `obj` for the call. +pub fn str_in(obj: *mut PyObject) -> R { + // SAFETY: the GIL is held, and `obj` is live. + unsafe { + if PyUnicode_Check(obj) == 0 { + return expected_named(obj, "str"); + } + let mut len = 0; + let ptr = PyUnicode_AsUTF8AndSize(obj, &mut len); + if ptr.is_null() { + return Err(Raised); + } + Ok(rt::Buf { + ptr: ptr.cast(), + len: len as usize, + owner: null_mut(), + }) + } +} + +/// A `str` of a buffer of UTF-8 Rust returned, freeing it. +pub fn str_out(buf: rt::Buf) -> R { + // SAFETY: a buffer Rust returned, freed once copied. + unsafe { + let str = Obj::new(PyUnicode_FromStringAndSize(buf.ptr.cast(), buf.len as Py_ssize_t)); + buf.free(); + str + } +} + +/// A view of any object with the buffer protocol, `bytes`, `bytearray`, or a +/// `memoryview` among them, held for the call: Rust reads it in place. +pub fn bytes_in(obj: *mut PyObject, s: &mut Scope) -> R { + // SAFETY: `Py_buffer` is plain data, which the call fills. + let mut view: Box = Box::new(unsafe { std::mem::zeroed() }); + // SAFETY: the GIL is held, and `obj` is live. + if unsafe { PyObject_GetBuffer(obj, &mut *view, PyBUF_SIMPLE) } != 0 { + return Err(Raised); + } + let buf = rt::Buf { + ptr: view.buf.cast_const().cast(), + len: view.len as usize, + owner: null_mut(), + }; + s.views.push(view); + Ok(buf) +} + +/// Bytes Rust returned, as a read-only `memoryview` that lends them without a copy until +/// it and every view of it are released. +pub fn bytes_out(buf: rt::Buf) -> R { + // A view Rust lent only for a call is copied into a buffer of its own. + let buf = match buf.owner.is_null() { + // SAFETY: a buffer Rust lent for the call. + true => rt::Buf::owned(bytes::Bytes::copy_from_slice(unsafe { buf.bytes("python") })), + false => buf, + }; + // SAFETY: the GIL is held, and the owner type's slots are this module's. + unsafe { + let owner = PyType_GenericAlloc(BYTES.get().expect("loaded").ptr().cast(), 0); + let Ok(owner) = Obj::new(owner) else { + buf.free(); + return Err(Raised); + }; + (*owner.ptr().cast::()).buf = buf; + Obj::new(PyMemoryView_FromObject(owner.ptr())) + } +} + +/// Bytes Rust lends a callback for its call, copied into `bytes`. +pub fn lent_bytes(buf: rt::Buf) -> R { + // SAFETY: the GIL is held, and a buffer Rust lent for the call. + unsafe { + let bytes = buf.bytes("python"); + Obj::new(PyBytes_FromStringAndSize(bytes.as_ptr().cast(), bytes.len() as Py_ssize_t)) + } +} + +/// A path's bytes, from a `str`, `bytes`, or `os.PathLike`, as `os.fsencode` encodes it, +/// so a name a Unix path holds that is not UTF-8 crosses as it came. +pub fn path_in(obj: *mut PyObject, s: &mut Scope) -> R { + let mut encoded: *mut PyObject = null_mut(); + // SAFETY: the GIL is held, and `obj` is live; the converter writes a new `bytes`. + unsafe { + if PyUnicode_FSConverter(obj, (&raw mut encoded).cast()) == 0 { + return Err(Raised); + } + let encoded = s.keep(Obj::new(encoded)?); + let mut ptr: *mut c_char = null_mut(); + let mut len = 0; + if PyBytes_AsStringAndSize(encoded, &mut ptr, &mut len) != 0 { + return Err(Raised); + } + Ok(rt::Buf { + ptr: ptr.cast_const().cast(), + len: len as usize, + owner: null_mut(), + }) + } +} + +/// A path Rust returned, as a `str` `os.fsdecode` decodes, freeing it. +pub fn path_out(buf: rt::Buf) -> R { + // SAFETY: a buffer Rust returned, freed once decoded. + unsafe { + let path = Obj::new(PyUnicode_DecodeFSDefaultAndSize(buf.ptr.cast(), buf.len as Py_ssize_t)); + buf.free(); + path + } +} + +static DAYS: Key = Key::new(c"days"); +static SECONDS: Key = Key::new(c"seconds"); +static MICROSECONDS: Key = Key::new(c"microseconds"); +static MAX: Key = Key::new(c"max"); + +/// A `Duration`, from a `datetime.timedelta` that is not negative. +pub fn duration_in(obj: *mut PyObject) -> R { + if !is(obj, T_TIMEDELTA) { + return expected(obj, T_TIMEDELTA); + } + let mut s = Scope::default(); + let days: u64 = match int_in::(attr(obj, &DAYS, &mut s)?)? { + days if days < 0 => return value_error("a negative timedelta has no Rust Duration"), + days => days as u64, + }; + let seconds: u64 = int_in(attr(obj, &SECONDS, &mut s)?)?; + let micros: u32 = int_in(attr(obj, &MICROSECONDS, &mut s)?)?; + Ok(rt::Duration { + secs: days * 86_400 + seconds, + nanos: micros * 1_000, + }) +} + +/// A `datetime.timedelta`, which holds whole microseconds up to 999,999,999 days: Rust's +/// nanoseconds are truncated, and a longer one is `timedelta.max`, as Rust's +/// `Duration::MAX` stands for forever. +pub fn duration_out(duration: rt::Duration) -> R { + let class = ty(T_TIMEDELTA); + // SAFETY: the GIL is held. + unsafe { + if duration.secs / 86_400 > 999_999_999 { + return Obj::new(PyObject_GetAttr(class, MAX.get()?)); + } + let args = tuple(vec![ + int_out(0), + uint_out(duration.secs), + uint_out(u64::from(duration.nanos / 1_000)), + ])?; + Obj::new(PyObject_CallObject(class, args.ptr())) + } +} + +/// Which addresses a socket address type holds. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Family { + Any, + V4, + V6, +} + +/// A socket address, from a tuple as Python's `socket` module writes one: `(host, port)`, +/// or for IPv6, `(host, port, flowinfo, scope_id)`, its host a numeric address. +pub fn addr_in(obj: *mut PyObject, family: Family) -> R { + // SAFETY: the GIL is held, and `obj` is live. + let len = unsafe { + if PyTuple_Check(obj) == 0 { + return expected_named(obj, "a (host, port) tuple"); + } + PyTuple_Size(obj) + }; + if len != 2 && len != 4 { + return value_error("a socket address is (host, port) or (host, port, flowinfo, scope_id)"); + } + // SAFETY: in range. + let item = |n| unsafe { PyTuple_GetItem(obj, n) }; + let host = str_in(item(0))?; + // SAFETY: a view of the host's UTF-8, which the tuple keeps. + let host = unsafe { host.str("python") }; + let Ok(ip) = host.parse::() else { + return value_error(&format!("{host:?} is no numeric IP address")); + }; + let port: u16 = int_in(item(1))?; + let (flowinfo, scope_id) = match len { + 4 => (int_in::(item(2))?, int_in::(item(3))?), + _ => (0, 0), + }; + let addr = match ip { + std::net::IpAddr::V4(_) if len == 4 => return value_error("an IPv4 address takes no flowinfo or scope_id"), + std::net::IpAddr::V4(ip) => std::net::SocketAddr::from((ip, port)), + std::net::IpAddr::V6(ip) => std::net::SocketAddrV6::new(ip, port, flowinfo, scope_id).into(), + }; + match (family, addr) { + (Family::V4, std::net::SocketAddr::V6(_)) => value_error("an IPv6 address has no Rust SocketAddrV4"), + (Family::V6, std::net::SocketAddr::V4(_)) => value_error("an IPv4 address has no Rust SocketAddrV6"), + _ => Ok(rt::SocketAddr::lower(addr)), + } +} + +/// A socket address as Python's `socket` module writes one. +pub fn addr_out(addr: rt::SocketAddr) -> R { + let text = |ip: &dyn std::fmt::Display| { + let text = ip.to_string(); + // SAFETY: the GIL is held. + unsafe { Obj::new(PyUnicode_FromStringAndSize(text.as_ptr().cast(), text.len() as Py_ssize_t)) } + }; + match addr.lift("python") { + std::net::SocketAddr::V4(v4) => tuple(vec![text(v4.ip()), uint_out(v4.port().into())]), + std::net::SocketAddr::V6(v6) => tuple(vec![ + text(v6.ip()), + uint_out(v6.port().into()), + uint_out(v6.flowinfo().into()), + uint_out(v6.scope_id().into()), + ]), + } +} + +static START: Key = Key::new(c"start"); +static STOP: Key = Key::new(c"stop"); +static STEP: Key = Key::new(c"step"); + +/// A `Range`, from a `range` of step 1. +pub fn range_in>(obj: *mut PyObject) -> R> { + // SAFETY: the GIL is held, and `obj` is live. + if unsafe { PyObject_TypeCheck(obj, &raw mut PyRange_Type) } == 0 { + return expected_named(obj, "range"); + } + let mut s = Scope::default(); + if int_in::(attr(obj, &STEP, &mut s)?)? != 1 { + return value_error("a range of a step other than 1 has no Rust Range"); + } + Ok(rt::Range { + start: int_in(attr(obj, &START, &mut s)?)?, + end: int_in(attr(obj, &STOP, &mut s)?)?, + }) +} + +pub fn range_out(start: R, end: R) -> R { + let args = tuple(vec![start, end])?; + // SAFETY: the GIL is held. + unsafe { Obj::new(PyObject_CallObject((&raw mut PyRange_Type).cast(), args.ptr())) } +} + +/// A tuple of `items`, each made before any error is returned, so each frees what Rust +/// handed over. +pub fn tuple(items: Vec>) -> R { + // SAFETY: the GIL is held. + unsafe { + let tuple = Obj::new(PyTuple_New(items.len() as Py_ssize_t)); + let mut raised = false; + for (n, item) in items.into_iter().enumerate() { + match (&tuple, item) { + (Ok(tuple), Ok(item)) => { + PyTuple_SetItem(tuple.ptr(), n as Py_ssize_t, item.into_ptr()); + } + (_, Err(Raised)) => raised = true, + (Err(Raised), Ok(_)) => {} + } + } + match raised { + true => Err(Raised), + false => tuple, + } + } +} + +/// The item `n` of a tuple of `len`, or a `TypeError`. +pub fn tuple_item(obj: *mut PyObject, n: usize, len: usize) -> R<*mut PyObject> { + // SAFETY: the GIL is held, and `obj` is live. + unsafe { + if PyTuple_Check(obj) == 0 || PyTuple_Size(obj) != len as Py_ssize_t { + return expected_named(obj, &format!("a tuple of {len}")); + } + Ok(PyTuple_GetItem(obj, n as Py_ssize_t)) + } +} + +/// A `Vec` of the items any iterable yields, each read by `each`, held for the call. +pub fn list_in( + obj: *mut PyObject, + s: &mut Scope, + mut each: impl FnMut(*mut PyObject, &mut Scope) -> R, +) -> R> { + // SAFETY: the GIL is held, and `obj` is live. + unsafe { + // A str iterates its characters, which no Rust list of strings means. + if PyUnicode_Check(obj) != 0 { + return expected_named(obj, "a list"); + } + let iter = Obj::new(PyObject_GetIter(obj))?; + let mut items = Vec::new(); + loop { + let item = PyIter_Next(iter.ptr()); + if item.is_null() { + if !PyErr_Occurred().is_null() { + return Err(Raised); + } + break; + } + let item = s.keep(Obj::new(item)?); + items.push(each(item, s)?); + } + let items = Box::new(items); + let list = rt::List { + ptr: items.as_ptr(), + len: items.len(), + owner: null_mut(), + }; + s.arrays.push(items); + Ok(list) + } +} + +/// A `list` of the elements of an array Rust returned, each made by `each`, which frees +/// what it holds, before the array is freed. +pub fn list_out(list: rt::List, mut each: impl FnMut(T) -> R) -> R { + // SAFETY: an array Rust returned, whose elements `each` takes. + let items: Vec> = unsafe { list.items("python") }.iter().map(|&item| each(item)).collect(); + // SAFETY: as above, with its elements taken. + unsafe { rt::List { len: 0, ..list }.free("python", |_| {}) }; + // SAFETY: the GIL is held. + unsafe { + let out = Obj::new(PyList_New(items.len() as Py_ssize_t)); + let mut raised = false; + for (n, item) in items.into_iter().enumerate() { + match (&out, item) { + (Ok(out), Ok(item)) => { + PyList_SetItem(out.ptr(), n as Py_ssize_t, item.into_ptr()); + } + (_, Err(Raised)) => raised = true, + (Err(Raised), Ok(_)) => {} + } + } + match raised { + true => Err(Raised), + false => out, + } + } +} + +/// An instance of the class registered at `index` with `fields` set, each made before +/// any error is returned, so each frees what Rust handed over. Made as `object.__new__` +/// makes one and set as `object.__setattr__` sets, so no `__init__` runs. +pub fn record(index: usize, fields: Vec<(&Key, R)>) -> R { + let class = ty(index); + // SAFETY: the GIL is held, and `class` a class of the generated Python. + unsafe { + let alloc: allocfunc = std::mem::transmute(PyType_GetSlot(class.cast(), Py_tp_alloc)); + let obj = Obj::new(alloc(class.cast(), 0)); + let mut raised = false; + for (key, value) in fields { + match (&obj, value, key.get()) { + (Ok(obj), Ok(value), Ok(key)) => { + raised |= PyObject_GenericSetAttr(obj.ptr(), key, value.ptr()) != 0; + } + _ => raised = true, + } + } + match raised { + true => Err(Raised), + false => obj, + } + } +} + +/// The field `key` of a record or variant, after checking `obj` is of the class +/// registered at `index`. +pub fn field(obj: *mut PyObject, index: usize, key: &Key, s: &mut Scope) -> R<*mut PyObject> { + match is(obj, index) { + true => attr(obj, key, s), + false => expected(obj, index), + } +} + +/// Check `obj` is of the class registered at `index`. +pub fn check(obj: *mut PyObject, index: usize) -> R<()> { + match is(obj, index) { + true => Ok(()), + false => expected(obj, index), + } +} + +static VALUE: Key = Key::new(c"value"); + +/// The members of the plain enum registered at `index`, with their discriminants. +fn members(index: usize) -> R<&'static [(i64, Obj)]> { + // SAFETY: the GIL is held. + let table = unsafe { MEMBERS.get() }; + if table[index].is_none() { + let mut members = Vec::new(); + // SAFETY: the GIL is held, and an `Enum` class iterates its members. + unsafe { + let iter = Obj::new(PyObject_GetIter(ty(index)))?; + loop { + let member = PyIter_Next(iter.ptr()); + if member.is_null() { + if !PyErr_Occurred().is_null() { + return Err(Raised); + } + break; + } + let member = Obj::new(member)?; + let value = Obj::new(PyObject_GetAttr(member.ptr(), VALUE.get()?))?; + members.push((int_in::(value.ptr())?, member)); + } + } + table[index] = Some(members); + } + Ok(table[index].as_deref().expect("set above")) +} + +/// The member of the plain enum registered at `index` whose discriminant is `value`. +pub fn member(index: usize, value: i64) -> R { + match members(index)?.iter().find(|(v, _)| *v == value) { + // SAFETY: a live member. + Some((_, member)) => Ok(unsafe { Obj::borrowed(member.ptr()) }), + None => rt::fail("python", "an enum's discriminant is invalid"), + } +} + +/// The discriminant of `obj`, a member of the plain enum registered at `index`. +pub fn member_in(obj: *mut PyObject, index: usize) -> R { + match members(index)?.iter().find(|(_, m)| m.ptr() == obj) { + Some((value, _)) => Ok(*value as i32), + None => expected(obj, index), + } +} + +/// Raise the exception class registered at `index`, holding `error`. +pub fn raise_error(index: usize, error: R) -> R { + let args = tuple(vec![error])?; + // SAFETY: the GIL is held, and the class an exception's. + unsafe { + let exc = Obj::new(PyObject_CallObject(ty(index), args.ptr()))?; + PyErr_SetObject(ty(index), exc.ptr()); + } + Err(Raised) +} + +/// Call `callable` with `args`, each made before any error is returned. +pub fn call_py(callable: *mut PyObject, args: Vec>) -> R { + let args = tuple(args)?; + // SAFETY: the GIL is held, and `callable` is live. + unsafe { Obj::new(PyObject_CallObject(callable, args.ptr())) } +} + +/// Call the method `key` of `obj` with `args`. +pub fn call_method(obj: *mut PyObject, key: &Key, args: Vec>) -> R { + // SAFETY: the GIL is held, and `obj` is live. + let method = unsafe { Obj::new(PyObject_GetAttr(obj, key.get()?)) }?; + call_py(method.ptr(), args) +} + +/// A Python implementation of a trait, given to Rust: a reference Rust drops through +/// [`free_callback`], or the scope drops if the call never happens. +pub fn callback(obj: *mut PyObject, s: &mut Scope) -> *mut c_void { + if is_none(obj) { + return null_mut(); + } + // SAFETY: the GIL is held, and `obj` is live. + let given = unsafe { Obj::borrowed(obj) }; + let ptr = given.ptr(); + s.given.push(given); + ptr.cast() +} + +static CLOSE: Key = Key::new(c"close"); + +/// Let go of a Python implementation of a trait, from any thread: closed first if it has +/// a `close` method. +/// +/// # Safety +/// `data` is a reference [`callback`] gave Rust, freed once. +pub unsafe extern "C" fn free_callback(data: *mut c_void) { + upcall("free", || { + // SAFETY: the reference Rust held. + let obj = unsafe { Obj::new(data.cast()) }?; + // SAFETY: the GIL is held. + if unsafe { PyObject_HasAttr(obj.ptr(), CLOSE.get()?) } == 1 { + call_method(obj.ptr(), &CLOSE, Vec::new())?; + } + Ok(()) + }) +} + +/// An object owning a handle Rust returned, or one Rust lends for a call: the instance +/// of a class of the generated Python, which subclasses `Object`. +#[repr(C)] +pub struct Handle { + ob_base: PyObject, + raw: *const c_void, + /// What frees the handle: none for one Rust lends, or once freed. + free: Option, + /// The calls holding it, each of which it outlives. + holds: usize, + closed: bool, +} + +/// A new object of the class registered at `index` owning `raw`, which `free` frees. +pub fn object(raw: *const c_void, index: usize, free: unsafe extern "C" fn(*const c_void)) -> R { + handle(raw, index, Some(free)) +} + +/// [`object`], or `None` for a null handle. +pub fn object_opt(raw: *const c_void, index: usize, free: unsafe extern "C" fn(*const c_void)) -> R { + match raw.is_null() { + true => none(), + false => object(raw, index, free), + } +} + +/// An object of the class registered at `index` for a handle Rust lends a callback, +/// closed once the scope `s` of the call ends. +pub fn lent(raw: *const c_void, index: usize, s: &mut Scope) -> R { + let obj = handle(raw, index, None)?; + // SAFETY: the GIL is held, and `obj` is live. + s.lent.push(unsafe { Obj::borrowed(obj.ptr()) }); + Ok(obj) +} + +fn handle(raw: *const c_void, index: usize, free: Option) -> R { + let class = ty(index); + // SAFETY: the GIL is held, and `class` subclasses `Object` with no fields of its own. + unsafe { + let obj = PyType_GenericAlloc(class.cast(), 0); + let Ok(obj) = Obj::new(obj) else { + if let Some(free) = free { + free(raw); + } + return Err(Raised); + }; + let handle = obj.ptr().cast::(); + (*handle).raw = raw; + (*handle).free = free; + (*handle).holds = 0; + (*handle).closed = false; + Ok(obj) + } +} + +/// The handle `obj` owns, an object of the class registered at `index`, held for the +/// call; `moved` when the call moves its value out, after which `obj` is closed. +pub fn hold(obj: *mut PyObject, index: usize, s: &mut Scope, moved: bool) -> R<*const c_void> { + check(obj, index)?; + let handle = obj.cast::(); + // SAFETY: an object of a class that subclasses `Object`, and the GIL is held. + unsafe { + if (*handle).closed { + return value_error(&format!("{} is closed", type_name(ty(index)))); + } + (*handle).holds += 1; + s.held.push(Obj::borrowed(obj)); + if moved { + s.moved.push(handle); + } + Ok((*handle).raw) + } +} + +/// [`hold`], or a null handle for `None`. +pub fn hold_opt(obj: *mut PyObject, index: usize, s: &mut Scope, moved: bool) -> R<*const c_void> { + match is_none(obj) { + true => Ok(null()), + false => hold(obj, index, s, moved), + } +} + +/// Let go of a call's hold, freeing the handle if it was closed meanwhile. +/// +/// # Safety +/// `handle` is live, and held. +unsafe fn release(handle: *mut Handle) { + // SAFETY: the caller's contract. + unsafe { + (*handle).holds -= 1; + if (*handle).closed && (*handle).holds == 0 { + free_now(handle); + } + } +} + +/// Close a handle: freed now, or once the last call holding it returns. +/// +/// # Safety +/// `handle` is live. +pub unsafe fn close(handle: *mut Handle) { + // SAFETY: the caller's contract. + unsafe { + if (*handle).closed { + return; + } + (*handle).closed = true; + if (*handle).holds == 0 { + free_now(handle); + } + } +} + +/// # Safety +/// `handle` is live and held by no call. +unsafe fn free_now(handle: *mut Handle) { + // SAFETY: the caller's contract. + unsafe { + if let Some(free) = (*handle).free.take() { + let raw = std::mem::replace(&mut (*handle).raw, null()); + call(|| free(raw)); + } + } +} + +unsafe extern "C" fn handle_close(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on an `Object`. + unsafe { close(obj.cast()) }; + none().map_or(null_mut(), Obj::into_ptr) +} + +unsafe extern "C" fn handle_enter(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a live object. + unsafe { Obj::borrowed(obj) }.into_ptr() +} + +unsafe extern "C" fn handle_exit(obj: *mut PyObject, _: *mut *mut PyObject, _: Py_ssize_t) -> *mut PyObject { + // SAFETY: Python calls it on an `Object`. + unsafe { close(obj.cast()) }; + none().map_or(null_mut(), Obj::into_ptr) +} + +unsafe extern "C" fn handle_dealloc(obj: *mut PyObject) { + // SAFETY: Python deallocates an `Object` once, holding the GIL; nothing holds it. + unsafe { + free_now(obj.cast()); + dealloc(obj); + } +} + +/// Free an instance of a heap type, and the reference to its type it held. +/// +/// # Safety +/// `obj` is being deallocated, holding the GIL. +unsafe fn dealloc(obj: *mut PyObject) { + // SAFETY: the caller's contract. + unsafe { + let class = Py_TYPE(obj); + let free: freefunc = std::mem::transmute(PyType_GetSlot(class, Py_tp_free)); + free(obj.cast()); + Py_DecRef(class.cast()); + } +} + +/// The owner of bytes Rust returned, which a `memoryview` reads in place. +#[repr(C)] +struct Bytes { + ob_base: PyObject, + buf: rt::Buf, +} + +static BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// A reference kept for the interpreter's life. +struct Static(Obj); + +// SAFETY: touched only with the GIL held. +unsafe impl Send for Static {} +// SAFETY: as above. +unsafe impl Sync for Static {} + +impl Static { + fn ptr(&self) -> *mut PyObject { + self.0.ptr() + } +} + +unsafe extern "C" fn bytes_getbuffer(obj: *mut PyObject, view: *mut Py_buffer, flags: c_int) -> c_int { + // SAFETY: Python asks a live `Bytes` for a view. + unsafe { + let buf = (*obj.cast::()).buf; + PyBuffer_FillInfo(view, obj, buf.ptr.cast_mut().cast(), buf.len as Py_ssize_t, 1, flags) + } +} + +unsafe extern "C" fn bytes_dealloc(obj: *mut PyObject) { + // SAFETY: Python deallocates a `Bytes` once no view of it is left. + unsafe { + (*obj.cast::()).buf.free(); + dealloc(obj); + } +} + +/// What a wake does: queue a token for the event loop, and write a byte it reads, once +/// until it takes them. +pub struct Queue { + tokens: Mutex>, + signaled: AtomicBool, + tx: Socket, + rx: Socket, +} + +#[cfg(unix)] +type Socket = std::os::unix::net::UnixStream; +#[cfg(windows)] +type Socket = std::net::TcpStream; + +/// A connected pair of sockets that never block. +#[cfg(unix)] +fn socket_pair() -> std::io::Result<(Socket, Socket)> { + let (tx, rx) = Socket::pair()?; + tx.set_nonblocking(true)?; + rx.set_nonblocking(true)?; + Ok((tx, rx)) +} + +/// A connected pair of sockets that never block, over loopback TCP, as Python's own +/// `socket.socketpair` makes one on Windows. +#[cfg(windows)] +fn socket_pair() -> std::io::Result<(Socket, Socket)> { + let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?; + let tx = Socket::connect(listener.local_addr()?)?; + let (rx, _) = listener.accept()?; + tx.set_nodelay(true)?; + tx.set_nonblocking(true)?; + rx.set_nonblocking(true)?; + Ok((tx, rx)) +} + +impl Queue { + fn push(&self, token: i64) { + self.tokens.lock().unwrap_or_else(|e| e.into_inner()).push(token); + if !self.signaled.swap(true, Ordering::SeqCst) { + // A full socket already signals. + let _ = (&self.tx).write(&[1]); + } + } + + /// The tokens queued, each once, in the order first queued: drained, then the signal + /// reset, then the queue taken, so a token queued after is signaled again. + fn take(&self) -> Vec { + let mut sink = [0u8; 64]; + while matches!((&self.rx).read(&mut sink), Ok(n) if n > 0) {} + self.signaled.store(false, Ordering::SeqCst); + let tokens = std::mem::take(&mut *self.tokens.lock().unwrap_or_else(|e| e.into_inner())); + let mut seen = std::collections::HashSet::new(); + tokens.into_iter().filter(|t| seen.insert(*t)).collect() + } +} + +/// What a waker Rust holds wakes: a future polled on an event loop, or a Python +/// `Waker`. A token names it to the loop; a `Waker`'s negative token says it is gone. +pub struct Node { + state: AtomicU8, + token: i64, + queue: Arc, + /// Whether it is a `Waker`'s, whose every wake reaches the loop. + waker: bool, +} + +const IDLE: u8 = 0; +const POLLING: u8 = 1; +const PARKED: u8 = 2; +const WOKEN: u8 = 3; + +static TOKENS: AtomicI64 = AtomicI64::new(1); + +impl Node { + fn new(queue: Arc, waker: bool) -> Arc { + Arc::new(Self { + state: AtomicU8::new(IDLE), + token: TOKENS.fetch_add(1, Ordering::Relaxed), + queue, + waker, + }) + } + + /// A C waker holding a new reference to the node. + fn give(self: &Arc) -> rt::Waker { + rt::Waker { + wake: Some(node_wake), + drop: Some(node_drop), + data: Arc::into_raw(self.clone()).cast_mut().cast(), + } + } + + /// The C waker [`Self::give`] gave, again: the C ABI reuses the one it holds. + fn lend(self: &Arc) -> rt::Waker { + rt::Waker { + wake: Some(node_wake), + drop: Some(node_drop), + data: Arc::as_ptr(self).cast_mut().cast(), + } + } +} + +impl Drop for Node { + fn drop(&mut self) { + if self.waker { + self.queue.push(-self.token); + } + } +} + +/// A wake, from any thread: a future's is latched while it polls, and queued once parked. +unsafe extern "C" fn node_wake(data: *mut c_void) { + // SAFETY: Rust holds a reference to the node while it may wake it. + let node = unsafe { &*data.cast::() }; + match node.waker { + true => node.queue.push(node.token), + false => { + if node.state.swap(WOKEN, Ordering::AcqRel) == PARKED { + node.queue.push(node.token); + } + } + } +} + +unsafe extern "C" fn node_drop(data: *mut c_void) { + // SAFETY: a reference `give` made, dropped once. + drop(unsafe { Arc::from_raw(data.cast::()) }); +} + +/// `Queue`: the queue of one event loop, with its socket's file descriptor to read. +#[repr(C)] +struct QueueObj { + ob_base: PyObject, + queue: Option>, +} + +unsafe extern "C" fn queue_new(class: *mut PyTypeObject, _: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + let (tx, rx) = match socket_pair() { + Ok(pair) => pair, + Err(err) => { + // SAFETY: a static exception type. + let _ = raise::<()>(unsafe { PyExc_OSError }, &err.to_string()); + return null_mut(); + } + }; + // SAFETY: the GIL is held, and `class` is `Queue`. + unsafe { + let obj = PyType_GenericAlloc(class, 0); + if !obj.is_null() { + std::ptr::write( + &raw mut (*obj.cast::()).queue, + Some(Arc::new(Queue { + tokens: Mutex::new(Vec::new()), + signaled: AtomicBool::new(false), + tx, + rx, + })), + ); + } + obj + } +} + +unsafe extern "C" fn queue_dealloc(obj: *mut PyObject) { + // SAFETY: Python deallocates a `Queue` once. + unsafe { + drop((*obj.cast::()).queue.take()); + dealloc(obj); + } +} + +/// The queue a `Queue` holds. +fn queue_of(obj: *mut PyObject) -> R> { + if !is_type(obj, &QUEUE) { + return expected_named(obj, "Queue"); + } + // SAFETY: a `Queue`, which holds its queue until deallocated. + Ok(unsafe { (*obj.cast::()).queue.clone() }.expect("a live queue")) +} + +unsafe extern "C" fn queue_fileno(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Queue`. + let queue = unsafe { (*obj.cast::()).queue.as_ref() }.expect("a live queue"); + #[cfg(unix)] + let fd = std::os::fd::AsRawFd::as_raw_fd(&queue.rx) as i64; + #[cfg(windows)] + let fd = std::os::windows::io::AsRawSocket::as_raw_socket(&queue.rx) as i64; + int_out(fd).map_or(null_mut(), Obj::into_ptr) +} + +unsafe extern "C" fn queue_take(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Queue`. + let queue = unsafe { (*obj.cast::()).queue.as_ref() }.expect("a live queue"); + let tokens = queue.take(); + wrap(|| { + let list = rt::List::owned(tokens); + list_out(list, int_out) + }) +} + +/// `Node`: a Python `Waker`'s node, made on its loop's queue. +#[repr(C)] +struct NodeObj { + ob_base: PyObject, + node: Option>, +} + +unsafe extern "C" fn node_new(class: *mut PyTypeObject, args: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + wrap(|| { + let queue = queue_of(tuple_item(args, 0, 1)?)?; + // SAFETY: the GIL is held, and `class` is `Node`. + unsafe { + let obj = Obj::new(PyType_GenericAlloc(class, 0))?; + std::ptr::write(&raw mut (*obj.ptr().cast::()).node, Some(Node::new(queue, true))); + Ok(obj) + } + }) +} + +unsafe extern "C" fn node_obj_dealloc(obj: *mut PyObject) { + // SAFETY: Python deallocates a `Node` once. + unsafe { + drop((*obj.cast::()).node.take()); + dealloc(obj); + } +} + +unsafe extern "C" fn node_token(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Node`. + let node = unsafe { (*obj.cast::()).node.as_ref() }.expect("a live node"); + int_out(node.token).map_or(null_mut(), Obj::into_ptr) +} + +static NODE_ATTR: Key = Key::new(c"_node"); + +/// A Python `Waker` passed by value: Rust holds a new reference to its node. +pub fn waker_in(obj: *mut PyObject, s: &mut Scope) -> R { + let node = attr(obj, &NODE_ATTR, s)?; + if !is_type(node, &NODE) { + return expected_named(obj, "Waker"); + } + // SAFETY: a `Node`, which holds its node until deallocated. + let node = unsafe { (*node.cast::()).node.clone() }.expect("a live node"); + let waker = node.give(); + s.nodes.push(waker); + Ok(waker) +} + +/// Poll a future once with a waker, returning its value once it finishes. +pub type Poll = unsafe fn(*mut c_void, rt::Waker) -> R>; + +/// `Future`: a C ABI future its first poll left pending, which `__await__` polls on its +/// loop's queue until it finishes, and frees once it does or is closed, cancelling it. +#[repr(C)] +pub struct Future { + ob_base: PyObject, + raw: *mut c_void, + poll: Option, + free: Option, + node: Option>, + token: i64, + /// The handles the call holds until the future ends. + held: Vec, +} + +static PENDING: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// A future the call `raw` made, polled at once unless `eager`, since a function that is +/// not `async` ran at the call and is polled when awaited: its value if ready, or else the +/// awaitable that polls it again, holding what the call held. +/// +/// # Safety +/// `raw` is a live future that `poll` and `free` take. +pub unsafe fn start(raw: *mut c_void, poll: Poll, free: unsafe fn(*mut c_void), s: &mut Scope, eager: bool) -> R { + if !eager { + let none = rt::Waker { + wake: None, + drop: None, + data: null_mut(), + }; + // SAFETY: the caller's contract. + match unsafe { poll(raw, none) } { + Ok(None) => {} + done => { + // SAFETY: as above, and finished or failed. + call(|| unsafe { free(raw) }); + return done.map(|value| value.expect("ready")); + } + } + } + // SAFETY: the GIL is held, and the registered class subclasses `Future`. + unsafe { + let obj = PyType_GenericAlloc(ty(T_FUTURE).cast(), 0); + let Ok(obj) = Obj::new(obj) else { + call(|| free(raw)); + return Err(Raised); + }; + // The allocation is zeroed, which is `None` for each option. + let future = obj.ptr().cast::(); + (*future).raw = raw; + (*future).poll = Some(poll); + (*future).free = Some(free); + (*future).token = TOKENS.fetch_add(1, Ordering::Relaxed); + std::ptr::write(&raw mut (*future).held, s.take_held()); + Ok(obj) + } +} + +/// Free a future's C future, cancelling it unless it finished, and let go of its holds. +/// +/// # Safety +/// `future` is live. +unsafe fn future_close(future: *mut Future) { + // SAFETY: the caller's contract. + unsafe { + if let Some(free) = (*future).free.take() { + let raw = std::mem::replace(&mut (*future).raw, null_mut()); + call(|| free(raw)); + } + (*future).poll = None; + (*future).node = None; + for held in std::mem::take(&mut (*future).held) { + release(held.ptr().cast()); + } + } +} + +/// `poll(queue)`: poll on the queue's loop, again at once if woken while polling: the +/// value, or `PENDING` once parked, after which a wake queues the future's token. +unsafe extern "C" fn future_poll(obj: *mut PyObject, args: *mut *mut PyObject, nargs: Py_ssize_t) -> *mut PyObject { + let future = obj.cast::(); + wrap(|| { + // SAFETY: Python's call. + let args = unsafe { self::args(args, nargs, 1, "poll") }?; + // SAFETY: Python calls it on a `Future`, holding the GIL. + let this = unsafe { &mut *future }; + let Some(poll) = this.poll else { + return value_error("the future ended"); + }; + let node = match &this.node { + Some(node) => node.clone(), + None => { + let node = Arc::new(Node { + state: AtomicU8::new(IDLE), + token: this.token, + queue: queue_of(args[0])?, + waker: false, + }); + this.node = Some(node.clone()); + node + } + }; + loop { + let first = node.state.swap(POLLING, Ordering::AcqRel) == IDLE; + // The first poll with the node gives the C ABI a reference, which it keeps + // while it reuses the waker. + let waker = match first { + true => node.give(), + false => node.lend(), + }; + // SAFETY: a live future, which `poll` takes. + let done = unsafe { poll(this.raw, waker) }; + match done { + Ok(None) => {} + done => { + // SAFETY: a live future, finished or failed. + unsafe { future_close(future) }; + return done.map(|value| value.expect("ready")); + } + } + if node + .state + .compare_exchange(POLLING, PARKED, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + // SAFETY: set when the module loaded. + return Ok(unsafe { Obj::borrowed(PENDING.get().expect("loaded").ptr()) }); + } + // Woken while it polled: the wake is for this poll's registration, so poll again. + } + }) +} + +unsafe extern "C" fn future_token(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Future`. + int_out(unsafe { (*obj.cast::()).token }).map_or(null_mut(), Obj::into_ptr) +} + +unsafe extern "C" fn future_close_py(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Future`, holding the GIL. + unsafe { future_close(obj.cast()) }; + none().map_or(null_mut(), Obj::into_ptr) +} + +unsafe extern "C" fn future_dealloc(obj: *mut PyObject) { + // SAFETY: Python deallocates a `Future` once. + unsafe { + future_close(obj.cast()); + std::ptr::drop_in_place(&raw mut (*obj.cast::()).held); + dealloc(obj); + } +} + +/// Whether `obj` is exactly the runtime's type in `slot`. +fn is_type(obj: *mut PyObject, slot: &std::sync::OnceLock) -> bool { + // SAFETY: `obj` is live. + slot.get().is_some_and(|t| unsafe { Py_TYPE(obj) }.cast() == t.ptr()) +} + +static OBJECT: std::sync::OnceLock = std::sync::OnceLock::new(); +static FUTURE: std::sync::OnceLock = std::sync::OnceLock::new(); +static QUEUE: std::sync::OnceLock = std::sync::OnceLock::new(); +static NODE: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// A method table entry. +pub const fn method(name: &'static CStr, f: PyCFunctionFast) -> PyMethodDef { + PyMethodDef { + ml_name: name.as_ptr(), + ml_meth: PyMethodDefPointer { PyCFunctionFast: f }, + ml_flags: METH_FASTCALL, + ml_doc: null(), + } +} + +const fn noargs(name: &'static CStr, f: PyCFunction) -> PyMethodDef { + PyMethodDef { + ml_name: name.as_ptr(), + ml_meth: PyMethodDefPointer { PyCFunction: f }, + ml_flags: METH_NOARGS, + ml_doc: null(), + } +} + +pub const END: PyMethodDef = PyMethodDef::zeroed(); + +/// A table of methods Python reads for the interpreter's life. +pub struct Methods(pub UnsafeCell<[PyMethodDef; N]>); + +// SAFETY: Python reads it, holding the GIL. +unsafe impl Sync for Methods {} + +impl Methods { + pub fn ptr(&self) -> *mut PyMethodDef { + self.0.get().cast() + } +} + +static OBJECT_METHODS: Methods<4> = Methods(UnsafeCell::new([ + noargs(c"close", handle_close), + noargs(c"__enter__", handle_enter), + method(c"__exit__", handle_exit), + END, +])); + +static FUTURE_METHODS: Methods<4> = Methods(UnsafeCell::new([ + method(c"poll", future_poll), + noargs(c"token", future_token), + noargs(c"close", future_close_py), + END, +])); + +static QUEUE_METHODS: Methods<3> = Methods(UnsafeCell::new([ + noargs(c"fileno", queue_fileno), + noargs(c"take", queue_take), + END, +])); + +static NODE_METHODS: Methods<2> = Methods(UnsafeCell::new([noargs(c"token", node_token), END])); + +static REGISTER: Methods<2> = Methods(UnsafeCell::new([method(c"_register", register), END])); + +/// Make a type from `slots` and add it to `module` by its name, noting it in `cell`. +/// +/// # Safety +/// The GIL is held, and `module` is the module being made. +unsafe fn add_type( + module: *mut PyObject, + cell: &std::sync::OnceLock, + name: &CStr, + size: usize, + flags: u64, + slots: &mut [PyType_Slot], +) -> R<()> { + let qualified = format!("{}.{}\0", module_name(module), name.to_str().expect("ASCII")); + // Python keeps the name for the type's life. + let qualified: &'static str = Box::leak(qualified.into_boxed_str()); + let mut spec = PyType_Spec { + name: qualified.as_ptr().cast(), + basicsize: size as c_int, + itemsize: 0, + flags: (Py_TPFLAGS_DEFAULT as u64 | flags) as _, + slots: slots.as_mut_ptr(), + }; + // SAFETY: the caller's contract, and `spec` a valid spec. + unsafe { + let class = Obj::new(PyType_FromSpec(&mut spec))?; + if PyModule_AddObjectRef(module, name.as_ptr(), class.ptr()) != 0 { + return Err(Raised); + } + let _ = cell.set(Static(class)); + } + Ok(()) +} + +fn module_name(module: *mut PyObject) -> String { + // SAFETY: the GIL is held, and `module` a module. + unsafe { + let name = PyModule_GetName(module); + CStr::from_ptr(name).to_string_lossy().into_owned() + } +} + +/// Make the extension module `def` names, with the runtime's types and functions beside +/// the glue's. +/// +/// # Safety +/// Python calls the module's `PyInit_` with the GIL held, once per interpreter. +pub unsafe fn module(def: *mut PyModuleDef) -> *mut PyObject { + // SAFETY: the caller's contract. + unsafe { + let Ok(module) = Obj::new(PyModule_Create(def)) else { + return null_mut(); + }; + match runtime(module.ptr()) { + Ok(()) => module.into_ptr(), + Err(Raised) => null_mut(), + } + } +} + +unsafe fn runtime(module: *mut PyObject) -> R<()> { + // SAFETY: the GIL is held, and `module` the module being made. + unsafe { + if PyModule_AddFunctions(module, REGISTER.ptr()) != 0 { + return Err(Raised); + } + let base = (Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION) as u64; + add_type(module, &OBJECT, c"Object", size_of::(), base, &mut [ + slot(Py_tp_dealloc, handle_dealloc as *mut c_void), + slot(Py_tp_methods, OBJECT_METHODS.ptr().cast()), + slot( + Py_tp_doc, + c"An object owning a Rust value's handle, freed once the last reference to it goes, or sooner by close() or a with block.".as_ptr().cast_mut().cast(), + ), + slot(0, null_mut()), + ])?; + add_type(module, &FUTURE, c"Future", size_of::(), base, &mut [ + slot(Py_tp_dealloc, future_dealloc as *mut c_void), + slot(Py_tp_methods, FUTURE_METHODS.ptr().cast()), + slot(0, null_mut()), + ])?; + add_type(module, &QUEUE, c"Queue", size_of::(), 0, &mut [ + slot(Py_tp_new, queue_new as *mut c_void), + slot(Py_tp_dealloc, queue_dealloc as *mut c_void), + slot(Py_tp_methods, QUEUE_METHODS.ptr().cast()), + slot(0, null_mut()), + ])?; + add_type(module, &NODE, c"Node", size_of::(), 0, &mut [ + slot(Py_tp_new, node_new as *mut c_void), + slot(Py_tp_dealloc, node_obj_dealloc as *mut c_void), + slot(Py_tp_methods, NODE_METHODS.ptr().cast()), + slot(0, null_mut()), + ])?; + let bytes_slots = &mut [ + slot(Py_bf_getbuffer, bytes_getbuffer as *mut c_void), + slot(Py_tp_dealloc, bytes_dealloc as *mut c_void), + slot(0, null_mut()), + ]; + add_type(module, &BYTES, c"Bytes", size_of::(), Py_TPFLAGS_DISALLOW_INSTANTIATION as u64, bytes_slots)?; + // A plain `object()`, which nothing else is. + let pending = Obj::new(PyObject_CallObject((&raw mut PyBaseObject_Type).cast(), null_mut()))?; + if PyModule_AddObjectRef(module, c"PENDING".as_ptr(), pending.ptr()) != 0 { + return Err(Raised); + } + let _ = PENDING.set(Static(pending)); + } + Ok(()) +} + +const fn slot(slot: c_int, pfunc: *mut c_void) -> PyType_Slot { + PyType_Slot { slot, pfunc } +} diff --git a/ffi/runtime/rt.py b/ffi/runtime/rt.py new file mode 100644 index 000000000..dabb4eaae --- /dev/null +++ b/ffi/runtime/rt.py @@ -0,0 +1,104 @@ +# The runtime every generated Python module includes: the awaitable a future that pends +# is, the queue each event loop reads its wakes from, and `Waker`. Its names start with +# `_` but `Waker`, so no exported name clashes with them. + + +class _Future(_n.Future): + """A Rust future its first poll left pending: awaiting it polls it on the running + event loop until it finishes. Cancelling the awaiting task frees it, which cancels it in + Rust and loses nothing, as does dropping it unawaited.""" + + __slots__ = () + + def __await__(self): + loop = _get_running_loop() + w = _wakeup(loop) + token = self.token() + try: + while True: + r = self.poll(w.queue) + if r is not _PENDING: + return r + parked = w.parked[token] = loop.create_future() + yield from parked + finally: + w.parked.pop(token, None) + self.close() + + +class _Wakeup: + """One event loop's wakes: Rust queues a token and writes a byte to a socket the loop + reads, from any thread and without the GIL, and the loop resumes what each names.""" + + __slots__ = ("queue", "parked", "wakers", "pump", "__weakref__") + + def __init__(self, loop): + self.queue = _n.Queue() + # The future each parked poll waits on, and each `Waker`'s callback, by token. + self.parked = {} + self.wakers = {} + self.pump = None + fd = self.queue.fileno() + try: + loop.add_reader(fd, self.dispatch) + except NotImplementedError: + # A proactor loop, Windows' default, reads only through its own calls. + sock = _socket.fromfd(fd, _socket.AF_INET if _sys.platform == "win32" else _socket.AF_UNIX, _socket.SOCK_STREAM) + sock.setblocking(False) + self.pump = loop.create_task(self._pump(loop, sock)) + + async def _pump(self, loop, sock): + with sock: + while True: + await loop.sock_recv(sock, 64) + self.dispatch() + + def dispatch(self): + loop = None + for token in self.queue.take(): + if token < 0: + self.wakers.pop(-token, None) + continue + parked = self.parked.pop(token, None) + if parked is not None and not parked.done(): + parked.set_result(None) + wake = self.wakers.get(token) + if wake is not None: + if loop is None: + loop = _get_running_loop() + loop.call_soon(wake) + + +_wakeups = _weakref.WeakKeyDictionary() + + +def _wakeup(loop): + w = _wakeups.get(loop) + if w is None: + w = _wakeups[loop] = _Wakeup(loop) + return w + + +class Waker: + """Wakes a future's poller: Rust may wake it from any thread, any number of times, and + each wake calls `wake` on the event loop it was made on, never inside Rust's wake. Wakes + that land before the loop gets to them call it once.""" + + __slots__ = ("_node",) + + def __init__(self, wake, loop=None): + w = _wakeup(_get_running_loop() if loop is None else loop) + self._node = _n.Node(w.queue) + w.wakers[self._node.token()] = wake + + +class _Const: + """A constant a class names, made anew each time it is read, as Rust makes it.""" + + __slots__ = ("get",) + + def __init__(self, get): + self.get = get + + def __get__(self, obj, cls=None): + return self.get() diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index c6e20ea3f..b3e3c1448 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -39,7 +39,8 @@ impl Crate<'_> { } /// Write the crate into `out`: `Cargo.toml`, `src/`, `include/.h` and `.hpp`, - /// and `kotlin/.kt`, whose JNI functions the crate's `kotlin` feature builds. + /// `kotlin/.kt`, whose JNI functions the crate's `kotlin` feature builds, and + /// `python/.py`, whose CPython extension its `python` feature builds. pub fn write(&self, out: &Path) -> Result<(), String> { // Cargo unifies in what the configured package enables in each other export. let dependencies: String = (self.exports.iter()) @@ -86,10 +87,13 @@ impl Crate<'_> { {dependencies}\ bytes = \"1\"\n\ jni-sys = {{ version = \"0.4\", optional = true }}\n\ + pyo3-ffi = {{ version = \"0.30\", optional = true, features = [\"abi3-py311\", \"extension-module\"] }}\n\ \n\ [features]\n\ # The JNI functions kotlin/{lib}.kt calls.\n\ kotlin = [\"dep:jni-sys\"]\n\ + # The CPython extension python/{lib}.py loads.\n\ + python = [\"dep:pyo3-ffi\"]\n\ \n\ [workspace]\n\ {patch}", @@ -101,6 +105,7 @@ impl Crate<'_> { let (header, contract) = crate::c::header(self.abi, &self.packages()); let cpp = crate::cpp::header(self.abi, &self.packages())?; let kotlin = crate::kotlin::generate(self.abi, &self.packages(), contract)?; + let python = crate::python::generate(self.abi, &self.packages(), contract)?; let mut generator = Gen { abi: self.abi, packages: self.packages(), @@ -123,6 +128,12 @@ impl Crate<'_> { crate::driver::write( &out.join("kotlin").join(format!("{}.pro", self.abi.prefix)), &kotlin.rules, + )?; + crate::driver::write(&out.join("src/python/mod.rs"), &python.glue)?; + crate::driver::write(&out.join("src/python/rt.rs"), crate::python::GLUE_RUNTIME)?; + crate::driver::write( + &out.join("python").join(format!("{}.py", self.abi.prefix)), + &python.python, ) } } @@ -459,6 +470,7 @@ impl Gen<'_> { self.line("#![allow(dead_code, reason = \"the runtime serves every API\")]\n"); self.line("mod rt;\n"); self.line("#[cfg(feature = \"kotlin\")]\nmod jni;\n"); + self.line("#[cfg(feature = \"python\")]\nmod python;\n"); self.line("use std::ffi::c_void;\n"); // An export may export nothing and only enable features in the others, as a // package standing in for a binding crate does, which cargo would call unused. diff --git a/ffi/src/kotlin.rs b/ffi/src/kotlin.rs index 3459e9ae3..4f024bfa2 100644 --- a/ffi/src/kotlin.rs +++ b/ffi/src/kotlin.rs @@ -1921,7 +1921,7 @@ fn jni_int(int: Int) -> &'static str { } /// The Rust type the C ABI's Rust side spells the C type `c` as, for a trampoline. -fn rust_c(abi: &Abi<'_>, c: &Ty) -> String { +pub(crate) fn rust_c(abi: &Abi<'_>, c: &Ty) -> String { match c { Ty::Void => "()".to_string(), Ty::Bool => "bool".to_string(), @@ -2008,7 +2008,7 @@ fn contract_literal(contract: u64) -> String { } /// `ty` returned: a borrow is returned as a clone, so owned. -fn strip_refs(ty: &Type) -> Type { +pub(crate) fn strip_refs(ty: &Type) -> Type { match ty { Type::Ref(_, inner) => strip_refs(inner), Type::Option(inner) => Type::Option(Box::new(strip_refs(inner))), @@ -2029,7 +2029,7 @@ fn ordinal(k: usize) -> String { } /// A snake-case name in Pascal case: `size_pair` is `SizePair`. -fn pascal(name: &str) -> String { +pub(crate) fn pascal(name: &str) -> String { name.split('_') .filter(|s| !s.is_empty()) .map(|s| { @@ -2179,7 +2179,7 @@ fn indent(code: &str) -> String { } /// Code indented by `n` tabs more, but for blank lines. -fn indent_by(code: &str, n: usize) -> String { +pub(crate) fn indent_by(code: &str, n: usize) -> String { let tabs = "\t".repeat(n); let mut out: String = code .lines() diff --git a/ffi/src/main.rs b/ffi/src/main.rs index 21ee5416f..68570be0e 100644 --- a/ffi/src/main.rs +++ b/ffi/src/main.rs @@ -29,6 +29,7 @@ mod driver; mod glue; mod ir; mod kotlin; +mod python; use std::process::ExitCode; diff --git a/ffi/src/python.rs b/ffi/src/python.rs new file mode 100644 index 000000000..ef4f640c6 --- /dev/null +++ b/ffi/src/python.rs @@ -0,0 +1,1358 @@ +//! Python over the C ABI, through a CPython extension: `python/.py`, and the +//! extension functions it calls, which the crate builds with its `python` feature on +//! CPython's stable ABI, so one build loads in every CPython from 3.11. +//! +//! Each name is its C name without the prefix, as Python spells it: `moq_track_producer` +//! is `moq.TrackProducer`, and its method `moq_track_producer_append_group` is +//! `TrackProducer.append_group`. A keyword, or a member Python gives every such class, +//! takes a trailing `_`. An object is a class owning its handle, freed once its last +//! reference goes or sooner by `close`; a record a dataclass; an enum an `Enum`, or for +//! one with data a class with a dataclass per variant; a trait an abstract class; a +//! closure any callable; a future an awaitable; an error an exception. The runtime they +//! share, [`runtime/rt.py`](../runtime/rt.py), holds the awaitable and the queue an event +//! loop reads its wakes from; the glue's, [`runtime/python.rs`](../runtime/python.rs), +//! the conversions and the types the extension defines. +//! +//! The glue converts each Python value to its C value and back itself, so a call is one +//! crossing and the Python is a thin layer naming each function: a record crosses as its +//! fields, read and set by name, and bytes returned as a `memoryview` lending Rust's +//! buffer. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; +use std::hash::Hasher; + +use rustc_data_structures::stable_hash::StableHasher; + +use crate::abi::{self, Abi, Ty}; +use crate::ir::{Access, Family, Function, Int, Literal, Receiver, Traits, Type}; +use crate::kotlin::{indent_by, pascal, rust_c, strip_refs}; + +const RUNTIME: &str = include_str!("../runtime/rt.py"); +pub const GLUE_RUNTIME: &str = include_str!("../runtime/python.rs"); + +/// What the generator writes: the Python module, and the Rust module of the extension it +/// loads. +pub struct Output { + pub python: String, + pub glue: String, +} + +/// The names the generated Python uses unqualified, which no exported item may take: the +/// runtime's own, each but `Waker` starting with `_`, and `Waker` and `Bound`, which the C +/// ABI claims as `_waker` and `_bound`. +const TAKEN: &[&str] = &[ + "_n", + "_load", + "_Future", + "_Wakeup", + "_wakeups", + "_wakeup", + "_Const", + "_PENDING", + "_consts", + "_abc", + "_asyncio", + "_dataclass", + "_datetime", + "_enum", + "_socket", + "_sys", + "_typing", + "_weakref", + "_get_running_loop", + "_property", + "_staticmethod", + "_isinstance", + "_str", + "_Exception", + "_AttributeError", +]; + +/// The registry's first classes, the runtime's: the glue's `T_FUTURE` and `T_TIMEDELTA`. +const RUNTIME_TYPES: &[&str] = &["_Future", "_datetime.timedelta"]; + +/// The Python for `abi`, whose header's contract is `contract`, or the names two items +/// claim alike. The Python's own contract hashes the header's with everything it and its +/// glue add, as their conversions change with no C declaration changing. +pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result { + let mut g = Gen { + abi, + py: String::new(), + top: String::new(), + consts: Vec::new(), + glue: String::new(), + table: Vec::new(), + types: RUNTIME_TYPES.iter().map(|t| t.to_string()).collect(), + index: BTreeMap::new(), + keys: BTreeSet::new(), + exceptions: BTreeMap::new(), + names: BTreeMap::new(), + errors: Vec::new(), + }; + for name in TAKEN { + g.names.insert(name.to_string(), "the runtime".to_string()); + } + g.write(); + if !g.errors.is_empty() { + return Err(g.errors.join("\n")); + } + let p = &abi.prefix; + let registry: String = g.types.iter().map(|t| format!("\t{t},\n")).collect(); + let consts: String = g.consts.iter().map(|(name, bare)| format!("\t\"{name}\": _n.{bare},\n")).collect(); + let mut body = format!( + "{RUNTIME}\n{}{}_n._register(\n{registry})\n\n_consts = {{\n{consts}}}\n\n\ndef __getattr__(name):\n\t\"\"\"A constant, made anew each time it is read, as Rust makes it.\"\"\"\n\tget = _consts.get(name)\n\tif get is None:\n\t\traise _AttributeError(f\"module {{__name__!r}} has no attribute {{name!r}}\")\n\treturn get()\n", + g.py, g.top + ); + let mut hasher = StableHasher::new(); + hasher.write_u64(contract); + for part in [&body, &g.glue, GLUE_RUNTIME] { + hasher.write(part.as_bytes()); + } + let contract = Hasher::finish(&hasher); + let head = format!( + "# Generated by rust2-ffi: Python over the C ABI of the pub API of {packages}. Do not edit.\n\ + \"\"\"The pub API of {packages}, over the CPython extension built into the library beside this file.\"\"\"\n\n\ + from __future__ import annotations\n\n\ + import abc as _abc\n\ + import asyncio as _asyncio\n\ + import datetime as _datetime\n\ + import enum as _enum\n\ + import socket as _socket\n\ + import sys as _sys\n\ + import typing as _typing\n\ + import weakref as _weakref\n\ + from dataclasses import dataclass as _dataclass\n\n\ + _get_running_loop = _asyncio.get_running_loop\n\ + _property = property\n\ + _staticmethod = staticmethod\n\ + _isinstance = isinstance\n\ + _str = str\n\ + _Exception = Exception\n\ + _AttributeError = AttributeError\n\n\n\ + def _load():\n\ + \timport importlib.machinery\n\ + \timport importlib.util\n\ + \timport os\n\n\ + \tname = {{\"win32\": \"{p}.dll\", \"darwin\": \"lib{p}.dylib\"}}.get(_sys.platform, \"lib{p}.so\")\n\ + \tpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), name)\n\ + \tloader = importlib.machinery.ExtensionFileLoader(\"_{p}\", path)\n\ + \tmodule = importlib.util.module_from_spec(importlib.util.spec_from_loader(\"_{p}\", loader, origin=path))\n\ + \tloader.exec_module(module)\n\ + \treturn module\n\n\n\ + _n = _load()\n\ + # Checked before any other call: a Python file and a library from different builds\n\ + # must not mix.\n\ + if _n.contract() != 0x{contract:016x}:\n\ + \traise ImportError(\"{p}.py and the library are from different builds\")\n\ + _PENDING = _n.PENDING\n\n\n" + ); + body.insert_str(0, &head); + let glue = format!( + "//! Generated by rust2-ffi: the CPython extension `python/{p}.py` loads, over the C ABI of {packages}.\n\n\ + #![allow(non_snake_case, non_upper_case_globals, unused_mut, unused_unsafe, unused_parens, unused_variables, clippy::all)]\n\n\ + mod rt;\n\n\ + use std::ffi::c_void;\n\ + use std::mem::MaybeUninit;\n\n\ + use rt::*;\n\n\ + unsafe extern \"C\" fn f_contract(_: *mut PyObject, _: *mut *mut PyObject, _: Py_ssize_t) -> *mut PyObject {{\n\ + \trt::wrap(|| rt::uint_out(0x{contract:016x}))\n\ + }}\n\n\ + {}\ + static METHODS: rt::Methods<{}> = rt::Methods(std::cell::UnsafeCell::new([\n\ + \trt::method(c\"contract\", f_contract),\n\ + {}\ + \trt::END,\n\ + ]));\n\n\ + static MODULE: rt::Gil = rt::Gil::new(PyModuleDef {{\n\ + \tm_base: PyModuleDef_HEAD_INIT,\n\ + \tm_name: c\"_{p}\".as_ptr(),\n\ + \tm_doc: std::ptr::null(),\n\ + \tm_size: -1,\n\ + \tm_methods: std::ptr::null_mut(),\n\ + \tm_slots: std::ptr::null_mut(),\n\ + \tm_traverse: None,\n\ + \tm_clear: None,\n\ + \tm_free: None,\n\ + }});\n\n\ + #[unsafe(no_mangle)]\n\ + pub unsafe extern \"C\" fn PyInit__{p}() -> *mut PyObject {{\n\ + \tunsafe {{\n\ + \t\tlet def = MODULE.get();\n\ + \t\tdef.m_methods = METHODS.ptr();\n\ + \t\trt::module(def)\n\ + \t}}\n\ + }}\n", + { + let keys: String = g + .keys + .iter() + .map(|k| format!("static K_{k}: rt::Key = rt::Key::new(c\"{k}\");\n")) + .collect(); + format!("{keys}\n{}{}", g.index_consts(), g.glue) + }, + g.table.len() + 2, + g.table.iter().map(|t| format!("\t{t},\n")).collect::(), + ); + Ok(Output { python: body, glue }) +} + +/// How a function's Python form is held. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Place { + /// A method of a class, taking `self`. + Member, + /// A static method of a class. + Static, + /// A module-level function. + Top, +} + +struct Gen<'a> { + abi: &'a Abi<'a>, + /// The classes, then the module-level functions. + py: String, + top: String, + /// The module-level constants that are no literal, by name, with their native function. + consts: Vec<(String, String)>, + glue: String, + /// The extension's method table entries. + table: Vec, + /// The classes the glue makes values of, in the order the Python registers them. + types: Vec, + /// Each registered class's index, by the C name or key it is registered under. + index: BTreeMap, + /// The attribute names the glue reads and sets. + keys: BTreeSet, + /// The exception class of each error type, by name. + exceptions: BTreeMap, + /// Each module-level Python name, with what claimed it. + names: BTreeMap, + errors: Vec, +} + +impl<'a> Gen<'a> { + /// A C name without the prefix. + fn bare<'n>(&self, c: &'n str) -> &'n str { + &c[self.abi.prefix.len() + 1..] + } + + /// The Python class name of the C type `c`. + fn class(&self, c: &str) -> String { + pascal(self.bare(c)) + } + + /// Claim the module-level Python name `name` for `what`. + fn claim(&mut self, name: &str, what: String) { + if let Some(first) = self.names.insert(name.to_string(), what.clone()) { + let clash = match first.as_str() { + "the runtime" => format!( + "{what} is `{}.{name}` in Python, which its runtime names: rename it in the config", + self.abi.prefix + ), + _ => format!( + "{first} and {what} are both `{}.{name}` in Python: rename one in the config", + self.abi.prefix + ), + }; + self.errors.push(clash); + } + } + + /// Register the class the Python expression `py` names under `key`, returning the + /// glue's constant for its index. + fn register(&mut self, key: &str, py: String) -> String { + if !self.index.contains_key(key) { + self.index.insert(key.to_string(), self.types.len()); + self.types.push(py); + } + format!("T_{key}") + } + + /// The glue's constant for the index of the class registered under `key`. + fn t(&self, key: &str) -> String { + format!("T_{key}") + } + + fn index_consts(&self) -> String { + let mut out = String::new(); + for (key, n) in &self.index { + let _ = writeln!(out, "const T_{key}: usize = {n};"); + } + out.push('\n'); + out + } + + /// The glue's interned attribute name `name`. + fn key(&mut self, name: &str) -> String { + self.keys.insert(name.to_string()); + format!("&K_{name}") + } + + /// The Python annotation of `ty` as a value, owning what it holds. + fn py_ty(&self, ty: &Type) -> String { + let abi = self.abi; + match ty { + Type::Unit => "None".to_string(), + Type::Bool => "bool".to_string(), + Type::Int(_) => "int".to_string(), + Type::F32 | Type::F64 => "float".to_string(), + Type::Char | Type::Str | Type::String | Type::Path | Type::PathBuf => "str".to_string(), + Type::Slice | Type::Vec | Type::Bytes => "memoryview".to_string(), + Type::Duration => "_datetime.timedelta".to_string(), + Type::SocketAddr(_) => "tuple".to_string(), + Type::List(inner) => format!("list[{}]", self.py_ty(inner)), + Type::Waker => "Waker".to_string(), + Type::Option(inner) => format!("{} | None", self.py_ty(inner)), + Type::Box(inner) | Type::Ref(_, inner) => self.py_ty(inner), + Type::Record(i) => self.class(&abi.record(*i)), + Type::Enum(i) => self.class(&abi.enumeration(*i)), + Type::Object(i) => self.class(&abi.object(*i)), + Type::Callback(i, _) => self.class(&abi.callback(*i)), + Type::Closure(params, ret) => { + let params: Vec = params.iter().map(|p| self.py_param(p)).collect(); + format!("_typing.Callable[[{}], {}]", params.join(", "), self.py_ty(ret)) + } + Type::Range(_) => "range".to_string(), + Type::Bound(_) | Type::Bounds(_) => self.class(&abi.range(ty)), + Type::Tuple(elements) => { + let elements: Vec = elements.iter().map(|e| self.py_ty(e)).collect(); + format!("tuple[{}]", elements.join(", ")) + } + } + } + + /// The Python annotation of a parameter of type `ty`, either way: its bytes are any + /// buffer, and its path any path. + fn py_param(&self, ty: &Type) -> String { + match ty { + Type::Slice | Type::Vec | Type::Bytes => "bytes".to_string(), + Type::Path | Type::PathBuf => "str".to_string(), + Type::List(inner) => format!("list[{}]", self.py_param(inner)), + Type::Option(inner) => format!("{} | None", self.py_param(inner)), + Type::Box(inner) | Type::Ref(_, inner) => self.py_param(inner), + Type::Tuple(elements) => { + let elements: Vec = elements.iter().map(|e| self.py_param(e)).collect(); + format!("tuple[{}]", elements.join(", ")) + } + _ => self.py_ty(ty), + } + } + + /// The glue's expression, an `R`, reading the Python object `o` as the C value of + /// `ty`, with what it borrows held in the scope `s`; `m` when the position moves an + /// object's value out. + fn lift(&mut self, ty: &Type, o: &str, m: bool) -> String { + let abi = self.abi; + match ty { + Type::Bool => format!("rt::bool_in({o})"), + Type::Int(int) => format!("rt::int_in::<{}>({o})", int.rust()), + Type::F32 => format!("rt::float_in({o}).map(|f| f as f32)"), + Type::F64 => format!("rt::float_in({o})"), + Type::Char => format!("rt::char_in({o})"), + Type::Str | Type::String => format!("rt::str_in({o})"), + Type::Slice | Type::Vec | Type::Bytes => format!("rt::bytes_in({o}, s)"), + Type::Path | Type::PathBuf => format!("rt::path_in({o}, s)"), + Type::Duration => format!("rt::duration_in({o})"), + Type::SocketAddr(family) => format!("rt::addr_in({o}, {})", family_of(*family)), + Type::List(_) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::Option(inner) => match inner.base() { + Type::Object(i) => format!( + "rt::hold_opt({o}, {}, s, {})", + self.t(&abi.object(*i)), + m && !abi.api.objects[*i].copy && !matches!(**inner, Type::Ref(..)) + ), + base if Abi::is_buf(base) => { + let some = self.lift(inner, o, m); + format!("(if rt::is_none({o}) {{ Ok(crate::rt::Buf::NONE) }} else {{ {some} }})") + } + Type::List(_) => { + let some = self.lift(inner, o, m); + format!("(if rt::is_none({o}) {{ Ok(crate::rt::List::NONE) }} else {{ {some} }})") + } + _ => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + }, + Type::Box(inner) => self.lift(inner, o, m), + Type::Ref(_, inner) => self.lift(inner, o, false), + Type::Record(_) => format!("r_{}({o}, s)", self.c_struct(ty)), + Type::Enum(i) => match abi.api.enums[*i].plain() { + true => format!("rt::member_in({o}, {})", self.t(&abi.enumeration(*i))), + false => format!("r_{}({o}, s)", abi.enumeration(*i)), + }, + Type::Object(i) => format!( + "rt::hold({o}, {}, s, {})", + self.t(&abi.object(*i)), + m && !abi.api.objects[*i].copy + ), + Type::Callback(i, _) => { + let c = abi.callback(*i); + format!("Ok(crate::{c} {{ vtable: &VTABLE_{c}, data: rt::callback({o}, s) }})") + } + Type::Waker => format!("rt::waker_in({o}, s)"), + Type::Range(int) => format!("rt::range_in::<{}>({o})", int.rust()), + Type::Bound(_) | Type::Bounds(_) => format!("r_{}({o}, s)", abi.range(ty)), + Type::Tuple(_) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::Unit | Type::Closure(..) => unreachable!("{ty:?} is no value Python passes"), + } + } + + /// The glue's expression, an `R`, of the C value `v` of `ty`, which Rust handed + /// over: what it owns is freed or moves into the Python value. + fn lower(&mut self, ty: &Type, v: &str) -> String { + let abi = self.abi; + match ty { + Type::Unit => "rt::none()".to_string(), + Type::Bool => format!("rt::bool_out({v})"), + Type::Int(int) if signed(*int) => format!("rt::int_out({v} as i64)"), + Type::Int(_) => format!("rt::uint_out({v} as u64)"), + Type::F32 | Type::F64 => format!("rt::float_out({v} as f64)"), + Type::Char => format!("rt::char_out({v})"), + Type::Str | Type::String => format!("rt::str_out({v})"), + Type::Slice | Type::Vec | Type::Bytes => format!("rt::bytes_out({v})"), + Type::Path | Type::PathBuf => format!("rt::path_out({v})"), + Type::Duration => format!("rt::duration_out({v})"), + Type::SocketAddr(_) => format!("rt::addr_out({v})"), + Type::List(_) => format!("w_{}({v})", self.c_struct(ty)), + Type::Option(inner) => match inner.base() { + Type::Object(i) => format!( + "rt::object_opt({v}, {}, crate::{}_free)", + self.t(&abi.object(*i)), + abi.object(*i) + ), + base if Abi::is_buf(base) || matches!(base, Type::List(_)) => { + let some = self.lower(inner, "v"); + format!("{{ let v = {v}; if v.is_none() {{ rt::none() }} else {{ {some} }} }}") + } + _ => format!("w_{}({v})", self.c_struct(ty)), + }, + Type::Box(inner) | Type::Ref(_, inner) => self.lower(inner, v), + Type::Record(_) => format!("w_{}({v})", self.c_struct(ty)), + Type::Enum(i) => match abi.api.enums[*i].plain() { + true => format!("rt::member({}, {v} as i64)", self.t(&abi.enumeration(*i))), + false => format!("w_{}({v})", abi.enumeration(*i)), + }, + Type::Object(i) => format!( + "rt::object({v}, {}, crate::{}_free)", + self.t(&abi.object(*i)), + abi.object(*i) + ), + Type::Range(int) => { + let start = self.lower(&Type::Int(*int), "r.start"); + let end = self.lower(&Type::Int(*int), "r.end"); + format!("{{ let r = {v}; rt::range_out({start}, {end}) }}") + } + Type::Bound(_) | Type::Bounds(_) => format!("w_{}({v})", abi.range(ty)), + Type::Tuple(_) => format!("w_{}({v})", self.c_struct(ty)), + Type::Callback(..) | Type::Closure(..) | Type::Waker => { + unreachable!("{ty:?} is no value Rust returns") + } + } + } + + /// The C name of the struct `ty` crosses as. + fn c_struct(&self, ty: &Type) -> String { + match self.abi.ty(&strip_refs(ty)) { + Ty::Struct(name) => name, + other => unreachable!("{ty:?} crosses as {other:?}, no struct"), + } + } + + fn write(&mut self) { + let abi = self.abi; + let api = abi.api; + + if abi.ranges.values().any(|ty| matches!(ty, Type::Bound(_))) { + let bound = abi.rt("bound"); + self.register(&bound, "Bound".to_string()); + self.py.push_str( + "class Bound(_enum.Enum):\n\t\"\"\"What a range's start or end bound does: nothing, include its value, or exclude it.\"\"\"\n\n\tUNBOUNDED = 0\n\tINCLUDED = 1\n\tEXCLUDED = 2\n\n\n", + ); + } + for (c, ty) in &abi.ranges.clone() { + self.range(c, ty); + } + for (c, inner) in &abi.options.clone() { + self.option(c, inner); + } + for (c, elements) in &abi.tuples.clone() { + self.tuple(c, elements); + } + for (c, elem) in &abi.lists.clone() { + self.list(c, elem); + } + for constant in &api.constants { + let name = self.bare(&abi.name(&constant.path)).to_uppercase(); + self.claim(&name, format!("const `{}`", constant.path.join("::"))); + let value = match constant.value { + Literal::Bool(true) => "True".to_string(), + Literal::Bool(false) => "False".to_string(), + Literal::Int(_, v) => v.to_string(), + }; + let _ = write!(self.top, "{name} = {value}\n{}\n", docstring(&constant.docs, "")); + } + for i in 0..api.records.len() { + self.record(i); + } + for i in 0..api.enums.len() { + self.enumeration(i); + } + for i in 0..api.objects.len() { + self.object(i); + } + for i in 0..api.callbacks.len() { + self.callback(i); + } + for function in &api.functions { + if let Some(code) = self.function(function, None, None, Place::Top) { + self.top.push_str(&code); + self.top.push_str("\n\n"); + } + } + for (name, ty) in &self.exceptions.clone() { + let py = self.py_ty(ty); + let _ = write!( + self.py, + "class {name}(_Exception):\n\t\"\"\"A `{py}` Rust returned as an error, in `error`, whose `str` is the exception's.\"\"\"\n\n\tdef __init__(self, error: {py}):\n\t\t_Exception.__init__(self, error)\n\t\tself.error = error\n\n\tdef __str__(self):\n\t\treturn _str(self.error)\n\n\n" + ); + } + } + + /// A range's bound or pair of bounds as a dataclass, and its glue; a `Range` is + /// Python's `range`. + fn range(&mut self, c: &str, ty: &Type) { + let name = self.class(c); + let (fields, read, write) = match ty { + Type::Range(_) => return, + Type::Bound(int) => { + let bound = self.abi.rt("bound"); + let (kind, value) = (self.key("kind"), self.key("value")); + let t_bound = self.t(&bound); + ( + "\tkind: Bound\n\tvalue: int\n".to_string(), + format!( + "let k = rt::field(o, {t}, {kind}, s)?;\n\tlet v = rt::attr(o, {value}, s)?;\n\tOk(crate::{c} {{ kind: rt::member_in(k, {t_bound})?, value: rt::int_in::<{r}>(v)? }})", + t = self.t(c), + r = int.rust() + ), + format!( + "rt::record({t}, vec![({kind}, rt::member({t_bound}, v.kind as i64)), ({value}, {})])", + self.lower(&Type::Int(*int), "v.value"), + t = self.t(c) + ), + ) + } + Type::Bounds(int) => { + let bound = self.abi.range(&Type::Bound(*int)); + let (start, end) = (self.key("start"), self.key("end")); + let class = self.class(&bound); + ( + format!("\tstart: {class}\n\tend: {class}\n"), + format!( + "let a = rt::field(o, {t}, {start}, s)?;\n\tlet b = rt::attr(o, {end}, s)?;\n\tOk(crate::{c} {{ start: r_{bound}(a, s)?, end: r_{bound}(b, s)? }})", + t = self.t(c) + ), + format!( + "rt::record({t}, vec![({start}, w_{bound}(v.start)), ({end}, w_{bound}(v.end))])", + t = self.t(c) + ), + ) + } + other => unreachable!("{other:?} is no range"), + }; + self.claim(&name, format!("`{c}`")); + self.register(c, name.clone()); + let _ = write!(self.py, "@_dataclass(slots=True)\nclass {name}:\n{fields}\n\n"); + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope) -> R {{\n\t{read}\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\t{write}\n}}\n\n" + ); + } + + /// The glue of an option struct. + fn option(&mut self, c: &str, inner: &Type) { + let read = self.lift(inner, "o", true); + let write = self.lower(inner, "v.value"); + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", + read.replace(", true)", ", m)") + ); + } + + /// The glue of a tuple struct, element by element. + fn tuple(&mut self, c: &str, elements: &[Type]) { + let n = elements.len(); + let mut reads = Vec::new(); + let mut fields = Vec::new(); + let mut writes = Vec::new(); + for (k, ty) in elements.iter().enumerate() { + let lift = self.lift(ty, &format!("x{k}"), true).replace(", true)", ", m)"); + reads.push(format!("let x{k} = rt::tuple_item(o, {k}, {n})?;")); + fields.push(format!("_{k}: {lift}?")); + writes.push(self.lower(ty, &format!("v._{k}"))); + } + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::tuple(vec![{}])\n}}\n\n", + reads.join("\n\t"), + fields.join(", "), + writes.join(", ") + ); + } + + /// The glue of a `Vec`'s array. + fn list(&mut self, c: &str, elem: &Type) { + let read = self.lift(elem, "o", true).replace(", true)", ", m)"); + let write = self.lower(elem, "e"); + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\trt::list_in(o, s, |o, s| {read})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::list_out(v, |e| {write})\n}}\n\n" + ); + } + + /// The Python members a type's std traits export beyond Python's own, with their glue: + /// `__str__` from `Display`, `__repr__` from `Debug`, a static `default` from + /// `Default`, and `clone` and `__eq__` for an object. + fn traits(&mut self, ty: &Type, c: &str, t: Traits, body: &mut Vec) { + let bare = self.bare(c).to_string(); + let class = self.class(c); + let object = matches!(ty, Type::Object(_)); + // The receiver as the glue reads it: a handle, held for the call, or a value. + let this = match object { + true => self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[0]", false), + false => self.lift(ty, "a[0]", false), + }; + let pass = match object { + true => "this", + false => "&this", + }; + let unary = |g: &mut Self, suffix: &str, lower: String| { + g.native( + &format!("{bare}_{suffix}"), + 1, + &format!("let this = {this}?;\n\t\tlet ret = rt::call(|| crate::{c}_{suffix}({pass}));\n\t\ts.done();\n\t\t{lower}"), + ); + }; + if t.display { + unary(self, "to_string", "rt::str_out(ret)".to_string()); + body.push(format!("\tdef __str__(self):\n\t\treturn _n.{bare}_to_string(self)\n")); + } + if t.debug { + unary(self, "debug", "rt::str_out(ret)".to_string()); + body.push(format!("\tdef __repr__(self):\n\t\treturn _n.{bare}_debug(self)\n")); + } + if object && t.clone { + let lower = self.lower(ty, "ret"); + unary(self, "clone", lower); + body.push(format!( + "\tdef clone(self) -> {class}:\n\t\t\"\"\"A new handle to a clone.\"\"\"\n\t\treturn _n.{bare}_clone(self)\n" + )); + } + if object && t.eq { + let other = self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[1]", false); + self.native( + &format!("{bare}_eq"), + 2, + &format!( + "let this = {this}?;\n\t\tlet other = {other}?;\n\t\tlet ret = rt::call(|| crate::{c}_eq(this, other));\n\t\ts.done();\n\t\trt::bool_out(ret)" + ), + ); + body.push(format!( + "\tdef __eq__(self, other):\n\t\treturn _isinstance(other, {class}) and _n.{bare}_eq(self, other)\n\n\t__hash__ = None\n" + )); + } + if t.default { + let lower = self.lower(ty, "ret"); + self.native( + &format!("{bare}_default"), + 0, + &format!("let ret = rt::call(|| crate::{c}_default());\n\t\ts.done();\n\t\t{lower}"), + ); + body.push(format!( + "\t@_staticmethod\n\tdef default() -> {class}:\n\t\t\"\"\"Rust's `Default`.\"\"\"\n\t\treturn _n.{bare}_default()\n" + )); + } + } + + /// The methods of a type, each a member or a static method by its receiver, with the + /// Python names they take, checked to differ. + fn methods(&mut self, methods: &[Function], ty: &Type, owner: &Ty, body: &mut Vec, rust: &str, reserved: &[&str]) { + for method in methods { + let place = match method.receiver { + Some(_) => Place::Member, + None => Place::Static, + }; + if let Some(code) = self.function(method, Some(owner), Some((ty, reserved)), place) { + body.push(indent_by(&code, 1)); + } + } + self.distinct(rust, body); + } + + fn record(&mut self, i: usize) { + let abi = self.abi; + let record = &abi.api.records[i]; + let c = abi.record(i); + let name = self.class(&c); + let rust = abi.api.rust(&record.path, &record.args); + self.claim(&name, format!("struct `{rust}`")); + let t = self.register(&c, name.clone()); + let ty = Type::Record(i); + let mut body = Vec::new(); + self.traits(&ty, &c, record.traits, &mut body); + let owner = Ty::Struct(c.clone()); + let mut members: Vec = record.fields.iter().map(|f| format!("\t{} =", field_name(&f.name))).collect(); + members.append(&mut body); + self.methods(&record.methods, &ty, &owner, &mut members, &rust, &[]); + let fields: String = record + .fields + .iter() + .map(|f| format!("\t{}: {}\n{}", field_name(&f.name), self.py_ty(&f.ty), docstring(&f.docs, "\t"))) + .collect(); + let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); + let repr = match record.traits.debug { + true => ", repr=False", + false => "", + }; + let _ = write!( + self.py, + "@_dataclass(slots=True{repr})\nclass {name}:\n{}{fields}{}\n\n", + docstring(&format!("{rust}\n\n{}", record.docs), "\t"), + members.iter().map(|m| format!("\n{m}")).collect::() + ); + // The glue. + let mut reads = Vec::new(); + let mut fields = Vec::new(); + let mut writes = Vec::new(); + for (n, field) in record.fields.iter().enumerate() { + let key = self.key(&field_name(&field.name)); + let read = match n { + 0 => format!("rt::field(o, {t}, {key}, s)?"), + _ => format!("rt::attr(o, {key}, s)?"), + }; + reads.push(format!("let x{n} = {read};")); + fields.push(format!("f{n}: {}?", self.lift(&field.ty, &format!("x{n}"), false))); + writes.push(format!("({key}, {})", self.lower(&field.ty, &format!("v.f{n}")))); + } + if record.fields.is_empty() { + reads.push(format!("rt::check(o, {t})?;")); + } + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::record({t}, vec![{}])\n}}\n\n", + reads.join("\n\t"), + fields.join(", "), + writes.join(", ") + ); + } + + fn enumeration(&mut self, i: usize) { + let abi = self.abi; + let e = &abi.api.enums[i]; + let c = abi.enumeration(i); + let name = self.class(&c); + let rust = abi.api.rust(&e.path, &e.args); + self.claim(&name, format!("enum `{rust}`")); + let t = self.register(&c, name.clone()); + let ty = Type::Enum(i); + let mut body = Vec::new(); + self.traits(&ty, &c, e.traits, &mut body); + let owner = abi.ty(&ty); + if e.plain() { + let mut members: Vec = e + .variants + .iter() + .map(|v| format!("\t{} =", abi::snake(&v.name).to_uppercase())) + .collect(); + members.append(&mut body); + // An `Enum`'s members have these. + self.methods(&e.methods, &ty, &owner, &mut members, &rust, &["name", "value"]); + let entries: String = e + .variants + .iter() + .map(|v| format!("\t{} = {}\n", abi::snake(&v.name).to_uppercase(), v.discriminant)) + .collect(); + let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); + let _ = write!( + self.py, + "class {name}(_enum.Enum):\n{}{entries}{}\n\n", + docstring(&format!("{rust}\n\n{}", e.docs), "\t"), + members.iter().map(|m| format!("\n{m}")).collect::() + ); + return; + } + let mut members: Vec = e + .variants + .iter() + .map(|v| format!("\t{} =", pascal(&abi::snake(&v.name)))) + .collect(); + members.append(&mut body); + self.methods(&e.methods, &ty, &owner, &mut members, &rust, &[]); + let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); + let _ = write!( + self.py, + "class {name}:\n{}\t__slots__ = ()\n{}\n\n", + docstring(&format!("{rust}\n\n{}", e.docs), "\t"), + members.iter().map(|m| format!("\n{m}")).collect::() + ); + let repr = match e.traits.debug { + true => ", repr=False", + false => "", + }; + let mut reads = Vec::new(); + let mut writes = Vec::new(); + for (n, variant) in e.variants.iter().enumerate() { + let vname = pascal(&abi::snake(&variant.name)); + let class = format!("_{name}_{vname}"); + let key = format!("{c}_{}", abi::snake(&variant.name)); + let tv = self.register(&key, class.clone()); + let fields: String = variant + .fields + .iter() + .map(|f| format!("\t{}: {}\n", field_name(&f.name), self.py_ty(&f.ty))) + .collect(); + let fields = match fields.is_empty() { + true => "\tpass\n".to_string(), + false => fields, + }; + let _ = write!( + self.py, + "@_dataclass(slots=True{repr})\nclass {class}({name}):\n{fields}\n\n{name}.{vname} = {class}\n{class}.__name__ = \"{vname}\"\n{class}.__qualname__ = \"{name}.{vname}\"\n\n\n" + ); + match variant.fields.is_empty() { + true => { + reads.push(format!( + "if rt::is(o, {tv}) {{\n\t\treturn Ok(crate::{c} {{ tag: {n}, u: unsafe {{ std::mem::zeroed() }} }});\n\t}}" + )); + writes.push(format!("\t\t{n} => rt::record({tv}, vec![]),")); + } + false => { + let payload = format!("{c}_{}", abi::snake(&variant.name)); + let mut binds = Vec::new(); + let mut fields = Vec::new(); + let mut lowers = Vec::new(); + for (k, f) in variant.fields.iter().enumerate() { + let key = self.key(&field_name(&f.name)); + binds.push(format!("let x{k} = rt::attr(o, {key}, s)?;")); + fields.push(format!("f{k}: {}?", self.lift(&f.ty, &format!("x{k}"), false))); + lowers.push(format!("({key}, {})", self.lower(&f.ty, &format!("p.f{k}")))); + } + reads.push(format!( + "if rt::is(o, {tv}) {{\n\t\t{}\n\t\treturn Ok(crate::{c} {{ tag: {n}, u: crate::{c}_union {{ v{n}: crate::{payload} {{ {} }} }} }});\n\t}}", + binds.join("\n\t\t"), + fields.join(", ") + )); + writes.push(format!( + "\t\t{n} => {{\n\t\t\tlet p = unsafe {{ v.u.v{n} }};\n\t\t\trt::record({tv}, vec![{}])\n\t\t}}", + lowers.join(", ") + )); + } + } + } + let _ = write!( + self.glue, + "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope) -> R {{\n\t{}\n\trt::check(o, {t})?;\n\tcrate::rt::fail(\"python\", \"a variant is of no class of its enum\")\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\tmatch v.tag {{\n{}\n\t\t_ => crate::rt::fail(\"python\", \"an enum's tag is invalid\"),\n\t}}\n}}\n\n", + reads.join("\n\t"), + writes.join("\n") + ); + } + + fn object(&mut self, i: usize) { + let abi = self.abi; + let object = &abi.api.objects[i]; + let c = abi.object(i); + let name = self.class(&c); + let rust = abi.api.rust(&object.path, &object.args); + self.claim(&name, format!("type `{rust}`")); + self.register(&c, name.clone()); + let ty = Type::Object(i); + let mut body = vec!["\t__slots__ = ()\n".to_string()]; + self.traits(&ty, &c, object.traits, &mut body); + let owner = Ty::Handle { + name: c.clone(), + constant: false, + }; + // What `Object` gives every object. + self.methods(&object.methods, &ty, &owner, &mut body, &rust, &["close"]); + let text = format!( + "{rust}\n\n{}\n\nIts handle is freed once its last reference goes, or sooner by `close()` or a `with` block, after which a call raises.", + object.docs + ); + let _ = write!( + self.py, + "class {name}(_n.Object):\n{}{}\n\n", + docstring(text.trim(), "\t"), + body.join("\n") + ); + } + + fn callback(&mut self, i: usize) { + let abi = self.abi; + let callback = &abi.api.callbacks[i]; + let c = abi.callback(i); + let name = self.class(&c); + let rust = callback.path.join("::"); + self.claim(&name, format!("trait `{rust}`")); + let mut body = Vec::new(); + let mut table = vec!["free: Some(rt::free_callback)".to_string()]; + for (n, method) in callback.methods.iter().enumerate() { + let member = member(&abi::snake(method.name()), &[]); + let params: Vec = std::iter::once("self".to_string()) + .chain( + method + .params + .iter() + .map(|p| format!("{}: {}", param_name(&p.name), self.py_lent(&p.ty))), + ) + .collect(); + body.push(format!( + "\t@_abc.abstractmethod\n\tdef {member}({}) -> {}:\n{}\t\t...\n", + params.join(", "), + self.py_ty(&method.ok), + docstring(&method.docs, "\t\t") + )); + let c_params: Vec = method.params.iter().map(|p| abi.ty(&p.ty)).collect(); + let c_ret = abi.ty(&method.ok); + let lent: Vec = method.params.iter().map(|p| p.ty.clone()).collect(); + let trampoline = format!("py_{c}_{n}"); + let key = self.key(&member); + self.trampoline( + &trampoline, + &format!("{rust}::{}", method.name()), + (&lent, &method.ok), + (&c_params, &c_ret), + Some(&key), + ); + table.push(format!("{}: Some({trampoline})", method.name())); + } + self.distinct(&rust, &body); + let text = format!( + "{rust}: implemented in Python. Rust calls its methods from any thread, and once it\ndrops the implementation, calls its `close()` if it has one.\n\n{}", + callback.docs + ); + let _ = write!( + self.py, + "class {name}(_abc.ABC):\n{}\t__slots__ = ()\n\n{}\n\n", + docstring(text.trim(), "\t"), + body.join("\n") + ); + let _ = write!( + self.glue, + "static VTABLE_{c}: crate::{c}_vtable = crate::{c}_vtable {{ {} }};\n\n", + table.join(", ") + ); + } + + /// The Python annotation of a value Rust lends a callback or closure: its bytes copied. + fn py_lent(&self, ty: &Type) -> String { + match ty.base() { + Type::Slice | Type::Vec | Type::Bytes => "bytes".to_string(), + Type::Option(inner) if matches!(inner.base(), Type::Slice | Type::Vec | Type::Bytes) => { + "bytes | None".to_string() + } + _ => self.py_ty(ty), + } + } + + /// The Rust function C calls for a callback's method or a closure, named `trampoline`, + /// which calls the Python value Rust holds: for a callback, its method `key`, and for a + /// closure, the callable itself. + fn trampoline( + &mut self, + trampoline: &str, + what: &str, + (lent, ret): (&[Type], &Type), + (c_params, c_ret): (&[Ty], &Ty), + key: Option<&str>, + ) { + let mut rust_params = vec!["data: *mut c_void".to_string()]; + let mut args = Vec::new(); + for (k, (ty, c)) in lent.iter().zip(c_params).enumerate() { + rust_params.push(format!("p{k}: {}", rust_c(self.abi, c))); + let p = format!("p{k}"); + args.push(match (c, strip_refs(&strip_box(ty))) { + (Ty::Handle { .. }, Type::Object(i)) => format!("rt::lent({p}, {}, s)", self.t(&self.abi.object(i))), + (Ty::Buf, _) => self.lent_buf(ty, &p), + _ => self.lower(ty, &p), + }); + } + let call = match key { + Some(key) => format!("rt::call_method(data.cast(), {key}, vec![{}])", args.join(", ")), + None => format!("rt::call_py(data.cast(), vec![{}])", args.join(", ")), + }; + let back = match c_ret { + Ty::Void => "Ok(())".to_string(), + _ => self.lift(ret, "r.ptr()", false), + }; + let rust_ret = rust_c(self.abi, c_ret); + let _ = write!( + self.glue, + "unsafe extern \"C\" fn {trampoline}({}) -> {rust_ret} {{\n\trt::upcall(\"{what}\", || unsafe {{\n\t\tlet s = &mut rt::Scope::default();\n\t\tlet r = {call}?;\n\t\t{back}\n\t}})\n}}\n\n", + rust_params.join(", "), + ); + } + + /// The glue's expression, an `R`, of a buffer `p` Rust lends for a call: text + /// is copied as when returned, and bytes into `bytes`, since a `memoryview` could + /// outlive the call. + fn lent_buf(&mut self, ty: &Type, p: &str) -> String { + match strip_refs(ty) { + Type::Slice | Type::Vec | Type::Bytes => format!("rt::lent_bytes({p})"), + Type::Option(inner) if matches!(*inner, Type::Slice | Type::Vec | Type::Bytes) => { + format!("(if {p}.is_none() {{ rt::none() }} else {{ rt::lent_bytes({p}) }})") + } + _ => self.lower(ty, p), + } + } + + /// A native function `bare`, taking `n` arguments as `a`, whose glue runs `body` with + /// a scope `s`, returning an `R`. + fn native(&mut self, bare: &str, n: usize, body: &str) { + let _ = write!( + self.glue, + "unsafe extern \"C\" fn f_{bare}(_: *mut PyObject, a: *mut *mut PyObject, n: Py_ssize_t) -> *mut PyObject {{\n\trt::wrap(|| unsafe {{\n\t\tlet a = rt::args(a, n, {n}, \"{bare}\")?;\n\t\tlet s = &mut rt::Scope::default();\n\t\t{body}\n\t}})\n}}\n\n" + ); + self.table.push(format!("rt::method(c\"{bare}\", f_{bare})")); + } + + /// Check that no two members of the type `rust` share a Python name. + fn distinct(&mut self, rust: &str, members: &[String]) { + let mut seen = BTreeSet::new(); + for member in members { + let Some(name) = member_name(member) else { continue }; + if !seen.insert(name.clone()) { + self.errors.push(format!( + "two members of `{rust}` are both `{name}` in Python: rename one in the config" + )); + } + } + } + + /// `function`'s Python form, a member of `owner` if it has one (whose IR type and + /// reserved member names are `owner_ty`), with its native function and glue. + fn function( + &mut self, + function: &Function, + owner: Option<&Ty>, + owner_ty: Option<(&Type, &[&str])>, + place: Place, + ) -> Option { + let abi = self.abi; + let export = abi.export(function, owner); + let c_name = export.call.name.clone(); + let bare = self.bare(&c_name).to_string(); + let owner_c = match owner { + Some(Ty::Struct(n) | Ty::Enum(n) | Ty::Handle { name: n, .. }) => Some(n.clone()), + _ => None, + }; + let own = match &owner_c { + Some(o) => c_name.strip_prefix(o.as_str()).and_then(|r| r.strip_prefix('_')), + None => None, + }; + let reserved = owner_ty.map_or(&[][..], |(_, r)| r); + let snake = own.unwrap_or(&bare); + let member = member(snake, reserved); + + // The glue: each argument read, the call, and its value lowered. + let mut stmts = Vec::new(); + let mut c_args = Vec::new(); + let mut py_params = Vec::new(); + let mut py_args = Vec::new(); + let mut n = 0; + let mut c_params = export.call.params.iter(); + let receiver = function.receiver.filter(|_| owner.is_some()); + if let Some(receiver) = receiver { + let (_, c) = c_params.next().expect("a method has its receiver"); + let ty = owner_ty.expect("a method has its type").0.clone(); + py_params.push("self".to_string()); + py_args.push("self".to_string()); + match c { + Ty::Handle { .. } => { + let ty = match receiver { + Receiver::Owned => ty, + Receiver::Ref => Type::Ref(crate::ir::Mutability::Not, Box::new(ty)), + Receiver::Mut => Type::Ref(crate::ir::Mutability::Mut, Box::new(ty)), + }; + stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), true))); + c_args.push(format!("p{n}")); + } + _ => { + stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), false))); + c_args.push(format!("&p{n}")); + } + } + n += 1; + } + let mut closures = Vec::new(); + for param in &function.params { + let (_, c) = c_params.next().expect("each parameter has its C one"); + let name = param_name(¶m.name); + py_params.push(format!("{name}: {}", self.py_param(¶m.ty))); + py_args.push(name.clone()); + match (¶m.ty, c) { + (Type::Closure(lent, ret), Ty::Fn(c_lent, c_ret)) => { + c_params.next().expect("a closure has its data"); + let trampoline = format!("py_{c_name}_{}", abi::escape(¶m.name)); + closures.push((lent.clone(), (**ret).clone(), c_lent.clone(), (**c_ret).clone(), trampoline.clone())); + c_args.push(format!("Some({trampoline})")); + c_args.push(format!("a[{n}].cast()")); + } + _ => { + stmts.push(format!("let p{n} = {}?;", self.lift(¶m.ty, &format!("a[{n}]"), true))); + c_args.push(format!("p{n}")); + } + } + n += 1; + } + for (lent, ret, c_lent, c_ret, trampoline) in closures { + self.trampoline(&trampoline, &c_name, (&lent, &ret), (&c_lent, &c_ret), None); + } + let outs: Vec<(String, Ty)> = c_params.cloned().collect(); + + // A tuple of one returned whole is written through its one out pointer. + let ok = strip_refs(&function.ok); + let err = function.err.as_ref().map(strip_refs); + let exception = err.as_ref().map(|e| { + let py = self.py_ty(e); + // Named for Rust's type, as Python's `str` reads as no type of its own. + let name = match e { + Type::Str | Type::String => "StringException".to_string(), + _ => format!("{}Exception", pascal_type(&py)), + }; + if !self.exceptions.contains_key(&name) { + self.claim(&name, format!("the exception of `{py}`")); + self.exceptions.insert(name.clone(), e.clone()); + self.register(&format!("exc_{name}"), name.clone()); + } + self.t(&format!("exc_{name}")) + }); + if let Some((_, poll, free)) = &export.future { + let poll_outs = &poll.params[2..]; + let (decl, ptrs, value) = self.outs(poll_outs, &ok, err.as_ref(), exception.as_deref(), "r == 1"); + let _ = write!( + self.glue, + "unsafe fn poll_{c_name}(f: *mut c_void, w: crate::rt::Waker) -> R> {{\n\tunsafe {{\n\t\t{decl}\n\t\tlet r = rt::call(|| crate::{}(f as _, w{}));\n\t\tif r == 0 {{\n\t\t\treturn Ok(None);\n\t\t}}\n\t\t{value}.map(Some)\n\t}}\n}}\n\nunsafe fn free_{c_name}(f: *mut c_void) {{\n\tunsafe {{ crate::{}(f as _) }}\n}}\n\n", + poll.name, + ptrs.iter().map(|p| format!(", {p}")).collect::(), + free.name, + ); + stmts.push(format!( + "let f = rt::call(|| crate::{c_name}({}));\n\t\ts.done();\n\t\trt::start(f.cast(), poll_{c_name}, free_{c_name}, s, {})", + c_args.join(", "), + function.eager + )); + } else { + let (decl, ptrs, value) = match outs.is_empty() { + true => { + let value = self.lower(&ok, "ret"); + (String::new(), Vec::new(), value) + } + false => self.outs(&outs, &ok, err.as_ref(), exception.as_deref(), "ret"), + }; + let args: Vec = c_args.iter().cloned().chain(ptrs).collect(); + stmts.push(format!( + "{decl}\n\t\tlet ret = rt::call(|| crate::{c_name}({}));\n\t\ts.done();\n\t\t{value}", + args.join(", ") + )); + } + self.native(&bare, n, &stmts.join("\n\t\t")); + + // The Python. + let native = format!("_n.{bare}({})", py_args.join(", ")); + let ret = self.py_ty(&ok); + let mut text = function.docs.clone(); + for param in &function.params { + if let Type::Closure(lent, _) = ¶m.ty { + let name = param_name(¶m.name); + text.push_str(&format!("\n\nRust calls `{name}` only during the call, from any thread.")); + if lent.iter().any(|p| matches!(p.base(), Type::Object(_))) { + text.push_str(&format!( + "\nAn object it lends lives only for that call of `{name}`: keep none past it." + )); + } + } + } + let docs = docstring(text.trim(), "\t"); + let params = py_params.join(", "); + let code = match (export.future.is_some(), function.eager) { + (true, false) => format!( + "async def {member}({params}) -> {ret}:\n{docs}\tr = {native}\n\treturn (await r) if r.__class__ is _Future else r" + ), + (true, true) => format!( + "def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\treturn {native}" + ), + (false, _) => format!("def {member}({params}) -> {ret}:\n{docs}\treturn {native}"), + }; + let code = match (function.access, place) { + (Access::Const, Place::Top) if function.params.is_empty() => { + let name = member.to_uppercase(); + self.claim(&name, format!("const `{}`", function.path.join("::"))); + self.consts.push((name, bare)); + return None; + } + (Access::Const, Place::Static) if function.params.is_empty() => { + format!("{} = _Const(_n.{bare})\n{}", member.to_uppercase(), docstring(text.trim(), "")) + } + (Access::Field { set: false, .. }, Place::Member) => format!("@_property\n{code}"), + (_, Place::Static) => format!("@_staticmethod\n{code}"), + _ => code, + }; + if place == Place::Top { + self.claim(&member, format!("fn `{}`", function.path.join("::"))); + } + Some(code) + } + + /// The out values of `outs`, for a function returning `ok` or raising `err` as + /// `exception`: their declarations, the pointers the C call takes after its arguments, + /// and the glue's expression of the value, an `R`, where `ok` says whether it + /// returned ok. + fn outs(&mut self, outs: &[(String, Ty)], ok: &Type, err: Option<&Type>, exception: Option<&str>, is_ok: &str) -> (String, Vec, String) { + let mut decl = Vec::new(); + let mut ptrs = Vec::new(); + let types: Vec<&Type> = match ok { + Type::Unit => Vec::new(), + Type::Tuple(elements) => elements.iter().collect(), + ok => vec![ok], + }; + let mut values = Vec::new(); + let mut error = String::new(); + for ((name, _), ty) in outs.iter().zip(types.into_iter().chain(err)) { + decl.push(format!("let mut o_{name} = MaybeUninit::zeroed();")); + ptrs.push(format!("o_{name}.as_mut_ptr()")); + let value = self.lower(ty, &format!("o_{name}.assume_init()")); + match name.as_str() { + "err" => error = value, + _ => values.push(value), + } + } + let value = match ok { + Type::Unit => "rt::none()".to_string(), + Type::Tuple(_) => format!("rt::tuple(vec![{}])", values.join(", ")), + _ => values.pop().expect("an out value"), + }; + let value = match exception { + Some(exception) => format!( + "(if {is_ok} {{ {value} }} else {{ rt::raise_error({exception}, {error}) }})" + ), + None => value, + }; + (decl.join("\n\t\t"), ptrs, value) + } +} + +/// The glue's name for a socket address family. +fn family_of(family: Family) -> &'static str { + match family { + Family::Any => "rt::Family::Any", + Family::V4 => "rt::Family::V4", + Family::V6 => "rt::Family::V6", + } +} + +fn signed(int: Int) -> bool { + matches!(int, Int::I8 | Int::I16 | Int::I32 | Int::I64 | Int::Isize) +} + +/// `ty` without its `Box`es. +fn strip_box(ty: &Type) -> Type { + match ty { + Type::Box(inner) => strip_box(inner), + ty => ty.clone(), + } +} + +/// A Python annotation as the start of a class name: `list[str] | None` is `ListStrNone`. +fn pascal_type(py: &str) -> String { + py.split(|c: char| !c.is_ascii_alphanumeric()) + .map(|word| { + let mut chars = word.chars(); + match chars.next() { + Some(first) => first.to_uppercase().chain(chars).collect(), + None => String::new(), + } + }) + .collect() +} + +/// Python's hard keywords, and the builtin constants. +const KEYWORDS: &[&str] = &[ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", + "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", +]; + +/// A name with `_` appended if Python reads it as a keyword. +fn kw(name: &str) -> String { + match KEYWORDS.contains(&name) { + true => format!("{name}_"), + false => name.to_string(), + } +} + +/// A member's Python name: its snake-case name, with `_` appended if a keyword or one +/// Python gives every such class. +fn member(snake: &str, reserved: &[&str]) -> String { + match reserved.contains(&snake) { + true => format!("{snake}_"), + false => kw(snake), + } +} + +/// A field's Python name: a positional one's is `_0`, `_1`, ... +fn field_name(name: &str) -> String { + match name.starts_with(|c: char| c.is_ascii_digit()) { + true => format!("_{name}"), + false => kw(&abi::snake(name)), + } +} + +/// A parameter's Python name, apart from the runtime's names a body reads. +fn param_name(name: &str) -> String { + let name = kw(&abi::snake(name)); + match TAKEN.contains(&name.as_str()) || name == "self" { + true => format!("{name}_"), + false => name, + } +} + +/// The name a member's code declares, for [`Gen::distinct`]. +fn member_name(code: &str) -> Option { + let line = code.lines().find(|l| { + let l = l.trim_start(); + !l.starts_with('@') && !l.is_empty() + })?; + let line = line.trim_start(); + let line = line.strip_prefix("async ").unwrap_or(line); + if let Some(rest) = line.strip_prefix("def ") { + return Some(rest.chars().take_while(|c| *c != '(').collect()); + } + if let Some((name, _)) = line.split_once(" =") { + return Some(name.split(':').next().unwrap_or(name).trim().to_string()); + } + None +} + +/// A docstring holding `docs`, each line indented by `indent`, or nothing for none. +fn docstring(docs: &str, indent: &str) -> String { + let docs = docs.trim(); + if docs.is_empty() { + return String::new(); + } + let escaped = docs.replace('\\', "\\\\").replace("\"\"\"", "\\\"\\\"\\\""); + let mut out = format!("{indent}\"\"\""); + for (n, line) in escaped.lines().enumerate() { + if n > 0 { + out.push('\n'); + if !line.is_empty() { + out.push_str(indent); + } + } + out.push_str(line); + } + if escaped.ends_with('"') { + out.push(' '); + } + out.push_str("\"\"\"\n"); + out +} diff --git a/ffi/tests/api.py b/ffi/tests/api.py new file mode 100644 index 000000000..cb4f71e0f --- /dev/null +++ b/ffi/tests/api.py @@ -0,0 +1,650 @@ +# The api fixture's Python test: the script fixtures/api/examples/transcript.rs runs +# natively, through the generated Python. tests/fixtures.rs compares the transcripts. +# +# With an argument, it instead checks the futures' semantics, what frees what nobody +# closed, or a path's bytes, or breaks a contract, which must fail. + +import asyncio +import gc +import os +import sys +import threading + +import api + + +class PrintLog(api.Log): + """The Log callback: keeps even levels.""" + + def log(self, level, line): + print(f"log {level} {line}") + return level % 2 == 0 + + # Rust closes it once it drops the implementation. + def close(self): + print("freed") + + +def bound(b): + match b.kind: + case api.Bound.INCLUDED: + return f"Included({b.value})" + case api.Bound.EXCLUDED: + return f"Excluded({b.value})" + case api.Bound.UNBOUNDED: + return "Unbounded" + + +def flag(value): + """A bool as Rust prints it.""" + return "true" if value else "false" + + +def data(view): + """The bytes a memoryview lends, releasing it.""" + with view: + return bytes(view) + + +def text(view): + return data(view).decode() + + +def debugs(values): + return "[" + ", ".join(repr(v) for v in values) + "]" + + +def quoted(values): + return "[" + ", ".join(f'"{v}"' for v in values) + "]" + + +def escaped(path): + """A path's bytes as Rust's `escape_ascii` prints them.""" + out = "" + for b in os.fsencode(path): + match b: + case 0x09: + out += "\\t" + case 0x0D: + out += "\\r" + case 0x0A: + out += "\\n" + case 0x5C | 0x27 | 0x22: + out += "\\" + chr(b) + case b if 0x20 <= b < 0x7F: + out += chr(b) + case b: + out += f"\\x{b:02x}" + return out + + +def check(ok, what): + if not ok: + print(f"failed: {what}", file=sys.stderr) + sys.exit(1) + + +def step(loop): + """Run the loop until what is ready has run.""" + loop.run_until_complete(asyncio.sleep(0)) + + +def until(loop, done): + """Run the loop until `done()`.""" + + async def wait(): + while not done(): + await asyncio.sleep(0) + + loop.run_until_complete(asyncio.wait_for(wait(), 5)) + + +def script(): + print(f"limit {api.LIMIT}") + + p = api.Point(1, 2).offset(2, 3) + print(f"offset {p.x} {p.y}") + print(f"debug {p!r}") + print(f"eq {flag(p == api.Point(3, 5))}") + d = api.Point.default() + print(f"default {d.x} {d.y}") + + c = api.Counter.new("c", 5) + print(f"get {c.get()}") + print(f"name {c.name()}") + print(f"add ok {c.add(10)}") + try: + c.add(2000) + except api.ErrorException as e: + print(f"add err {e} {e.error!r}") + try: + c.sub(20) + except api.ErrorException as e: + print(f"sub err {e} {e.error!r}") + c.rename("d") + print(f"name {c.name()}") + print(f"is d {flag(c.is_('d'))}") + with c.clone() as copy: + copy.add(1) + print(f"clone {copy.get()} original {c.get()}") + print(f"debug {c!r}") + label = c.into_label(p) + print(f"label {label!r}") + print(f"note {flag(api.has_text(api.Note('')))} {flag(api.has_text(api.Note(None)))}") + + below, above = api.Counter.new("s", 7).split(5) + print(f"split {below.name()} {below.get()} {above.name()} {above.get()}") + (only,) = api.only(api.Counter.new("o", 3)) + print(f"only {only.name()} {only.get()}") + w8, w16, w32, w64 = api.widths(98305) + print(f"widths {w8} {w16} {w32} {w64}") + for b in [2, 0]: + try: + q, r = api.divide(7, b) + print(f"divide {q} {r}") + except api.ErrorException as e: + print(f"divide err {e}") + for n in [98305, 0]: + spread = api.spread(n) + print("spread none" if spread is None else f"spread {spread[0]} {spread[1]} {spread[2]} {spread[3]}") + for needle in [b"lo", b"x"]: + found = api.find(b"hello", needle) + print("find none" if found is None else f"find {found[0]} {found[1]}") + stat_rows = api.rows([api.Point(1, 2), api.Point(3, 3)]) + print("rows " + ", ".join(f"{name} {shape!r} {p.x} {p.y}" for name, shape, p in stat_rows)) + # Each handle in a tuple moves in, as one taken by value does. + with api.Board.new([("low", api.Counter.new("l", 1)), ("high", api.Counter.new("h", 9))]) as board: + ranked = board.ranked() + print("ranked " + ", ".join(f"{name} {counter.name()} {counter.get()}" for name, counter in ranked)) + span = api.Span((2, 10), ("s", api.Shape.SQUARE)) + lower_half, upper_half = api.halves(span) + print(f"halves {lower_half[0]} {lower_half[1]} {upper_half[0]} {upper_half[1]} {span!r}") + swapped_n, swapped_text = api.swapped(("t", 4)) + print(f"swapped {swapped_n} {swapped_text}") + for value in [None, (None,), (3,)]: + doubled = api.doubled(value) + if doubled is None: + print("doubled None") + else: + print(f"doubled Some(({'None' if doubled[0] is None else f'Some({doubled[0]})'},))") + # A borrowed or boxed tuple of one is a tuple, not its element. + lent_one = list(data(api.lent_back((bytes([1, 2]),))[0])) + boxed_one = list(data(api.boxed_tuple(bytes([3]))[0])) + print(f"lent back {lent_one} boxed {boxed_one}") + # A borrowed string and an owned one share a tuple struct. + print( + f'title Some(("{api.title(True)[0]}",)) {"None" if api.title(False) is None else "Some"} ' + f'named Some(("{api.named("x")[0]}",))' + ) + + # Each lent counter is a new handle to a clone. + registry = api.Registry.new(api.Counter.new("first", 1)) + print(f"last {flag(registry.last() is not None)}") + registry.push(api.Counter.new("last", 2)) + first = registry.first + first.add(10) + print(f"first {registry.first.get()} lent {first.get()}") + last = registry.last() + last.add(10) + print(f"last {registry.last().get()} lent {last.get()}") + # A closure lent for the call edits the first counter through a handle. + by = 5 + + def edit(counter): + counter.add(by) + return counter.edited() and counter.get() == 6 + + print(f"edit {flag(registry.edit(edit))} {registry.first.get()}") + # A setter moves an object in, as assignment does, closing it. + registry.set_first(api.Counter.new("second", 3)) + print(f"set first {registry.first.get()} {registry.first.name()}") + print(f"walk {api.walk(3, lambda cursor: cursor.at() * 2)}") + print(f"each {api.each(2, lambda at: print(f'visit {at}'))}") + print(f"offset {api.offset(lambda: 7, 3)}") + print(f"shift {api.shift(lambda: 7, 4)}") + + name = api.Name.owned("a") + joined = name.join(api.Name.owned("b")) + print(f"joined {joined.as_str()} {joined!r} {flag(joined == name)}") + joined.rename("c") + print(f"renamed {joined.as_str()}") + long = api.longer(joined, api.Name.owned("dd")) + print(f"longer {long.as_str()}") + print(f"unnamed [{api.unnamed().as_str()}]") + + print(f"total {api.total(range(2, 5))}") + wide = api.widen(range(2, 5), 1) + print(f"widen {wide.start} {wide.stop}") + inc, exc, unb = api.Bound.INCLUDED, api.Bound.EXCLUDED, api.Bound.UNBOUNDED + counts = [ + api.count(api.BoundsU64(api.BoundU64(start, s), api.BoundU64(end, e))) + for (start, s), (end, e) in [ + ((inc, 2), (exc, 5)), + ((inc, 2), (inc, 5)), + ((inc, 3), (unb, 0)), + ((unb, 0), (unb, 0)), + ((exc, 2), (inc, 4)), + ] + ] + print("count " + " ".join(str(n) for n in counts)) + print(f"end {bound(api.end(3))} {bound(api.end(0))}") + ends_start, ends_end = api.ends(3) + print(f"ends {bound(ends_start)} {bound(ends_end)}") + for n in [2, 5]: + try: + print(f"below {api.below(n, 3)}") + except api.BoundsU64Exception as e: + print(f"below err {bound(e.error.start)} {bound(e.error.end)}") + window = api.Window(range(2, 8), api.end(5)) + print(f"window {flag(window.contains(4))} {flag(window.contains(6))} {window!r}") + + print(api.greet("x", api.Shape.SQUARE)) + print(f"len {api.len(b'abc')}") + binary = bytes([0xFF, 0, ord("a")]) + print(f"hex {api.hex(binary)}") + print(f"timed hex {api.timed_hex(binary)}") + print(f"sample hex {api.sample_hex(binary)}") + print(f"queued hex {api.queued_hex(binary)}") + print(f"tick {api.tick_name(api.Label('tick', None, None))}") + print(f"rooted {api.rooted_name(api.Label('rooted', None, None))}") + print(f"stored {api.stored_text('stored')}") + print(f"layer {api.layer_name(api.Label('layer', None, None))}") + for event in [api.Event.Moved(p), api.Event.Named("n"), api.Event.Done()]: + print(api.describe(event)) + + # A `Copy` handle passed by value is copied, so it stays usable. + stamp = api.Stamp.from_millis(2500) + print(f"secs {stamp.secs()}") + print(f"later {flag(api.Stamp.from_millis(3000).later(stamp))}") + print(f"secs {stamp.secs()}") + message = api.Message.new(stamp, b"body") + print(f"payload {text(message.payload)}") + print(f"stamp {message.stamp!r}") + print(f"sent {flag(message.sent())}") + # Setters assign in place: the bytes are copied in, and a `Copy` handle copied. + reset = api.Stamp.from_millis(4000) + message.set_stamp(reset) + message.set_payload(b"edited") + print(f"payload {text(message.payload)}") + print(f"stamp {message.stamp!r}") + print(f"secs {reset.secs()}") + message.set_stamp(None) + print(f"stamp {flag(message.stamp is not None)}") + + print(f"kept {api.emit(PrintLog(), bytes([1, 2, 3, 4]))}") + with api.Logger.new(PrintLog()) as logger: + print(f"write {flag(logger.write('hi'))}") + print("dropped") + # Rust calls it on another thread, before the call returns. + threshold = 2 + + def keep(level, line): + print(f"filter {level} {line}") + return level > threshold + + print(f"filtered {api.filter(bytes([1, 2, 3, 4]), keep)}") + + loop = asyncio.new_event_loop() + channel = api.Channel.new() + wakes = 0 + + def wake(): + nonlocal wakes + wakes += 1 + + waker = api.Waker(wake, loop) + # On a loop the script steps, so it sees the receive pend, and its wake resume it. + got = None + + async def recv(): + nonlocal got + got = await channel.recv() + + task = loop.create_task(recv()) + step(loop) + print(f"recv pending {flag(got is None)}") + channel.send(b"hi") + check(got is None, "recv never resumes inside the wake") + loop.run_until_complete(task) + wakes += 1 + print(f"woken {wakes}") + print(f"recv {text(got)}") + + async def block(): + channel.send(b"") + try: + with await channel.recv_nonempty() as payload: + print(f"nonempty {payload.nbytes}") + except api.ErrorException as e: + print(f"nonempty err {e}") + # The send happens at the call: the future is dropped unawaited. + channel.send_later(b"eager") + print(f"recv {text(await channel.recv())}") + channel.send(b"sized") + payload, size = await channel.recv_sized() + print(f"sized {text(payload)} {size}") + + loop.run_until_complete(block()) + # The waker moves into Rust, which keeps it until the send wakes it. + channel.notify(waker) + channel.send(b"rung") + until(loop, lambda: wakes == 2) + print(f"woken {wakes}") + loop.close() + + nodes = api.Node.new(1, None) + nodes.push(2) + print(f"node {nodes.sum()}") + + tally = api.Tally.new() + tally.bump() + print(f"tally {tally.bump()} {flag(tally == tally)}") + print(f"both {api.bump_both(tally, tally)}") + print(f"peek {api.peek(tally, lambda lent: lent.bump())}") + + area = api.grow(api.Area.new(api.Size(2, 3)), 1) + size = area.size() + print(f"area {size.width} {size.height} {area.pixels()}") + + # Generic types, each at the instances it infers, named by their arguments. + pair = api.PairU32(2, 5) + swapped = pair.swap() + print(f"pair {swapped.left} {swapped.right} sum {pair.sum()} {pair!r} {flag(pair == swapped)}") + print(f"even {flag(pair.even())} {flag(api.PairU32(7, 7).even())}") + # An instance renamed alone, beside another of `Pair` that keeps its name. + sized = api.tag(api.Size(1, 2), 3) + flipped = sized.swap() + print(f"sized {flipped.left} {flipped.right.width} {flipped!r}") + beside = api.PairU32(258, 9).beside_size(sized.left) + print(f"beside {beside.left} {beside!r}") + # At its default, named as the type itself. + stack = api.Stack.new() + for i, n in enumerate(["a", "b"]): + stack.push(api.Counter.new(n, i + 1)) + top = stack.pop().get() + print(f"stack {stack.depth()} Some({top}) {flag(stack.pop() is not None)} {flag(stack.pop() is None)}") + # Generic functions and methods, at each implementation of the trait bounding them, by + # reference, which leaves Python its object. + from_point = api.Counter.seeded_point("p", api.Point(-3, 4)) + from_size = api.Counter.from_size("s", api.Size(5, 6)) + from_counter = api.Counter.seeded_counter("c", from_point) + from_polar = api.Counter.seeded_polar_point("q", api.PolarPoint(1, 2)) + print( + f"seeded {from_point.get()} {from_size.get()} {from_counter.get()} {from_polar.get()} " + f"past {stack.depth_past_point(api.Point(1, 2))} seeds {api.seeds_point_size(api.Size(2, 3))}" + ) + + # A trait implemented for `Point` and `&Point` apart: one instance. + print(f"weigh {api.weigh_point(api.Point(-1, 2))}") + # Instances that name each other, and one made only by an impl of it. + cross = api.cross(api.LeftRightU8(api.RightU8(1)), api.RightLeftU8(api.LeftU8(2))) + print(f"cross {cross} outer {api.OuterInnerU8.new(4).get()}") + # A bound that outlives the call takes a value, and an instance only a constant names. + print(f"kept {api.kept_point(api.Point(1, -2))} boxed {api.BOXED.value}") + print(f"keep {api.keep_point(api.Point(1, -2))}") + # Inferred instances taking a lent closure and a foreign implementation, and one only + # its impl of a trait names. + sides = api.PairU32(2, 5) + print(f"apply {api.apply_u32(sides, lambda side: side * 3)}") + print(f"report {api.report_u32(sides, PrintLog())}") + print(f"wrapped {api.kept_wrap_u32(api.WrapU32(4))}") + print(f"keeps {api.keeps(PrintLog()).value}") + + # Its Drop runs when Python closes it, not before. + token = api.Token.new(7) + print(f"token {token.id} dropped {api.dropped()}") + token.close() + print(f"dropped {api.dropped()}") + + # A field and the method of its name, each its own member. + retry = api.Retry.new(5) + limit = retry.limit_setting + print(f"retry {0 if limit is None else 1} {limit or 0} {retry.limit()} {retry.delay()}") + # The renamed field's setter, named as its getter. + retry.set_limit_setting(None) + print(f"retry {retry.limit()}") + print(f"slot {api.SlotU8.new(200).value()}") + + # Foreign types: a URL parsed from text, socket addresses, OS paths, and `Vec`s. + base = api.Url.parse("https://example.com/a/b") + host = api.host(base) + host = "None" if host is None else f'Some("{host}")' + print(f"url {base} host {host}") + with api.resolve(base.clone(), "c?d") as url: + print(f"resolve {url} {flag(url == base)}") + try: + api.Url.parse("no scheme").close() + except api.StringException as e: + print(f"parse err {e.error}") + v4 = ("127.0.0.1", 8080) + v6 = ("::1", 443, 0, 3) + print(f"ports {api.ports([v4, v6])}") + peer = api.Peer(v6, ["a", "b"]).next("c") + print(f"peer {peer!r} {quoted(peer.aliases())}") + file = api.file("dir", "f\\'\"\t") + print(f"file {escaped(file)}") + print("parents " + " ".join("none" if p is None else escaped(p) for p in api.parents([file, "/"]))) + words = api.words(" one two three ") + print(f"words {len(words)} {quoted(words)}") + print(f"sentence {api.sentence(words)} {api.sentence(None)}") + print(f"sentence {api.sentence([])}|") + print(f"chunks {[list(data(chunk)) for chunk in api.chunks(b'abcde', 2)]}") + counters = api.counters(["x", "y"]) + print("counters " + ", ".join(f"{c.name()} {c.get()}" for c in counters)) + # Each counter moves in, so Python closes it. + print(f"worth {api.worth([counters[0]], [counters[1]])} {api.worth(None, [])}") + for counter in counters: + try: + counter.get() + check(False, "a counter moved in is closed") + except ValueError: + pass + print(f"sums {api.sums([[1, 2], [], [3]])}") + summed = api.summed([api.Point(1, 2), api.Point(-4, 6)]) + none_summed = api.summed([]) + print(f"summed {summed.x} {summed.y} {none_summed.x} {none_summed.y}") + print(f"longest {api.longest(['a', 'abc'])} {api.longest([])}") + doc = api.Json.parse('{"b": [1, 2.5, null], "a": "é"}') + print(f"json {api.encode(doc)}") + print(f"timed text {api.timed_text(doc)}") + with api.tagged(doc, "t") as tagged: + print(f"tagged {tagged.to_text()}") + try: + api.Json.parse('{"a":').close() + except api.StringException as e: + print(f"json parse err {e.error}") + site = api.Site.new(base) + with site.home as a, site.home as b: + print(f"site {a} {b}") + + # Slices, passed in and returned as clones, and the socket address of each family. + print(f"shapes {debugs(api.Shape.ALL)} greetings {quoted(api.GREETINGS)}") + with api.Trail.new([api.Point(1, 2), api.Point(3, 4)], ["a", "b"]) as trail: + print(f"trail {debugs(trail.points())} {debugs(trail.labels())}") + # Returned iterators, each collected at the call: drained labels are gone after. + print(f"iterated {debugs(trail.iter())} {quoted(trail.names())} {list(data(trail.bytes()))}") + print(f"drained {debugs(trail.drain())} then {debugs(trail.labels())}") + numbered = api.numbered(["p", "q"]) + print("numbered " + ", ".join(f"{c.name()} {c.get()}" for c in numbered)) + next4, next6 = api.nexts(v4, v6) + print(f"nexts {next4[0]} {next4[1]} {next6[1]} {next6[3]}") + + +def futures(): + """The futures' semantics beyond the transcript, each checked by an assertion.""" + channel = api.Channel.new() + + # The send lands from another thread while the receive waits. + async def threaded(): + sender = threading.Thread(target=lambda: channel.send(b"blocked")) + sender.start() + payload = await asyncio.wait_for(channel.recv(), 5) + sender.join() + check(text(payload) == "blocked", "recv returns the payload another thread sent") + + asyncio.run(threaded()) + + loop = asyncio.new_event_loop() + + # Resumed on its loop, never inside the wake. + got = None + + async def recv(): + nonlocal got + got = text(await channel.recv()) + + task = loop.create_task(recv()) + step(loop) + check(got is None, "recv waits for a send") + channel.send(b"then") + check(got is None, "recv never polls inside the wake") + loop.run_until_complete(asyncio.wait_for(task, 5)) + check(got == "then", "recv resumes with the payload") + + # A wake that lands while the future polls, before it returns pending, is kept: the + # future polls again rather than waiting for a wake that already happened. + check(loop.run_until_complete(asyncio.wait_for(api.yielded(7), 5)) == 7, "a wake during the poll polls again") + + # Cancelled between its wake and its poll, a receive loses nothing: the payload stays + # for the next. + called = False + + async def cancelled(): + nonlocal called + await channel.recv() + called = True + + task = loop.create_task(cancelled()) + step(loop) + channel.send(b"kept") + task.cancel() + until(loop, task.done) + check(not called and task.cancelled(), "a cancelled receive never resumes") + payload = loop.run_until_complete(asyncio.wait_for(channel.recv(), 5)) + check(text(payload) == "kept", "a cancelled receive loses nothing") + + # A handle closed while a call awaits on it is freed once the call ends. + other = api.Channel.new() + got = None + + async def held(): + nonlocal got + got = text(await other.recv()) + + task = loop.create_task(held()) + step(loop) + sender = other.clone() + other.close() + sender.send(b"held") + loop.run_until_complete(asyncio.wait_for(task, 5)) + check(got == "held", "a closed object stays held by the call awaiting on it") + sender.close() + + # A waker woken from another thread calls back on its loop, once for wakes that land + # before the loop gets to them. + wakes = [] + waker = api.Waker(lambda: wakes.append(threading.get_ident()), loop) + channel.notify(waker) + sender = threading.Thread(target=lambda: channel.send(b"rung")) + sender.start() + sender.join() + until(loop, lambda: wakes) + check(wakes == [threading.get_ident()], "a waker calls back on its loop's thread") + loop.close() + print("futures ok") + + +def owned(): + """What frees an object or bytes nobody closed: its last reference going, or the + cycle collector, and only what closing did not free already. Rust's bytes cross lent, + until released.""" + for n in range(1023, 1026): + payload = bytes(i % 256 for i in range(n)) + packet = api.next_packet(api.Packet(1, payload)) + check(packet.seq == 2 and data(packet.payload) == payload, f"a record's {n} bytes cross both ways") + freed = api.dropped() + for n in range(1023, 1026): + view = api.token_bytes(n, n) + check(api.dropped() == freed, f"{n} bytes are lent, not copied") + check(bytes(view) == bytes([n % 256]) * n, f"{n} bytes read back") + view.release() + freed += 1 + check(api.dropped() == freed, f"releasing {n} bytes frees them") + view.release() + check(api.dropped() == freed, f"releasing {n} bytes again frees nothing") + try: + bytes(view) + check(False, f"released {n} bytes refuse a read") + except ValueError: + pass + + before = api.dropped() + api.Token.new(1).close() + api.token_bytes(2, 1025).release() + check(api.dropped() == before + 2, "closing frees at once") + unclosed() + check(api.dropped() == before + 4, "the last reference going frees what nobody closed") + cycle = [api.Token.new(5), api.token_bytes(6, 1)] + cycle.append(cycle) + del cycle + gc.collect() + check(api.dropped() == before + 6, "the cycle collector frees what a cycle held") + print("owned ok") + + +def unclosed(): + api.Token.new(3) + api.token_bytes(4, 1025) + + +def stray(): + """A Unix path whose bytes are not UTF-8 crosses as `os.fsdecode` decodes it, and back + as it came.""" + path = api.stray() + check(os.fsencode(path) == b"\xff", "a path that is not UTF-8 keeps its bytes") + check(api.parents([path, os.fsdecode(b"\xfe/\xff")])[1] == os.fsdecode(b"\xfe"), "and crosses back") + print("stray ok") + + +def broken(which): + """Each breaks a contract, which must fail with a message naming it.""" + match which: + case "moved": + c = api.Counter.new("c", 0) + c.into_label(None) + c.get() + case "boxed moved": + tail = api.Node.new(1, None) + api.Node.new(2, tail) + tail.sum() + case "lent moved": + api.Registry.new(api.Counter.new("c", 0)).edit(lambda counter: counter.into_label(None) and False) + case "lent kept": + kept = [] + api.Registry.new(api.Counter.new("c", 0)).edit(lambda counter: kept.append(counter) or False) + kept[0].get() + # An IPv6 address where Rust takes `SocketAddrV4`. + case "family": + api.nexts(("::", 1, 0, 0), ("::", 1, 0, 0)) + case "callback threw": + + class Threw(api.Log): + def log(self, level, line): + raise RuntimeError("a log failed") + + api.emit(Threw(), bytes([1])) + case _: + raise ValueError(f"no case {which}") + + +match sys.argv[1:]: + case []: + script() + case ["futures"]: + futures() + case ["owned"]: + owned() + case ["stray"]: + stray() + case [which]: + broken(which) From 4f5804df82b9d1b5db83e3e4f710288afdc2fcf3 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:05:54 -0800 Subject: [PATCH 03/13] wip(ffi): run the Python script in the fixture tests Co-Authored-By: Claude Opus 5.5 --- ffi/tests/fixtures.rs | 109 +++++++++++++++++++++++++++++++++++++----- flake.nix | 5 +- 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index b9165ab03..1a38765f1 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -1,7 +1,7 @@ //! The api fixture is exported, built as a static library, and driven from C by -//! `tests/api.c`, from C++ by `tests/api.cpp`, and from Kotlin by `tests/api.kt` on the JVM -//! and, shrunk by R8, on Android, whose transcripts must match the fixture's own script run -//! natively. +//! `tests/api.c`, from C++ by `tests/api.cpp`, from Kotlin by `tests/api.kt` on the JVM +//! and, shrunk by R8, on Android, and from Python by `tests/api.py`, whose transcripts must +//! match the fixture's own script run natively. //! The reject fixture's export must fail at each item it cannot export. use std::env::consts::EXE_SUFFIX; @@ -85,16 +85,27 @@ fn api() -> &'static Api { // The workspace's lockfile pins what the export depends on. std::fs::copy(manifest_dir().join("Cargo.lock"), out.join("Cargo.lock")).unwrap(); let target = scratch().join("api").join("glue"); - let built = run(cargo() - .arg("build") - .arg("--quiet") - .arg("--manifest-path") - .arg(out.join("Cargo.toml")) - .arg("--target-dir") - .arg(&target) - .args(["--features", "kotlin"]) - // Glue the exporter writes builds without warnings. - .env("RUSTFLAGS", "-D warnings --print=native-static-libs")); + let build = |feature: &str| { + run(cargo() + .arg("build") + .arg("--quiet") + .arg("--manifest-path") + .arg(out.join("Cargo.toml")) + .arg("--target-dir") + .arg(&target) + .args(["--features", feature]) + // Glue the exporter writes builds without warnings. + .env("RUSTFLAGS", "-D warnings --print=native-static-libs")) + }; + // The CPython extension's symbols would leave the C programs' links undefined, so + // it builds apart, before the library they link, and moves beside the Python. + build("python"); + std::fs::rename( + target.join("debug").join(PYTHON_LIB), + out.join("python").join(PYTHON_LIB), + ) + .unwrap(); + let built = build("kotlin"); let native = String::from_utf8_lossy(&built.stderr) .lines() .find_map(|line| line.strip_prefix("note: native-static-libs: ")) @@ -474,6 +485,78 @@ fn kotlin(native: &str, vm: &dyn Fn(&[&str]) -> Command) { } } +/// The library `python/api.py` loads from beside itself. +const PYTHON_LIB: &str = match (cfg!(windows), cfg!(target_os = "macos")) { + (true, _) => "api.dll", + (_, true) => "libapi.dylib", + _ => "libapi.so", +}; + +/// `tests/api.py`'s script prints the native transcript; then its futures, what frees +/// what nobody closed, a path that is not UTF-8, and each contract it fails loud on. +#[test] +fn api_python() { + let api = api(); + let python = |dir: &Path, args: &[&str]| { + let mut cmd = Command::new("python3"); + // `-P` keeps `tests/` off the path, where `api.py` is this script; dev mode and + // warnings as errors fail on what the runtime only warns of. + cmd.args(["-P", "-X", "dev", "-W", "error"]) + .arg(manifest_dir().join("tests").join("api.py")) + .args(args) + .env("PYTHONPATH", dir); + cmd + }; + let dir = api.out.join("python"); + let py = run(&mut python(&dir, &[])); + assert_eq!( + String::from_utf8_lossy(&py.stdout), + api.transcript, + "the Python transcript differs from the native one" + ); + for case in ["futures", "owned"].into_iter().chain(cfg!(unix).then_some("stray")) { + run(&mut python(&dir, &[case])); + } + for (case, message) in [ + ("moved", "ValueError: Counter is closed"), + ("boxed moved", "ValueError: Node is closed"), + ("lent kept", "ValueError: Counter is closed"), + ( + "lent moved", + "api_counter_into_label: a lent handle's value cannot move out", + ), + ("callback threw", "api_fixture::Log::log: a call into Python raised"), + ("family", "ValueError: an IPv6 address has no Rust SocketAddrV4"), + ] { + let output = python(&dir, &[case]).output().unwrap(); + assert!(!output.status.success(), "Python {case} did not fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains(message), "Python {case} printed {stderr}"); + } + + // The same Python, with another contract than the library's in place of its own, + // refuses to load it. + let mismatched = scratch().join("api").join("python-mismatched"); + let _ = std::fs::remove_dir_all(&mismatched); + std::fs::create_dir_all(&mismatched).unwrap(); + std::fs::copy(dir.join(PYTHON_LIB), mismatched.join(PYTHON_LIB)).unwrap(); + let source = std::fs::read_to_string(dir.join("api.py")).unwrap(); + let at = source.find("_n.contract() != 0x").expect("the Python checks its contract") + 19; + let flipped = match &source[at..at + 1] { + "0" => "1", + _ => "0", + }; + let source = format!("{}{flipped}{}", &source[..at], &source[at + 1..]); + std::fs::write(mismatched.join("api.py"), source).unwrap(); + let output = python(&mismatched, &[]).output().unwrap(); + assert!(!output.status.success(), "a mismatched contract did not fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("api.py and the library are from different builds"), + "a mismatched contract printed {stderr}" + ); +} + /// The contract a Kotlin file checks its library against. fn kotlin_contract(kotlin: &Path) -> u64 { let kotlin = std::fs::read_to_string(kotlin).unwrap(); diff --git a/flake.nix b/flake.nix index 2c3c1400a..fe4bb4760 100644 --- a/flake.nix +++ b/flake.nix @@ -297,8 +297,10 @@ typescript bun esbuild - # rust2-ffi's tests compile its generated Kotlin and run it on the JVM. + # rust2-ffi's tests compile its generated Kotlin and run it on the JVM, and run + # its generated Python. kotlin + python3 ]; RUST2_TS_TOOLCHAIN = tsToolchain; RUST2_TS_SYSROOT = rust2-ts-std; @@ -334,7 +336,6 @@ # Every foreign toolchain the UniFFI fixture suite runs, apart for their size. devShells.uniffi = self.devShells.${system}.default.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.python3 pkgs.kotlin pkgs.swift pkgs.go From 6137c1373dcce3ae7eb2ae4b4afc7df7911f9604 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:17:49 -0800 Subject: [PATCH 04/13] wip(ffi): run a moq session from Python Co-Authored-By: Claude Opus 5.5 --- ffi/justfile | 16 +- ffi/moq/moq.py | 365 ++++++++++++++++++++++++++++++++++++++++++++++ ffi/src/python.rs | 83 +++++------ 3 files changed, 419 insertions(+), 45 deletions(-) create mode 100644 ffi/moq/moq.py diff --git a/ffi/justfile b/ffi/justfile index 6944613d9..028d9271a 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -64,7 +64,7 @@ bench checkout="": (moq checkout) kotlin -Djava.library.path="$out/target/release" -cp "target/bench-kt:$KOTLINX_COROUTINES" ExportKt done -# Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, and moq/moq.kt against it, from the moq revision `just ts moq` pins or a checkout. +# Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, moq/moq.kt, and moq/moq.py against it, from the moq revision `just ts moq` pins or a checkout. moq checkout="": (moq-cpp checkout) #!/usr/bin/env bash set -euo pipefail @@ -76,6 +76,8 @@ moq checkout="": (moq-cpp checkout) rm -rf target/moq-kt kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/moq.kt -d target/moq-kt kotlin -Djava.library.path="$out/target/release" -cp "target/moq-kt:$KOTLINX_COROUTINES" MoqKt + # The Python, through the CPython extension the python feature builds. + PYTHONPATH="$out/python" python3 -P -X dev -W error moq/moq.py # Export and build moq's crates' C ABI as `just ffi moq` does, and run only moq/moq.cpp against it, built by $CXX or the platform's compiler: MSVC's on Windows, from a developer environment. moq-cpp checkout="": (moq-export checkout) @@ -104,7 +106,17 @@ moq-export checkout="": (standin checkout) cp target/moq-ffi/Cargo.lock "$out/Cargo.lock" # A later nightly makes this future-compat lint a hard error; fail on it in moq's crates # and the glue now, while the pin can still move. rustc prints the native libraries. - RUSTFLAGS="-D recursion_depth_exceeding_limit --print=native-static-libs" cargo build --quiet --release --manifest-path "$out/Cargo.toml" --features kotlin 2>&1 | tee target/moq-build.log + export RUSTFLAGS="-D recursion_depth_exceeding_limit --print=native-static-libs" + # The CPython extension's symbols would leave the C programs' links undefined, so it + # builds apart, before the library they link, and moves beside the Python. + case "{{ os() }}" in + windows) lib=moq.dll ;; + macos) lib=libmoq.dylib ;; + *) lib=libmoq.so ;; + esac + cargo build --quiet --release --manifest-path "$out/Cargo.toml" --features python + mv "$out/target/release/$lib" "$out/python/$lib" + cargo build --quiet --release --manifest-path "$out/Cargo.toml" --features kotlin 2>&1 | tee target/moq-build.log sed -n 's/^note: native-static-libs: //p' target/moq-build.log > target/moq-native # Write target/moq-ffi, an empty library on every crate moq/ffi.toml exports, from the moq revision `just ts moq` pins or a checkout: the check builds with all of them, as moq-ffi's remainder will. diff --git a/ffi/moq/moq.py b/ffi/moq/moq.py new file mode 100644 index 000000000..7738d65bb --- /dev/null +++ b/ffi/moq/moq.py @@ -0,0 +1,365 @@ +# moq.c's script through the generated Python: a broadcast's track, published and +# subscribed to in one process, with every read awaited, and an origin, its driver run on +# asyncio's loop, with moq-pattern, moq-clock, moq-tokio, hang, and moq-mux exported into +# the same library, and the catalog lent to Python for a call. +# +# A local read that must wait resumes once the write that wakes it lands, on the loop the +# script steps. A session over QUIC runs on moq-tokio's fallback runtime, whose threads +# wake the loop's. + +import asyncio +import sys + +import moq + +failures = 0 + + +def expect(ok, what): + global failures + if not ok: + print(f"failed: {what}", file=sys.stderr) + failures += 1 + + +def text(view): + """The bytes a memoryview lends as text, releasing it.""" + with view: + return bytes(view).decode() + + +def step(loop): + """Run the loop until what is ready has run.""" + loop.run_until_complete(asyncio.sleep(0)) + + +async def drive(driver): + """Poll `driver` as asyncio's loop would: on each wake, and at the deadline each poll + returns, with one waiter held across polls, since what a poll registers lives only as + long as its waiter. It returns once the driver finishes, raising its error.""" + woken = asyncio.Event() + waiter = moq.Waiter.new(moq.Waker(woken.set)) + while True: + woken.clear() + deadline = driver.poll(moq.now(), waiter) + timeout = None if deadline is None else deadline.saturating_duration_since(moq.now()).total_seconds() + try: + await asyncio.wait_for(woken.wait(), timeout) + except TimeoutError: + pass + + +def local(loop): + broadcast = moq.BroadcastInfo.new().produce() + track = broadcast.create_track("video", None) + expect(track.name() == "video", "a track has its name") + + # A second track of the same name is an error, printed as Rust prints it. + try: + broadcast.create_track("video", None) + expect(False, "a duplicate track is an error") + except moq.ErrorException as e: + expect(str(e) == "duplicate", "a duplicate track's error is Rust's") + + subscriber = loop.run_until_complete(track.consume().subscribe(None)) + + # A group's arrival wakes the subscriber's pending receive. + recv = loop.create_task(subscriber.recv_group()) + step(loop) + expect(not recv.done(), "a receive waits for a group") + writer = track.append_group() + expect(not recv.done(), "a receive never resumes inside the wake") + first = loop.run_until_complete(asyncio.wait_for(recv, 5)) + expect(first.index() == 0, "the first group is 0") + + # So does a frame's. + read = loop.create_task(first.read_frame()) + step(loop) + expect(not read.done(), "a read waits for a frame") + writer.write_frame(None, b"hello") + hello = loop.run_until_complete(asyncio.wait_for(read, 5)) + expect(text(hello.payload) == "hello", "a frame has its payload") + expect(hello.timestamp is None, "an untimed frame has no timestamp") + # When it arrived: moq-clock's `Instant`, from moq-clock's own export. + expect(moq.now().checked_duration_since(hello.arrived()) is not None, "a frame arrived before now") + + # A timestamp is a `Copy` value behind a handle: passing it copies it out. The track is + # untimed, so a timestamped frame is refused. + at = moq.Timestamp.from_millis(1500) + expect(at.as_secs() == 1, "1500 ms is 1 s") + try: + writer.write_frame(at, b"late") + expect(False, "a timestamped frame on an untimed track is an error") + except moq.ErrorException as e: + expect(str(e) == "frame timestamp doesn't fit its track", "the late frame's error is Rust's") + expect(at.as_secs() == 1, "a copied timestamp stays usable") + + # The last frame, bytes that are not UTF-8, then the group's end, which reads as None. + world = b"\xffworld" + writer.write_frame(None, world) + writer.finish() + last = loop.run_until_complete(first.read_frame()) + expect(last is not None and bytes(last.payload) == world, "the last frame reads") + expect(loop.run_until_complete(first.read_frame()) is None, "a finished group reads None") + + +def origin(loop): + # An origin and the driver that runs it, a pair. Its root, which it lends, is returned + # as a new object of a clone. + origin, driver = moq.OriginProducer.new(moq.OriginConfig.default()) + expect(origin.root().as_str() == "", "an origin's root is empty") + + # Scoped beneath a root to patterns, which moq-pattern's own export builds. + with moq.Patterns.new() as patterns: + expect(patterns.insert(moq.Pattern.subtree("alice")), "a pattern inserts") + with origin.scope("room", patterns) as scoped: + expect(scoped.root().as_str() == "room", "a scope has its root") + expect(scoped.absolute("alice").as_str() == "room/alice", "a scope's paths are absolute") + allowed = scoped.allowed() + expect(allowed.matches("alice/cam"), "a scope allows its patterns") + expect(not allowed.matches("bob"), "a scope allows nothing else") + with scoped.consume() as consumer: + expect(consumer.absolute("bob").as_str() == "room/bob", "a scope's consumer shares its root") + + # A broadcast the origin creates is stamped with its path, which its info lends. Each + # object of it is closed, since an open one keeps it, and so the origin, alive. + with origin.create_broadcast("room/alice") as publication: + with publication.broadcast() as broadcast: + path = broadcast.info().path + + expect(path.as_str() == "room/alice", "a broadcast has its path") + + # Paths own their segments, so Python builds them, and relative ones, from a string. + try: + moq.Path.new("/cam/") + expect(False, "a path with an empty segment is an error") + except moq.PathInvalidException as e: + expect(e.error == moq.PathInvalid.EMPTY_SEGMENT, "an empty segment is its own error") + cam = moq.Path.new("cam") + expect(cam.as_str() == "cam", "a path builds from its segments") + joined = path.join(cam) + expect(joined.as_str() == "room/alice/cam", "paths join") + expect(joined.strip_prefix(path) == cam, "a path strips its prefix") + expect(path.strip_prefix(joined) is None, "a path strips no longer prefix") + expect(joined.resolve(moq.Relative.new("../bob")).as_str() == "room/bob", "a relative path resolves") + + # The driver, run on asyncio's loop until every owner of the origin is gone. + driving = loop.create_task(drive(driver)) + step(loop) + # A client publishes the origin through moq-net's generic `with_publisher`, at each + # type that implements its bound, by reference, so Python keeps its object. + moq.Client.new().with_publisher_producer(origin).close() + expect(not driving.done(), "a driver runs while its origin has an owner") + origin.close() + # The last owner's close wakes the driver, which then finishes, cleanly. + try: + loop.run_until_complete(asyncio.wait_for(driving, 5)) + expect(False, "a driver whose origin is gone finishes") + except moq.ErrorException as e: + expect(str(e) == "closed", "a finished driver's error is Rust's") + + +def catalog(loop): + # moq-mux reads an OpusHead into hang's catalog config: 44.1 kHz in, stereo, which the + # catalog lists at the decoder's 48 kHz, keeping the head as its description. + head = b"OpusHead" + bytes([1, 2, 0x38, 0x01, 0x44, 0xAC, 0, 0, 0, 0, 0]) + audio = moq.codec_opus_audio_config(head) + expect(audio.sample_rate == 48000, "Opus decodes at 48 kHz") + expect(audio.channel_count == 2, "the head is stereo") + expect(audio.codec == moq.CatalogAudioCodec.Opus(), "the codec is Opus") + expect(audio.description is not None and bytes(audio.description) == head, "the head is the description") + # A truncated head is moq-mux's error. + try: + moq.codec_opus_audio_config(head[:8]) + expect(False, "a truncated head is an error") + except moq.MuxErrorException as e: + expect(str(e) != "", "a truncated head's error has a message") + + # A subscription's ranges, from any iterable of them: a list. + b = moq.Bound + groups = moq.TrackRange.groups(moq.BoundsU64(moq.BoundU64(b.INCLUDED, 2), moq.BoundU64(b.EXCLUDED, 6))) + ranged = moq.TrackSubscription.default().with_ranges([moq.TrackRange.live(), groups]).ranges + expect(ranged == [moq.TrackRange.live(), groups], "a subscription keeps its ranges") + expect(groups.start == moq.TrackAnchor.Absolute(2), "groups start at their first") + expect(groups.end == moq.TrackAnchor.Absolute(6), "groups end before their end") + + # A catalog at moq-mux's untyped extension, from the config Python builds, whose generic + # `is_claimed` is named by the last segment of its argument, and whose `mutate` lends + # the catalog to Python. + publisher = moq.BroadcastInfo.new().produce() + extra = moq.CatalogConfig.default().with_catalog_extra(moq.HangCatalogExtra.default()) + producer = moq.CatalogProducerExtra.new(publisher, extra) + expect(not producer.is_claimed_binary_config("state"), "an unclaimed name is not claimed") + read = 0 + + def count(lent): + nonlocal read + if lent.to_json().startswith("{"): + read += 1 + + producer.mutate(count) + expect(read == 1, "mutate lends the catalog once") + + # A JSON track the catalog lists, published through moq-mux's generic `json_snapshot` at + # a JSON document, and a section Python parses from text, set in the lent catalog and + # read back as text. + status = producer.json_snapshot_config( + publisher.create_track("status", None), moq.JsonConfig.default(), moq.JsonOptions.default() + ) + expect(status.name() == "status", "a JSON snapshot has its track's name") + expect(producer.is_claimed_json_config("status"), "the JSON snapshot claims its name") + # A value Python parses is published as the track's latest, which a subscriber reads + # back as JSON text. + with moq.Json.parse('{"on": true}') as on: + status.update(on) + + async def latest(): + group = await status.consume().recv_group() + return None if group is None else await group.read_frame() + + frame = loop.run_until_complete(latest()) + expect(frame is not None and text(frame.payload) == '{"on":true}', "a subscriber reads the latest value") + + def section(lent): + sections = lent.ext + sections.set("app", moq.Json.parse('{"v": [1, true]}')) + lent.set_ext(sections) + + producer.mutate(section) + app = None + + def back(lent): + nonlocal app + expect('"status"' in lent.to_json(), "the catalog lists the JSON track") + value = lent.ext.get("app") + app = None if value is None else value.to_text() + + producer.mutate(back) + expect(app == '{"v":[1,true]}', "the catalog reads its section back as text") + # Malformed text fails to parse, and a reserved section is moq-mux's error. + try: + moq.Json.parse('{"v":').close() + expect(False, "malformed JSON fails to parse") + except moq.StringException as e: + expect(e.error == "EOF while parsing a value at line 1 column 5", "a parse error is serde_json's") + try: + with moq.CatalogHangExtra.default() as hang: + hang.set("json", moq.Json.parse("1")) + expect(False, "a reserved section is an error") + except moq.MuxErrorException as e: + expect(str(e) == "reserved catalog section: json", "a reserved section is moq-mux's error") + + +def config(): + # moq-tokio's terminal connect errors, which moq-net's `Error` shares no names with. + expect(moq.ConnectError.UNAUTHORIZED.is_auth(), "unauthorized is an auth error") + expect(str(moq.ConnectError.UNAUTHORIZED) == "unauthorized", "a connect error prints as Rust's") + + # Session config's foreign types. A URL is parsed from text, failing with why. + try: + moq.Url.parse("relay.example").close() + expect(False, "a relative URL is an error") + except moq.StringException as e: + expect(str(e) == "relative URL without a base", "a URL's parse error is Rust's") + # A session's config, its URL and pinned fingerprints set as Rust assigns them. + connect = moq.ConnectConfig.default() + with connect.tls as tls: + tls.set_fingerprint(["ab:cd"]) + connect.set_tls(tls) + connect.set_url(moq.Url.parse("https://relay.example:4443/demo")) + with connect.tls as tls: + expect(tls.fingerprint == ["ab:cd"], "a TLS config keeps its fingerprints") + url = connect.url + connect.close() + versions = moq.Versions.default() + with moq.AuthTokens.new([]) as tokens, tokens.dial(url, versions) as dial, dial.url as dialed: + expect(dialed == url, "a dial keeps its URL") + expect(str(dialed) == "https://relay.example:4443/demo", "a URL prints as Rust's") + url.close() + # A `Vec`, as a list. + expect(len(moq.ALPNS) > 0, "a constant slice crosses as a list") + alpns = versions.alpns() + expect(len(alpns) > 0 and all(alpns), "each version has an ALPN") + versions.close() + # A socket address, crossing in a listener's bind. + bind = moq.ListenBind.Addr(("::1", 4443, 0, 0)) + expect(str(bind) == "[::1]:4443", "an IPv6 bind prints as Rust's") + v4 = moq.ListenBind.Addr(("127.0.0.1", 0)) + expect(str(v4) == "127.0.0.1:0", "an IPv4 bind prints as Rust's") + with moq.ListenConfig.default() as listen: + listen.set_bind(bind) + # The preferred address of each family, which takes only that family. + listen.set_preferred_v4(v4._0) + expect(listen.preferred_v4 == v4._0 and listen.preferred_v6 is None, "a listener keeps its preferred IPv4 address") + expect(listen.bind == bind, "a listener's config keeps its bind") + # TLS files, as OS paths, and fingerprints, as strings: none by default. + with moq.TlsConnect.default() as tls: + expect(tls.root == [] and tls.fingerprint == [], "a default TLS config lists none") + + +async def session(): + """A session, as moq.c runs it: a server bound to port 0 reads back the port it got + and serves with one call, in a task of its own, and a client dials it with one call and + reads a frame back, each await resuming once moq-tokio's fallback runtime wakes it from + its threads.""" + served = moq.origin_spawn() + live = served.create_broadcast("live") + live.announce(moq.OriginRoute.default()) + live_track = live.broadcast().create_track("video", None) + live_group = live_track.append_group() + live_group.write_frame(None, b"hello") + live_group.finish() + + server_config = moq.ServerConfig.default() + listen = server_config.listen + listen.set_bind(moq.ListenBind.Addr(("::", 0, 0, 0))) + tls = listen.tls + tls.set_generate(["localhost"]) + listen.set_tls(tls) + server_config.set_listen(listen) + server = server_config.init() + local = server.local_addr() + expect(local[1] != 0, "a server bound to port 0 reads back the port it got") + serving = asyncio.create_task(server.serve_publish(served.consume())) + + connect = moq.ConnectConfig.default() + connect.set_url(moq.Url.parse(f"moqt://localhost:{local[1]}")) + insecure = connect.tls + insecure.set_insecure(True) + connect.set_tls(insecure) + client_config = moq.ClientConfig.default() + client_config.set_connect(connect) + client = client_config.init() + received = moq.origin_spawn() + remote = received.consume() + connection = client.clone().with_reconnect(False).consume(received) + expect(connection is not None, "a client with a URL dials it") + established = await connection.established() + + broadcast = await remote.request_broadcast("live", None) + subscriber = await broadcast.track("video").subscribe(None) + group = await subscriber.recv_group() + frame = None if group is None else await group.read_frame() + expect(frame is not None and text(frame.payload) == "hello", "a session reads the frame the server serves") + + await established.close_() + await client.close_() + serving.cancel() + try: + await serving + except asyncio.CancelledError: + pass + + +loop = asyncio.new_event_loop() +local(loop) +origin(loop) +catalog(loop) +config() +# A stall fails with the timeout's exception. +loop.run_until_complete(asyncio.wait_for(session(), 30)) +loop.close() + +if failures == 0: + print("ok") +sys.exit(0 if failures == 0 else 1) diff --git a/ffi/src/python.rs b/ffi/src/python.rs index ef4f640c6..eebc54d47 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -674,19 +674,40 @@ impl<'a> Gen<'a> { } } - /// The methods of a type, each a member or a static method by its receiver, with the - /// Python names they take, checked to differ. - fn methods(&mut self, methods: &[Function], ty: &Type, owner: &Ty, body: &mut Vec, rust: &str, reserved: &[&str]) { + /// The methods of a type, each a member or a static method by its receiver, after its + /// members `body`, checked to differ from them and its data's names `data`, which a + /// method's name takes a trailing `_` to keep apart from, as from those of `reserved`. + fn methods( + &mut self, + methods: &[Function], + (ty, owner, rust): (&Type, &Ty, &str), + data: &[String], + reserved: &[&str], + body: &mut Vec, + ) { + let reserved: Vec<&str> = reserved.iter().copied().chain(data.iter().map(String::as_str)).collect(); for method in methods { let place = match method.receiver { Some(_) => Place::Member, None => Place::Static, }; - if let Some(code) = self.function(method, Some(owner), Some((ty, reserved)), place) { + if let Some(code) = self.function(method, Some(owner), Some((ty, &reserved)), place) { body.push(indent_by(&code, 1)); } } - self.distinct(rust, body); + self.distinct(rust, data, body); + } + + /// Check that the members `body` of the type `rust` and its data's names `data` differ. + fn distinct(&mut self, rust: &str, data: &[String], body: &[String]) { + let mut seen = BTreeSet::new(); + for name in data.iter().cloned().chain(body.iter().filter_map(|code| member_name(code))) { + if !seen.insert(name.clone()) { + self.errors.push(format!( + "two members of `{rust}` are both `{name}` in Python: rename one in the config" + )); + } + } } fn record(&mut self, i: usize) { @@ -701,15 +722,13 @@ impl<'a> Gen<'a> { let mut body = Vec::new(); self.traits(&ty, &c, record.traits, &mut body); let owner = Ty::Struct(c.clone()); - let mut members: Vec = record.fields.iter().map(|f| format!("\t{} =", field_name(&f.name))).collect(); - members.append(&mut body); - self.methods(&record.methods, &ty, &owner, &mut members, &rust, &[]); + let data: Vec = record.fields.iter().map(|f| field_name(&f.name)).collect(); + self.methods(&record.methods, (&ty, &owner, &rust), &data, &[], &mut body); let fields: String = record .fields .iter() .map(|f| format!("\t{}: {}\n{}", field_name(&f.name), self.py_ty(&f.ty), docstring(&f.docs, "\t"))) .collect(); - let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); let repr = match record.traits.debug { true => ", repr=False", false => "", @@ -718,7 +737,7 @@ impl<'a> Gen<'a> { self.py, "@_dataclass(slots=True{repr})\nclass {name}:\n{}{fields}{}\n\n", docstring(&format!("{rust}\n\n{}", record.docs), "\t"), - members.iter().map(|m| format!("\n{m}")).collect::() + body.iter().map(|m| format!("\n{m}")).collect::() ); // The glue. let mut reads = Vec::new(); @@ -759,41 +778,32 @@ impl<'a> Gen<'a> { self.traits(&ty, &c, e.traits, &mut body); let owner = abi.ty(&ty); if e.plain() { - let mut members: Vec = e - .variants - .iter() - .map(|v| format!("\t{} =", abi::snake(&v.name).to_uppercase())) - .collect(); - members.append(&mut body); + let data: Vec = e.variants.iter().map(|v| abi::snake(&v.name).to_uppercase()).collect(); // An `Enum`'s members have these. - self.methods(&e.methods, &ty, &owner, &mut members, &rust, &["name", "value"]); + self.methods(&e.methods, (&ty, &owner, &rust), &data, &["name", "value"], &mut body); let entries: String = e .variants .iter() .map(|v| format!("\t{} = {}\n", abi::snake(&v.name).to_uppercase(), v.discriminant)) .collect(); - let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); let _ = write!( self.py, "class {name}(_enum.Enum):\n{}{entries}{}\n\n", docstring(&format!("{rust}\n\n{}", e.docs), "\t"), - members.iter().map(|m| format!("\n{m}")).collect::() + body.iter().map(|m| format!("\n{m}")).collect::() ); return; } - let mut members: Vec = e - .variants - .iter() - .map(|v| format!("\t{} =", pascal(&abi::snake(&v.name)))) - .collect(); - members.append(&mut body); - self.methods(&e.methods, &ty, &owner, &mut members, &rust, &[]); - let members: Vec<&String> = members.iter().filter(|m| !m.ends_with(" =")).collect(); + // Each variant's class, and its fields, which on an instance hide the enum's members. + let mut data: Vec = e.variants.iter().map(|v| pascal(&abi::snake(&v.name))).collect(); + let fields: BTreeSet = e.variants.iter().flat_map(|v| &v.fields).map(|f| field_name(&f.name)).collect(); + data.extend(fields); + self.methods(&e.methods, (&ty, &owner, &rust), &data, &[], &mut body); let _ = write!( self.py, "class {name}:\n{}\t__slots__ = ()\n{}\n\n", docstring(&format!("{rust}\n\n{}", e.docs), "\t"), - members.iter().map(|m| format!("\n{m}")).collect::() + body.iter().map(|m| format!("\n{m}")).collect::() ); let repr = match e.traits.debug { true => ", repr=False", @@ -873,7 +883,7 @@ impl<'a> Gen<'a> { constant: false, }; // What `Object` gives every object. - self.methods(&object.methods, &ty, &owner, &mut body, &rust, &["close"]); + self.methods(&object.methods, (&ty, &owner, &rust), &[], &["close"], &mut body); let text = format!( "{rust}\n\n{}\n\nIts handle is freed once its last reference goes, or sooner by `close()` or a `with` block, after which a call raises.", object.docs @@ -925,7 +935,7 @@ impl<'a> Gen<'a> { ); table.push(format!("{}: Some({trampoline})", method.name())); } - self.distinct(&rust, &body); + self.distinct(&rust, &[], &body); let text = format!( "{rust}: implemented in Python. Rust calls its methods from any thread, and once it\ndrops the implementation, calls its `close()` if it has one.\n\n{}", callback.docs @@ -1015,19 +1025,6 @@ impl<'a> Gen<'a> { self.table.push(format!("rt::method(c\"{bare}\", f_{bare})")); } - /// Check that no two members of the type `rust` share a Python name. - fn distinct(&mut self, rust: &str, members: &[String]) { - let mut seen = BTreeSet::new(); - for member in members { - let Some(name) = member_name(member) else { continue }; - if !seen.insert(name.clone()) { - self.errors.push(format!( - "two members of `{rust}` are both `{name}` in Python: rename one in the config" - )); - } - } - } - /// `function`'s Python form, a member of `owner` if it has one (whose IR type and /// reserved member names are `owner_ty`), with its native function and glue. fn function( From 97907969b528a146a1f260d3d8b50de29c77c647 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:21:21 -0800 Subject: [PATCH 05/13] wip(ffi): bench the Python Co-Authored-By: Claude Opus 5.5 --- ffi/justfile | 3 +- ffi/moq/bench/export.py | 126 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 ffi/moq/bench/export.py diff --git a/ffi/justfile b/ffi/justfile index 028d9271a..40bfb6b2f 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -41,7 +41,7 @@ run *args: census checkout="": (standin checkout) cargo run --locked --quiet -- --config moq/ffi.toml --manifest-path target/moq-ffi/Cargo.toml --census target/census -# Time frame writes through the export, from C, C++, and Kotlin, and through moq's hand-written rs/moq-c, in the bench shell: nix develop .#bench. +# Time frame writes through the export, from C, C++, Kotlin, and Python, and through moq's hand-written rs/moq-c, in the bench shell: nix develop .#bench. bench checkout="": (moq checkout) #!/usr/bin/env bash set -euo pipefail @@ -62,6 +62,7 @@ bench checkout="": (moq checkout) target/bench-export target/bench-export-cpp kotlin -Djava.library.path="$out/target/release" -cp "target/bench-kt:$KOTLINX_COROUTINES" ExportKt + PYTHONPATH="$out/python" python3 -P moq/bench/export.py done # Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, moq/moq.kt, and moq/moq.py against it, from the moq revision `just ts moq` pins or a checkout. diff --git a/ffi/moq/bench/export.py b/ffi/moq/bench/export.py new file mode 100644 index 000000000..253b8a7f6 --- /dev/null +++ b/ffi/moq/bench/export.py @@ -0,0 +1,126 @@ +# bench/export.kt's frame writes and reads through the generated Python, so the two differ +# by what CPython and the Python wrapper add, with the calls bench/ffi/README.md compares +# BoltFFI's Python on. Each round runs twice, the first to warm CPython's caches. + +import asyncio +import time + +import moq + +GROUPS = 1000 +FRAMES = 100 + + +def per_frame(start): + """Nanoseconds per frame since `start`.""" + return (time.perf_counter_ns() - start) / (GROUPS * FRAMES) + + +def round(loop): + broadcast = moq.BroadcastInfo.new().produce() + with moq.Timescale.MICRO as micro: + track = broadcast.create_track("bench", moq.TrackInfo.default().with_timescale(micro)) + + # A frame of 100 zero bytes, as bench/export.c writes. + frame = bytes(100) + micros = 0 + + # Each frame stamped through a fresh timestamp object, as a caller holding an int must. + start = time.perf_counter_ns() + for _ in range(GROUPS): + with track.append_group() as group: + for _ in range(FRAMES): + with moq.Timestamp.from_micros(micros) as at: + group.write_frame(at, frame) + micros += 1 + group.finish() + stamped = per_frame(start) + + # The same, reusing one timestamp object: what the object costs. + at = moq.Timestamp.from_micros(micros) + start = time.perf_counter_ns() + for _ in range(GROUPS): + with track.append_group() as group: + for _ in range(FRAMES): + group.write_frame(at, frame) + group.finish() + reused = per_frame(start) + + # The floor: a hold of a handle and a bool back. + start = time.perf_counter_ns() + closed = 0 + for _ in range(GROUPS * FRAMES): + if track.is_closed(): + closed += 1 + call = per_frame(start) + + # A record returned: the track's info, its fields made as Python objects. + start = time.perf_counter_ns() + for _ in range(GROUPS * FRAMES): + track.info() + info = per_frame(start) + + # Reads: each frame written, then read through an awaitable ready at its first poll, + # so it never suspends. + async def ready(): + with track.consume() as consumer: + subscriber = await consumer.subscribe(None) + last = None + start = time.perf_counter_ns() + for g in range(GROUPS): + with track.append_group() as group, await subscriber.recv_group() as reader: + for f in range(FRAMES): + group.write_frame(at, frame) + read = await reader.read_frame() + if g == GROUPS - 1 and f == FRAMES - 1: + last = read + else: + read.close() + group.finish() + subscriber.close() + return per_frame(start), last + + read, last = loop.run_until_complete(ready()) + + # Woken reads: each read awaited before its frame is written, so it pends, and the + # write's wake resumes it through the loop. + async def woken(): + with track.consume() as consumer: + subscriber = await consumer.subscribe(None) + start = time.perf_counter_ns() + for _ in range(GROUPS // 10): + with track.append_group() as group, await subscriber.recv_group() as reader: + for _ in range(FRAMES): + pending = asyncio.ensure_future(reader.read_frame()) + await asyncio.sleep(0) + group.write_frame(at, frame) + (await pending).close() + group.finish() + subscriber.close() + return per_frame(start) * 10 + + wake = loop.run_until_complete(woken()) + + # Payload reads: the last frame's 100 bytes, lent without a copy, then released. + start = time.perf_counter_ns() + with last: + for _ in range(GROUPS * FRAMES): + with last.payload as payload: + closed += payload[0] + lent = per_frame(start) + at.close() + track.close() + broadcast.close() + + bang = "!" if closed != 0 else "" + return ( + f"export (Python): {stamped:.0f} ns per stamped 100 B frame, {reused:.0f} reusing the timestamp, " + f"{call:.1f} ns per handle call, {info:.0f} ns per record return, {read:.0f} ns per frame written and read, " + f"{wake:.0f} ns per frame written and read after a wake, {lent:.0f} ns per payload read{bang}" + ) + + +loop = asyncio.new_event_loop() +round(loop) +print(round(loop)) +loop.close() From a6f9846b34d6a2067647afab41f90851a23b40df Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:26:32 -0800 Subject: [PATCH 06/13] wip(ffi): check callbacks and closures, await a Rust future once Co-Authored-By: Claude Opus 5.5 --- ffi/runtime/python.rs | 40 ++++++++++++++++++------- ffi/runtime/rt.py | 5 ++-- ffi/src/python.rs | 70 +++++++++++++++++++++++-------------------- ffi/tests/api.py | 14 +++++++++ 4 files changed, 84 insertions(+), 45 deletions(-) diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 8c4962970..89da5696d 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -885,17 +885,26 @@ pub fn call_method(obj: *mut PyObject, key: &Key, args: Vec>) -> R { call_py(method.ptr(), args) } -/// A Python implementation of a trait, given to Rust: a reference Rust drops through -/// [`free_callback`], or the scope drops if the call never happens. -pub fn callback(obj: *mut PyObject, s: &mut Scope) -> *mut c_void { - if is_none(obj) { - return null_mut(); - } +/// A Python implementation of a trait, an instance of the abstract class registered at +/// `index`, given to Rust: a reference Rust drops through [`free_callback`], or the scope +/// drops if the call never happens. +pub fn callback(obj: *mut PyObject, index: usize, s: &mut Scope) -> R<*mut c_void> { + check(obj, index)?; // SAFETY: the GIL is held, and `obj` is live. let given = unsafe { Obj::borrowed(obj) }; let ptr = given.ptr(); s.given.push(given); - ptr.cast() + Ok(ptr.cast()) +} + +/// A callable Rust calls only during the call, as a closure's data: the caller's reference +/// outlives it. +pub fn closure(obj: *mut PyObject) -> R<*mut c_void> { + // SAFETY: the GIL is held, and `obj` is live. + match unsafe { PyCallable_Check(obj) } { + 0 => expected_named(obj, "a callable"), + _ => Ok(obj.cast()), + } } static CLOSE: Key = Key::new(c"close"); @@ -1382,6 +1391,9 @@ pub struct Future { free: Option, node: Option>, token: i64, + /// Whether an `await` took it, after which another raises, as awaiting a coroutine + /// twice does. + claimed: bool, /// The handles the call holds until the future ends. held: Vec, } @@ -1504,9 +1516,15 @@ unsafe extern "C" fn future_poll(obj: *mut PyObject, args: *mut *mut PyObject, n }) } -unsafe extern "C" fn future_token(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { - // SAFETY: Python calls it on a `Future`. - int_out(unsafe { (*obj.cast::()).token }).map_or(null_mut(), Obj::into_ptr) +/// `claim()`: take the future for one `await`, returning the token its wakes queue. +unsafe extern "C" fn future_claim(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { + // SAFETY: Python calls it on a `Future`, holding the GIL. + let future = unsafe { &mut *obj.cast::() }; + wrap(|| match std::mem::replace(&mut future.claimed, true) { + // SAFETY: a static exception type. + true => raise(unsafe { PyExc_RuntimeError }, "a Rust future is awaited already"), + false => int_out(future.token), + }) } unsafe extern "C" fn future_close_py(obj: *mut PyObject, _: *mut PyObject) -> *mut PyObject { @@ -1577,7 +1595,7 @@ static OBJECT_METHODS: Methods<4> = Methods(UnsafeCell::new([ static FUTURE_METHODS: Methods<4> = Methods(UnsafeCell::new([ method(c"poll", future_poll), - noargs(c"token", future_token), + noargs(c"claim", future_claim), noargs(c"close", future_close_py), END, ])); diff --git a/ffi/runtime/rt.py b/ffi/runtime/rt.py index dabb4eaae..3b66d0dd8 100644 --- a/ffi/runtime/rt.py +++ b/ffi/runtime/rt.py @@ -6,14 +6,15 @@ class _Future(_n.Future): """A Rust future its first poll left pending: awaiting it polls it on the running event loop until it finishes. Cancelling the awaiting task frees it, which cancels it in - Rust and loses nothing, as does dropping it unawaited.""" + Rust and loses nothing, as does dropping it unawaited. Like a coroutine, it is awaited + once: a second `await` raises `RuntimeError`.""" __slots__ = () def __await__(self): loop = _get_running_loop() w = _wakeup(loop) - token = self.token() + token = self.claim() try: while True: r = self.poll(w.queue) diff --git a/ffi/src/python.rs b/ffi/src/python.rs index eebc54d47..ef86c2d63 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -38,9 +38,9 @@ pub struct Output { pub glue: String, } -/// The names the generated Python uses unqualified, which no exported item may take: the -/// runtime's own, each but `Waker` starting with `_`, and `Waker` and `Bound`, which the C -/// ABI claims as `_waker` and `_bound`. +/// The runtime's names the generated Python uses unqualified, which no exported item may +/// take. Its `Waker` and `Bound` need no entry: the C ABI claims `_waker` and +/// `_bound` already. const TAKEN: &[&str] = &[ "_n", "_load", @@ -340,9 +340,9 @@ impl<'a> Gen<'a> { } /// The glue's expression, an `R`, reading the Python object `o` as the C value of - /// `ty`, with what it borrows held in the scope `s`; `m` when the position moves an - /// object's value out. - fn lift(&mut self, ty: &Type, o: &str, m: bool) -> String { + /// `ty`, with what it borrows held in the scope `s`; `m` is the glue's `bool`, whether + /// the position moves an object's value out, closing it. + fn lift(&mut self, ty: &Type, o: &str, m: &str) -> String { let abi = self.abi; match ty { Type::Bool => format!("rt::bool_in({o})"), @@ -357,11 +357,13 @@ impl<'a> Gen<'a> { Type::SocketAddr(family) => format!("rt::addr_in({o}, {})", family_of(*family)), Type::List(_) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), Type::Option(inner) => match inner.base() { - Type::Object(i) => format!( - "rt::hold_opt({o}, {}, s, {})", - self.t(&abi.object(*i)), - m && !abi.api.objects[*i].copy && !matches!(**inner, Type::Ref(..)) - ), + Type::Object(i) => { + let m = match abi.api.objects[*i].copy || matches!(**inner, Type::Ref(..)) { + true => "false", + false => m, + }; + format!("rt::hold_opt({o}, {}, s, {m})", self.t(&abi.object(*i))) + } base if Abi::is_buf(base) => { let some = self.lift(inner, o, m); format!("(if rt::is_none({o}) {{ Ok(crate::rt::Buf::NONE) }} else {{ {some} }})") @@ -373,20 +375,22 @@ impl<'a> Gen<'a> { _ => format!("r_{}({o}, s, {m})", self.c_struct(ty)), }, Type::Box(inner) => self.lift(inner, o, m), - Type::Ref(_, inner) => self.lift(inner, o, false), + Type::Ref(_, inner) => self.lift(inner, o, "false"), Type::Record(_) => format!("r_{}({o}, s)", self.c_struct(ty)), Type::Enum(i) => match abi.api.enums[*i].plain() { true => format!("rt::member_in({o}, {})", self.t(&abi.enumeration(*i))), false => format!("r_{}({o}, s)", abi.enumeration(*i)), }, - Type::Object(i) => format!( - "rt::hold({o}, {}, s, {})", - self.t(&abi.object(*i)), - m && !abi.api.objects[*i].copy - ), + Type::Object(i) => { + let m = match abi.api.objects[*i].copy { + true => "false", + false => m, + }; + format!("rt::hold({o}, {}, s, {m})", self.t(&abi.object(*i))) + } Type::Callback(i, _) => { let c = abi.callback(*i); - format!("Ok(crate::{c} {{ vtable: &VTABLE_{c}, data: rt::callback({o}, s) }})") + format!("Ok(crate::{c} {{ vtable: &VTABLE_{c}, data: rt::callback({o}, {}, s)? }})", self.t(&c)) } Type::Waker => format!("rt::waker_in({o}, s)"), Type::Range(int) => format!("rt::range_in::<{}>({o})", int.rust()), @@ -570,12 +574,12 @@ impl<'a> Gen<'a> { /// The glue of an option struct. fn option(&mut self, c: &str, inner: &Type) { - let read = self.lift(inner, "o", true); + let read = self.lift(inner, "o", "m"); let write = self.lower(inner, "v.value"); let _ = write!( self.glue, "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", - read.replace(", true)", ", m)") + read ); } @@ -586,7 +590,7 @@ impl<'a> Gen<'a> { let mut fields = Vec::new(); let mut writes = Vec::new(); for (k, ty) in elements.iter().enumerate() { - let lift = self.lift(ty, &format!("x{k}"), true).replace(", true)", ", m)"); + let lift = self.lift(ty, &format!("x{k}"), "m"); reads.push(format!("let x{k} = rt::tuple_item(o, {k}, {n})?;")); fields.push(format!("_{k}: {lift}?")); writes.push(self.lower(ty, &format!("v._{k}"))); @@ -602,7 +606,7 @@ impl<'a> Gen<'a> { /// The glue of a `Vec`'s array. fn list(&mut self, c: &str, elem: &Type) { - let read = self.lift(elem, "o", true).replace(", true)", ", m)"); + let read = self.lift(elem, "o", "m"); let write = self.lower(elem, "e"); let _ = write!( self.glue, @@ -619,8 +623,8 @@ impl<'a> Gen<'a> { let object = matches!(ty, Type::Object(_)); // The receiver as the glue reads it: a handle, held for the call, or a value. let this = match object { - true => self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[0]", false), - false => self.lift(ty, "a[0]", false), + true => self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[0]", "false"), + false => self.lift(ty, "a[0]", "false"), }; let pass = match object { true => "this", @@ -649,7 +653,7 @@ impl<'a> Gen<'a> { )); } if object && t.eq { - let other = self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[1]", false); + let other = self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[1]", "false"); self.native( &format!("{bare}_eq"), 2, @@ -750,7 +754,7 @@ impl<'a> Gen<'a> { _ => format!("rt::attr(o, {key}, s)?"), }; reads.push(format!("let x{n} = {read};")); - fields.push(format!("f{n}: {}?", self.lift(&field.ty, &format!("x{n}"), false))); + fields.push(format!("f{n}: {}?", self.lift(&field.ty, &format!("x{n}"), "false"))); writes.push(format!("({key}, {})", self.lower(&field.ty, &format!("v.f{n}")))); } if record.fields.is_empty() { @@ -844,7 +848,7 @@ impl<'a> Gen<'a> { for (k, f) in variant.fields.iter().enumerate() { let key = self.key(&field_name(&f.name)); binds.push(format!("let x{k} = rt::attr(o, {key}, s)?;")); - fields.push(format!("f{k}: {}?", self.lift(&f.ty, &format!("x{k}"), false))); + fields.push(format!("f{k}: {}?", self.lift(&f.ty, &format!("x{k}"), "false"))); lowers.push(format!("({key}, {})", self.lower(&f.ty, &format!("p.f{k}")))); } reads.push(format!( @@ -903,6 +907,7 @@ impl<'a> Gen<'a> { let name = self.class(&c); let rust = callback.path.join("::"); self.claim(&name, format!("trait `{rust}`")); + self.register(&c, name.clone()); let mut body = Vec::new(); let mut table = vec!["free: Some(rt::free_callback)".to_string()]; for (n, method) in callback.methods.iter().enumerate() { @@ -992,7 +997,7 @@ impl<'a> Gen<'a> { }; let back = match c_ret { Ty::Void => "Ok(())".to_string(), - _ => self.lift(ret, "r.ptr()", false), + _ => self.lift(ret, "r.ptr()", "false"), }; let rust_ret = rust_c(self.abi, c_ret); let _ = write!( @@ -1070,11 +1075,11 @@ impl<'a> Gen<'a> { Receiver::Ref => Type::Ref(crate::ir::Mutability::Not, Box::new(ty)), Receiver::Mut => Type::Ref(crate::ir::Mutability::Mut, Box::new(ty)), }; - stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), true))); + stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), "true"))); c_args.push(format!("p{n}")); } _ => { - stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), false))); + stmts.push(format!("let p{n} = {}?;", self.lift(&ty, &format!("a[{n}]"), "false"))); c_args.push(format!("&p{n}")); } } @@ -1091,11 +1096,12 @@ impl<'a> Gen<'a> { c_params.next().expect("a closure has its data"); let trampoline = format!("py_{c_name}_{}", abi::escape(¶m.name)); closures.push((lent.clone(), (**ret).clone(), c_lent.clone(), (**c_ret).clone(), trampoline.clone())); + stmts.push(format!("let p{n} = rt::closure(a[{n}])?;")); c_args.push(format!("Some({trampoline})")); - c_args.push(format!("a[{n}].cast()")); + c_args.push(format!("p{n}")); } _ => { - stmts.push(format!("let p{n} = {}?;", self.lift(¶m.ty, &format!("a[{n}]"), true))); + stmts.push(format!("let p{n} = {}?;", self.lift(¶m.ty, &format!("a[{n}]"), "true"))); c_args.push(format!("p{n}")); } } diff --git a/ffi/tests/api.py b/ffi/tests/api.py index cb4f71e0f..c2a887e75 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -540,6 +540,20 @@ async def held(): check(got == "held", "a closed object stays held by the call awaiting on it") sender.close() + # An awaitable from Rust is awaited once, as a coroutine is. + later = channel.send_later(b"once") + + async def wait(awaitable): + return await awaitable + + loop.run_until_complete(asyncio.wait_for(wait(later), 5)) + try: + loop.run_until_complete(wait(later)) + check(False, "a second await raises") + except RuntimeError: + pass + check(text(loop.run_until_complete(asyncio.wait_for(channel.recv(), 5))) == "once", "the send landed once") + # A waker woken from another thread calls back on its loop, once for wakes that land # before the loop gets to them. wakes = [] From 0622f96a51ada749933f83ce21b339c6a34b2233 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:28:35 -0800 Subject: [PATCH 07/13] wip(ffi): document the Python, and check the values its types bound Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 82 ++++++++++++++++++++++++++++++++----- ffi/fixtures/api/src/lib.rs | 5 +++ ffi/runtime/python.rs | 13 ++++-- ffi/tests/api.py | 42 ++++++++++++++----- ffi/tests/fixtures.rs | 4 +- 5 files changed, 121 insertions(+), 25 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 22996c12d..89e30bf27 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -27,7 +27,8 @@ A `[replace]` from the source it replaces, which no `[patch]` can carry, is an e Beside the C header, `include/.hpp` holds the same API in C++, and `kotlin/.kt` holds it in Kotlin, over JNI functions the crate builds with `--features kotlin`, and `kotlin/.pro` the R8 and ProGuard rules keeping what -those functions look up by name. +those functions look up by name. `python/.py` holds it in Python, over a CPython +extension the crate builds with `--features python`. ```toml # ffi.toml, beside the package's Cargo.toml unless --manifest-path says otherwise. @@ -272,6 +273,58 @@ in as a `ByteArray`, bytes returned as their owner, and anything else encoded in Android library ships it as a consumer rule (`consumerProguardFiles`), which its AAR carries to each app that shrinks it. +## Python + +`.py` is Python over a CPython extension that `--features python` builds into the +same library on the stable ABI, so one build loads in CPython 3.11 and later. The module +loads the library from beside itself: `lib.so`, `lib.dylib`, or +`.dll`. Its glue converts each Python value to the C value and calls the C ABI +function in-process, so a call is one crossing, which releases the GIL while Rust runs. + +- **Names** are in the module: types in PascalCase without the prefix, members in + snake_case without their type's name (`moq.TrackProducer.append_group`). A keyword, a + member Python gives every such class, or a method named as a field takes a trailing `_`. + Constructors such as `new` stay static methods: `Counter.new(1)`. +- **Objects** are classes owning their handle, freed once their last reference goes, or + sooner by `close()` or a `with` block once no call holds it; a call after it raises + `ValueError`. A `self` method, or passing the object by value, closes it once the call + returns. `Clone` is `clone()`, and `PartialEq` is `==`, unhashable. A field is a + property, and its setter a method, `set_first(value)`, since setting an object moves it + in and closes it, which an assignment would hide. +- **Records** are slotted dataclasses, compared by their fields, and **enums** an `Enum` + of upper-case members, or for one with data a class of a dataclass per variant, + `Event.Moved(p)`. `Display` is `str()`, `Debug` `repr()`, and `Default` a static + `default()`. Constants are module or class attributes, made anew on each read unless a + literal. +- **Bytes** passed in are any buffer, such as `bytes`, read in place. Bytes Rust returns, + or a record holds, are a read-only `memoryview` lending Rust's buffer until it and its + views are released. Those Rust lends a closure or trait method are copied into `bytes`. + Strings are `str`, `Option` `T | None`, a tuple a `tuple`, `Duration` + `datetime.timedelta`, truncated to the microsecond, with `timedelta.max` for any longer and +for `Duration::MAX`, a `Range` a `range`, and an `Err` a raised + `Exception` holding it in `error`, whose `str` is its `Display`. +- A **path** is a `str`, or any path passed in, which crosses as `os.fsencode` encodes it, so + one that is not UTF-8 keeps its bytes. A `SocketAddr` is the `socket` module's tuple, + `(host, port)` or `(host, port, flowinfo, scope_id)`, of that family for `V4` or `V6`. + A `Vec` or `&[T]` is a `list`, and passing a `Vec` by value closes each object in it. + An integer out of its Rust type's range raises `OverflowError`. +- **Traits** are abstract classes an implementation subclasses; once Rust drops one, its + `close()` is called if it has one. +- **Closures** lent for a call are any callable, which Rust calls only during the call, + from any thread. An object it lends is closed once the call returns. An exception a + closure or trait method raises prints its traceback and aborts, since Rust has nowhere + to unwind it to. +- **Futures** are `async def`s, which run nothing until awaited, or for a function that is + not `async`, an awaitable the call returns. They resume on the running event loop and + never inside a wake: a wake from any thread queues the future's token and writes a byte + to a socket the loop reads, without the GIL. Cancelling the task frees the future and + loses nothing. The call polls it once before suspending, and a future is awaited once, + as a coroutine is. +- A **waker** is a `Waker(callback)`, which Rust may wake from any thread, calling + `callback` on its event loop. +- **The contract** is checked when the module imports, before any other call: a Python + file and a library from different builds raise `ImportError`. + ## IR [`src/ir.rs`](src/ir.rs) is the API as each generator reads it, with no language decided: @@ -280,10 +333,13 @@ its Rust path, its span, and its docs. A type is in the exact form Rust takes or (`&str` apart from `String`, `Box` apart from `impl Trait`), so a generator can borrow where Rust borrows. [`src/abi.rs`](src/abi.rs) decides the C ABI from it once, names and signatures, so [`src/c.rs`](src/c.rs), [`src/cpp.rs`](src/cpp.rs), -[`src/kotlin.rs`](src/kotlin.rs), and [`src/glue.rs`](src/glue.rs) agree; the runtime the -glue includes is [`runtime/rt.rs`](runtime/rt.rs), the one the C++ header includes -[`runtime/rt.hpp`](runtime/rt.hpp), and the ones Kotlin and its JNI glue include -[`runtime/rt.kt`](runtime/rt.kt) and [`runtime/jni.rs`](runtime/jni.rs). +[`src/kotlin.rs`](src/kotlin.rs), [`src/python.rs`](src/python.rs), and +[`src/glue.rs`](src/glue.rs) agree; the runtime the glue includes is +[`runtime/rt.rs`](runtime/rt.rs), the one the C++ header includes +[`runtime/rt.hpp`](runtime/rt.hpp), the ones Kotlin and its JNI glue include +[`runtime/rt.kt`](runtime/rt.kt) and [`runtime/jni.rs`](runtime/jni.rs), and the ones +Python and its extension include [`runtime/rt.py`](runtime/rt.py) and +[`runtime/python.rs`](runtime/python.rs). ## Census @@ -296,12 +352,12 @@ refusal, which is a bug in the exporter's inference rather than in the crate. ## Tests ```bash -just ffi test # export the fixtures, build them, and drive api from C, C++, and Kotlin -just ffi moq # export moq's crates at the pinned revision, build them, and run moq/moq.c, moq/moq.cpp, and moq/moq.kt +just ffi test # export the fixtures, build them, and drive api from C, C++, Kotlin, and Python +just ffi moq # export moq's crates at the pinned revision, build them, and run moq/moq.c, moq/moq.cpp, moq/moq.kt, and moq/moq.py just ffi moq-cpp # the same export, running only moq/moq.cpp just ffi census # census each into target/census nix develop .#android --command just ffi android # the Kotlin test, shrunk by R8, on the Android device adb reaches -nix develop .#bench --command just ffi bench # frame writes and reads, through the export from C, C++, and Kotlin, and moq's rs/moq-c +nix develop .#bench --command just ffi bench # frame writes and reads, through the export from C, C++, Kotlin, and Python, and moq's rs/moq-c ``` The `api` fixture covers each rule once, and names the `dep` fixture's types, which @@ -316,7 +372,11 @@ each runtime check, which must throw or abort with its message. `just ffi androi same on ART: the Kotlin and `tests/api.kt`, shrunk by R8 into an app's dex with only the generated rules keeping what JNI looks up, against the library built by the NDK for the device's ABI, which ANDROID_HOME and ANDROID_NDK_HOME locate. Nightly runs it on an -x86_64 emulator at API 24, the oldest moq's Android library supports, and at API 35. The `reject` +x86_64 emulator at API 24, the oldest moq's Android library supports, and at API 35. +`tests/api.py` runs the script through the Python, in CPython's development mode with +warnings as errors, checks its futures the same way, what frees an object or bytes nobody +closed, and the values Python's types bound, and breaks each runtime check, which must raise or +abort with its message; the Python with another contract must refuse the library. The `reject` fixture's expected errors, each with its notes, are `fixtures/reject/expected.txt`. moq's crates export through `target/moq-ffi`, an empty library on each that `just ffi moq` writes in place of moq-ffi's remainder. `moq/moq.c` publishes and subscribes to a moq-net @@ -327,7 +387,9 @@ serves a broadcast from a moq-tokio server bound to port 0 to a client that read QUIC, both on moq-tokio's fallback runtime, whose threads wake C's. `moq/moq.cpp` does the same through the C++ header, short of the catalog's field, at C++17 without exceptions or RTTI, as OBS builds it, serving on an executor's thread and blocking on each read, and -`moq/moq.kt` through the Kotlin, serving in a coroutine and suspending on each read. +`moq/moq.kt` through the Kotlin, serving in a coroutine and suspending on each read, and +`moq/moq.py` through the Python, running the origin's driver and the server as asyncio +tasks and awaiting each read. The C++ builds take `CXX`, or `c++`, with every warning an error and `-Wshadow`, which stands in for MSVC's warnings that a name hides another. On Windows they take MSVC's `cl` diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index e68769f70..01036aef2 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -1292,6 +1292,11 @@ pub fn ports(addrs: Vec) -> Vec { addrs.iter().map(std::net::SocketAddr::port).collect() } +/// `ttl` doubled, saturating, so `Duration::MAX` stays forever. +pub fn doubled_ttl(ttl: Duration) -> Duration { + ttl.saturating_mul(2) +} + /// `name` in `dir`. pub fn file(dir: &std::path::Path, name: &str) -> std::path::PathBuf { dir.join(name) diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 89da5696d..53483d985 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -548,7 +548,11 @@ static SECONDS: Key = Key::new(c"seconds"); static MICROSECONDS: Key = Key::new(c"microseconds"); static MAX: Key = Key::new(c"max"); -/// A `Duration`, from a `datetime.timedelta` that is not negative. +/// The days in `timedelta.max`, past which [`duration_out`] gives it. +const MAX_DAYS: u64 = 999_999_999; + +/// A `Duration`, from a `datetime.timedelta` that is not negative: `timedelta.max`, which +/// [`duration_out`] gives for any longer one, is `Duration::MAX`. pub fn duration_in(obj: *mut PyObject) -> R { if !is(obj, T_TIMEDELTA) { return expected(obj, T_TIMEDELTA); @@ -560,6 +564,9 @@ pub fn duration_in(obj: *mut PyObject) -> R { }; let seconds: u64 = int_in(attr(obj, &SECONDS, &mut s)?)?; let micros: u32 = int_in(attr(obj, &MICROSECONDS, &mut s)?)?; + if (days, seconds, micros) == (MAX_DAYS, 86_399, 999_999) { + return Ok(rt::Duration::lower(std::time::Duration::MAX)); + } Ok(rt::Duration { secs: days * 86_400 + seconds, nanos: micros * 1_000, @@ -568,12 +575,12 @@ pub fn duration_in(obj: *mut PyObject) -> R { /// A `datetime.timedelta`, which holds whole microseconds up to 999,999,999 days: Rust's /// nanoseconds are truncated, and a longer one is `timedelta.max`, as Rust's -/// `Duration::MAX` stands for forever. +/// `Duration::MAX` stands for forever, which [`duration_in`] gives back. pub fn duration_out(duration: rt::Duration) -> R { let class = ty(T_TIMEDELTA); // SAFETY: the GIL is held. unsafe { - if duration.secs / 86_400 > 999_999_999 { + if duration.secs / 86_400 > MAX_DAYS { return Obj::new(PyObject_GetAttr(class, MAX.get()?)); } let args = tuple(vec![ diff --git a/ffi/tests/api.py b/ffi/tests/api.py index c2a887e75..5b8be7b75 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -2,9 +2,10 @@ # natively, through the generated Python. tests/fixtures.rs compares the transcripts. # # With an argument, it instead checks the futures' semantics, what frees what nobody -# closed, or a path's bytes, or breaks a contract, which must fail. +# closed, or the values only Python's types bound, or breaks a contract, which must fail. import asyncio +import datetime import gc import os import sys @@ -611,13 +612,34 @@ def unclosed(): api.token_bytes(4, 1025) -def stray(): - """A Unix path whose bytes are not UTF-8 crosses as `os.fsdecode` decodes it, and back - as it came.""" - path = api.stray() - check(os.fsencode(path) == b"\xff", "a path that is not UTF-8 keeps its bytes") - check(api.parents([path, os.fsdecode(b"\xfe/\xff")])[1] == os.fsdecode(b"\xfe"), "and crosses back") - print("stray ok") +def values(): + """The values Python's types bound: integers checked against their Rust type's range, + durations truncated to the microsecond up to `timedelta.max`, which stands for longer, + and on Unix, a path whose bytes are not UTF-8, which crosses as `os.fsdecode` decodes it, + and back as it came.""" + + def raises(error, f, what): + try: + f() + check(False, what) + except error: + pass + + raises(OverflowError, lambda: api.widths(2**64), "an int past u64 raises") + raises(OverflowError, lambda: api.widths(-1), "a negative int for a u64 raises") + raises(TypeError, lambda: api.title(1), "an int is no bool") + + td = datetime.timedelta + check(api.doubled_ttl(td(seconds=1, microseconds=5)) == td(seconds=2, microseconds=10), "a duration crosses") + check(api.doubled_ttl(td(days=600_000_000)) == td.max, "a duration past timedelta's is its max") + check(api.doubled_ttl(td.max) == td.max, "timedelta's max is Duration::MAX") + raises(ValueError, lambda: api.doubled_ttl(td(seconds=-1)), "a negative duration raises") + + if os.name == "posix": + path = api.stray() + check(os.fsencode(path) == b"\xff", "a path that is not UTF-8 keeps its bytes") + check(api.parents([path, os.fsdecode(b"\xfe/\xff")])[1] == os.fsdecode(b"\xfe"), "and crosses back") + print("values ok") def broken(which): @@ -658,7 +680,7 @@ def log(self, level, line): futures() case ["owned"]: owned() - case ["stray"]: - stray() + case ["values"]: + values() case [which]: broken(which) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 1a38765f1..b28872aaf 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -493,7 +493,7 @@ const PYTHON_LIB: &str = match (cfg!(windows), cfg!(target_os = "macos")) { }; /// `tests/api.py`'s script prints the native transcript; then its futures, what frees -/// what nobody closed, a path that is not UTF-8, and each contract it fails loud on. +/// what nobody closed, the values Python's types bound, and each contract it fails loud on. #[test] fn api_python() { let api = api(); @@ -514,7 +514,7 @@ fn api_python() { api.transcript, "the Python transcript differs from the native one" ); - for case in ["futures", "owned"].into_iter().chain(cfg!(unix).then_some("stray")) { + for case in ["futures", "owned", "values"] { run(&mut python(&dir, &[case])); } for (case, message) in [ From dfe8a117752d93a18b7827bf8e4143faeaff0e21 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:42:26 -0800 Subject: [PATCH 08/13] wip(ffi): link the Python extension on macOS through pyo3-build-config Co-Authored-By: Claude Opus 5.5 --- ffi/src/glue.rs | 14 ++++++- ffi/src/python.rs | 95 ++++++++++++++++++++++++++++++++++--------- ffi/tests/fixtures.rs | 5 ++- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index b3e3c1448..cd9cc9f3e 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -15,6 +15,14 @@ use crate::ir::{Access, Family, Function, Mutability, Receiver, Traits, Type}; const RUNTIME: &str = include_str!("../runtime/rt.rs"); +/// The crate's build script: on macOS, a CPython extension leaves Python's symbols for the +/// interpreter that loads it to resolve. +const BUILD: &str = "// Generated by rust2-ffi. Do not edit.\n\n\ + fn main() {\n\ + \t#[cfg(feature = \"python\")]\n\ + \tpyo3_build_config::add_extension_module_link_args();\n\ + }\n"; + /// The name the crate depends on an export by, its root suffixed: an export may be /// named `bytes` or `std`, which the crate names too. fn dependency(krate: &str) -> String { @@ -89,11 +97,14 @@ impl Crate<'_> { jni-sys = {{ version = \"0.4\", optional = true }}\n\ pyo3-ffi = {{ version = \"0.30\", optional = true, features = [\"abi3-py311\", \"extension-module\"] }}\n\ \n\ + [build-dependencies]\n\ + pyo3-build-config = {{ version = \"0.30\", optional = true }}\n\ + \n\ [features]\n\ # The JNI functions kotlin/{lib}.kt calls.\n\ kotlin = [\"dep:jni-sys\"]\n\ # The CPython extension python/{lib}.py loads.\n\ - python = [\"dep:pyo3-ffi\"]\n\ + python = [\"dep:pyo3-ffi\", \"dep:pyo3-build-config\"]\n\ \n\ [workspace]\n\ {patch}", @@ -116,6 +127,7 @@ impl Crate<'_> { generator.lib(); crate::driver::write(&out.join("Cargo.toml"), &manifest)?; crate::driver::write(&out.join("src/lib.rs"), &generator.out)?; + crate::driver::write(&out.join("build.rs"), BUILD)?; crate::driver::write(&out.join("src/rt.rs"), RUNTIME)?; crate::driver::write(&out.join("include").join(format!("{}.h", self.abi.prefix)), &header)?; crate::driver::write(&out.join("include").join(format!("{}.hpp", self.abi.prefix)), &cpp)?; diff --git a/ffi/src/python.rs b/ffi/src/python.rs index ef86c2d63..5738eadf9 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -99,7 +99,11 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result Gen<'a> { } Type::Callback(i, _) => { let c = abi.callback(*i); - format!("Ok(crate::{c} {{ vtable: &VTABLE_{c}, data: rt::callback({o}, {}, s)? }})", self.t(&c)) + format!( + "Ok(crate::{c} {{ vtable: &VTABLE_{c}, data: rt::callback({o}, {}, s)? }})", + self.t(&c) + ) } Type::Waker => format!("rt::waker_in({o}, s)"), Type::Range(int) => format!("rt::range_in::<{}>({o})", int.rust()), @@ -623,7 +630,11 @@ impl<'a> Gen<'a> { let object = matches!(ty, Type::Object(_)); // The receiver as the glue reads it: a handle, held for the call, or a value. let this = match object { - true => self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[0]", "false"), + true => self.lift( + &Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), + "a[0]", + "false", + ), false => self.lift(ty, "a[0]", "false"), }; let pass = match object { @@ -634,7 +645,9 @@ impl<'a> Gen<'a> { g.native( &format!("{bare}_{suffix}"), 1, - &format!("let this = {this}?;\n\t\tlet ret = rt::call(|| crate::{c}_{suffix}({pass}));\n\t\ts.done();\n\t\t{lower}"), + &format!( + "let this = {this}?;\n\t\tlet ret = rt::call(|| crate::{c}_{suffix}({pass}));\n\t\ts.done();\n\t\t{lower}" + ), ); }; if t.display { @@ -653,7 +666,11 @@ impl<'a> Gen<'a> { )); } if object && t.eq { - let other = self.lift(&Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), "a[1]", "false"); + let other = self.lift( + &Type::Ref(crate::ir::Mutability::Not, Box::new(ty.clone())), + "a[1]", + "false", + ); self.native( &format!("{bare}_eq"), 2, @@ -689,7 +706,11 @@ impl<'a> Gen<'a> { reserved: &[&str], body: &mut Vec, ) { - let reserved: Vec<&str> = reserved.iter().copied().chain(data.iter().map(String::as_str)).collect(); + let reserved: Vec<&str> = reserved + .iter() + .copied() + .chain(data.iter().map(String::as_str)) + .collect(); for method in methods { let place = match method.receiver { Some(_) => Place::Member, @@ -705,7 +726,11 @@ impl<'a> Gen<'a> { /// Check that the members `body` of the type `rust` and its data's names `data` differ. fn distinct(&mut self, rust: &str, data: &[String], body: &[String]) { let mut seen = BTreeSet::new(); - for name in data.iter().cloned().chain(body.iter().filter_map(|code| member_name(code))) { + for name in data + .iter() + .cloned() + .chain(body.iter().filter_map(|code| member_name(code))) + { if !seen.insert(name.clone()) { self.errors.push(format!( "two members of `{rust}` are both `{name}` in Python: rename one in the config" @@ -731,7 +756,14 @@ impl<'a> Gen<'a> { let fields: String = record .fields .iter() - .map(|f| format!("\t{}: {}\n{}", field_name(&f.name), self.py_ty(&f.ty), docstring(&f.docs, "\t"))) + .map(|f| { + format!( + "\t{}: {}\n{}", + field_name(&f.name), + self.py_ty(&f.ty), + docstring(&f.docs, "\t") + ) + }) .collect(); let repr = match record.traits.debug { true => ", repr=False", @@ -800,7 +832,12 @@ impl<'a> Gen<'a> { } // Each variant's class, and its fields, which on an instance hide the enum's members. let mut data: Vec = e.variants.iter().map(|v| pascal(&abi::snake(&v.name))).collect(); - let fields: BTreeSet = e.variants.iter().flat_map(|v| &v.fields).map(|f| field_name(&f.name)).collect(); + let fields: BTreeSet = e + .variants + .iter() + .flat_map(|v| &v.fields) + .map(|f| field_name(&f.name)) + .collect(); data.extend(fields); self.methods(&e.methods, (&ty, &owner, &rust), &data, &[], &mut body); let _ = write!( @@ -1095,13 +1132,22 @@ impl<'a> Gen<'a> { (Type::Closure(lent, ret), Ty::Fn(c_lent, c_ret)) => { c_params.next().expect("a closure has its data"); let trampoline = format!("py_{c_name}_{}", abi::escape(¶m.name)); - closures.push((lent.clone(), (**ret).clone(), c_lent.clone(), (**c_ret).clone(), trampoline.clone())); + closures.push(( + lent.clone(), + (**ret).clone(), + c_lent.clone(), + (**c_ret).clone(), + trampoline.clone(), + )); stmts.push(format!("let p{n} = rt::closure(a[{n}])?;")); c_args.push(format!("Some({trampoline})")); c_args.push(format!("p{n}")); } _ => { - stmts.push(format!("let p{n} = {}?;", self.lift(¶m.ty, &format!("a[{n}]"), "true"))); + stmts.push(format!( + "let p{n} = {}?;", + self.lift(¶m.ty, &format!("a[{n}]"), "true") + )); c_args.push(format!("p{n}")); } } @@ -1167,7 +1213,9 @@ impl<'a> Gen<'a> { for param in &function.params { if let Type::Closure(lent, _) = ¶m.ty { let name = param_name(¶m.name); - text.push_str(&format!("\n\nRust calls `{name}` only during the call, from any thread.")); + text.push_str(&format!( + "\n\nRust calls `{name}` only during the call, from any thread." + )); if lent.iter().any(|p| matches!(p.base(), Type::Object(_))) { text.push_str(&format!( "\nAn object it lends lives only for that call of `{name}`: keep none past it." @@ -1181,9 +1229,7 @@ impl<'a> Gen<'a> { (true, false) => format!( "async def {member}({params}) -> {ret}:\n{docs}\tr = {native}\n\treturn (await r) if r.__class__ is _Future else r" ), - (true, true) => format!( - "def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\treturn {native}" - ), + (true, true) => format!("def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\treturn {native}"), (false, _) => format!("def {member}({params}) -> {ret}:\n{docs}\treturn {native}"), }; let code = match (function.access, place) { @@ -1194,7 +1240,11 @@ impl<'a> Gen<'a> { return None; } (Access::Const, Place::Static) if function.params.is_empty() => { - format!("{} = _Const(_n.{bare})\n{}", member.to_uppercase(), docstring(text.trim(), "")) + format!( + "{} = _Const(_n.{bare})\n{}", + member.to_uppercase(), + docstring(text.trim(), "") + ) } (Access::Field { set: false, .. }, Place::Member) => format!("@_property\n{code}"), (_, Place::Static) => format!("@_staticmethod\n{code}"), @@ -1210,7 +1260,14 @@ impl<'a> Gen<'a> { /// `exception`: their declarations, the pointers the C call takes after its arguments, /// and the glue's expression of the value, an `R`, where `ok` says whether it /// returned ok. - fn outs(&mut self, outs: &[(String, Ty)], ok: &Type, err: Option<&Type>, exception: Option<&str>, is_ok: &str) -> (String, Vec, String) { + fn outs( + &mut self, + outs: &[(String, Ty)], + ok: &Type, + err: Option<&Type>, + exception: Option<&str>, + is_ok: &str, + ) -> (String, Vec, String) { let mut decl = Vec::new(); let mut ptrs = Vec::new(); let types: Vec<&Type> = match ok { @@ -1235,9 +1292,7 @@ impl<'a> Gen<'a> { _ => values.pop().expect("an out value"), }; let value = match exception { - Some(exception) => format!( - "(if {is_ok} {{ {value} }} else {{ rt::raise_error({exception}, {error}) }})" - ), + Some(exception) => format!("(if {is_ok} {{ {value} }} else {{ rt::raise_error({exception}, {error}) }})"), None => value, }; (decl.join("\n\t\t"), ptrs, value) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index b28872aaf..11781b0f4 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -541,7 +541,10 @@ fn api_python() { std::fs::create_dir_all(&mismatched).unwrap(); std::fs::copy(dir.join(PYTHON_LIB), mismatched.join(PYTHON_LIB)).unwrap(); let source = std::fs::read_to_string(dir.join("api.py")).unwrap(); - let at = source.find("_n.contract() != 0x").expect("the Python checks its contract") + 19; + let at = source + .find("_n.contract() != 0x") + .expect("the Python checks its contract") + + 19; let flipped = match &source[at..at + 1] { "0" => "1", _ => "0", From e251d8503f6091e3e30502e3afc106ca7371690e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:42:57 -0800 Subject: [PATCH 09/13] chore(quest): finish quest/m1/ffi-export/python.md Co-Authored-By: Claude Opus 5.5 --- quest/m1/ffi-export/README.md | 1 - quest/m1/ffi-export/python.md | 26 -------------------------- 2 files changed, 27 deletions(-) delete mode 100644 quest/m1/ffi-export/python.md diff --git a/quest/m1/ffi-export/README.md b/quest/m1/ffi-export/README.md index 89163a335..84bfc4eae 100644 --- a/quest/m1/ffi-export/README.md +++ b/quest/m1/ffi-export/README.md @@ -58,6 +58,5 @@ than `ctypes`; the others call C natively. moq's packages switch to them in moq' - [C++ bindings on MSVC](/quest/m1/ffi-export/cpp.md) - the nightly's MSVC job passes the C++ checks - [Waiting: moq-tokio within the recursion limit](/quest/m1/ffi-export/recursion-limit.md) - the exporter's moq pin moves past moq-dev/moq#7032 once moq-tokio's `Send` check fits the limit - [Swift bindings](/quest/m1/ffi-export/swift.md) - Swift over the C ABI as a clang module, tested on Linux and Apple -- [Python bindings](/quest/m1/ffi-export/python.md) - a generated CPython extension, skipping the C ABI as Kotlin's JNI does - [Go bindings](/quest/m1/ffi-export/go.md) - Go over the C ABI through cgo - [Dart bindings](/quest/m1/ffi-export/dart.md) - Dart over the C ABI through `dart:ffi` diff --git a/quest/m1/ffi-export/python.md b/quest/m1/ffi-export/python.md deleted file mode 100644 index 10395cab8..000000000 --- a/quest/m1/ffi-export/python.md +++ /dev/null @@ -1,26 +0,0 @@ -# [L] Python bindings - -## Goal - -The exporter generates a CPython extension for the crate, built into the same library, meeting -the [epic's](/quest/m1/ffi-export/README.md) rules for every generator, native futures -included. `tests/api.py` runs the shared script, and `moq/moq.py` runs a moq session. - -## Plan - -Decided with the maintainer on 2026-11-01: - -- Python skips the C ABI, as Kotlin does through JNI: generated Rust glue on the CPython API, - built with `--features python`. moq's - [bench](/bench/ffi/README.md#boltffi-comparison) measured `noop()` at 834 ns through UniFFI's - `ctypes` and 21.6 ns through BoltFFI's CPython extension. Not `ctypes` or `cffi`. -- The glue uses [pyo3-ffi](https://crates.io/crates/pyo3-ffi) and the stable ABI (`abi3`), one - build for every supported CPython, unless `just ffi bench` measures the version-specific API - faster by enough to pay for a build per version. -- The generated API is the crate's `pub` API in Python's idiom, as Kotlin's is, not UniFFI's. - moq's `py/` packages adapt in moq's [switch](https://github.com/moq-dev/moq/blob/main/quest/m2/uniffi-fork.md). -- An object frees its Rust value when its last reference goes, and an explicit close or - context manager frees it sooner. -- Futures are `asyncio` awaitables. - -Public API: a new generator. From 514b883afd454ae02c5dcf34f7d45479b4adbad4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:43:14 -0800 Subject: [PATCH 10/13] docs(ffi): note the Python's new name clashes in MIGRATE.md Co-Authored-By: Claude Opus 5.5 --- ffi/MIGRATE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ffi/MIGRATE.md b/ffi/MIGRATE.md index ab8ae0f12..c65f0df61 100644 --- a/ffi/MIGRATE.md +++ b/ffi/MIGRATE.md @@ -2,6 +2,11 @@ ## Unreleased +- Every export writes `python/.py`, so an export fails when two items take the same + Python name, such as a type `Foo` beside an error type's `FooException`, or when one + takes a name the Python's runtime uses unqualified, each starting with `_`: give it + another name with a `rename`. The crate it writes has a build script, and a `python` + feature that builds the CPython extension the Python loads. - Kotlin returns bytes as a `Bytes`, not a `ByteArray`: Rust's buffer as `bytes.buffer`, a read-only `ByteBuffer`, copied up to 1 KiB and otherwise a direct one lent uncopied until `bytes.close()` frees it. Read it in `use { }`, or copy it out with From 24e4ddf875aaa8719356c90583b2c8134a679769 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:44:01 -0800 Subject: [PATCH 11/13] ci(ffi): run the Python test nightly on Windows and macOS Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 10 +++++++++- ffi/tests/fixtures.rs | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8ff80b40f..f2e96123f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -181,7 +181,8 @@ jobs: - name: Clang run: nix develop .#clang --command bash -c 'export CC=gcc; just ffi test cpp && just ffi moq-cpp' - # The C++ header with the compilers the job above lacks, through the same checks. + # The C++ header with the compilers the job above lacks, through the same checks, and the + # Python on the platforms the job above lacks. ffi-cpp: name: Export moq-net to C++ (${{ matrix.name }}) runs-on: ${{ matrix.os }} @@ -231,6 +232,13 @@ jobs: - name: Test run: just ffi test cpp + # The Python, on the runner's CPython, which Windows names `python`: its proactor + # loop reads wakes through a task rather than a reader callback. + - name: Python + run: just ffi test python + env: + PYTHON: ${{ runner.os == 'Windows' && 'python' || 'python3' }} + - name: Export run: just ffi moq-cpp diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 11781b0f4..3b8af52de 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -492,13 +492,14 @@ const PYTHON_LIB: &str = match (cfg!(windows), cfg!(target_os = "macos")) { _ => "libapi.so", }; -/// `tests/api.py`'s script prints the native transcript; then its futures, what frees -/// what nobody closed, the values Python's types bound, and each contract it fails loud on. +/// `tests/api.py`'s script, run by `PYTHON` or `python3`, prints the native transcript; then +/// its futures, what frees what nobody closed, the values Python's types bound, and each +/// contract it fails loud on. #[test] fn api_python() { let api = api(); let python = |dir: &Path, args: &[&str]| { - let mut cmd = Command::new("python3"); + let mut cmd = Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())); // `-P` keeps `tests/` off the path, where `api.py` is this script; dev mode and // warnings as errors fail on what the runtime only warns of. cmd.args(["-P", "-X", "dev", "-W", "error"]) From b17d4532af589c2417032236862c186dd7ac0d32 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:54:01 -0800 Subject: [PATCH 12/13] fix(ffi): copy mutable buffers, defer a close during a poll, and return borrowed tuples whole in Python Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 3 +- ffi/fixtures/api/src/lib.rs | 14 ++++ ffi/runtime/python.rs | 155 +++++++++++++++++++++--------------- ffi/src/python.rs | 6 +- ffi/tests/api.py | 7 +- 5 files changed, 118 insertions(+), 67 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 89e30bf27..9ec1bf8a2 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -296,7 +296,8 @@ function in-process, so a call is one crossing, which releases the GIL while Rus `Event.Moved(p)`. `Display` is `str()`, `Debug` `repr()`, and `Default` a static `default()`. Constants are module or class attributes, made anew on each read unless a literal. -- **Bytes** passed in are any buffer, such as `bytes`, read in place. Bytes Rust returns, +- **Bytes** passed in are any buffer: `bytes` is read in place, and others, which Python + may change while Rust reads, are copied. Bytes Rust returns, or a record holds, are a read-only `memoryview` lending Rust's buffer until it and its views are released. Those Rust lends a closure or trait method are copied into `bytes`. Strings are `str`, `Option` `T | None`, a tuple a `tuple`, `Duration` diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index 01036aef2..1ec432187 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -179,6 +179,20 @@ pub fn divide(a: u64, b: u64) -> Result<(u64, u64), Error> { } } +/// The unit corner, borrowed for good, failing unless `unit`: a borrowed tuple returns as +/// one value, not an out per element. +pub fn corner(unit: bool) -> Result<&'static (u32, u32), Error> { + match unit { + true => Ok(&(1, 1)), + false => Err(Error::Overflow), + } +} + +/// [`corner`]'s, from a future. +pub async fn corner_later() -> &'static (u32, u32) { + &(1, 1) +} + /// `n` narrowed as [`widths`] does, unless 0: a tuple inside an `Option` is a struct. pub fn spread(n: u64) -> Option<(u8, i16, u32, u64)> { (n != 0).then(|| widths(n)) diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 53483d985..e218ad7a7 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -191,13 +191,12 @@ impl Key { } } -/// What a call borrows from Python until it returns: the objects whose memory it reads, -/// the buffers it views, the handles it holds and those it moves out of, and what it +/// What a call borrows from Python until it returns: the objects and copies whose memory +/// it reads, the handles it holds and those it moves out of, and what it /// gives Rust, taken back if the call never happens. #[derive(Default)] pub struct Scope { keep: Vec, - views: Vec>, arrays: Vec>, held: Vec, moved: Vec<*mut Handle>, @@ -246,10 +245,6 @@ impl Drop for Scope { // SAFETY: an object [`lent`] made. unsafe { close(lent.ptr().cast()) }; } - for view in &mut self.views { - // SAFETY: a view `PyObject_GetBuffer` filled, released once. - unsafe { PyBuffer_Release(&mut **view) }; - } for held in self.held.drain(..) { // SAFETY: a handle this scope holds. unsafe { release(held.ptr().cast()) }; @@ -462,22 +457,43 @@ pub fn str_out(buf: rt::Buf) -> R { } } -/// A view of any object with the buffer protocol, `bytes`, `bytearray`, or a -/// `memoryview` among them, held for the call: Rust reads it in place. +/// The bytes of `bytes`, read in place, or of any other object with the buffer protocol, +/// such as a `bytearray` or a `memoryview`, copied: the call runs without the GIL, so +/// another thread could write storage that is not immutable while Rust reads it. pub fn bytes_in(obj: *mut PyObject, s: &mut Scope) -> R { - // SAFETY: `Py_buffer` is plain data, which the call fills. - let mut view: Box = Box::new(unsafe { std::mem::zeroed() }); // SAFETY: the GIL is held, and `obj` is live. - if unsafe { PyObject_GetBuffer(obj, &mut *view, PyBUF_SIMPLE) } != 0 { - return Err(Raised); + unsafe { + if PyBytes_Check(obj) != 0 { + let mut ptr: *mut c_char = null_mut(); + let mut len = 0; + if PyBytes_AsStringAndSize(obj, &mut ptr, &mut len) != 0 { + return Err(Raised); + } + // A `bytes` never changes, and the caller keeps it for the call. + return Ok(rt::Buf { + ptr: ptr.cast_const().cast(), + len: len as usize, + owner: null_mut(), + }); + } + // SAFETY: `Py_buffer` is plain data, which the call fills. + let mut view: Py_buffer = std::mem::zeroed(); + if PyObject_GetBuffer(obj, &mut view, PyBUF_SIMPLE) != 0 { + return Err(Raised); + } + let copy = match view.len { + 0 => Vec::new(), + len => std::slice::from_raw_parts(view.buf.cast_const().cast::(), len as usize).to_vec(), + }; + PyBuffer_Release(&mut view); + let buf = rt::Buf { + ptr: copy.as_ptr(), + len: copy.len(), + owner: null_mut(), + }; + s.arrays.push(Box::new(copy)); + Ok(buf) } - let buf = rt::Buf { - ptr: view.buf.cast_const().cast(), - len: view.len as usize, - owner: null_mut(), - }; - s.views.push(view); - Ok(buf) } /// Bytes Rust returned, as a read-only `memoryview` that lends them without a copy until @@ -1401,6 +1417,10 @@ pub struct Future { /// Whether an `await` took it, after which another raises, as awaiting a coroutine /// twice does. claimed: bool, + /// Whether a poll runs, without the GIL, so a close from another thread waits for it. + polling: bool, + /// Whether it was closed while it polled, which the poll then does. + closing: bool, /// The handles the call holds until the future ends. held: Vec, } @@ -1448,13 +1468,18 @@ pub unsafe fn start(raw: *mut c_void, poll: Poll, free: unsafe fn(*mut c_void), } } -/// Free a future's C future, cancelling it unless it finished, and let go of its holds. +/// Free a future's C future, cancelling it unless it finished, and let go of its holds: +/// once its poll returns, if one runs. /// /// # Safety /// `future` is live. unsafe fn future_close(future: *mut Future) { // SAFETY: the caller's contract. unsafe { + if (*future).polling { + (*future).closing = true; + return; + } if let Some(free) = (*future).free.take() { let raw = std::mem::replace(&mut (*future).raw, null_mut()); call(|| free(raw)); @@ -1474,51 +1499,57 @@ unsafe extern "C" fn future_poll(obj: *mut PyObject, args: *mut *mut PyObject, n wrap(|| { // SAFETY: Python's call. let args = unsafe { self::args(args, nargs, 1, "poll") }?; - // SAFETY: Python calls it on a `Future`, holding the GIL. - let this = unsafe { &mut *future }; - let Some(poll) = this.poll else { - return value_error("the future ended"); - }; - let node = match &this.node { - Some(node) => node.clone(), - None => { - let node = Arc::new(Node { - state: AtomicU8::new(IDLE), - token: this.token, - queue: queue_of(args[0])?, - waker: false, - }); - this.node = Some(node.clone()); - node + // SAFETY: Python calls it on a `Future`, holding the GIL. Each access goes through + // the pointer, as a thread holding the GIL may close it while the poll runs without. + unsafe { + let Some(poll) = (*future).poll else { + return value_error("the future ended"); + }; + if (*future).polling { + return raise(PyExc_RuntimeError, "a Rust future is polled already"); } - }; - loop { - let first = node.state.swap(POLLING, Ordering::AcqRel) == IDLE; - // The first poll with the node gives the C ABI a reference, which it keeps - // while it reuses the waker. - let waker = match first { - true => node.give(), - false => node.lend(), + let node = match &(*future).node { + Some(node) => node.clone(), + None => { + let node = Arc::new(Node { + state: AtomicU8::new(IDLE), + token: (*future).token, + queue: queue_of(args[0])?, + waker: false, + }); + (*future).node = Some(node.clone()); + node + } }; - // SAFETY: a live future, which `poll` takes. - let done = unsafe { poll(this.raw, waker) }; - match done { - Ok(None) => {} - done => { - // SAFETY: a live future, finished or failed. - unsafe { future_close(future) }; - return done.map(|value| value.expect("ready")); + loop { + let first = node.state.swap(POLLING, Ordering::AcqRel) == IDLE; + // The first poll with the node gives the C ABI a reference, which it keeps + // while it reuses the waker. + let waker = match first { + true => node.give(), + false => node.lend(), + }; + (*future).polling = true; + let done = poll((*future).raw, waker); + (*future).polling = false; + if std::mem::take(&mut (*future).closing) { + future_close(future); + // What the poll returned is dropped with the future. + return value_error("the future was closed while it polled"); } + if let Some(done) = done.transpose() { + future_close(future); + return done; + } + if node + .state + .compare_exchange(POLLING, PARKED, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Ok(Obj::borrowed(PENDING.get().expect("loaded").ptr())); + } + // Woken while it polled: the wake is for this poll's registration, so poll again. } - if node - .state - .compare_exchange(POLLING, PARKED, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - // SAFETY: set when the module loaded. - return Ok(unsafe { Obj::borrowed(PENDING.get().expect("loaded").ptr()) }); - } - // Woken while it polled: the wake is for this poll's registration, so poll again. } }) } diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 5738eadf9..9bfbbe385 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -1158,7 +1158,7 @@ impl<'a> Gen<'a> { } let outs: Vec<(String, Ty)> = c_params.cloned().collect(); - // A tuple of one returned whole is written through its one out pointer. + // The outs follow the unstripped return: a borrowed tuple is one struct, not its elements. let ok = strip_refs(&function.ok); let err = function.err.as_ref().map(strip_refs); let exception = err.as_ref().map(|e| { @@ -1177,7 +1177,7 @@ impl<'a> Gen<'a> { }); if let Some((_, poll, free)) = &export.future { let poll_outs = &poll.params[2..]; - let (decl, ptrs, value) = self.outs(poll_outs, &ok, err.as_ref(), exception.as_deref(), "r == 1"); + let (decl, ptrs, value) = self.outs(poll_outs, &function.ok, err.as_ref(), exception.as_deref(), "r == 1"); let _ = write!( self.glue, "unsafe fn poll_{c_name}(f: *mut c_void, w: crate::rt::Waker) -> R> {{\n\tunsafe {{\n\t\t{decl}\n\t\tlet r = rt::call(|| crate::{}(f as _, w{}));\n\t\tif r == 0 {{\n\t\t\treturn Ok(None);\n\t\t}}\n\t\t{value}.map(Some)\n\t}}\n}}\n\nunsafe fn free_{c_name}(f: *mut c_void) {{\n\tunsafe {{ crate::{}(f as _) }}\n}}\n\n", @@ -1196,7 +1196,7 @@ impl<'a> Gen<'a> { let value = self.lower(&ok, "ret"); (String::new(), Vec::new(), value) } - false => self.outs(&outs, &ok, err.as_ref(), exception.as_deref(), "ret"), + false => self.outs(&outs, &function.ok, err.as_ref(), exception.as_deref(), "ret"), }; let args: Vec = c_args.iter().cloned().chain(ptrs).collect(); stmts.push(format!( diff --git a/ffi/tests/api.py b/ffi/tests/api.py index 5b8be7b75..532ccde87 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -505,6 +505,7 @@ async def recv(): # A wake that lands while the future polls, before it returns pending, is kept: the # future polls again rather than waiting for a wake that already happened. check(loop.run_until_complete(asyncio.wait_for(api.yielded(7), 5)) == 7, "a wake during the poll polls again") + check(loop.run_until_complete(api.corner_later()) == (1, 1), "a future's borrowed tuple returns whole") # Cancelled between its wake and its poll, a receive loses nothing: the payload stays # for the next. @@ -616,7 +617,7 @@ def values(): """The values Python's types bound: integers checked against their Rust type's range, durations truncated to the microsecond up to `timedelta.max`, which stands for longer, and on Unix, a path whose bytes are not UTF-8, which crosses as `os.fsdecode` decodes it, - and back as it came.""" + and back as it came. Also a borrowed tuple, returned whole.""" def raises(error, f, what): try: @@ -629,6 +630,10 @@ def raises(error, f, what): raises(OverflowError, lambda: api.widths(-1), "a negative int for a u64 raises") raises(TypeError, lambda: api.title(1), "an int is no bool") + # A borrowed tuple is one C value, not one out per element. + check(api.corner(True) == (1, 1), "a borrowed tuple returns whole") + raises(api.ErrorException, lambda: api.corner(False), "a borrowed tuple's error raises") + td = datetime.timedelta check(api.doubled_ttl(td(seconds=1, microseconds=5)) == td(seconds=2, microseconds=10), "a duration crosses") check(api.doubled_ttl(td(days=600_000_000)) == td.max, "a duration past timedelta's is its max") From fb66b632e0e71b57d57e97c3fa8a73da34e5dcd1 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:05:22 -0800 Subject: [PATCH 13/13] fix(ffi): end a Python future before its free releases the GIL Co-Authored-By: Claude Opus 5.5 --- ffi/fixtures/api/src/lib.rs | 6 ++++++ ffi/runtime/python.rs | 3 ++- ffi/tests/api.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index 1ec432187..f500d6503 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -574,6 +574,12 @@ pub fn filter(lines: Vec, mut keep: impl FnMut(u8, &str) -> bool + Send) -> }) } +/// Logs a line once polled, returning whether it was kept: the future holds `log` until it +/// is dropped. +pub fn log_later(log: Box) -> impl Future + use<> { + async move { log.log(0, "later") } +} + /// Holds a foreign implementation as a trait object, dropping it with itself. pub struct Logger { sink: Box, diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index e218ad7a7..f1be554d4 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -1480,11 +1480,12 @@ unsafe fn future_close(future: *mut Future) { (*future).closing = true; return; } + // Ended before the free releases the GIL, as Python may run during it. + (*future).poll = None; if let Some(free) = (*future).free.take() { let raw = std::mem::replace(&mut (*future).raw, null_mut()); call(|| free(raw)); } - (*future).poll = None; (*future).node = None; for held in std::mem::take(&mut (*future).held) { release(held.ptr().cast()); diff --git a/ffi/tests/api.py b/ffi/tests/api.py index 532ccde87..906535ba8 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -556,6 +556,21 @@ async def wait(awaitable): pass check(text(loop.run_until_complete(asyncio.wait_for(channel.recv(), 5))) == "once", "the send landed once") + # Python run while a closing future is freed, as by its drop's callback, finds it ended. + class Polls(api.Log): + def log(self, level, line): + return True + + def close(self): + try: + later.poll(api._n.Queue()) + check(False, "a future polled while it is freed raises") + except ValueError: + pass + + later = api.log_later(Polls()) + later.close() + # A waker woken from another thread calls back on its loop, once for wakes that land # before the loop gets to them. wakes = []