From 383e0734090d28db7c2ab74eead486a34d0139f1 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:27:22 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m3/ffi-python-methods.md Quest-Claim: ba1fa15d5a23de21 From f69e09a50b375f22ac66c60f541f8bf47878e818 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:51:07 -0800 Subject: [PATCH 2/5] perf(ffi): call the extension's Python methods with no wrapper frame Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 5 + ffi/runtime/python.rs | 157 ++++++++++++++++++++++++- ffi/src/python.rs | 262 +++++++++++++++++++++++++++++++++--------- ffi/tests/api.py | 62 +++++++++- ffi/tests/fixtures.rs | 2 +- 5 files changed, 426 insertions(+), 62 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index dd4cf2e51..c765b8cfe 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -314,6 +314,11 @@ same library on the stable ABI, so one build loads in CPython 3.11 and later. Th loads the library from beside itself: `lib.so`, `lib.dylib`, or `.dll`. Its glue converts each Python value to the C value and calls the C ABI function in-process, so a call is one crossing, which releases the GIL while Rust runs. +Each function, method, and property but an `async def` is the extension's own, bound in +the module or set on its class as it loads, so a call enters it with no Python frame +between; the Python declares them under `typing.TYPE_CHECKING`, for type checkers. They +take arguments by position or keyword, raise as a `def` would, and carry their +docstrings and signatures. - **Names** are in the module: types in PascalCase without the prefix, members in snake_case without their type's name (`moq.TrackProducer.append_group`). A keyword, a diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 85ebdc5fa..951c858b3 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -139,6 +139,79 @@ pub unsafe fn args<'a>(args: *mut *mut PyObject, nargs: Py_ssize_t, n: usize, na } } +/// The arguments of a call of `name`, a function Python calls directly whose parameters +/// are `names`, in their order after `this`, its `self` if it is a method: `n` +/// positional ones at `a`, then one for each name in `kw`. A wrong call raises as Python +/// raises for a `def` with these parameters. +/// +/// # Safety +/// `a` points at `n` live objects, then one for each name in `kw`, a tuple of `str` or +/// null. +pub unsafe fn params( + this: Option<*mut PyObject>, + a: *const *mut PyObject, + n: Py_ssize_t, + kw: *mut PyObject, + names: &[&str], + name: &str, +) -> R<[*mut PyObject; N]> { + let mut out = [null_mut(); N]; + let skip = match this { + Some(this) => { + out[0] = this; + 1 + } + None => 0, + }; + let n = n as usize; + if n > names.len() { + let (takes, given) = (names.len() + skip, n + skip); + let s = if takes == 1 { "" } else { "s" }; + let was = if given == 1 { "was" } else { "were" }; + return type_error(&format!("{name}() takes {takes} positional argument{s} but {given} {was} given")); + } + // SAFETY: the caller's contract. + unsafe { + out[skip..skip + n].copy_from_slice(std::slice::from_raw_parts(a, n)); + if !kw.is_null() { + for k in 0..PyTuple_Size(kw) { + let mut len = 0; + let key = PyUnicode_AsUTF8AndSize(PyTuple_GetItem(kw, k), &mut len); + if key.is_null() { + return Err(Raised); + } + let key = std::slice::from_raw_parts(key.cast::(), len as usize); + let key = String::from_utf8_lossy(key); + let Some(p) = names.iter().position(|name| *name == key) else { + return type_error(&format!("{name}() got an unexpected keyword argument '{key}'")); + }; + if !out[skip + p].is_null() { + return type_error(&format!("{name}() got multiple values for argument '{key}'")); + } + out[skip + p] = *a.add(n + k as usize); + } + } + } + let missing: Vec = names + .iter() + .zip(&out[skip..]) + .filter(|(_, arg)| arg.is_null()) + .map(|(name, _)| format!("'{name}'")) + .collect(); + if let Some((last, rest)) = missing.split_last() { + let (s, list) = match rest.len() { + 0 => ("", last.clone()), + 1 => ("s", format!("{} and {last}", rest[0])), + _ => ("s", format!("{}, and {last}", rest.join(", "))), + }; + return type_error(&format!( + "{name}() missing {} required positional argument{s}: {list}", + missing.len() + )); + } + Ok(out) +} + /// Call into the C ABI with the GIL released, so Rust may call Python back from any /// thread while this one waits, then resume what it woke on this thread's loop. pub fn call(f: impl FnOnce() -> T) -> T { @@ -356,7 +429,8 @@ pub fn ty(index: usize) -> *mut PyObject { (unsafe { TYPES.get() })[index].ptr() } -/// `_register(*classes)`: note the classes the glue makes values of. +/// `_register(*classes)`: note the classes the glue makes values of, and set the glue's +/// members on them. unsafe extern "C" fn register(_: *mut PyObject, args: *mut *mut PyObject, nargs: Py_ssize_t) -> *mut PyObject { // SAFETY: Python's call. let args = unsafe { std::slice::from_raw_parts(args, nargs as usize) }; @@ -365,7 +439,40 @@ unsafe extern "C" fn register(_: *mut PyObject, args: *mut *mut PyObject, nargs: *TYPES.get() = args.iter().map(|&class| Obj::borrowed(class)).collect(); *MEMBERS.get() = (0..args.len()).map(|_| None).collect(); } - none().map_or(null_mut(), Obj::into_ptr) + wrap(|| { + // SAFETY: the GIL is held, and each member's index one `_register` was passed. + unsafe { install() }?; + none() + }) +} + +/// Set each of the glue's members on its class: a method as a method descriptor, which +/// CPython calls without binding it, as it does a builtin type's. +unsafe fn install() -> R<()> { + // SAFETY: the GIL is held, and `INSTALL` lives as long as the interpreter. + unsafe { + let mut staticmethod = None; + for member in INSTALL.get().iter_mut() { + let (class, name, value) = match member { + Member::Method(class, def) => (*class, def.ml_name, PyDescr_NewMethod(ty(*class).cast(), def)), + Member::Get(class, def) => (*class, def.name, PyDescr_NewGetSet(ty(*class).cast(), def)), + Member::Static(class, def) => { + let f = Obj::new(PyCMethod_New(def, null_mut(), null_mut(), null_mut())); + if staticmethod.is_none() { + let builtins = Obj::new(PyImport_ImportModule(c"builtins".as_ptr()))?; + staticmethod = Some(Obj::new(PyObject_GetAttrString(builtins.ptr(), c"staticmethod".as_ptr()))?); + } + let wrap = staticmethod.as_ref().expect("just set").ptr(); + (*class, def.ml_name, call_py(wrap, vec![f]).map_or(null_mut(), Obj::into_ptr)) + } + }; + let value = Obj::new(value)?; + if PyObject_SetAttrString(ty(class), name, value.ptr()) != 0 { + return Err(Raised); + } + } + } + Ok(()) } pub fn none() -> R { @@ -1726,6 +1833,47 @@ const fn onearg(name: &'static CStr, f: PyCFunction) -> PyMethodDef { pub const END: PyMethodDef = PyMethodDef::zeroed(); +/// A member the glue defines natively, which `_register` sets on the class registered +/// at its index, so Python calls into the extension with no frame of its own between. +pub enum Member { + /// A method taking `self`. + Method(usize, PyMethodDef), + /// A static method. + Static(usize, PyMethodDef), + /// A read-only property. + Get(usize, PyGetSetDef), +} + +/// A method or function Python calls directly, by position or keyword, whose `doc` +/// starts with its signature, as `inspect.signature` reads it. +pub const fn function(name: &'static CStr, doc: &'static CStr, f: PyCFunctionFastWithKeywords) -> PyMethodDef { + PyMethodDef { + ml_name: name.as_ptr(), + ml_meth: PyMethodDefPointer { + PyCFunctionFastWithKeywords: f, + }, + ml_flags: METH_FASTCALL | METH_KEYWORDS, + ml_doc: doc.as_ptr(), + } +} + +/// A read-only property. +pub const fn property(name: &'static CStr, doc: Option<&'static CStr>, f: getter) -> PyGetSetDef { + PyGetSetDef { + name: name.as_ptr(), + get: Some(f), + set: None, + doc: match doc { + Some(doc) => doc.as_ptr(), + None => null(), + }, + closure: null_mut(), + } +} + +/// The glue's members, which `_register` sets on their classes. +static INSTALL: Gil<&'static mut [Member]> = Gil::new(&mut []); + /// A table of methods Python reads for the interpreter's life. pub struct Methods(pub UnsafeCell<[PyMethodDef; N]>); @@ -1797,13 +1945,14 @@ fn module_name(module: *mut PyObject) -> String { } /// Make the extension module `def` names, with the runtime's types and functions beside -/// the glue's. +/// the glue's, whose `members` `_register` sets on their classes. /// /// # Safety /// Python calls the module's `PyInit_` with the GIL held, once per interpreter. -pub unsafe fn module(def: *mut PyModuleDef) -> *mut PyObject { +pub unsafe fn module(def: *mut PyModuleDef, members: &'static mut [Member]) -> *mut PyObject { // SAFETY: the caller's contract. unsafe { + *INSTALL.get() = members; let Ok(module) = Obj::new(PyModule_Create(def)) else { return null_mut(); }; diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 41b4321cb..b4bab41b4 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -14,9 +14,10 @@ //! the conversions and the types the extension defines. //! //! The glue converts each Python value to its C value and back itself, so a call is one -//! crossing and the Python is a thin layer naming each function: a record crosses as its -//! fields, read and set by name, and bytes returned as a `memoryview` lending Rust's -//! buffer. +//! crossing: a record crosses as its fields, read and set by name, and bytes returned as a +//! `memoryview` lending Rust's buffer. Each function but an `async def` is the +//! extension's own, which `_register` sets on its class or the module binds, so Python +//! calls it with no frame of its own; the Python only declares it, for type checkers. use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Write; @@ -85,6 +86,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result, packages: &str, contract: u64) -> Result = rt::Gil::new([\n\ + {}\ + ]);\n\n\ static MODULE: rt::Gil = rt::Gil::new(PyModuleDef {{\n\ \tm_base: PyModuleDef_HEAD_INIT,\n\ \tm_name: c\"_{p}\".as_ptr(),\n\ @@ -184,7 +189,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result, packages: &str, contract: u64) -> Result(), + g.members.len(), + g.members.iter().map(|m| format!("\t{m},\n")).collect::(), ); Ok(Output { python: body, glue }) } @@ -212,6 +219,44 @@ enum Place { Top, } +/// What a native function Python calls directly is. +struct Direct<'s> { + /// Its Python name. + member: &'s str, + place: Place, + /// Whether it is a read-only property. + property: bool, + /// The C name of the class it is a member of, which `_register` sets it on. + owner: Option<&'s str>, + /// Its parameters' Python names, after its `self`. + params: &'s [String], + docs: &'s str, +} + +/// A class's members: those Python defines, and those the glue does, which the class +/// declares only for type checkers. +#[derive(Default)] +struct Body { + python: Vec, + native: Vec, +} + +impl Body { + fn codes(&self) -> impl Iterator { + self.python.iter().chain(&self.native) + } + + /// Each member's code, the glue's in one block. + fn members(&self) -> Vec { + let mut members = self.python.clone(); + if !self.native.is_empty() { + let native: Vec = self.native.iter().map(|m| indent_by(m, 1)).collect(); + members.push(format!("\tif _typing.TYPE_CHECKING:\n{}", native.join("\n"))); + } + members + } +} + struct Gen<'a> { abi: &'a Abi<'a>, /// The classes, then the module-level functions. @@ -222,6 +267,8 @@ struct Gen<'a> { glue: String, /// The extension's method table entries. table: Vec, + /// The members `_register` sets on their classes. + members: Vec, /// The classes the glue makes values of, in the order the Python registers them. types: Vec, /// Each registered class's index, by the C name or key it is registered under. @@ -513,12 +560,30 @@ impl<'a> Gen<'a> { for i in 0..api.callbacks.len() { self.callback(i); } + // The glue's functions, declared for type checkers and bound to its own. + let (mut declared, mut bound) = (Vec::new(), Vec::new()); for function in &api.functions { - if let Some(code) = self.function(function, None, None, Place::Top) { - self.top.push_str(&code); - self.top.push_str("\n\n"); + match self.function(function, None, None, Place::Top) { + Some((code, true)) => { + let name = member_name(&code).expect("a function's name"); + bound.push(format!("\t{name} = _n.{name}\n")); + declared.push(indent_by(&code, 1)); + } + Some((code, false)) => { + self.top.push_str(&code); + self.top.push_str("\n\n"); + } + None => {} } } + if !declared.is_empty() { + let _ = write!( + self.top, + "if _typing.TYPE_CHECKING:\n{}\nelse:\n{}\n\n", + declared.join("\n\n"), + bound.concat() + ); + } for (name, ty) in &self.exceptions.clone() { let py = self.py_ty(ty); let _ = write!( @@ -624,7 +689,7 @@ impl<'a> Gen<'a> { /// The Python members a type's std traits export beyond Python's own, with their glue: /// `__str__` from `Display`, `__repr__` from `Debug`, a static `default` from /// `Default`, and `clone` and `__eq__` for an object. - fn traits(&mut self, ty: &Type, c: &str, t: Traits, body: &mut Vec) { + fn traits(&mut self, ty: &Type, c: &str, t: Traits, body: &mut Body) { let bare = self.bare(c).to_string(); let class = self.class(c); let object = matches!(ty, Type::Object(_)); @@ -641,29 +706,36 @@ impl<'a> Gen<'a> { true => "this", false => "&this", }; - let unary = |g: &mut Self, suffix: &str, lower: String| { - g.native( + let unary = |g: &mut Self, suffix: &str, member: &str, docs: &str, lower: String| { + let d = Direct { + member, + place: Place::Member, + property: false, + owner: Some(c), + params: &[], + docs, + }; + g.direct( &format!("{bare}_{suffix}"), - 1, + d, &format!( "let this = {this}?;\n\t\tlet ret = rt::call(|| crate::{c}_{suffix}({pass}));\n\t\ts.done();\n\t\t{lower}" ), ); }; if t.display { - unary(self, "to_string", "rt::str_out(ret)".to_string()); - body.push(format!("\tdef __str__(self):\n\t\treturn _n.{bare}_to_string(self)\n")); + unary(self, "to_string", "__str__", "", "rt::str_out(ret)".to_string()); + body.native.push("\tdef __str__(self) -> str: ...\n".to_string()); } if t.debug { - unary(self, "debug", "rt::str_out(ret)".to_string()); - body.push(format!("\tdef __repr__(self):\n\t\treturn _n.{bare}_debug(self)\n")); + unary(self, "debug", "__repr__", "", "rt::str_out(ret)".to_string()); + body.native.push("\tdef __repr__(self) -> str: ...\n".to_string()); } if object && t.clone { let lower = self.lower(ty, "ret"); - unary(self, "clone", lower); - body.push(format!( - "\tdef clone(self) -> {class}:\n\t\t\"\"\"A new handle to a clone.\"\"\"\n\t\treturn _n.{bare}_clone(self)\n" - )); + let docs = "A new handle to a clone."; + unary(self, "clone", "clone", docs, lower); + body.native.push(format!("\tdef clone(self) -> {class}:\n\t\t\"\"\"{docs}\"\"\"\n\t\t...\n")); } if object && t.eq { let other = self.lift( @@ -678,19 +750,28 @@ impl<'a> Gen<'a> { "let this = {this}?;\n\t\tlet other = {other}?;\n\t\tlet ret = rt::call(|| crate::{c}_eq(this, other));\n\t\ts.done();\n\t\trt::bool_out(ret)" ), ); - body.push(format!( + body.python.push(format!( "\tdef __eq__(self, other):\n\t\treturn _isinstance(other, {class}) and _n.{bare}_eq(self, other)\n\n\t__hash__ = None\n" )); } if t.default { let lower = self.lower(ty, "ret"); - self.native( + let docs = "Rust's `Default`."; + let d = Direct { + member: "default", + place: Place::Static, + property: false, + owner: Some(c), + params: &[], + docs, + }; + self.direct( &format!("{bare}_default"), - 0, + d, &format!("let ret = rt::call(|| crate::{c}_default());\n\t\ts.done();\n\t\t{lower}"), ); - body.push(format!( - "\t@_staticmethod\n\tdef default() -> {class}:\n\t\t\"\"\"Rust's `Default`.\"\"\"\n\t\treturn _n.{bare}_default()\n" + body.native.push(format!( + "\t@_staticmethod\n\tdef default() -> {class}:\n\t\t\"\"\"{docs}\"\"\"\n\t\t...\n" )); } } @@ -704,7 +785,7 @@ impl<'a> Gen<'a> { (ty, owner, rust): (&Type, &Ty, &str), data: &[String], reserved: &[&str], - body: &mut Vec, + body: &mut Body, ) { let reserved: Vec<&str> = reserved .iter() @@ -716,21 +797,20 @@ impl<'a> Gen<'a> { Some(_) => Place::Member, None => Place::Static, }; - if let Some(code) = self.function(method, Some(owner), Some((ty, &reserved)), place) { - body.push(indent_by(&code, 1)); + if let Some((code, native)) = self.function(method, Some(owner), Some((ty, &reserved)), place) { + match native { + true => body.native.push(indent_by(&code, 1)), + false => body.python.push(indent_by(&code, 1)), + } } } - self.distinct(rust, data, body); + self.distinct(rust, data, body.codes()); } - /// Check that the members `body` of the type `rust` and its data's names `data` differ. - fn distinct(&mut self, rust: &str, data: &[String], body: &[String]) { + /// Check that the members `codes` of the type `rust` and its data's names `data` differ. + fn distinct<'c>(&mut self, rust: &str, data: &[String], codes: impl Iterator) { let mut seen = BTreeSet::new(); - for name in data - .iter() - .cloned() - .chain(body.iter().filter_map(|code| member_name(code))) - { + for name in data.iter().cloned().chain(codes.filter_map(|code| member_name(code))) { if !seen.insert(name.clone()) { self.errors.push(format!( "two members of `{rust}` are both `{name}` in Python: rename one in the config" @@ -748,7 +828,7 @@ impl<'a> Gen<'a> { self.claim(&name, format!("struct `{rust}`")); let t = self.register(&c, name.clone()); let ty = Type::Record(i); - let mut body = Vec::new(); + let mut body = Body::default(); self.traits(&ty, &c, record.traits, &mut body); let owner = Ty::Struct(c.clone()); let data: Vec = record.fields.iter().map(|f| field_name(&f.name)).collect(); @@ -773,7 +853,7 @@ impl<'a> Gen<'a> { self.py, "@_dataclass(slots=True{repr})\nclass {name}:\n{}{fields}{}\n\n", docstring(&format!("{rust}\n\n{}", record.docs), "\t"), - body.iter().map(|m| format!("\n{m}")).collect::() + body.members().iter().map(|m| format!("\n{m}")).collect::() ); // The glue. let mut reads = Vec::new(); @@ -810,7 +890,7 @@ impl<'a> Gen<'a> { self.claim(&name, format!("enum `{rust}`")); let t = self.register(&c, name.clone()); let ty = Type::Enum(i); - let mut body = Vec::new(); + let mut body = Body::default(); self.traits(&ty, &c, e.traits, &mut body); let owner = abi.ty(&ty); if e.plain() { @@ -826,7 +906,7 @@ impl<'a> Gen<'a> { self.py, "class {name}(_enum.Enum):\n{}{entries}{}\n\n", docstring(&format!("{rust}\n\n{}", e.docs), "\t"), - body.iter().map(|m| format!("\n{m}")).collect::() + body.members().iter().map(|m| format!("\n{m}")).collect::() ); return; } @@ -844,7 +924,7 @@ impl<'a> Gen<'a> { self.py, "class {name}:\n{}\t__slots__ = ()\n{}\n\n", docstring(&format!("{rust}\n\n{}", e.docs), "\t"), - body.iter().map(|m| format!("\n{m}")).collect::() + body.members().iter().map(|m| format!("\n{m}")).collect::() ); let repr = match e.traits.debug { true => ", repr=False", @@ -917,7 +997,10 @@ impl<'a> Gen<'a> { self.claim(&name, format!("type `{rust}`")); self.register(&c, name.clone()); let ty = Type::Object(i); - let mut body = vec!["\t__slots__ = ()\n".to_string()]; + let mut body = Body { + python: vec!["\t__slots__ = ()\n".to_string()], + native: Vec::new(), + }; self.traits(&ty, &c, object.traits, &mut body); let owner = Ty::Handle { name: c.clone(), @@ -932,7 +1015,7 @@ impl<'a> Gen<'a> { self.py, "class {name}(_n.Object):\n{}{}\n\n", docstring(text.trim(), "\t"), - body.join("\n") + body.members().join("\n") ); } @@ -976,7 +1059,7 @@ impl<'a> Gen<'a> { ); table.push(format!("{}: Some({trampoline})", method.name())); } - self.distinct(&rust, &[], &body); + self.distinct(&rust, &[], body.iter()); let text = format!( "{rust}: implemented in Python. Rust calls its methods from any thread, and once it\ndrops the implementation, calls its `close()` if it has one.\n\n{}", callback.docs @@ -1056,8 +1139,8 @@ impl<'a> Gen<'a> { } } - /// A native function `bare`, taking `n` arguments as `a`, whose glue runs `body` with - /// a scope `s`, returning an `R`. + /// A native function `bare`, which the generated Python calls by position with `n` + /// arguments as `a`, whose glue runs `body` with a scope `s`, returning an `R`. fn native(&mut self, bare: &str, n: usize, body: &str) { let _ = write!( self.glue, @@ -1066,6 +1149,49 @@ impl<'a> Gen<'a> { self.table.push(format!("rt::method(c\"{bare}\", f_{bare})")); } + /// A native function `bare` that Python calls as `d` describes, with no Python + /// frame between. Its glue runs `body` like a [`Gen::native`]'s, with `a` holding + /// `self` first if it has one. + fn direct(&mut self, bare: &str, d: Direct<'_>, body: &str) { + let member = d.member; + let class = d.owner.map(|c| (self.t(c), self.class(c))); + if d.property { + let (t, _) = class.expect("a property has its class"); + let _ = write!( + self.glue, + "unsafe extern \"C\" fn f_{bare}(this: *mut PyObject, _: *mut std::ffi::c_void) -> *mut PyObject {{\n\trt::wrap(|| unsafe {{\n\t\tlet a = [this];\n\t\tlet s = &mut rt::Scope::default();\n\t\t{body}\n\t}})\n}}\n\n" + ); + let doc = match d.docs.trim() { + "" => "None".to_string(), + docs => format!("Some({})", c_str(docs)), + }; + self.members.push(format!("rt::Member::Get({t}, rt::property(c\"{member}\", {doc}, f_{bare}))")); + return; + } + let method = d.place == Place::Member; + let qual = match &class { + Some((_, name)) => format!("{name}.{member}"), + None => member.to_string(), + }; + let names: Vec = d.params.iter().map(|p| format!("\"{p}\"")).collect(); + let _ = write!( + self.glue, + "unsafe extern \"C\" fn f_{bare}(this: *mut PyObject, a: *const *mut PyObject, n: Py_ssize_t, kw: *mut PyObject) -> *mut PyObject {{\n\trt::wrap(|| unsafe {{\n\t\tlet a = rt::params::<{}>({}, a, n, kw, &[{}], \"{qual}\")?;\n\t\tlet s = &mut rt::Scope::default();\n\t\t{body}\n\t}})\n}}\n\n", + d.params.len() + method as usize, + if method { "Some(this)" } else { "None" }, + names.join(", "), + ); + // The signature `inspect.signature` reads, before the docstring. + let params: Vec<&str> = method.then_some("$self").into_iter().chain(d.params.iter().map(String::as_str)).collect(); + let doc = c_str(&format!("{member}({})\n--\n\n{}", params.join(", "), d.docs.trim())); + let def = format!("rt::function(c\"{member}\", {doc}, f_{bare})"); + match (class, d.place) { + (Some((t, _)), Place::Member) => self.members.push(format!("rt::Member::Method({t}, {def})")), + (Some((t, _)), Place::Static) => self.members.push(format!("rt::Member::Static({t}, {def})")), + _ => self.table.push(def), + } + } + /// `function`'s Python form, a member of `owner` if it has one (whose IR type and /// reserved member names are `owner_ty`), with its native function and glue. fn function( @@ -1074,7 +1200,7 @@ impl<'a> Gen<'a> { owner: Option<&Ty>, owner_ty: Option<(&Type, &[&str])>, place: Place, - ) -> Option { + ) -> Option<(String, bool)> { let abi = self.abi; let export = abi.export(function, owner); let c_name = export.call.name.clone(); @@ -1203,11 +1329,7 @@ impl<'a> Gen<'a> { args.join(", ") )); } - self.native(&bare, n, &stmts.join("\n\t\t")); - // The Python. - let native = format!("_n.{bare}({})", py_args.join(", ")); - let ret = self.py_ty(&ok); let mut text = function.docs.clone(); for param in &function.params { if let Type::Closure(lent, _) = ¶m.ty { @@ -1222,37 +1344,60 @@ impl<'a> Gen<'a> { } } } + let body = stmts.join("\n\t\t"); + let constant = function.access == Access::Const && function.params.is_empty() && place != Place::Member; + // An `async def` stays Python's, as callers may need a coroutine. + let direct = !constant && (export.future.is_none() || function.eager); + let property = matches!(function.access, Access::Field { set: false, .. }) && place == Place::Member; + match direct { + true => { + let names: Vec = function.params.iter().map(|p| param_name(&p.name)).collect(); + let d = Direct { + member: &member, + place, + property, + owner: owner_c.as_deref(), + params: &names, + docs: &text, + }; + self.direct(&bare, d, &body); + } + false => self.native(&bare, n, &body), + } + let native = format!("_n.{bare}({})", py_args.join(", ")); + let ret = self.py_ty(&ok); let docs = docstring(text.trim(), "\t"); let params = py_params.join(", "); + // The glue's own is only declared, for type checkers. let code = match (export.future.is_some(), function.eager) { (true, false) => format!( "async def {member}({params}) -> {ret}:\n{docs}\tr = {native}\n\treturn (await r) if r.__class__ is _Future else r" ), - (true, true) => format!("def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\treturn {native}"), - (false, _) => format!("def {member}({params}) -> {ret}:\n{docs}\treturn {native}"), + (true, true) => format!("def {member}({params}) -> _typing.Awaitable[{ret}]:\n{docs}\t..."), + (false, _) => format!("def {member}({params}) -> {ret}:\n{docs}\t..."), }; let code = match (function.access, place) { - (Access::Const, Place::Top) if function.params.is_empty() => { + (Access::Const, Place::Top) if constant => { let name = member.to_uppercase(); self.claim(&name, format!("const `{}`", function.path.join("::"))); self.consts.push((name, bare)); return None; } - (Access::Const, Place::Static) if function.params.is_empty() => { + (Access::Const, Place::Static) if constant => { format!( "{} = _Const(_n.{bare})\n{}", member.to_uppercase(), docstring(text.trim(), "") ) } - (Access::Field { set: false, .. }, Place::Member) => format!("@_property\n{code}"), + _ if property => format!("@_property\n{code}"), (_, Place::Static) => format!("@_staticmethod\n{code}"), _ => code, }; if place == Place::Top { self.claim(&member, format!("fn `{}`", function.path.join("::"))); } - Some(code) + Some((code, direct)) } /// The out values of `outs`, for a function returning `ok` or raising `err` as @@ -1390,6 +1535,11 @@ fn member_name(code: &str) -> Option { None } +/// A C string literal of `text`. +fn c_str(text: &str) -> String { + format!("c{text:?}") +} + /// A docstring holding `docs`, each line indented by `indent`, or nothing for none. fn docstring(docs: &str, indent: &str) -> String { let docs = docs.trim(); diff --git a/ffi/tests/api.py b/ffi/tests/api.py index 6a4df8396..f81092213 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -2,7 +2,8 @@ # natively, through the generated Python. tests/fixtures.rs compares the transcripts. # # With an argument, it instead checks the futures' semantics, what frees what nobody -# closed, or the values only Python's types bound, or breaks a contract, which must fail. +# closed, the values only Python's types bound, or how Python calls into the extension, +# or breaks a contract, which must fail. import asyncio import datetime @@ -749,6 +750,63 @@ def raises(error, f, what): print("values ok") +def calls(): + """Calls Python makes straight into the extension, with no Python frame between, as a + `def` of the same signature takes them: by position or keyword, raising its errors, + with its docstring and the signature `inspect` reads.""" + import inspect + + def raises(f, message): + try: + f() + check(False, message) + except TypeError as e: + check(str(e) == message, f"{e!r} is {message!r}") + + kinds = { + (api.Counter, "get"): "method_descriptor", + (api.Counter, "new"): "staticmethod", + (api.Counter, "__repr__"): "method_descriptor", + (api.Registry, "first"): "getset_descriptor", + (api.Point, "offset"): "method_descriptor", + (api.Channel, "send_later"): "method_descriptor", + # An `async def` stays a coroutine function. + (api.Gauge, "later"): "function", + } + for (cls, name), kind in kinds.items(): + got = type(inspect.getattr_static(cls, name)).__name__ + check(got == kind, f"{cls.__name__}.{name} is a {got}") + check(type(api.has_text).__name__ == "builtin_function_or_method", "a function is the extension's") + + c = api.Counter.new(start=2, name="c") + check(c.add(n=3) == 5 and c.sub(1) == 4, "a method takes its arguments by keyword") + check(api.Point(1, 2).offset(dy=3, dx=4) == api.Point(5, 5), "a record's method takes keywords") + check(api.has_text(note=api.Note.default()) is False, "a function takes keywords") + check(api.Registry.new(first=c).edit(edit=lambda counter: True), "a closure passes by keyword") + loop = asyncio.new_event_loop() + check(loop.run_until_complete(api.halved(n=4)) == 2, "an async def takes keywords") + channel = api.Channel.new() + loop.run_until_complete(channel.send_later(payload=b"kw")) + check(bytes(loop.run_until_complete(channel.recv())) == b"kw", "an eager future takes keywords") + loop.close() + + raises(lambda: c.add(), "Counter.add() missing 1 required positional argument: 'n'") + raises(lambda: c.add(1, 2), "Counter.add() takes 2 positional arguments but 3 were given") + raises(lambda: c.add(1, n=2), "Counter.add() got multiple values for argument 'n'") + raises(lambda: c.add(m=1), "Counter.add() got an unexpected keyword argument 'm'") + raises(lambda: c.get(1), "Counter.get() takes 1 positional argument but 2 were given") + raises(lambda: api.Counter.new(), "Counter.new() missing 2 required positional arguments: 'name' and 'start'") + raises(lambda: api.has_text(1, 2), "has_text() takes 1 positional argument but 2 were given") + + check(str(inspect.signature(c.add)) == "(n)", "a bound method's signature") + check(str(inspect.signature(api.Counter.new)) == "(name, start)", "a static method's signature") + check(api.has_text.__doc__ == "Whether `note` has text, even empty.", "a function's docstring") + check(api.Gauge.set.__doc__ == "Set the level each clone reads.", "a method's docstring") + check(api.Counter.get.__doc__ is None, "a method without docs has no docstring") + check(api.Registry.first.__doc__ is None, "a property without docs has no docstring") + print("calls ok") + + def broken(which): """Each breaks a contract, which must fail with a message naming it.""" match which: @@ -809,5 +867,7 @@ def log(self, level, line): owned() case ["values"]: values() + case ["calls"]: + calls() case [which]: broken(which) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index f697584ab..b1887cb4f 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -525,7 +525,7 @@ fn api_python() { api.transcript, "the Python transcript differs from the native one" ); - for case in ["futures", "owned", "values"] { + for case in ["futures", "owned", "values", "calls"] { run(&mut python(&dir, &[case])); } for (case, message) in [ From a0429f18d073ae24a1cd1d60664c3ece24613e7d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:51:43 -0800 Subject: [PATCH 3/5] chore(quest): finish ffi-python-methods Co-Authored-By: Claude Opus 5.5 --- quest/m3/README.md | 1 - quest/m3/ffi-python-loop-lookup.md | 4 ---- quest/m3/ffi-python-methods.md | 25 ------------------------- 3 files changed, 30 deletions(-) delete mode 100644 quest/m3/ffi-python-methods.md diff --git a/quest/m3/README.md b/quest/m3/README.md index 2e02a3495..2df158d06 100644 --- a/quest/m3/README.md +++ b/quest/m3/README.md @@ -29,5 +29,4 @@ generic std gap is planned here even when moq never calls it. - [Every panic of a hooked program has its location](/quest/m3/ts-panic-sites.md) - no panic in a program that sets a hook lacks the location native code gives it - [The chunk a BufMut lends](/quest/m3/ts-chunk-mut.md) - `BufMut::chunk_mut` and its `UninitSlice`'s writes translate, reaching the buffer's storage - [Slices not yet written](/quest/m3/ts-uninit-slices.md) - `Box::new_uninit_slice` and `write_clone_of_slice` translate -- [Python methods without a wrapper frame](/quest/m3/ffi-python-methods.md) - a method call on a generated Python object enters the extension with no wrapper frame - [Python calls skip the loop lookup](/quest/m3/ffi-python-loop-lookup.md) - a call while an asyncio loop runs costs what one outside does unless something waits diff --git a/quest/m3/ffi-python-loop-lookup.md b/quest/m3/ffi-python-loop-lookup.md index ffcb6a4ac..350ff3151 100644 --- a/quest/m3/ffi-python-loop-lookup.md +++ b/quest/m3/ffi-python-loop-lookup.md @@ -16,7 +16,3 @@ lookup unless something on the thread is waiting; #564's PR body explains why re only the thread that last ran the loop is unsafe (a stale loop can deadlock a blocking call). moq's Python calls run inside a loop, so moq pays this cost. Public API: none. - -## Related - -- [Python methods without a wrapper frame](/quest/m3/ffi-python-methods.md) - the other per-call cost diff --git a/quest/m3/ffi-python-methods.md b/quest/m3/ffi-python-methods.md deleted file mode 100644 index a3ed47e5e..000000000 --- a/quest/m3/ffi-python-methods.md +++ /dev/null @@ -1,25 +0,0 @@ -# [M] Python methods without a wrapper frame - -## Goal - -A method call on an object the exporter's [Python](/ffi/README.md) generates enters the -extension directly, with no Python wrapper frame in between, saving about 25 ns of a call's -104 ns. `just ffi bench` reports the call against main and -[BoltFFI's](/bench/ffi/README.md#boltffi-comparison) in the same run. The Python API and the -GIL release during a call stay as they are. - -## Plan - -Decided with the maintainer on 2026-11-03: planned from the alternatives -[#560](https://github.com/moq-dev/rust2/pull/560) measured and dropped. That PR found the wrapper -frame costs about 25 ns; native methods (a `PyType_FromSpec` method table, or whatever -CPython offers that is faster) would remove it. It changes how the generated classes are -built, so keep what callers see the same: signatures, keyword arguments, defaults, -docstrings, and the type stubs. Holding the GIL stays rejected, since a call blocking on a -lock a Rust thread holds while calling Python back would deadlock. - -moq's Python bindings are the consumer. Public API: none. - -## Related - -- [Python calls skip the loop lookup](/quest/m3/ffi-python-loop-lookup.md) - the other per-call cost left after #564 From 05ef3913146b0ea9885fae5a0894893c28446e4c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:52:21 -0800 Subject: [PATCH 4/5] style(ffi): rustfmt Co-Authored-By: Claude Opus 5.5 --- ffi/src/python.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ffi/src/python.rs b/ffi/src/python.rs index b4bab41b4..195caa1e2 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -735,7 +735,9 @@ impl<'a> Gen<'a> { let lower = self.lower(ty, "ret"); let docs = "A new handle to a clone."; unary(self, "clone", "clone", docs, lower); - body.native.push(format!("\tdef clone(self) -> {class}:\n\t\t\"\"\"{docs}\"\"\"\n\t\t...\n")); + body.native.push(format!( + "\tdef clone(self) -> {class}:\n\t\t\"\"\"{docs}\"\"\"\n\t\t...\n" + )); } if object && t.eq { let other = self.lift( @@ -1165,7 +1167,9 @@ impl<'a> Gen<'a> { "" => "None".to_string(), docs => format!("Some({})", c_str(docs)), }; - self.members.push(format!("rt::Member::Get({t}, rt::property(c\"{member}\", {doc}, f_{bare}))")); + self.members.push(format!( + "rt::Member::Get({t}, rt::property(c\"{member}\", {doc}, f_{bare}))" + )); return; } let method = d.place == Place::Member; @@ -1182,7 +1186,11 @@ impl<'a> Gen<'a> { names.join(", "), ); // The signature `inspect.signature` reads, before the docstring. - let params: Vec<&str> = method.then_some("$self").into_iter().chain(d.params.iter().map(String::as_str)).collect(); + let params: Vec<&str> = method + .then_some("$self") + .into_iter() + .chain(d.params.iter().map(String::as_str)) + .collect(); let doc = c_str(&format!("{member}({})\n--\n\n{}", params.join(", "), d.docs.trim())); let def = format!("rt::function(c\"{member}\", {doc}, f_{bare})"); match (class, d.place) { From 19995aed19794ecf16b69afbe529b62324586e4e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:56:12 -0800 Subject: [PATCH 5/5] fix(ffi): take a call with no argument array Co-Authored-By: Claude Opus 5.5 --- ffi/runtime/python.rs | 6 ++++-- ffi/tests/api.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 951c858b3..3ec5bcd89 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -170,9 +170,11 @@ pub unsafe fn params( let was = if given == 1 { "was" } else { "were" }; return type_error(&format!("{name}() takes {takes} positional argument{s} but {given} {was} given")); } - // SAFETY: the caller's contract. + // SAFETY: the caller's contract; `a` may be null when it points at nothing. unsafe { - out[skip..skip + n].copy_from_slice(std::slice::from_raw_parts(a, n)); + if n != 0 { + out[skip..skip + n].copy_from_slice(std::slice::from_raw_parts(a, n)); + } if !kw.is_null() { for k in 0..PyTuple_Size(kw) { let mut len = 0; diff --git a/ffi/tests/api.py b/ffi/tests/api.py index f81092213..a72b41b86 100644 --- a/ffi/tests/api.py +++ b/ffi/tests/api.py @@ -790,6 +790,13 @@ def raises(f, message): check(bytes(loop.run_until_complete(channel.recv())) == b"kw", "an eager future takes keywords") loop.close() + # C callers may pass no argument array at all for no arguments. + import ctypes + + no_args = ctypes.pythonapi.PyObject_CallNoArgs + no_args.restype, no_args.argtypes = ctypes.py_object, [ctypes.py_object] + check(no_args(api.Note.default) == api.Note.default(), "a call with no argument array") + raises(lambda: c.add(), "Counter.add() missing 1 required positional argument: 'n'") raises(lambda: c.add(1, 2), "Counter.add() takes 2 positional arguments but 3 were given") raises(lambda: c.add(1, n=2), "Counter.add() got multiple values for argument 'n'")