From 1567ff9cd8e969385e96b6399fcead77c1b57379 Mon Sep 17 00:00:00 2001 From: Jamie Wells Date: Sun, 11 Oct 2026 07:51:06 +0000 Subject: [PATCH 1/2] Add script-owned audio recording and hot microphone APIs --- Engine/Cargo.lock | 2 + Engine/crates/engine-play/Cargo.toml | 2 + .../crates/engine-play/src/audio_capture.rs | 354 ++++++++++++++++++ Engine/crates/engine-play/src/audio_input.rs | 196 ++++++++++ Engine/crates/engine-play/src/lib.rs | 3 + .../crates/engine-play/src/script_runtime.rs | 126 ++++++- .../engine-scripting/src/sdk/gameplay.lua | 39 ++ Engine/docs/Scripting/API/audio.md | 102 +++++ 8 files changed, 823 insertions(+), 1 deletion(-) create mode 100644 Engine/crates/engine-play/src/audio_capture.rs create mode 100644 Engine/crates/engine-play/src/audio_input.rs diff --git a/Engine/Cargo.lock b/Engine/Cargo.lock index 7bd78922..60c02743 100644 --- a/Engine/Cargo.lock +++ b/Engine/Cargo.lock @@ -1184,11 +1184,13 @@ dependencies = [ name = "engine-play" version = "0.1.0" dependencies = [ + "cpal", "engine-assets", "engine-core", "engine-scripting", "engine-world", "glam 0.33.6", + "hound", "interprocess", "rodio", "ron", diff --git a/Engine/crates/engine-play/Cargo.toml b/Engine/crates/engine-play/Cargo.toml index fd4bab6f..5d2f2252 100644 --- a/Engine/crates/engine-play/Cargo.toml +++ b/Engine/crates/engine-play/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] serde_json.workspace = true +hound.workspace = true engine-assets = { path = "../engine-assets", version = "0.1.0" } engine-core = { path = "../engine-core", version = "0.1.0" } engine-scripting = { path = "../engine-scripting", version = "0.1.0" } @@ -27,4 +28,5 @@ tempfile.workspace = true workspace = true [target.'cfg(windows)'.dependencies] +cpal = "0.16" rodio = { version = "0.21", default-features = false, features = ["playback"] } diff --git a/Engine/crates/engine-play/src/audio_capture.rs b/Engine/crates/engine-play/src/audio_capture.rs new file mode 100644 index 00000000..cba8e358 --- /dev/null +++ b/Engine/crates/engine-play/src/audio_capture.rs @@ -0,0 +1,354 @@ +//! Script-owned audio recording and local microphone streams. +use crate::audio_input::AudioInput; +use engine_core::gameplay::Owner; +use std::{collections::BTreeMap, fs::File, io::BufWriter, path::Path}; + +const MAX_SAMPLES: usize = 16 * 1024 * 1024; // 64 MiB per in-memory recording. +const READ_SAMPLES: usize = 4096; +type Writer = hound::WavWriter>; + +struct Capture { + owner: Owner, + input: Option, + microphone: bool, + recording: bool, + active: bool, + sample_rate: u32, + channels: u16, + samples: Vec, + cursor: usize, + writer: Option, + error: Option, + dropped: usize, +} +impl Capture { + fn append(&mut self, samples: &[f32]) { + if !self.active { + return; + } + if self.samples.len() + samples.len() > MAX_SAMPLES { + self.error = + Some("recording exceeded the 64 MiB memory limit; stop and unload it".into()); + self.active = false; + self.input = None; + return; + } + if let Some(writer) = &mut self.writer { + for sample in samples { + if let Err(error) = writer.write_sample(*sample) { + self.error = Some(format!("cannot write recording: {error}")); + self.active = false; + self.input = None; + break; + } + } + } + self.samples.extend_from_slice(samples); + } + fn pump(&mut self) -> Result { + let Some(input) = &self.input else { + return Ok(0); + }; + let (samples, dropped) = input.read(usize::MAX)?; + self.append(&samples); + self.dropped += dropped; + Ok(dropped) + } + fn stop(&mut self) -> Result<(), String> { + let input = self.input.take(); + if let Some(mut input) = input { + input.close(); + match input.read(usize::MAX) { + Ok((samples, dropped)) => { + self.dropped += dropped; + if self.recording { + self.append(&samples); + } + } + Err(error) => self.error = Some(error), + } + } + self.active = false; + if let Some(writer) = self.writer.take() { + writer.finalize().map_err(|e| e.to_string())?; + } + if let Some(error) = &self.error { + return Err(error.clone()); + } + Ok(()) + } +} +impl Drop for Capture { + fn drop(&mut self) { + let _ = self.stop(); + } +} + +#[derive(Default)] +pub(crate) struct AudioCapture { + captures: BTreeMap, +} +impl AudioCapture { + pub fn start( + &mut self, + owner: Owner, + recording: bool, + microphone: bool, + path: Option<&Path>, + ) -> Result { + if self.captures.values().filter(|c| c.owner == owner).count() >= 4 { + return Err( + "a script may retain at most four capture handles; unload an old handle first" + .into(), + ); + } + let input = if microphone { + Some(AudioInput::open()?) + } else { + None + }; + let sample_rate = input.as_ref().map_or(48_000, |i| i.sample_rate); + let channels = input.as_ref().map_or(2, |i| i.channels); + let writer = path + .map(|path| { + // Never overwrite an existing file. Parent directories must already exist. + let file = File::options() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| e.to_string())?; + hound::WavWriter::new( + BufWriter::new(file), + hound::WavSpec { + channels, + sample_rate, + bits_per_sample: 32, + sample_format: hound::SampleFormat::Float, + }, + ) + .map_err(|e| e.to_string()) + }) + .transpose()?; + let id = uuid::Uuid::new_v4().to_string(); + self.captures.insert( + id.clone(), + Capture { + owner, + input, + microphone, + recording, + active: true, + sample_rate, + channels, + samples: Vec::new(), + cursor: 0, + writer, + error: None, + dropped: 0, + }, + ); + Ok(id) + } + fn get(&mut self, owner: Owner, id: &str) -> Result<&mut Capture, String> { + self.captures + .get_mut(id) + .filter(|c| c.owner == owner) + .ok_or_else(|| "capture handle is unavailable or belongs to another script".into()) + } + pub fn stop(&mut self, owner: Owner, id: Option<&str>, recording: bool) -> Result<(), String> { + if let Some(id) = id { + let capture = self.get(owner, id)?; + if capture.recording != recording { + return Err("wrong capture handle type".into()); + } + return capture.stop(); + } + let mut error = None; + for capture in self + .captures + .values_mut() + .filter(|c| c.owner == owner && c.recording == recording) + { + if let Err(e) = capture.stop() { + error = Some(e); + } + } + error.map_or(Ok(()), Err) + } + pub fn read(&mut self, owner: Owner, id: &str) -> Result { + let capture = self.get(owner, id)?; + if capture.recording { + capture.pump()?; + } else { + let (samples, dropped) = capture + .input + .as_ref() + .map_or(Ok((Vec::new(), 0)), |i| i.read(READ_SAMPLES))?; + return Ok( + serde_json::json!({"samples":samples,"sampleRate":capture.sample_rate, + "channels":capture.channels,"active":capture.active,"droppedSamples":dropped}), + ); + } + let dropped = std::mem::take(&mut capture.dropped); + if let Some(error) = &capture.error { + return Err(error.clone()); + } + let count = READ_SAMPLES - READ_SAMPLES % usize::from(capture.channels); + let end = (capture.cursor + count).min(capture.samples.len()); + let packet = serde_json::json!({"samples":&capture.samples[capture.cursor..end], + "sampleRate":capture.sample_rate,"channels":capture.channels, + "active":capture.active,"droppedSamples":dropped}); + capture.cursor = end; + Ok(packet) + } + pub fn unload(&mut self, owner: Owner, id: &str) -> Result<(), String> { + self.get(owner, id)?; + self.captures.remove(id); + Ok(()) + } + pub fn submit(&mut self, samples: &[f32]) { + for capture in self + .captures + .values_mut() + .filter(|c| c.recording && c.active) + { + if capture.microphone { + if let Err(error) = capture.pump() { + capture.error = Some(error); + capture.active = false; + capture.input = None; + } + } else { + capture.append(samples); + } + } + } + pub fn cleanup(&mut self, mut predicate: impl FnMut(Owner) -> bool) { + self.captures.retain(|_, c| !predicate(c.owner)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn owner() -> Owner { + Owner { + entity_bound: true, + scene: engine_core::SceneId::new(), + entity: engine_core::EntityId::new(), + script: engine_core::ScriptId::new(), + } + } + #[test] + fn hot_mic_reads_live_pcm_and_stop_releases_input() { + let owner = owner(); + let mut service = AudioCapture::default(); + let id = service.start(owner, true, false, None).unwrap(); + let capture = service.get(owner, &id).unwrap(); + capture.recording = false; + capture.microphone = true; + capture.input = Some(AudioInput::fixture(vec![0.5; READ_SAMPLES + 2], 2, 4)); + let packet = service.read(owner, &id).unwrap(); + assert_eq!(packet["samples"].as_array().unwrap().len(), READ_SAMPLES); + assert_eq!(packet["droppedSamples"], 4); + assert_eq!(packet["active"], true); + assert_eq!( + service.read(owner, &id).unwrap()["samples"], + serde_json::json!([0.5, 0.5]) + ); + service.stop(owner, None, false).unwrap(); + assert!(service.get(owner, &id).unwrap().input.is_none()); + let packet = service.read(owner, &id).unwrap(); + assert_eq!(packet["active"], false); + assert_eq!(packet["samples"], serde_json::json!([])); + } + + #[test] + fn recording_retains_pcm_after_stop_and_finalizes_wav() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("record.wav"); + let owner = owner(); + let mut service = AudioCapture::default(); + let id = service.start(owner, true, false, Some(&path)).unwrap(); + service.submit(&[0.25, -0.25, 0.5, -0.5]); + assert!(service.read(super::tests::owner(), &id).is_err()); + assert!(service.stop(owner, Some(&id), false).is_err()); + service.stop(owner, Some(&id), true).unwrap(); + service.stop(owner, Some(&id), true).unwrap(); + service.submit(&[1.0, 1.0]); + let packet = service.read(owner, &id).unwrap(); + assert_eq!( + packet["samples"], + serde_json::json!([0.25, -0.25, 0.5, -0.5]) + ); + assert_eq!(packet["sampleRate"], 48_000); + assert_eq!(packet["channels"], 2); + assert_eq!(packet["active"], false); + assert_eq!( + service.read(owner, &id).unwrap()["samples"], + serde_json::json!([]) + ); + let mut reader = hound::WavReader::open(&path).unwrap(); + assert_eq!( + reader + .samples::() + .collect::, _>>() + .unwrap(), + vec![0.25, -0.25, 0.5, -0.5] + ); + assert!(service.start(owner, true, false, Some(&path)).is_err()); + service.unload(owner, &id).unwrap(); + assert!(service.read(owner, &id).is_err()); + } + #[test] + fn owner_cleanup_finalizes_files_and_read_packets_are_bounded() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("cleanup.wav"); + let owner = owner(); + let mut service = AudioCapture::default(); + let id = service.start(owner, true, false, Some(&path)).unwrap(); + let other = service + .start(super::tests::owner(), true, false, None) + .unwrap(); + service.submit(&vec![0.25; READ_SAMPLES * 2]); + assert_eq!( + service.read(owner, &id).unwrap()["samples"] + .as_array() + .unwrap() + .len(), + READ_SAMPLES + ); + service.cleanup(|o| o == owner); + assert!(service.read(owner, &id).is_err()); + assert!(service.captures.contains_key(&other)); + assert_eq!( + hound::WavReader::open(path).unwrap().duration(), + u32::try_from(READ_SAMPLES).unwrap() + ); + } + #[test] + fn memory_limit_and_handle_limit_are_reported() { + let owner = owner(); + let mut service = AudioCapture::default(); + let id = service.start(owner, true, false, None).unwrap(); + // Fill directly to avoid writing a 64 MiB fixture to disk. + service + .get(owner, &id) + .unwrap() + .samples + .resize(MAX_SAMPLES, 0.0); + service.submit(&[1.0, 1.0]); + assert!( + service + .stop(owner, Some(&id), true) + .unwrap_err() + .contains("64 MiB") + ); + for _ in 0..3 { + service.start(owner, true, false, None).unwrap(); + } + assert!(service.start(owner, true, false, None).is_err()); + service.unload(owner, &id).unwrap(); + assert!(service.start(owner, true, false, None).is_ok()); + } +} diff --git a/Engine/crates/engine-play/src/audio_input.rs b/Engine/crates/engine-play/src/audio_input.rs new file mode 100644 index 00000000..260c0701 --- /dev/null +++ b/Engine/crates/engine-play/src/audio_input.rs @@ -0,0 +1,196 @@ +//! Native microphone capture never takes scene or scripting locks. +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct InputBuffer { + samples: VecDeque, + dropped: usize, + error: Option, +} + +pub(crate) struct AudioInput { + buffer: Arc>, + pub sample_rate: u32, + pub channels: u16, + #[cfg(windows)] + stop: std::sync::mpsc::Sender<()>, + #[cfg(windows)] + worker: Option>, +} + +impl AudioInput { + pub fn open() -> Result { + #[cfg(not(windows))] + return Err("microphone capture is currently supported on Windows only".into()); + #[cfg(windows)] + { + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + let buffer = Arc::new(Mutex::new(InputBuffer::default())); + let shared = Arc::clone(&buffer); + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let (stop, stopped) = std::sync::mpsc::channel(); + let worker = std::thread::Builder::new() + .name("rustic-microphone".into()) + .spawn(move || { + let open = || -> Result<(cpal::Stream, u32, u16), String> { + let device = cpal::default_host() + .default_input_device() + .ok_or("no default microphone is available")?; + let supported = device.default_input_config().map_err(|e| e.to_string())?; + let config: cpal::StreamConfig = supported.clone().into(); + let stream = match supported.sample_format() { + cpal::SampleFormat::F32 => build::(&device, &config, &shared), + cpal::SampleFormat::I16 => build::(&device, &config, &shared), + cpal::SampleFormat::U16 => build::(&device, &config, &shared), + format => { + Err(format!("unsupported microphone sample format: {format:?}")) + } + }?; + stream.play().map_err(|e| e.to_string())?; + Ok((stream, config.sample_rate.0, config.channels)) + }; + match open() { + Ok((stream, rate, channels)) => { + let _ = ready_tx.send(Ok((rate, channels))); + let _ = stopped.recv(); + drop(stream); + } + Err(error) => { + let _ = ready_tx.send(Err(error)); + } + } + }) + .map_err(|e| e.to_string())?; + let (sample_rate, channels) = ready_rx + .recv_timeout(std::time::Duration::from_secs(2)) + .map_err(|e| format!("microphone startup failed: {e}"))??; + Ok(Self { + buffer, + sample_rate, + channels, + stop, + worker: Some(worker), + }) + } + } + + #[cfg_attr( + not(windows), + allow( + clippy::unused_self, + reason = "only the Windows backend owns a native capture worker" + ) + )] + pub fn close(&mut self) { + #[cfg(windows)] + { + let _ = self.stop.send(()); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } + } + + pub fn read(&self, limit: usize) -> Result<(Vec, usize), String> { + let mut buffer = self + .buffer + .lock() + .map_err(|_| "microphone buffer lock poisoned")?; + if let Some(error) = &buffer.error { + return Err(error.clone()); + } + let count = limit.min(buffer.samples.len()); + let count = count - count % usize::from(self.channels); + let samples = buffer.samples.drain(..count).collect(); + let dropped = std::mem::take(&mut buffer.dropped); + Ok((samples, dropped)) + } +} + +#[cfg(windows)] +impl Drop for AudioInput { + fn drop(&mut self) { + self.close(); + } +} + +#[cfg(windows)] +fn build( + device: &cpal::Device, + config: &cpal::StreamConfig, + buffer: &Arc>, +) -> Result +where + T: cpal::SizedSample, + f32: cpal::FromSample, +{ + use cpal::traits::DeviceTrait; + let data = Arc::clone(buffer); + let errors = Arc::clone(buffer); + let capacity = config.sample_rate.0 as usize * usize::from(config.channels) * 2; + device + .build_input_stream( + config, + move |samples: &[T], _| { + // Bound the queue to two seconds; a slow script gets recent input. + if let Ok(mut buffer) = data.lock() { + for sample in samples { + if buffer.samples.len() == capacity { + buffer.samples.pop_front(); + buffer.dropped += 1; + } + let value = sample.to_sample::(); + buffer.samples.push_back(if value.is_finite() { + value.clamp(-1.0, 1.0) + } else { + 0.0 + }); + } + } + }, + move |error| { + if let Ok(mut buffer) = errors.lock() { + buffer.error = Some(error.to_string()); + } + }, + None, + ) + .map_err(|e| e.to_string()) +} + +#[cfg(test)] +impl AudioInput { + pub fn fixture(samples: Vec, channels: u16, dropped: usize) -> Self { + #[cfg(windows)] + let (stop, _) = std::sync::mpsc::channel(); + Self { + buffer: Arc::new(Mutex::new(InputBuffer { + samples: samples.into(), + dropped, + error: None, + })), + sample_rate: 48_000, + channels, + #[cfg(windows)] + stop, + #[cfg(windows)] + worker: None, + } + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn reads_preserve_channel_frames_and_report_dropped_input_once() { + let input = AudioInput::fixture(vec![0.25; 12], 3, 6); + let (samples, dropped) = input.read(8).unwrap(); + assert_eq!(samples.len(), 6); + assert_eq!(dropped, 6); + assert_eq!(input.read(usize::MAX).unwrap(), (vec![0.25; 6], 0)); + assert!(input.read(4096).unwrap().0.is_empty()); + input.buffer.lock().unwrap().error = Some("device disconnected".into()); + assert_eq!(input.read(4096).unwrap_err(), "device disconnected"); + } +} diff --git a/Engine/crates/engine-play/src/lib.rs b/Engine/crates/engine-play/src/lib.rs index 3b9fc232..2b031f51 100644 --- a/Engine/crates/engine-play/src/lib.rs +++ b/Engine/crates/engine-play/src/lib.rs @@ -43,3 +43,6 @@ pub use supervisor::{RuntimeLaunch, SupervisedRuntime, SupervisorError, Supervis mod audio_output; mod gameplay_world; + +mod audio_capture; +mod audio_input; diff --git a/Engine/crates/engine-play/src/script_runtime.rs b/Engine/crates/engine-play/src/script_runtime.rs index c0d2263d..cea4d6a0 100644 --- a/Engine/crates/engine-play/src/script_runtime.rs +++ b/Engine/crates/engine-play/src/script_runtime.rs @@ -220,6 +220,7 @@ struct GameplayContext { physics: crate::physics::PhysicsWorld, audio: engine_core::gameplay::audio::Mixer, audio_output: crate::audio_output::AudioOutput, + audio_capture: crate::audio_capture::AudioCapture, clips: BTreeMap<(EntityId, String), engine_core::gameplay::animation::Clip>, frame_delta: f64, fixed_delta: f64, @@ -233,6 +234,7 @@ impl Default for GameplayContext { physics: crate::physics::PhysicsWorld::default(), audio: engine_core::gameplay::audio::Mixer::default(), audio_output: crate::audio_output::AudioOutput::new(), + audio_capture: crate::audio_capture::AudioCapture::default(), clips: BTreeMap::new(), frame_delta: 1.0 / 60.0, fixed_delta: 1.0 / 60.0, @@ -509,6 +511,9 @@ impl RuntimeScripts { gameplay .audio .cleanup(|owner| owner.entity_bound && !world.contains(owner.entity)); + gameplay + .audio_capture + .cleanup(|owner| owner.entity_bound && !world.contains(owner.entity)); gameplay .clips .retain(|(entity, _), _| world.contains(*entity)); @@ -536,7 +541,10 @@ impl RuntimeScripts { context.runtime.time_scale() }; match context.audio.mix_scaled(delta, scale) { - Ok(samples) => context.audio_output.submit(samples), + Ok(samples) => { + context.audio_capture.submit(&samples); + context.audio_output.submit(samples); + } Err(error) => push_log(&self.logs, &self.dropped_logs, "error", &error), } } @@ -1094,6 +1102,73 @@ impl GameplayHost for RuntimeHost { action.validate()?; return serde_json::to_value(action).map_err(|e| e.to_string()); } + if op.starts_with("capture_") { + let mut context = self.gameplay.lock().map_err(|_| "gameplay lock poisoned")?; + let owner = Owner { + entity_bound: self.entity_bound, + scene: context.scene, + entity: self.entity, + script: self.owner_script, + }; + let id = query + .get("id") + .filter(|v| !v.is_null()) + .map(|v| v.as_str().ok_or("capture id must be a string")) + .transpose()?; + match op { + "capture_recording_start" | "capture_mic_start" => { + let recording = op == "capture_recording_start"; + let source = query + .get("source") + .map(|v| v.as_str().ok_or("recording source must be a string")) + .transpose()? + .unwrap_or("game"); + if source != "game" && source != "microphone" { + return Err("recording source must be game or microphone".into()); + } + let path = match query.get("path").filter(|v| !v.is_null()) { + Some(value) => { + let path = value + .as_str() + .filter(|s| !s.is_empty()) + .ok_or("recording path must be a nonempty string")?; + if !path.to_ascii_lowercase().ends_with(".wav") { + return Err("recording path must end in .wav".into()); + } + let path = PathBuf::from(path); + Some(if path.is_absolute() { + path + } else { + self.snapshot_root.join(path) + }) + } + None => None, + }; + let id = context.audio_capture.start( + owner, + recording, + !recording || source == "microphone", + path.as_deref(), + )?; + return Ok(serde_json::json!(id)); + } + "capture_recording_stop" | "capture_mic_stop" => { + context + .audio_capture + .stop(owner, id, op == "capture_recording_stop")?; + } + "capture_read" => { + return context + .audio_capture + .read(owner, id.ok_or("capture id is required")?); + } + "capture_unload" => context + .audio_capture + .unload(owner, id.ok_or("capture id is required")?)?, + _ => return Err("unknown capture operation".into()), + } + return Ok(serde_json::Value::Null); + } if op.starts_with("audio_") { let mut context = self.gameplay.lock().map_err(|_| "gameplay lock poisoned")?; let owner = Owner { @@ -1447,6 +1522,9 @@ impl GameplayHost for RuntimeHost { context .audio .cleanup(|owner| owner.script == self.owner_script); + context + .audio_capture + .cleanup(|owner| owner.script == self.owner_script); } } fn set_current_camera(&mut self, source: &str) -> Result<(), String> { @@ -2025,6 +2103,52 @@ mod tests { }; use engine_world::{EntitySnapshot, SceneDocument, ScriptComponent}; + #[test] + fn lua_and_luau_recording_handles_retain_samples_and_save_wav() { + for language in [ScriptLanguage::Lua54, ScriptLanguage::Luau] { + let temp = tempfile::tempdir().unwrap(); + let world = Arc::new(Mutex::new(SceneWorld::new())); + let mut host = environment_host(&world); + host.snapshot_root = temp.path().into(); + let context = Arc::clone(&host.gameplay); + let mut behavior = RuntimeBehavior::load(language, ScriptId::new(), br" + local rec, file + return { + Start=function() + assert(not pcall(function() rustic.game:startRecording('bad.mp3') end)) + assert(not pcall(function() rustic.game:startRecording(nil, {source='bad'}) end)) + rec=rustic.game:startRecording() + file=rustic.game:startRecording('capture.wav') + end, + Update=function() + assert(rec:stopRecording()==rec) + rustic.game:stopRecording(file) + local packet=rec:read() + assert(packet.sampleRate==48000 and packet.channels==2 and not packet.active) + assert(#packet.samples==4 and packet.samples[1]==0.25 and packet.samples[4]==-0.5) + assert(#rec:read().samples==0) + rec:stopRecording() + rec:unload() + assert(not pcall(function() rec:read() end)) + file:unload() + end + } + ", "capture.lua", Box::new(host)).unwrap(); + behavior.on_start().unwrap(); + context + .lock() + .unwrap() + .audio_capture + .submit(&[0.25, -0.25, 0.5, -0.5]); + behavior.update(0.01).unwrap(); + let mut wav = hound::WavReader::open(temp.path().join("capture.wav")).unwrap(); + assert_eq!( + wav.samples::().collect::, _>>().unwrap(), + vec![0.25, -0.25, 0.5, -0.5] + ); + } + } + #[test] fn compressed_audio_formats_load_play_and_release_from_lua() { let temp = tempfile::tempdir().unwrap(); diff --git a/Engine/crates/engine-scripting/src/sdk/gameplay.lua b/Engine/crates/engine-scripting/src/sdk/gameplay.lua index 2d268954..8dd22ef0 100644 --- a/Engine/crates/engine-scripting/src/sdk/gameplay.lua +++ b/Engine/crates/engine-scripting/src/sdk/gameplay.lua @@ -300,3 +300,42 @@ Camera.fov=Camera.zoom function Animation.value(keys,fn) return start(rustic.query({op="keyframes",keys=keys,token=callback(fn,true)})) end function Animation.addMarker(o,clip,time,name) return rustic.query({op="animation_marker",entity=entity(o),clip=clip,time=time,name=name}) end + +-- Capture handles retain recorded PCM until unload or owner cleanup. +local function captureHandle(id, recording) + local handle={id=id} + function handle:read() return rustic.query({op='capture_read',id=id}) end + function handle:unload() rustic.query({op='capture_unload',id=id}) end + local function stop() + rustic.query({op=recording and 'capture_recording_stop' or 'capture_mic_stop',id=id}) + return handle + end + handle.stop=stop + if recording then + handle.stopRecording=stop; handle.stop_recording=stop + else + handle.stopHotMic=stop; handle.stop_hot_mic=stop + end + return handle +end +function rustic.game:startRecording(path, options) + assert(path==nil or type(path)=='string','recording path must be a string or nil') + options=options or {} + assert(type(options)=='table','recording options must be a table') + return captureHandle(rustic.query({op='capture_recording_start',path=path,source=options.source or 'game'}),true) +end +function rustic.game:stopRecording(recording) + rustic.query({op='capture_recording_stop',id=recording and recording.id}) + return recording +end +function rustic.game:startHotMic() + return captureHandle(rustic.query({op='capture_mic_start'}),false) +end +function rustic.game:stopHotMic(mic) + rustic.query({op='capture_mic_stop',id=mic and mic.id}) + return mic +end +rustic.game.start_recording=rustic.game.startRecording +rustic.game.stop_recording=rustic.game.stopRecording +rustic.game.start_hot_mic=rustic.game.startHotMic +rustic.game.stop_hot_mic=rustic.game.stopHotMic diff --git a/Engine/docs/Scripting/API/audio.md b/Engine/docs/Scripting/API/audio.md index 702b2ef6..a30f3791 100644 --- a/Engine/docs/Scripting/API/audio.md +++ b/Engine/docs/Scripting/API/audio.md @@ -136,3 +136,105 @@ globalThis.behavior = { - Voices are owner-bound, with at most 1024 retained voices. - Check script attachment, enabled state and the Console when an operation fails. Asynchronous errors also appear in operation state. Callback exceptions disable the owner and clean up its actions. - Owner destruction, disable, successful reload or Stop cleans up owned actions and subscriptions. Failed reload retains the previous behavior's operations. See [Operation handles](/docs/api/operation-handles) for controls and lifetime details. + +## Record audio and read a hot microphone (Lua and Luau) + +Create a Lua or Luau Object Component script in `scripts/`, attach it to an +**enabled** scene object, and press **Play**. Start capture inside `Start` or +another callback. Capture methods use camel case and colon syntax: + +```lua +local rec +return { + Start = function() + rec = rustic.game:startRecording() + end, + Stop = function() + rec:stopRecording() + local packet = rec:read() + -- packet.samples contains interleaved floating-point PCM. + -- Keep reading until #packet.samples == 0 to consume the recording. + print(packet.sampleRate, packet.channels, #packet.samples) + rec:unload() + end +} +``` + +`rustic.game:startRecording(path?, options?)` returns a script-owned recording +handle. With no path it retains audio in memory, so the variable remains usable +after `:stopRecording()`. The default source is game mixer output: 48,000 Hz, +stereo, interleaved float PCM. This records audio, not video. Scene clock scaling +and paused playback affect the game audio being recorded. + +To write a WAV file as well, pass a new `.wav` filename: + +```lua +local rec = rustic.game:startRecording("C:/Users/YourName/Recordings/session.wav") +-- Later, inside a callback: +rec:stopRecording() -- finalizes the WAV header; returns rec +``` + +Use an absolute path to keep the file outside the temporary play snapshot. +Relative paths resolve inside the runtime snapshot directory. Parent directories +must already exist. Capture creates a new file and refuses to overwrite an +existing file. The output is a 32-bit floating-point WAV, retaining the same PCM +in the handle. A missing directory, invalid extension or unwritable destination +raises a script error at start; catch expected failures with `pcall`. + +On Windows, microphone recording uses the default input device: + +```lua +local rec = rustic.game:startRecording(nil, {source = "microphone"}) +-- Or pass a .wav path as the first argument to also write it to disk. +rec:stopRecording() +``` + +Hot mic capture exposes local input immediately, without retaining a full recording: + +```lua +local mic +return { + Start = function() + mic = rustic.game:startHotMic() + end, + Update = function() + local packet = mic:read() + local peak = 0 + for _, sample in ipairs(packet.samples) do + peak = math.max(peak, math.abs(sample)) + end + -- Feed packet.samples, packet.sampleRate and packet.channels to your + -- audio processing or relay code. peak can drive a microphone meter. + end, + Stop = function() + mic:stopHotMic() + mic:unload() + end +} +``` + +- `rec:read()` and `mic:read()` return `{samples, sampleRate, channels, active, + droppedSamples}`. Each call consumes at most 4,096 samples, preserving complete + channel frames. An empty sample list means no queued data is available. +- Samples are interleaved by channel. Microphone rate/channel count come from the + default device; input is converted to finite float samples in `[-1, 1]`. +- Hot mic queues at most two seconds of input. Read regularly, using multiple + reads if needed. `droppedSamples` reports input discarded since the preceding + read. Stopping hot mic closes the device and discards unread input. +- `rustic.game:stopRecording(rec)` and `rustic.game:stopHotMic(mic)` stop one + handle. Omit the handle to stop all matching captures owned by the script. + Handles also support `:stop()`; stopping repeatedly is safe. +- Recordings retain at most 64 MiB of PCM per handle (about 175 seconds of game + audio), including recordings written to files. Exceeding the limit stops + capture; `read`/`stopRecording` report the error. Split long recordings into + separate handles and call `:unload()` after consuming them. A script may retain + at most four capture handles at once. +- Destroying an owner, ending Play or successful reload closes its microphone, + finalizes its WAV files and releases its capture handles. Other scripts cannot + use those handles. Explicit stop callbacks let you read samples before cleanup. +- Microphone capture currently requires Windows, an available default microphone, + and OS microphone access. Default F32, I16 and U16 device formats are supported. + Missing devices, permission failures, unsupported formats, startup timeouts and + device disconnection raise errors. Game mixer recording works headlessly too. +- These convenience bindings are available in Lua 5.4 and Luau. Hot mic supplies + local PCM; automatic network transport and speaker monitoring are not provided. From 2b3dc3c82eea581f2e2aa8f41f4703e20c561f75 Mon Sep 17 00:00:00 2001 From: Jamie Wells Date: Sun, 11 Oct 2026 08:19:35 +0000 Subject: [PATCH 2/2] Expose bundled Luau host to engine CI tests --- .github/workflows/quality.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index d14a615b..76816ddd 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -55,7 +55,13 @@ jobs: cargo clippy --locked --manifest-path apps/luau-host/Cargo.toml --all-targets -- -D warnings - name: Build bundled Luau host if: ${{ !cancelled() }} - run: cargo build --locked --manifest-path apps/luau-host/Cargo.toml --target-dir target + run: | + cargo build --locked --manifest-path apps/luau-host/Cargo.toml --target-dir target + if [[ "$RUNNER_OS" == "Windows" ]]; then + cygpath -w "$PWD/target/debug" >> "$GITHUB_PATH" + else + echo "$PWD/target/debug" >> "$GITHUB_PATH" + fi - name: Initialize .NET before parallel adapter tests if: ${{ !cancelled() }} run: | @@ -66,6 +72,7 @@ jobs: - name: Unit, integration, and documentation tests if: ${{ !cancelled() }} run: | + rustic-luau-host --version cargo test --locked --workspace cargo test --locked --manifest-path apps/luau-host/Cargo.toml - name: Verify rendered lighting and viewport