Publish harness and TUI open-source

initial sync from the monorepo
This commit is contained in:
grokkybara[bot] 2026-07-16 06:46:02 +01:00
commit c68e39f604
2734 changed files with 1437016 additions and 0 deletions

View file

@ -0,0 +1,53 @@
[package]
license = "Apache-2.0"
name = "xai-grok-voice"
version = "0.1.0"
edition.workspace = true
description = "Voice dictation (streaming STT) for Grok Build CLI"
[dependencies]
anyhow = { workspace = true }
futures-util = { workspace = true }
# Used by the standalone `voice-probe` bin to install the process-level crypto
# provider (`ring`, matching the pager's choice in its own main). The lib itself
# only reaches rustls through tokio-tungstenite.
rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread"] }
tokio-tungstenite = { workspace = true, features = ["rustls-tls-webpki-roots"] }
toml = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
url = { workspace = true }
[features]
# `audio` = "microphone capture is compiled in". It's enabled on every OS for
# production builds: macOS/Windows link `cpal`; Linux shells out to a system
# recorder (see `src/audio/capture_linux.rs`) and links no audio library, so
# `cpal` -> `alsa-sys` never reaches the fully-static musl release binary.
# `cpal` is target-gated to non-Linux below, so enabling `audio` on Linux pulls
# no extra dependency — it only compiles the subprocess backend.
default = ["audio"]
# Alternate default feature set that opts out of audio. CI builds the voice
# crate WITHOUT `audio` so the pager binary has no mic capture in the test
# sandbox (it has no audio device), which is what the pager's local PTY e2e
# test needs.
default-bazel = []
audio = ["dep:cpal"]
# cpal is only used by the non-Linux capture backend. Gating it to non-Linux
# keeps `alsa-sys` (a NEEDED `libasound.so.2`) out of the static musl build; on
# Linux the `audio` feature compiles the subprocess recorder backend instead.
[target.'cfg(not(target_os = "linux"))'.dependencies.cpal]
version = "0.15"
optional = true
[[bin]]
name = "voice-probe"
path = "src/bin/voice_probe.rs"
required-features = ["audio"]
[lints]
workspace = true

View file

@ -0,0 +1,464 @@
//! PCM16 mono capture via cpal for streaming STT.
//!
//! Prefers a device input config that natively supports the target rate (16 kHz)
//! so no resampling is needed; otherwise it uses the device default (typically
//! 48 kHz stereo F32 on macOS) and downmixes + resamples to 16 kHz mono for the
//! STT API. cpal streams are not `Send` on all platforms; capture runs on a
//! dedicated std thread and forwards PCM chunks through a sync channel.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc::TrySendError;
use std::thread::{self, JoinHandle};
use std::time::Duration;
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{FromSample, Sample, SampleFormat, SizedSample};
use tokio::sync::mpsc as async_mpsc;
use crate::error::VoiceError;
/// Stop handle for the cpal input stream (owned by a background thread).
pub struct CaptureHandle {
stop: Arc<AtomicBool>,
thread: Option<JoinHandle<()>>,
bridge: tokio::task::JoinHandle<()>,
}
impl CaptureHandle {
/// Stop capture and wait for the thread to exit.
///
/// Dropping a `CaptureHandle` also stops capture (see the `Drop` impl), but
/// without joining; call `stop()` when you need to be sure the device is
/// released before continuing.
pub fn stop(mut self) {
self.stop.store(true, Ordering::Release);
if let Some(thread) = self.thread.take() {
let _ = thread.join();
}
// `Drop` runs next and aborts the bridge task.
}
}
impl Drop for CaptureHandle {
fn drop(&mut self) {
// Always signal the capture thread to exit so the mic is released even
// when `stop()` was never called — e.g. the STT session ended on its
// own (server close / error) or the pipeline shut down mid-utterance.
// The thread observes the flag within one poll interval and exits,
// dropping the cpal stream. We deliberately do not join here so `Drop`
// never blocks (it may run on an async executor).
self.stop.store(true, Ordering::Release);
self.bridge.abort();
}
}
/// Spawn cpal capture; PCM16 LE chunks are forwarded to `pcm_tx`.
pub fn spawn_pcm_capture(
sample_rate: u32,
pcm_tx: async_mpsc::Sender<Vec<u8>>,
) -> Result<CaptureHandle, VoiceError> {
let (sync_tx, sync_rx) = std::sync::mpsc::sync_channel::<Vec<u8>>(64);
// Bridge the cpal callback's std sync channel to the async STT sender. The
// `recv()` blocks between audio chunks for the whole session, so it runs on
// the blocking pool (via `spawn_blocking` + `blocking_send`) instead of a
// core runtime worker — parking a worker here would shrink executor capacity
// under pager load. The loop exits on its own when capture stops (sync_tx is
// dropped) or the STT consumer goes away (`blocking_send` errors), so the
// `abort()` in `CaptureHandle`'s teardown is only a backstop.
let bridge = tokio::task::spawn_blocking(move || {
while let Ok(bytes) = sync_rx.recv() {
if pcm_tx.blocking_send(bytes).is_err() {
break;
}
}
});
let stop = Arc::new(AtomicBool::new(false));
let stop_flag = Arc::clone(&stop);
let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel::<Result<(), VoiceError>>(1);
let thread = thread::spawn(move || {
run_capture_loop(sample_rate, sync_tx, stop_flag, ready_tx);
});
// Wait briefly for the device to actually open (mirrors the STT
// `wait_ready` handshake) so device/permission failures propagate to the
// caller — and on to a `VoiceEvent::Error` toast — instead of leaving the
// session "listening" with no audio.
match ready_rx.recv_timeout(Duration::from_secs(5)) {
Ok(Ok(())) => {}
Ok(Err(e)) => {
let _ = thread.join();
bridge.abort();
return Err(e);
}
Err(_) => {
stop.store(true, Ordering::Release);
let _ = thread.join();
bridge.abort();
return Err(VoiceError::Config(
"voice capture did not start within 5s".into(),
));
}
}
Ok(CaptureHandle {
stop,
thread: Some(thread),
bridge,
})
}
/// Record mono PCM16 LE for a fixed duration (probe / diagnostics).
pub fn capture_pcm_for_duration(
sample_rate: u32,
seconds: u32,
) -> Result<(Vec<u8>, u32), VoiceError> {
let (sync_tx, sync_rx) = std::sync::mpsc::sync_channel::<Vec<u8>>(256);
let stop = Arc::new(AtomicBool::new(false));
let stop_flag = Arc::clone(&stop);
let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel::<Result<(), VoiceError>>(1);
let thread = thread::spawn(move || {
run_capture_loop(sample_rate, sync_tx, stop_flag, ready_tx);
});
// Surface device-open failures before recording instead of returning empty.
match ready_rx.recv_timeout(Duration::from_secs(2)) {
Ok(Ok(())) => {}
Ok(Err(e)) => {
let _ = thread.join();
return Err(e);
}
Err(_) => {
stop.store(true, Ordering::Release);
let _ = thread.join();
return Err(VoiceError::Config(
"voice capture did not start within 2s".into(),
));
}
}
thread::sleep(Duration::from_secs(seconds.max(1) as u64));
stop.store(true, Ordering::Release);
let _ = thread.join();
let mut pcm = Vec::new();
let mut chunks = 0u32;
while let Ok(chunk) = sync_rx.try_recv() {
chunks += 1;
pcm.extend_from_slice(&chunk);
}
Ok((pcm, chunks))
}
struct CaptureStreamParams<'a> {
device: &'a cpal::Device,
stream_config: cpal::StreamConfig,
in_channels: u16,
stream_rate: u32,
target_rate: u32,
sync_tx: std::sync::mpsc::SyncSender<Vec<u8>>,
stop: Arc<AtomicBool>,
/// Count of PCM chunks dropped because the channel was full. Logged off the
/// audio thread by `run_capture_loop`.
dropped: Arc<AtomicUsize>,
}
fn run_capture_loop(
sample_rate: u32,
sync_tx: std::sync::mpsc::SyncSender<Vec<u8>>,
stop: Arc<AtomicBool>,
ready_tx: std::sync::mpsc::SyncSender<Result<(), VoiceError>>,
) {
let dropped = Arc::new(AtomicUsize::new(0));
// Open the device first and report success/failure to the caller (the
// capture-side equivalent of the STT `wait_ready` handshake) so that
// device/permission errors surface as a `VoiceError` instead of being
// logged silently here.
let (stream, device_name) = match open_capture_stream(
sample_rate,
sync_tx,
Arc::clone(&stop),
Arc::clone(&dropped),
) {
Ok(v) => {
let _ = ready_tx.send(Ok(()));
v
}
Err(e) => {
tracing::warn!(error = %e, "voice capture failed to start");
let _ = ready_tx.send(Err(e));
return;
}
};
run_capture_poll_loop(stream, stop, dropped, device_name);
}
/// Open the input device, build, and start the cpal capture stream. All
/// device/config/permission failures surface here as a `VoiceError` so the
/// caller can report them before entering the steady-state loop.
fn open_capture_stream(
sample_rate: u32,
sync_tx: std::sync::mpsc::SyncSender<Vec<u8>>,
stop: Arc<AtomicBool>,
dropped: Arc<AtomicUsize>,
) -> Result<(cpal::Stream, String), VoiceError> {
let host = cpal::default_host();
let device = host
.default_input_device()
.ok_or_else(|| VoiceError::Config("no default input audio device".into()))?;
let device_name = device.name().unwrap_or_else(|_| "<unknown>".to_string());
let default_config = device.default_input_config().map_err(|e| {
VoiceError::Config(format!(
"default input config for {device_name}: {e} (grant mic permission in System Settings)"
))
})?;
// Prefer a device-native `sample_rate` (e.g. a mic that supports 16 kHz
// directly) so we can skip resampling entirely; fall back to the device
// default and the linear resampler when no native config matches.
let supported = native_rate_config(&device, sample_rate).unwrap_or(default_config);
let stream_rate = supported.sample_rate().0;
let in_channels = supported.channels();
let sample_format = supported.sample_format();
let stream_config: cpal::StreamConfig = supported.into();
tracing::info!(
device = %device_name,
stream_rate,
channels = in_channels,
?sample_format,
target_rate = sample_rate,
"voice capture stream"
);
let params = CaptureStreamParams {
device: &device,
stream_config,
in_channels,
stream_rate,
target_rate: sample_rate,
sync_tx,
stop,
dropped,
};
let stream = match sample_format {
SampleFormat::F32 => build_capture_stream::<f32>(params)?,
SampleFormat::F64 => build_capture_stream::<f64>(params)?,
SampleFormat::I8 => build_capture_stream::<i8>(params)?,
SampleFormat::I16 => build_capture_stream::<i16>(params)?,
SampleFormat::I32 => build_capture_stream::<i32>(params)?,
SampleFormat::I64 => build_capture_stream::<i64>(params)?,
SampleFormat::U8 => build_capture_stream::<u8>(params)?,
SampleFormat::U16 => build_capture_stream::<u16>(params)?,
SampleFormat::U32 => build_capture_stream::<u32>(params)?,
SampleFormat::U64 => build_capture_stream::<u64>(params)?,
other => {
return Err(VoiceError::Config(format!(
"unsupported input sample format {other:?} on {device_name} \
(supported: f32/f64/i8/i16/i32/i64/u8/u16/u32/u64)"
)));
}
};
stream
.play()
.map_err(|e| VoiceError::Config(format!("play input stream: {e}")))?;
Ok((stream, device_name))
}
/// Steady-state loop: wait for shutdown and report dropped frames off the
/// real-time audio thread (logging here keeps the callback allocation/lock-free).
fn run_capture_poll_loop(
stream: cpal::Stream,
stop: Arc<AtomicBool>,
dropped: Arc<AtomicUsize>,
device_name: String,
) {
let mut last_reported = 0usize;
let mut ticks: u32 = 0;
while !stop.load(Ordering::Acquire) {
thread::sleep(Duration::from_millis(50));
ticks += 1;
// ~once per second
if ticks.is_multiple_of(20) {
let total = dropped.load(Ordering::Relaxed);
if total > last_reported {
tracing::warn!(
device = %device_name,
dropped_total = total,
dropped_since_last = total - last_reported,
"voice capture dropping PCM chunks (consumer not keeping up)"
);
last_reported = total;
}
}
}
drop(stream);
let total = dropped.load(Ordering::Relaxed);
if total > 0 {
tracing::warn!(
device = %device_name,
dropped_total = total,
"voice capture finished with dropped PCM chunks"
);
}
}
/// Find a supported input config whose range includes `target_rate`, so capture
/// runs at the STT rate with no resampling. Prefers a config matching the
/// device's default sample format; returns `None` when nothing matches.
fn native_rate_config(
device: &cpal::Device,
target_rate: u32,
) -> Option<cpal::SupportedStreamConfig> {
let target = cpal::SampleRate(target_rate);
let preferred_format = device
.default_input_config()
.ok()
.map(|c| c.sample_format());
let configs: Vec<_> = device.supported_input_configs().ok()?.collect();
let contains_target = |c: &cpal::SupportedStreamConfigRange| {
c.min_sample_rate() <= target && target <= c.max_sample_rate()
};
configs
.iter()
.find(|c| contains_target(c) && Some(c.sample_format()) == preferred_format)
.or_else(|| configs.iter().find(|c| contains_target(c)))
.map(|c| c.with_sample_rate(target))
}
fn build_capture_stream<T>(params: CaptureStreamParams<'_>) -> Result<cpal::Stream, VoiceError>
where
T: Sample + SizedSample,
i16: FromSample<T>,
{
let CaptureStreamParams {
device,
stream_config,
in_channels,
stream_rate,
target_rate,
sync_tx,
stop,
dropped,
} = params;
let channels = in_channels as usize;
let stop_cb = Arc::clone(&stop);
let stream = device
.build_input_stream(
&stream_config,
move |data: &[T], _: &cpal::InputCallbackInfo| {
if stop_cb.load(Ordering::Acquire) {
return;
}
let mono = frames_to_mono_i16(data, channels);
let pcm = if stream_rate == target_rate {
mono
} else {
resample_mono_i16(&mono, stream_rate, target_rate)
};
if pcm.is_empty() {
return;
}
let bytes: Vec<u8> = pcm.iter().flat_map(|s| s.to_le_bytes()).collect();
// Never block the real-time audio thread: shed load if the
// consumer is behind. Dropped chunks are counted and logged by
// `run_capture_loop`.
match sync_tx.try_send(bytes) {
Ok(()) => {}
Err(TrySendError::Full(_)) => {
dropped.fetch_add(1, Ordering::Relaxed);
}
Err(TrySendError::Disconnected(_)) => {}
}
},
|err| {
tracing::warn!(error = %err, "voice capture stream error");
},
None,
)
.map_err(|e| VoiceError::Config(format!("build input stream: {e}")))?;
Ok(stream)
}
fn frames_to_mono_i16<T>(data: &[T], channels: usize) -> Vec<i16>
where
T: Sample,
i16: FromSample<T>,
{
if channels == 0 {
return Vec::new();
}
if channels == 1 {
return data.iter().map(|s| i16::from_sample(*s)).collect();
}
let mut mono = Vec::with_capacity(data.len() / channels);
for frame in data.chunks_exact(channels) {
let mut sum: i32 = 0;
for sample in frame {
sum += i16::from_sample(*sample) as i32;
}
let avg = (sum / channels as i32).clamp(i16::MIN as i32, i16::MAX as i32);
mono.push(avg as i16);
}
mono
}
fn resample_mono_i16(samples: &[i16], input_rate: u32, output_rate: u32) -> Vec<i16> {
if samples.is_empty() || input_rate == 0 || output_rate == 0 {
return Vec::new();
}
if input_rate == output_rate {
return samples.to_vec();
}
let output_len =
((samples.len() as u64 * output_rate as u64) / input_rate as u64).max(1) as usize;
let step = input_rate as f64 / output_rate as f64;
let mut output = Vec::with_capacity(output_len);
for i in 0..output_len {
let src_pos = i as f64 * step;
let idx = src_pos.floor() as usize;
let frac = src_pos - idx as f64;
let s0 = samples[idx] as f64;
let s1 = *samples.get(idx + 1).unwrap_or(&samples[idx]) as f64;
let sample = s0 + (s1 - s0) * frac;
let clamped = sample.round().max(i16::MIN as f64).min(i16::MAX as f64);
output.push(clamped as i16);
}
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resample_halves_rate() {
let input: Vec<i16> = (0..48).map(|i| (i * 100) as i16).collect();
let out = resample_mono_i16(&input, 48_000, 16_000);
assert_eq!(out.len(), 16);
}
#[test]
fn downmix_stereo_to_mono() {
let stereo = [i16::MAX, i16::MIN];
let mono = frames_to_mono_i16(&stereo, 2);
assert_eq!(mono.len(), 1);
assert_eq!(mono[0], 0);
}
}

View file

@ -0,0 +1,431 @@
//! Microphone capture on Linux via a subprocess recorder.
//!
//! The release CLI ships as a fully-static `*-unknown-linux-musl` binary, so it
//! cannot link `cpal` -> `alsa-sys` (a `NEEDED libasound.so.2`) without losing
//! the static guarantee enforced by the release build. Statically linking ALSA
//! is no help either: it reaches the user's real device (PulseAudio/PipeWire)
//! through plugins it loads via `dlopen`, which a static musl binary can't do.
//!
//! Instead, capture mic audio by spawning the system recorder (`pw-record`,
//! `parec`, or `arecord`) and reading raw PCM16 mono from its stdout — no native
//! audio library is linked into the binary at all. The recorders are asked for
//! signed 16-bit little-endian mono at the STT sample rate, which is exactly the
//! format the pipeline forwards, so there is no downmix/resample step.
//!
//! This module exposes the same interface as the `cpal` backend
//! (`spawn_pcm_capture`, `capture_pcm_for_duration`, `CaptureHandle`) so the
//! pipeline and probe are backend-agnostic.
use std::io::Read;
use std::process::{Child, ChildStdout, Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use tokio::sync::mpsc as async_mpsc;
use crate::error::VoiceError;
/// PCM read size from the recorder's stdout (bytes) — ~64 ms at 16 kHz mono
/// PCM16. Small enough to stream responsively, large enough to avoid syscall
/// churn on the reader thread.
const READ_CHUNK: usize = 2048;
/// How long to wait after spawning before deciding the recorder started cleanly.
/// A missing device or a stopped audio server makes the recorder exit within a
/// few ms; this surfaces that as an error instead of a session that "listens"
/// but never produces audio (mirrors the `cpal` backend's open handshake).
const START_GRACE: Duration = Duration::from_millis(300);
/// A system audio recorder that can stream raw PCM16 mono to stdout.
#[derive(Clone, Copy)]
enum Recorder {
/// PipeWire's `pw-record`.
PwRecord,
/// PulseAudio's `parec`.
Parec,
/// ALSA's `arecord` (alsa-utils).
Arecord,
}
impl Recorder {
fn program(self) -> &'static str {
match self {
Recorder::PwRecord => "pw-record",
Recorder::Parec => "parec",
Recorder::Arecord => "arecord",
}
}
/// Args that emit signed 16-bit little-endian mono PCM at `rate` Hz to
/// stdout. (`pw-record`/`pw-cat` and `arecord` take an explicit `-` stdout
/// target; `parec` writes raw to stdout by default.)
fn args(self, rate: u32) -> Vec<String> {
let rate = rate.to_string();
match self {
Recorder::PwRecord => vec![
"--rate".into(),
rate,
"--channels".into(),
"1".into(),
"--format".into(),
"s16".into(),
"-".into(),
],
Recorder::Parec => vec![
"--raw".into(),
"--format=s16le".into(),
format!("--rate={rate}"),
"--channels=1".into(),
],
Recorder::Arecord => vec![
"-q".into(),
"-t".into(),
"raw".into(),
"-f".into(),
"S16_LE".into(),
"-c".into(),
"1".into(),
"-r".into(),
rate,
"-".into(),
],
}
}
}
/// First recorder found on `PATH`, preferring PipeWire > PulseAudio > ALSA so we
/// go through the user's configured audio server (and its default input device)
/// rather than grabbing a raw ALSA `hw:` device.
fn detect_recorder() -> Option<Recorder> {
detect_recorder_with(binary_on_path)
}
/// [`detect_recorder`] with the `PATH` probe injected, so the preference order
/// is unit-testable without process-global `PATH` mutation.
fn detect_recorder_with(available: impl Fn(&str) -> bool) -> Option<Recorder> {
[Recorder::PwRecord, Recorder::Parec, Recorder::Arecord]
.into_iter()
.find(|r| available(r.program()))
}
/// Whether `name` resolves to an executable regular file on any `PATH` entry
/// (so a stray non-executable file can't shadow a working recorder).
fn binary_on_path(name: &str) -> bool {
use std::os::unix::fs::PermissionsExt;
let Some(path) = std::env::var_os("PATH") else {
return false;
};
std::env::split_paths(&path).any(|dir| {
dir.join(name)
.metadata()
.map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0)
.unwrap_or(false)
})
}
/// Spawn the chosen recorder with stdout/stderr piped, and confirm it didn't
/// exit immediately (no device, audio server down). On success the child is
/// running with `stdout` available for reading.
fn spawn_recorder(sample_rate: u32) -> Result<(Recorder, Child), VoiceError> {
let recorder = detect_recorder().ok_or_else(|| {
VoiceError::Config(
"no microphone recorder found on PATH: install pipewire (pw-record), \
pulseaudio-utils (parec), or alsa-utils (arecord)"
.into(),
)
})?;
let mut child = Command::new(recorder.program())
.args(recorder.args(sample_rate))
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| VoiceError::Config(format!("failed to start {}: {e}", recorder.program())))?;
thread::sleep(START_GRACE);
match child.try_wait() {
Ok(Some(status)) => {
let mut stderr = String::new();
if let Some(mut err) = child.stderr.take() {
let _ = err.read_to_string(&mut stderr);
}
let stderr = stderr.trim();
Err(VoiceError::Config(format!(
"{} exited immediately ({status}){}",
recorder.program(),
if stderr.is_empty() {
String::new()
} else {
format!(": {stderr}")
},
)))
}
Ok(None) => Ok((recorder, child)),
Err(e) => Err(VoiceError::Config(format!(
"failed to poll {}: {e}",
recorder.program()
))),
}
}
/// Stop handle for the recorder subprocess (owns the child + reader thread).
pub struct CaptureHandle {
/// `Some` until `stop()` or `Drop` consumes it (kill + reap).
child: Option<Child>,
stop: Arc<AtomicBool>,
reader: Option<JoinHandle<()>>,
}
impl CaptureHandle {
/// Stop capture: kill the recorder, reap it, and join the reader thread so
/// the input device is released before returning.
///
/// Dropping a `CaptureHandle` also kills and reaps the recorder (see
/// `Drop`), but without joining the reader; call `stop()` when you must be
/// sure the device is freed before continuing.
pub fn stop(mut self) {
self.stop.store(true, Ordering::Release);
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
if let Some(reader) = self.reader.take() {
let _ = reader.join();
}
}
}
impl Drop for CaptureHandle {
fn drop(&mut self) {
// Always kill the recorder so the mic is released even when `stop()` was
// never called — e.g. the STT session ended on its own (server close /
// error). Killing closes the child's stdout, so the reader thread's
// blocking `read` returns 0 and it exits. `Drop` must never block (it
// may run on an async executor), so the reap happens on a detached
// thread — without it every drop-path teardown (session supersede, STT
// error, connect failure) would leave a zombie until the pager exits.
self.stop.store(true, Ordering::Release);
if let Some(mut child) = self.child.take() {
let _ = child.kill();
// `Builder::spawn` (not `thread::spawn`) so spawn failure under
// thread exhaustion degrades to kill-without-reap instead of a
// panic — a panicking `Drop` during unwind would abort.
let _ = thread::Builder::new()
.name("voice-capture-reap".into())
.spawn(move || {
let _ = child.wait();
});
}
}
}
/// Spawn subprocess capture; PCM16 LE chunks are forwarded to `pcm_tx`.
pub fn spawn_pcm_capture(
sample_rate: u32,
pcm_tx: async_mpsc::Sender<Vec<u8>>,
) -> Result<CaptureHandle, VoiceError> {
let (recorder, mut child) = spawn_recorder(sample_rate)?;
let stdout = child
.stdout
.take()
.ok_or_else(|| VoiceError::Config(format!("{} produced no stdout", recorder.program())))?;
drain_stderr(&mut child, recorder.program());
let stop = Arc::new(AtomicBool::new(false));
let stop_reader = Arc::clone(&stop);
let device = recorder.program();
let reader = thread::spawn(move || forward_pcm(stdout, pcm_tx, stop_reader, device));
tracing::info!(
recorder = recorder.program(),
sample_rate,
"voice capture stream (subprocess)"
);
Ok(CaptureHandle {
child: Some(child),
stop,
reader: Some(reader),
})
}
/// Drain the recorder's stderr to EOF on a detached thread so a chatty recorder
/// (xrun/underrun warnings, etc.) can't fill the pipe buffer and block its own
/// writes — which would stall capture, since the hot path never reads stderr.
/// Non-empty output is logged at debug for diagnostics. The thread ends on its
/// own when the child exits (EOF), so it is not joined.
fn drain_stderr(child: &mut Child, device: &'static str) {
let Some(mut stderr) = child.stderr.take() else {
return;
};
thread::spawn(move || {
let mut buf = String::new();
if stderr.read_to_string(&mut buf).is_ok() {
let msg = buf.trim();
if !msg.is_empty() {
tracing::debug!(device, stderr = msg, "voice recorder stderr");
}
}
});
}
/// Forward raw PCM from the recorder's stdout to the async STT sender until the
/// recorder stops (EOF on kill), the consumer goes away, or `stop` is set.
fn forward_pcm(
mut stdout: ChildStdout,
pcm_tx: async_mpsc::Sender<Vec<u8>>,
stop: Arc<AtomicBool>,
device: &'static str,
) {
let mut buf = vec![0u8; READ_CHUNK];
let mut dropped = 0u64;
loop {
if stop.load(Ordering::Acquire) {
break;
}
match stdout.read(&mut buf) {
// EOF: the recorder closed stdout (killed by teardown or exited).
Ok(0) => break,
Ok(n) => {
// Never park this thread on the channel: `stop()` joins it, so a
// send that waits on a stalled STT consumer would turn teardown
// into a hang. Shed load instead when the consumer is behind —
// the same strategy as the cpal backend's real-time callback.
// (`read` itself is unblocked by the kill-on-stop path: killing
// the recorder closes stdout, so a waiting `read` returns 0.)
match pcm_tx.try_send(buf[..n].to_vec()) {
Ok(()) => {}
Err(async_mpsc::error::TrySendError::Full(_)) => dropped += 1,
// Consumer is gone: the session ended; stop capturing.
Err(async_mpsc::error::TrySendError::Closed(_)) => break,
}
}
Err(e) => {
tracing::warn!(device, error = %e, "voice capture read error");
break;
}
}
}
if dropped > 0 {
tracing::warn!(
device,
dropped,
"voice capture dropped PCM chunks (slow consumer)"
);
}
}
/// Record mono PCM16 LE for a fixed duration (probe / diagnostics).
pub fn capture_pcm_for_duration(
sample_rate: u32,
seconds: u32,
) -> Result<(Vec<u8>, u32), VoiceError> {
let (recorder, mut child) = spawn_recorder(sample_rate)?;
let mut stdout = child
.stdout
.take()
.ok_or_else(|| VoiceError::Config(format!("{} produced no stdout", recorder.program())))?;
drain_stderr(&mut child, recorder.program());
let duration = Duration::from_secs(seconds.max(1) as u64);
let deadline = Instant::now() + duration;
// Watchdog: kill the recorder at the deadline so a `read` that is blocked
// waiting for PCM (recorder alive but idle / stalled pipe) gets EOF instead
// of running past the requested duration. Killing at the deadline also ends
// a healthy capture, so the read loop below needs no between-read deadline
// check beyond its backstop.
// Deliberately not joined: if the recorder dies early we return without
// waiting out the full duration, and the watchdog's late `kill` on an
// already-reaped `Child` is a harmless `InvalidInput` (std tracks the reap,
// so no PID-reuse hazard).
let child = Arc::new(Mutex::new(child));
let watchdog_child = Arc::clone(&child);
thread::spawn(move || {
thread::sleep(duration);
let mut child = watchdog_child.lock().expect("watchdog lock poisoned");
let _ = child.kill();
});
let mut pcm = Vec::new();
let mut chunks = 0u32;
let mut buf = vec![0u8; READ_CHUNK];
// Small slack past the deadline: the kill's EOF (`Ok(0)`) is the intended
// exit; the time check is a backstop against a pathological pipe.
while Instant::now() < deadline + Duration::from_secs(1) {
match stdout.read(&mut buf) {
Ok(0) => break,
Ok(n) => {
chunks += 1;
pcm.extend_from_slice(&buf[..n]);
}
Err(_) => break,
}
}
{
let mut child = child.lock().expect("child lock poisoned");
let _ = child.kill();
let _ = child.wait();
}
Ok((pcm, chunks))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn arecord_args_are_raw_s16_mono() {
let args = Recorder::Arecord.args(16_000);
assert!(args.contains(&"S16_LE".to_string()));
assert!(args.contains(&"raw".to_string()));
// mono
let c = args.iter().position(|a| a == "-c").unwrap();
assert_eq!(args[c + 1], "1");
// rate
let r = args.iter().position(|a| a == "-r").unwrap();
assert_eq!(args[r + 1], "16000");
// stdout target
assert_eq!(args.last().unwrap(), "-");
}
#[test]
fn parec_and_pw_args_carry_rate_format_and_mono() {
let parec = Recorder::Parec.args(24_000);
assert!(parec.contains(&"--raw".to_string()));
assert!(parec.contains(&"--format=s16le".to_string()));
assert!(parec.contains(&"--rate=24000".to_string()));
assert!(parec.contains(&"--channels=1".to_string()));
let pw = Recorder::PwRecord.args(48_000);
let r = pw.iter().position(|a| a == "--rate").unwrap();
assert_eq!(pw[r + 1], "48000");
let f = pw.iter().position(|a| a == "--format").unwrap();
assert_eq!(pw[f + 1], "s16");
let c = pw.iter().position(|a| a == "--channels").unwrap();
assert_eq!(pw[c + 1], "1");
assert_eq!(pw.last().unwrap(), "-"); // stdout target
}
#[test]
fn recorder_preference_is_pipewire_then_pulse_then_alsa() {
// All present: PipeWire wins (routes through the user's audio server).
let all = detect_recorder_with(|_| true);
assert!(matches!(all, Some(Recorder::PwRecord)));
// No PipeWire: PulseAudio next.
let no_pw = detect_recorder_with(|p| p != "pw-record");
assert!(matches!(no_pw, Some(Recorder::Parec)));
// alsa-utils only: arecord is the last resort.
let alsa_only = detect_recorder_with(|p| p == "arecord");
assert!(matches!(alsa_only, Some(Recorder::Arecord)));
assert!(detect_recorder_with(|_| false).is_none());
}
}

View file

@ -0,0 +1,18 @@
//! Microphone capture (optional `audio` feature).
//!
//! Two backends share one interface (`spawn_pcm_capture`,
//! `capture_pcm_for_duration`, `CaptureHandle`):
//! - non-Linux (macOS/Windows): `cpal` (coreaudio/wasapi), linked into the binary;
//! - Linux: a subprocess recorder (`pw-record`/`parec`/`arecord`), because the
//! static-musl release binary cannot link `cpal` -> `alsa-sys`. See
//! [`capture_linux`] for the full rationale.
#[cfg(not(target_os = "linux"))]
mod capture;
#[cfg(not(target_os = "linux"))]
pub use capture::{CaptureHandle, capture_pcm_for_duration, spawn_pcm_capture};
#[cfg(target_os = "linux")]
mod capture_linux;
#[cfg(target_os = "linux")]
pub use capture_linux::{CaptureHandle, capture_pcm_for_duration, spawn_pcm_capture};

View file

@ -0,0 +1,79 @@
//! Bearer resolution for voice STT requests.
//! The voice clients are long-lived: a single voice session opens many STT
//! WebSocket connections over its lifetime, and an OAuth/session bearer rotates
//! (~15 min). Capturing a token once at startup would 401 mid-session. So
//! instead of a static `String`, the clients hold a [`SharedVoiceAuth`] and
//! resolve a fresh bearer at the point of each connection.
//!
//! This crate stays dependency-light: it defines its own minimal async trait
//! rather than depending on the shell's `AuthManager` / tools' `ApiKeyProvider`.
//! The pager adapts the shell's refreshing provider onto this trait.
use std::future::{Future, ready};
use std::pin::Pin;
use std::sync::Arc;
#[cfg(feature = "audio")]
use crate::error::VoiceError;
pub trait VoiceAuthProvider: std::fmt::Debug + Send + Sync + 'static {
fn bearer(&self) -> Pin<Box<dyn Future<Output = Option<String>> + Send + '_>>;
}
/// Shared provider handed to the voice pipeline.
pub type SharedVoiceAuth = Arc<dyn VoiceAuthProvider>;
#[cfg(feature = "audio")]
pub(crate) async fn require_bearer(auth: &SharedVoiceAuth) -> Result<String, VoiceError> {
auth.bearer().await.ok_or_else(|| {
VoiceError::Auth("not signed in — run `grok login` or set XAI_API_KEY".into())
})
}
/// A fixed bearer that never refreshes.
///
/// Used by the standalone `voice-probe` binary and tests, where there is no
/// `AuthManager` — only a raw `XAI_API_KEY`.
pub struct StaticVoiceAuth(pub String);
impl std::fmt::Debug for StaticVoiceAuth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("StaticVoiceAuth")
.field(&"<redacted>")
.finish()
}
}
impl VoiceAuthProvider for StaticVoiceAuth {
fn bearer(&self) -> Pin<Box<dyn Future<Output = Option<String>> + Send + '_>> {
Box::pin(ready(Some(self.0.clone())))
}
}
impl StaticVoiceAuth {
/// Build a [`SharedVoiceAuth`] from a static key, trimming whitespace and
/// rejecting an empty value.
pub fn shared(key: impl Into<String>) -> Option<SharedVoiceAuth> {
let key = key.into().trim().to_string();
if key.is_empty() {
return None;
}
Some(Arc::new(Self(key)))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn static_provider_resolves() {
let provider = StaticVoiceAuth::shared(" sk-test ").unwrap();
assert_eq!(provider.bearer().await.as_deref(), Some("sk-test"));
}
#[test]
fn static_provider_rejects_empty() {
assert!(StaticVoiceAuth::shared(" ").is_none());
}
}

View file

@ -0,0 +1,157 @@
//! Standalone voice debug harness: mic → streaming STT → transcript.
//!
//! ```bash
//! export XAI_API_KEY=...
//! cargo run -p xai-grok-voice --bin voice-probe -- --seconds 5
//! ```
use std::path::PathBuf;
use xai_grok_voice::{
StaticVoiceAuth, VoiceConfig, VoiceProbeOptions, format_probe_report, run_streaming_probe,
};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// Standalone binary: install the process-level rustls provider (the pager
// does this in its own main), or the first TLS/WSS connect panics with
// "Could not automatically determine the process-level CryptoProvider".
let _ = rustls::crypto::ring::default_provider().install_default();
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "info,xai_grok_voice=debug".into()),
)
.init();
let args = parse_args(std::env::args().skip(1).collect());
let auth = std::env::var("XAI_API_KEY")
.ok()
.and_then(StaticVoiceAuth::shared)
.ok_or_else(|| {
anyhow::anyhow!("set XAI_API_KEY (standalone probe has no login session)")
})?;
let config = load_config(args.config_path.as_deref());
eprintln!(
"Voice probe: listening {}s (sample_rate={})",
args.seconds, config.sample_rate
);
eprintln!("Speak now...\n");
if args.mic_only {
#[cfg(feature = "audio")]
{
let (bytes, chunks) =
xai_grok_voice::run_mic_only_probe(config.sample_rate, args.seconds)?;
println!("Mic-only OK: {bytes} bytes in {chunks} chunks");
if bytes == 0 {
println!("WARNING: no audio — grant mic access to the terminal");
}
return Ok(());
}
#[cfg(not(feature = "audio"))]
anyhow::bail!("built without `audio` feature");
}
let report = run_streaming_probe(VoiceProbeOptions {
config,
auth,
capture_secs: args.seconds,
})
.await?;
print!("{}", format_probe_report(&report));
if report
.transcript
.as_ref()
.is_none_or(|t| t.trim().is_empty())
{
std::process::exit(1);
}
Ok(())
}
struct Args {
seconds: u32,
config_path: Option<PathBuf>,
mic_only: bool,
}
fn parse_args(argv: Vec<String>) -> Args {
let mut out = Args {
seconds: 5,
config_path: None,
mic_only: false,
};
let mut i = 0;
while i < argv.len() {
match argv[i].as_str() {
"--seconds" | "-s" => {
i += 1;
if i < argv.len() {
out.seconds = argv[i].parse().unwrap_or(5);
}
}
"--config" => {
i += 1;
if i < argv.len() {
out.config_path = Some(PathBuf::from(&argv[i]));
}
}
"--mic-only" => out.mic_only = true,
"--help" | "-h" => {
print_help();
std::process::exit(0);
}
other if !other.starts_with('-') => {}
_ => eprintln!("unknown arg: {}", argv[i]),
}
i += 1;
}
out
}
fn load_config(path: Option<&std::path::Path>) -> VoiceConfig {
if let Some(path) = path
&& let Ok(raw) = std::fs::read_to_string(path)
&& let Ok(table) = toml::from_str::<toml::Table>(&raw)
{
return VoiceConfig::from_config_table(&table);
}
if let Ok(home) = std::env::var("GROK_HOME")
&& let Ok(raw) = std::fs::read_to_string(PathBuf::from(home).join("config.toml"))
&& let Ok(table) = toml::from_str::<toml::Table>(&raw)
{
return VoiceConfig::from_config_table(&table);
}
if let Ok(raw) = std::fs::read_to_string(
std::env::var("HOME")
.map(PathBuf::from)
.unwrap_or_default()
.join(".grok/config.toml"),
) && let Ok(table) = toml::from_str::<toml::Table>(&raw)
{
return VoiceConfig::from_config_table(&table);
}
VoiceConfig::default()
}
fn print_help() {
eprintln!(
r#"voice-probe — debug mic + STT outside the pager
Usage:
voice-probe [--seconds 5] [--mic-only]
Environment:
XAI_API_KEY required
RUST_LOG optional (default info,xai_grok_voice=debug)
Reads [voice] from ~/.grok/config.toml unless --config PATH is set.
"#
);
}

View file

@ -0,0 +1,182 @@
use serde::{Deserialize, Serialize};
use crate::error::VoiceError;
/// Voice settings for the STT transport.
///
/// Parsed from optional `[voice]` in config (URL, language, sample rate,
/// endpointing) plus pager-stamped request-identity fields. Availability is
/// owned by the pager (`GROK_VOICE_MODE` / `[features] voice_mode` / remote);
/// this table has no enable/disable knob.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(default)]
pub struct VoiceConfig {
pub api_base: String,
pub stt_ws_path: String,
/// Preferred STT language: a catalog code from [`crate::STT_LANGUAGES`], or
/// the client-only sentinel `"auto"` (system locale). Resolved to a concrete
/// API code via [`crate::language_for_api`] at connect time — never send the
/// raw field when it may be `"auto"`.
pub language: String,
pub sample_rate: u32,
pub stt_endpointing_ms: u32,
pub stt_interim_results: bool,
/// Request-identity headers attached to every STT handshake so the backend
/// can attribute and meter voice usage by client — mirroring the
/// `x-grok-client-identifier` / `User-Agent` headers the sampler and imagine
/// request paths send. These are **runtime identity, not user config**:
/// `#[serde(skip)]` keeps them out of the parsed `[voice]` table (a user
/// can't spoof them) and the pager fills them in after parsing. Empty →
/// the corresponding header is omitted.
///
/// `x-grok-client-identifier` value (e.g. `"grok-shell"`).
#[serde(skip)]
pub client_identifier: String,
/// `User-Agent` value (e.g. `"grok-shell/1.2.3 (macos; aarch64)"`).
#[serde(skip)]
pub user_agent: String,
}
impl Default for VoiceConfig {
fn default() -> Self {
Self {
api_base: "https://api.x.ai".into(),
stt_ws_path: "/v1/stt".into(),
language: "en".into(),
sample_rate: 16_000,
stt_endpointing_ms: 400,
stt_interim_results: true,
client_identifier: String::new(),
user_agent: String::new(),
}
}
}
impl VoiceConfig {
/// Build the streaming-STT WebSocket URL.
///
/// Only TLS endpoints are allowed: an `https://` / `wss://` (or scheme-less)
/// `api_base` maps to `wss://`. An `http://`/`ws://` `api_base` is rejected with a
/// [`VoiceError::Config`] rather than silently downgrading, since the bearer
/// token is sent as a header on this connection and must never traverse a
/// plaintext socket.
pub fn stt_ws_url(&self) -> Result<String, VoiceError> {
ws_url(&self.api_base, &self.stt_ws_path)
}
/// Parse `[voice]` from the root of an effective config document.
pub fn from_config_table(root: &toml::Table) -> Self {
root.get("voice")
.and_then(|v| v.clone().try_into().ok())
.unwrap_or_default()
}
}
fn ws_url(api_base: &str, path: &str) -> Result<String, VoiceError> {
let base = api_base.trim_end_matches('/');
let path = path.trim_start_matches('/');
if base.starts_with("http://") || base.starts_with("ws://") {
return Err(VoiceError::Config(format!(
"insecure voice api_base {api_base:?}: voice requires a TLS endpoint \
(https:// / wss://). Refusing to send the bearer token over a \
plaintext connection."
)));
}
let host = base
.strip_prefix("https://")
.or_else(|| base.strip_prefix("wss://"))
.unwrap_or(base);
Ok(format!("wss://{host}/{path}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_stt_ws_uses_wss() {
let cfg = VoiceConfig::default();
assert_eq!(cfg.stt_ws_url().unwrap(), "wss://api.x.ai/v1/stt");
}
#[test]
fn scheme_less_api_base_uses_wss() {
let cfg = VoiceConfig {
api_base: "api.x.ai".into(),
..VoiceConfig::default()
};
assert_eq!(cfg.stt_ws_url().unwrap(), "wss://api.x.ai/v1/stt");
}
#[test]
fn wss_api_base_is_not_doubled() {
let cfg = VoiceConfig {
api_base: "wss://api.x.ai".into(),
..VoiceConfig::default()
};
assert_eq!(cfg.stt_ws_url().unwrap(), "wss://api.x.ai/v1/stt");
}
#[test]
fn http_api_base_is_rejected_not_downgraded() {
let cfg = VoiceConfig {
api_base: "http://localhost:8080".into(),
..VoiceConfig::default()
};
let err = cfg.stt_ws_url().unwrap_err();
assert!(matches!(err, VoiceError::Config(_)), "got {err:?}");
}
#[test]
fn ws_api_base_is_rejected() {
let cfg = VoiceConfig {
api_base: "ws://localhost:8080".into(),
..VoiceConfig::default()
};
assert!(cfg.stt_ws_url().is_err());
}
/// Legacy / unknown keys — including the removed local `enabled` opt-out —
/// must be ignored without failing the parse (no `deny_unknown_fields`), so
/// old configs still load (the key is now a silent no-op; the pager owns the
/// voice gate — default on, remote kill switch / `GROK_VOICE_MODE`).
#[test]
fn ignores_additional_fields() {
let raw = r#"
[voice]
enabled = false
push_to_talk = true
language = "es"
"#;
let table: toml::Table = toml::from_str(raw).unwrap();
let cfg = VoiceConfig::from_config_table(&table);
// Known fields still apply; unknown/legacy keys are dropped silently.
assert_eq!(cfg.language, "es");
assert_eq!(cfg.sample_rate, 16_000);
}
/// `client_identifier` / `user_agent` are `#[serde(skip)]` runtime identity,
/// not user config: a value placed in `[voice]` must be ignored so a user
/// can't spoof the attribution headers. The pager stamps them after parsing.
#[test]
fn identity_fields_are_not_parsed_from_config() {
let raw = r#"
[voice]
client_identifier = "spoofed"
user_agent = "malicious/9.9"
language = "es"
"#;
let table: toml::Table = toml::from_str(raw).unwrap();
let cfg = VoiceConfig::from_config_table(&table);
assert_eq!(cfg.language, "es", "ordinary fields still parse");
assert!(
cfg.client_identifier.is_empty(),
"client_identifier must not be settable via config"
);
assert!(
cfg.user_agent.is_empty(),
"user_agent must not be settable via config"
);
}
}

View file

@ -0,0 +1,16 @@
use thiserror::Error;
#[derive(Debug, Error)]
pub enum VoiceError {
#[error("configuration: {0}")]
Config(String),
#[error("STT: {0}")]
Stt(String),
#[error("auth: {0}")]
Auth(String),
#[error("WebSocket: {0}")]
WebSocket(String),
}

View file

@ -0,0 +1,12 @@
/// Events emitted by [`crate::pipeline::run_voice_pipeline`] to the pager event loop.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VoiceEvent {
/// Partial transcript while the user is speaking (`interim_results` / non-final chunks).
InterimTranscript { text: String },
/// Utterance complete (`speech_final` on streaming STT, or batch result).
UtteranceFinal { text: String },
/// Non-fatal or fatal error from STT.
Error { message: String },
}

View file

@ -0,0 +1,311 @@
//! Grok Speech-to-Text language codes.
//!
//! Source of truth for the `language` query/form parameter on
//! `https://api.x.ai/v1/stt` and `wss://api.x.ai/v1/stt`.
//!
//! Official catalog (25 languages):
//! <https://docs.x.ai/developers/model-capabilities/audio/speech-to-text#supported-languages>
//!
//! Per the docs, the model can transcribe these languages regardless of the
//! parameter; setting `language` enables Inverse Text Normalization (numbers,
//! currencies, units → written form) for that language. The STT API does **not**
//! accept `auto` (unlike TTS) — clients must send a concrete code. Use
//! [`language_for_api`] to resolve a stored preference (including the client-only
//! `auto` sentinel) before connecting.
/// One supported STT language from the public API catalog.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SttLanguage {
/// ISO / BCP-47 primary code sent as the `language` parameter (e.g. `en`).
pub code: &'static str,
/// English display name for UIs.
pub name: &'static str,
}
/// Client-only sentinel meaning “resolve from the process locale at connect time”.
/// Never send this value to the STT API — use [`language_for_api`].
pub const STT_LANGUAGE_AUTO: &str = "auto";
/// Default STT language when unset or unrecognized.
pub const STT_LANGUAGE_DEFAULT: &str = "en";
/// Official Grok STT languages (docs.x.ai), sorted by English name.
///
/// Keep this list in lockstep with the public docs. Adding a code that the API
/// does not list will not break transcription, but ITN formatting may not apply.
pub const STT_LANGUAGES: &[SttLanguage] = &[
SttLanguage {
code: "ar",
name: "Arabic",
},
SttLanguage {
code: "cs",
name: "Czech",
},
SttLanguage {
code: "da",
name: "Danish",
},
SttLanguage {
code: "nl",
name: "Dutch",
},
SttLanguage {
code: "en",
name: "English",
},
SttLanguage {
code: "fil",
name: "Filipino",
},
SttLanguage {
code: "fr",
name: "French",
},
SttLanguage {
code: "de",
name: "German",
},
SttLanguage {
code: "hi",
name: "Hindi",
},
SttLanguage {
code: "id",
name: "Indonesian",
},
SttLanguage {
code: "it",
name: "Italian",
},
SttLanguage {
code: "ja",
name: "Japanese",
},
SttLanguage {
code: "ko",
name: "Korean",
},
SttLanguage {
code: "mk",
name: "Macedonian",
},
SttLanguage {
code: "ms",
name: "Malay",
},
SttLanguage {
code: "fa",
name: "Persian",
},
SttLanguage {
code: "pl",
name: "Polish",
},
SttLanguage {
code: "pt",
name: "Portuguese",
},
SttLanguage {
code: "ro",
name: "Romanian",
},
SttLanguage {
code: "ru",
name: "Russian",
},
SttLanguage {
code: "es",
name: "Spanish",
},
SttLanguage {
code: "sv",
name: "Swedish",
},
SttLanguage {
code: "th",
name: "Thai",
},
SttLanguage {
code: "tr",
name: "Turkish",
},
SttLanguage {
code: "vi",
name: "Vietnamese",
},
];
/// Look up a catalog entry by exact (case-sensitive) code.
pub fn stt_language_by_code(code: &str) -> Option<&'static SttLanguage> {
STT_LANGUAGES.iter().find(|l| l.code == code)
}
/// Map a user/config string to a catalog code or [`STT_LANGUAGE_AUTO`].
///
/// - `None` / blank / unknown → [`STT_LANGUAGE_DEFAULT`] (`en`)
/// - `auto` (any case) → [`STT_LANGUAGE_AUTO`]
/// - Exact catalog code (any case) → that code
/// - BCP-47 / locale forms (`en-US`, `pt_BR.UTF-8`) → primary subtag when supported
/// - Common aliases: `tl` → `fil` (Tagalog → Filipino)
pub fn canonicalize_stt_language(value: Option<&str>) -> &'static str {
let raw = value.unwrap_or_default().trim();
if raw.is_empty() {
return STT_LANGUAGE_DEFAULT;
}
if raw.eq_ignore_ascii_case(STT_LANGUAGE_AUTO) {
return STT_LANGUAGE_AUTO;
}
if let Some(code) = match_supported_code(raw) {
return code;
}
// Primary subtag of BCP-47 / POSIX locales.
let primary = primary_language_subtag(raw);
if let Some(code) = match_supported_code(primary) {
return code;
}
if let Some(aliased) = alias_to_supported(primary) {
return aliased;
}
STT_LANGUAGE_DEFAULT
}
/// Concrete language code to send on the STT wire.
///
/// Resolves [`STT_LANGUAGE_AUTO`] from the process locale; never returns `auto`.
pub fn language_for_api(stored: &str) -> &'static str {
let canonical = canonicalize_stt_language(Some(stored));
if canonical == STT_LANGUAGE_AUTO {
system_stt_language().unwrap_or(STT_LANGUAGE_DEFAULT)
} else {
canonical
}
}
/// Best-effort system locale → supported STT code (`None` if unset/unsupported).
///
/// POSIX precedence, treating set-but-empty vars as unset (an empty `LC_ALL`
/// must not mask a usable `LANG`).
fn system_stt_language() -> Option<&'static str> {
let loc = ["LC_ALL", "LC_MESSAGES", "LANG"]
.into_iter()
.find_map(|var| std::env::var(var).ok().filter(|v| !v.is_empty()))?;
if loc.eq_ignore_ascii_case("C") || loc.eq_ignore_ascii_case("POSIX") {
return None;
}
let primary = primary_language_subtag(&loc);
match_supported_code(primary).or_else(|| alias_to_supported(primary))
}
fn primary_language_subtag(raw: &str) -> &str {
raw.split(['_', '-', '.']).next().unwrap_or("").trim()
}
fn match_supported_code(raw: &str) -> Option<&'static str> {
STT_LANGUAGES
.iter()
.map(|l| l.code)
.find(|&code| raw.eq_ignore_ascii_case(code))
}
/// Map common non-catalog primaries onto a supported code.
fn alias_to_supported(primary: &str) -> Option<&'static str> {
// Tagalog (`tl`) is the usual system locale; API uses Filipino (`fil`).
if primary.eq_ignore_ascii_case("tl") {
return Some("fil");
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
/// Pins the public docs catalog (25 languages as of docs last-updated May 2026).
const DOCS_CODES: &[&str] = &[
"ar", "cs", "da", "nl", "en", "fil", "fr", "de", "hi", "id", "it", "ja", "ko", "mk", "ms",
"fa", "pl", "pt", "ro", "ru", "es", "sv", "th", "tr", "vi",
];
#[test]
fn catalog_matches_public_docs_exactly() {
let ours: HashSet<&str> = STT_LANGUAGES.iter().map(|l| l.code).collect();
let docs: HashSet<&str> = DOCS_CODES.iter().copied().collect();
assert_eq!(
ours, docs,
"STT_LANGUAGES drifted from docs.x.ai supported languages"
);
}
#[test]
fn catalog_codes_are_unique_and_names_nonempty() {
let mut seen = HashSet::new();
for lang in STT_LANGUAGES {
assert!(
seen.insert(lang.code),
"duplicate STT language code {}",
lang.code
);
assert!(!lang.name.is_empty());
assert!(!lang.code.is_empty());
assert!(
!lang.code.contains('-'),
"use primary codes only: {}",
lang.code
);
}
}
#[test]
fn catalog_sorted_by_english_name() {
let names: Vec<&str> = STT_LANGUAGES.iter().map(|l| l.name).collect();
let mut sorted = names.clone();
sorted.sort_unstable();
assert_eq!(
names, sorted,
"STT_LANGUAGES must stay sorted by English name"
);
}
#[test]
fn canonicalize_known_and_unknown() {
assert_eq!(canonicalize_stt_language(None), "en");
assert_eq!(canonicalize_stt_language(Some("")), "en");
assert_eq!(canonicalize_stt_language(Some(" ")), "en");
assert_eq!(canonicalize_stt_language(Some("en")), "en");
assert_eq!(canonicalize_stt_language(Some("ES")), "es");
assert_eq!(canonicalize_stt_language(Some(" fr ")), "fr");
assert_eq!(canonicalize_stt_language(Some("auto")), "auto");
assert_eq!(canonicalize_stt_language(Some("AUTO")), "auto");
assert_eq!(canonicalize_stt_language(Some("en-US")), "en");
assert_eq!(canonicalize_stt_language(Some("pt_BR.UTF-8")), "pt");
assert_eq!(canonicalize_stt_language(Some("fil")), "fil");
assert_eq!(canonicalize_stt_language(Some("tl")), "fil");
assert_eq!(canonicalize_stt_language(Some("tl-PH")), "fil");
// Chinese is not in the STT formatting catalog.
assert_eq!(canonicalize_stt_language(Some("zh")), "en");
assert_eq!(canonicalize_stt_language(Some("zh-Hans")), "en");
assert_eq!(canonicalize_stt_language(Some("nope")), "en");
}
#[test]
fn language_for_api_never_returns_auto() {
assert_ne!(language_for_api("auto"), "auto");
assert_eq!(language_for_api("ja"), "ja");
assert_eq!(language_for_api("EN"), "en");
assert_eq!(language_for_api(""), "en");
assert_eq!(language_for_api("xx"), "en");
}
#[test]
fn lookup_is_exact_code() {
assert!(stt_language_by_code("en").is_some());
assert!(stt_language_by_code("EN").is_none());
assert!(stt_language_by_code("auto").is_none());
assert!(stt_language_by_code("zh").is_none());
}
}

View file

@ -0,0 +1,39 @@
//! Voice input for Grok Build CLI: an xAI streaming STT client and the
//! [`run_voice_pipeline`] task that emits [`VoiceEvent`]s for the pager.
//!
//! Voice is dictation only: mic → streaming STT → transcript into the prompt box.
#[cfg(feature = "audio")]
pub mod audio;
pub mod auth;
pub mod config;
pub mod error;
pub mod event;
pub mod language;
pub mod pipeline;
pub mod probe;
pub mod stt;
pub use auth::{SharedVoiceAuth, StaticVoiceAuth, VoiceAuthProvider};
pub use config::VoiceConfig;
pub use error::VoiceError;
pub use event::VoiceEvent;
pub use language::{
STT_LANGUAGE_AUTO, STT_LANGUAGE_DEFAULT, STT_LANGUAGES, SttLanguage, canonicalize_stt_language,
language_for_api, stt_language_by_code,
};
pub use pipeline::{VoiceCommand, run_voice_pipeline};
#[cfg(feature = "audio")]
pub use probe::run_mic_only_probe;
pub use probe::{VoiceProbeOptions, VoiceProbeReport, format_probe_report, run_streaming_probe};
/// Whether this build can capture microphone audio (the `audio` feature).
/// Production CLI builds enable it on every OS: macOS/Windows link `cpal`
/// (coreaudio/wasapi), while Linux shells out to a system recorder
/// (`pw-record`/`parec`/`arecord`) so the static-musl binary links no audio
/// library. Bazel builds drop `audio` (no capture in the test sandbox).
///
/// On Linux a `true` value means capture is *compiled in*; whether a recorder
/// is actually installed is reported when a session starts. Consumers gate voice
/// on this so a no-audio build never advertises a mic it can't open.
pub const AUDIO_SUPPORTED: bool = cfg!(feature = "audio");

View file

@ -0,0 +1,407 @@
//! Voice pipeline: mic → streaming STT → pager events.
//!
//! The pager drives capture with press/release commands. They back both a
//! toggle (`/voice`, `Ctrl+Shift+M`) and true push-to-talk (F12 hold) — hence
//! the `Ptt*` names — so a press may be followed by a release after a long hold
//! or, for a toggle, a later stop.
#[cfg(feature = "audio")]
use std::collections::VecDeque;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::auth::SharedVoiceAuth;
use crate::config::VoiceConfig;
use crate::error::VoiceError;
use crate::event::VoiceEvent;
#[cfg(feature = "audio")]
use crate::stt::{StreamingSttEvent, StreamingSttSession};
/// Commands from the pager event loop (toggle start/stop, or F12 push-to-talk).
#[derive(Debug)]
pub enum VoiceCommand {
/// Begin streaming audio to STT (mic open until [`VoiceCommand::PttRelease`]).
PttPress,
/// End the current capture session (`audio.done`, release mic).
PttRelease,
/// Tear down the pipeline task.
Shutdown,
}
struct ActivePtt {
finish_tx: mpsc::Sender<()>,
reader: JoinHandle<()>,
}
/// Run until [`VoiceCommand::Shutdown`].
pub async fn run_voice_pipeline(
config: VoiceConfig,
auth: SharedVoiceAuth,
mut cmd_rx: mpsc::Receiver<VoiceCommand>,
event_tx: mpsc::Sender<VoiceEvent>,
) {
let mut active: Option<ActivePtt> = None;
while let Some(cmd) = cmd_rx.recv().await {
match cmd {
VoiceCommand::Shutdown => break,
VoiceCommand::PttPress => {
// Supersede any prior session — including one still draining its
// trailing final after a `PttRelease` — rather than ignoring the
// press. A rapid stop→start would otherwise be dropped here while
// the pager already flipped to "listening", leaving a dead mic
// behind a recording UI and letting the old session's final land
// on the new target. Aborting drops the old reader's capture +
// STT session, releasing the mic and socket at once. (The pager
// always sends a `PttRelease` between presses, so an `active`
// session here is one that's stopping, never a live duplicate.)
// We don't join the old reader, so its stream may still be
// releasing as the new one opens — a brief overlap cpal handles.
if let Some(prev) = active.take() {
prev.reader.abort();
}
// Connect + device-open take hundreds of ms. Race them against
// the next command so a release/stop (or shutdown) arriving
// mid-connect cancels the start — otherwise a quick tap-and-
// release would open a hot mic and append a spurious final after
// the user already let go. `biased` polls the start first so a
// just-completed session is always kept (dropping it would leak
// its reader). Dropping an unfinished start cancels the connect;
// the concurrent mic-open still completes but its handle is then
// dropped, releasing the device right away.
tokio::select! {
biased;
session = open_session(&config, &auth, &event_tx) => {
active = session;
}
next = cmd_rx.recv() => match next {
// Released before capture was ready → cancel the start.
Some(VoiceCommand::PttRelease) => {}
Some(VoiceCommand::Shutdown) | None => break,
// Unreachable per the release-between-presses contract;
// start fresh defensively.
Some(VoiceCommand::PttPress) => {
active = open_session(&config, &auth, &event_tx).await;
}
},
}
}
VoiceCommand::PttRelease => {
let Some(session) = active.as_ref() else {
continue;
};
// The reader task owns the capture handle; signalling it lets the
// reader stop the mic and send `audio.done` in a single place,
// matching the silence-guard teardown below.
let _ = session.finish_tx.send(()).await;
}
}
}
if let Some(session) = active {
session.reader.abort();
}
}
/// Open a capture session, emitting a `VoiceEvent::Error` (and returning `None`)
/// on failure. Extracted so the `PttPress` start can be raced against an
/// incoming release in `select!` and reused for the defensive restart path.
async fn open_session(
config: &VoiceConfig,
auth: &SharedVoiceAuth,
event_tx: &mpsc::Sender<VoiceEvent>,
) -> Option<ActivePtt> {
match start_capture_session(config, auth, event_tx).await {
Ok(session) => Some(session),
Err(e) => {
let _ = event_tx
.send(VoiceEvent::Error {
message: e.to_string(),
})
.await;
None
}
}
}
#[cfg(not(feature = "audio"))]
async fn start_capture_session(
_config: &VoiceConfig,
_auth: &SharedVoiceAuth,
_event_tx: &mpsc::Sender<VoiceEvent>,
) -> Result<ActivePtt, VoiceError> {
Err(VoiceError::Config(
"voice audio capture disabled (build without `audio` feature)".into(),
))
}
/// Hard cap on the pre-connect PCM backlog (memory safety). Sized far above any
/// real connect — the STT connect timeout aborts long before this is reached, so
/// in practice it never drops; it only bounds a pathological hang.
#[cfg(feature = "audio")]
const BACKLOG_MAX_CHUNKS: usize = 1024;
/// Bridge mic PCM into the STT socket across the connect handshake.
///
/// Until `audio_tx_rx` yields the live STT sender, captured chunks accumulate in
/// a bounded backlog (so the mic never backpressures while the socket connects);
/// once it arrives the backlog is flushed in order and capture streams live.
/// Holding the sender also defers the writer's `audio.done` until the backlog is
/// drained on teardown. Returns when the mic stops (`mic_rx` closed), the socket
/// goes away (`audio_tx` closed), or connect fails (`audio_tx_rx` dropped).
#[cfg(feature = "audio")]
async fn forward_pcm(
mut mic_rx: mpsc::Receiver<Vec<u8>>,
mut audio_tx_rx: tokio::sync::oneshot::Receiver<mpsc::Sender<Vec<u8>>>,
) {
let mut backlog: VecDeque<Vec<u8>> = VecDeque::new();
let audio_tx = loop {
tokio::select! {
chunk = mic_rx.recv() => match chunk {
// A normal connect stays well under the cap, so the lead-in is
// kept intact; only a pathologically slow connect (which the
// connect timeout aborts anyway) drops its oldest chunks rather
// than letting the buffer grow unbounded.
Some(c) => {
if backlog.len() == BACKLOG_MAX_CHUNKS {
backlog.pop_front();
}
backlog.push_back(c);
}
None => return, // mic stopped before the socket was ready
},
tx = &mut audio_tx_rx => match tx {
Ok(tx) => break tx,
Err(_) => return, // connect failed → sender dropped
},
}
};
for chunk in backlog {
if audio_tx.send(chunk).await.is_err() {
return;
}
}
while let Some(chunk) = mic_rx.recv().await {
if audio_tx.send(chunk).await.is_err() {
break;
}
}
}
#[cfg(feature = "audio")]
async fn start_capture_session(
config: &VoiceConfig,
auth: &SharedVoiceAuth,
event_tx: &mpsc::Sender<VoiceEvent>,
) -> Result<ActivePtt, VoiceError> {
// Open the mic concurrently with the bearer + connect handshake (TLS +
// WebSocket + `transcript.created`). Both legs take hundreds of ms and used
// to run in series before any capture, clipping the first word of a hold.
let (mic_tx, mic_rx) = mpsc::channel::<Vec<u8>>(64);
let sample_rate = config.sample_rate;
// `spawn_pcm_capture` blocks until the device opens; keep it off the runtime.
let capture_task =
tokio::task::spawn_blocking(move || crate::audio::spawn_pcm_capture(sample_rate, mic_tx));
// Start draining the mic *now* — before connect resolves — so the capture
// chain never backpressures (and cpal never drops chunks) while the socket
// comes up. `forward_pcm` buffers until the STT sender arrives, then flushes
// and streams live.
let (audio_tx_tx, audio_tx_rx) = tokio::sync::oneshot::channel::<mpsc::Sender<Vec<u8>>>();
tokio::spawn(forward_pcm(mic_rx, audio_tx_rx));
let connect = async {
let bearer = crate::auth::require_bearer(auth).await?;
StreamingSttSession::connect(config, &bearer).await
};
let (connect_res, capture_res) = tokio::join!(connect, capture_task);
// Resolve the mic first so a device/permission failure wins over a socket
// error; the `?` on `connect_res` then drops `capture`, releasing the mic.
let capture = match capture_res {
Ok(Ok(handle)) => handle,
Ok(Err(e)) => return Err(e),
Err(join_err) => {
return Err(VoiceError::Config(format!(
"voice capture task failed: {join_err}"
)));
}
};
let mut stt = connect_res?;
// Hand the live sender to the forwarder; it flushes the backlog then streams.
let audio_tx = stt
.audio_sender()
.ok_or_else(|| VoiceError::Stt("STT audio sender unavailable".into()))?;
let _ = audio_tx_tx.send(audio_tx);
let (finish_tx, mut finish_rx) = mpsc::channel::<()>(1);
let mut capture = Some(capture);
let out = event_tx.clone();
let reader = tokio::spawn(async move {
// Stop the mic (releasing the device and dropping the capture thread's
// clone of the audio sender) before signalling end-of-utterance, so no
// stray PCM is queued after `audio.done`. Idempotent via `Option::take`.
let stop_capture = |capture: &mut Option<crate::audio::CaptureHandle>| {
if let Some(handle) = capture.take() {
handle.stop();
}
};
// Surface a hint if nothing is transcribed within the first 10s of a
// session — the usual sign of a denied mic permission, a muted mic, or
// the wrong input device (on macOS a "healthy" stream still delivers
// silence until access is granted). The check applies only before the
// first transcript: once the user starts talking, pauses can be as long
// as they like.
let silence_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
let mut silence_check = true;
// Chunk-final (`is_final && !speech_final`) text is locked: the server
// sends it as a delta of the turn. Stitch those deltas into the live
// preview so a long pauseless utterance keeps accumulating on screen
// instead of resetting to the latest ~3s chunk. The committed prompt
// text never comes from here — only from `speech_final`, which the
// server produces as a clean one-pass re-transcription of the whole
// turn (better than stitched deltas). Reset on each `speech_final`.
let mut locked_prefix = String::new();
loop {
tokio::select! {
msg = finish_rx.recv() => {
if msg.is_some() {
// User ended the turn; stop watching for initial silence.
silence_check = false;
stop_capture(&mut capture);
stt.finish_audio();
} else {
return;
}
}
_ = tokio::time::sleep_until(silence_deadline), if silence_check => {
// Give up on this turn: stop the mic and flush `audio.done`
// so the session tears down instead of streaming silence
// until the pager sends stop (toggle off).
stop_capture(&mut capture);
stt.finish_audio();
let _ = out
.send(VoiceEvent::Error {
message: "no audio detected in 10s: check that your \
terminal has microphone permission"
.into(),
})
.await;
return;
}
ev = stt.recv() => {
match ev {
Some(StreamingSttEvent::Partial(p)) => {
let text = p.text.trim();
if text.is_empty() {
continue;
}
// Real speech arrived: disarm the initial-silence guard.
silence_check = false;
let event = if p.speech_final {
locked_prefix.clear();
VoiceEvent::UtteranceFinal { text: p.text }
} else if p.is_final {
// Lock this chunk's delta into the running preview.
if !locked_prefix.is_empty() {
locked_prefix.push(' ');
}
locked_prefix.push_str(text);
VoiceEvent::InterimTranscript {
text: locked_prefix.clone(),
}
} else if locked_prefix.is_empty() {
VoiceEvent::InterimTranscript {
text: text.to_owned(),
}
} else {
VoiceEvent::InterimTranscript {
text: format!("{locked_prefix} {text}"),
}
};
// Receiver gone (pager dropped the channel): tear down.
if out.send(event).await.is_err() {
return;
}
}
Some(StreamingSttEvent::Done { text }) => {
locked_prefix.clear();
if !text.trim().is_empty() {
silence_check = false;
let _ = out.send(VoiceEvent::UtteranceFinal { text }).await;
}
}
Some(StreamingSttEvent::Error { message }) => {
let _ = out.send(VoiceEvent::Error { message }).await;
return;
}
Some(StreamingSttEvent::Ready) | None => return,
}
}
}
}
});
Ok(ActivePtt { finish_tx, reader })
}
#[cfg(all(test, feature = "audio"))]
mod tests {
use super::*;
/// Chunks captured before the STT sender arrives are flushed (in order)
/// ahead of the live stream, with nothing reordered or dropped across the
/// handoff.
#[tokio::test]
async fn forward_pcm_delivers_buffered_then_live_in_order() {
let (mic_tx, mic_rx) = mpsc::channel::<Vec<u8>>(8);
let (tx_tx, tx_rx) = tokio::sync::oneshot::channel();
let (audio_tx, mut audio_rx) = mpsc::channel::<Vec<u8>>(8);
let task = tokio::spawn(forward_pcm(mic_rx, tx_rx));
// Buffered before the live sender is handed over, then flushed once it
// arrives. (Keep `mic_tx` open across the handoff: a mic that closes
// before the socket is ready means "abandoned", and discards the
// backlog — see the separate test.)
mic_tx.send(vec![1]).await.unwrap();
mic_tx.send(vec![2]).await.unwrap();
tx_tx.send(audio_tx).unwrap();
assert_eq!(audio_rx.recv().await, Some(vec![1]));
assert_eq!(audio_rx.recv().await, Some(vec![2]));
// Streamed live afterward, still in order.
mic_tx.send(vec![3]).await.unwrap();
assert_eq!(audio_rx.recv().await, Some(vec![3]));
drop(mic_tx);
assert_eq!(audio_rx.recv().await, None, "ends when the mic closes");
task.await.unwrap();
}
/// Mic stops before the socket is ready → the forwarder exits cleanly.
#[tokio::test]
async fn forward_pcm_returns_when_mic_closes_before_connect() {
let (mic_tx, mic_rx) = mpsc::channel::<Vec<u8>>(8);
let (_tx_tx, tx_rx) = tokio::sync::oneshot::channel::<mpsc::Sender<Vec<u8>>>();
let task = tokio::spawn(forward_pcm(mic_rx, tx_rx));
drop(mic_tx);
task.await.unwrap();
}
/// Connect fails (oneshot sender dropped without a value) → forwarder exits
/// and the buffered audio is discarded.
#[tokio::test]
async fn forward_pcm_returns_when_connect_fails() {
let (mic_tx, mic_rx) = mpsc::channel::<Vec<u8>>(8);
let (tx_tx, tx_rx) = tokio::sync::oneshot::channel::<mpsc::Sender<Vec<u8>>>();
let task = tokio::spawn(forward_pcm(mic_rx, tx_rx));
mic_tx.send(vec![1]).await.unwrap();
drop(tx_tx);
task.await.unwrap();
}
}

View file

@ -0,0 +1,166 @@
//! End-to-end voice probe: mic → streaming STT → transcript (for local debugging).
#[cfg(feature = "audio")]
use std::sync::Arc;
#[cfg(feature = "audio")]
use std::sync::atomic::{AtomicUsize, Ordering};
#[cfg(feature = "audio")]
use std::time::Duration;
#[cfg(feature = "audio")]
use tokio::time::timeout;
use crate::auth::SharedVoiceAuth;
use crate::config::VoiceConfig;
use crate::error::VoiceError;
#[cfg(feature = "audio")]
use crate::stt::{StreamingSttEvent, StreamingSttSession};
/// Options for [`run_streaming_probe`].
#[derive(Debug, Clone)]
pub struct VoiceProbeOptions {
pub config: VoiceConfig,
pub auth: SharedVoiceAuth,
/// How long to capture microphone audio before `audio.done`.
pub capture_secs: u32,
}
/// Collected probe output.
#[derive(Debug)]
pub struct VoiceProbeReport {
pub pcm_bytes: usize,
pub stt_log: Vec<String>,
pub transcript: Option<String>,
}
/// Capture mic audio and stream it to xAI STT, reporting the transcript.
#[cfg(feature = "audio")]
pub async fn run_streaming_probe(opts: VoiceProbeOptions) -> Result<VoiceProbeReport, VoiceError> {
let bearer = crate::auth::require_bearer(&opts.auth).await?;
let mut stt = StreamingSttSession::connect(&opts.config, &bearer).await?;
let stt_tx = stt
.audio_sender()
.ok_or_else(|| VoiceError::Stt("STT audio sender unavailable".into()))?;
let byte_count = Arc::new(AtomicUsize::new(0));
let byte_count_cb = Arc::clone(&byte_count);
let (pcm_tx, pcm_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(64);
let forward = tokio::spawn(async move {
let mut pcm_rx = pcm_rx;
while let Some(chunk) = pcm_rx.recv().await {
byte_count_cb.fetch_add(chunk.len(), Ordering::Relaxed);
if stt_tx.send(chunk).await.is_err() {
break;
}
}
});
let sample_rate = opts.config.sample_rate;
let secs = opts.capture_secs.max(1);
let capture = crate::audio::spawn_pcm_capture(sample_rate, pcm_tx)?;
tracing::info!(secs, "speak now — probe is listening");
tokio::time::sleep(Duration::from_secs(secs as u64)).await;
capture.stop();
let _ = forward.await;
stt.finish_audio();
let mut stt_log = Vec::new();
let mut transcript = None;
let deadline = Duration::from_secs(30);
loop {
let ev = match timeout(deadline, stt.recv()).await {
Ok(Some(ev)) => ev,
Ok(None) => {
stt_log.push("STT channel closed".into());
break;
}
Err(_) => {
stt_log.push("STT recv timed out (30s)".into());
break;
}
};
match &ev {
StreamingSttEvent::Ready => stt_log.push("STT: ready (transcript.created)".into()),
StreamingSttEvent::Partial(p) => {
stt_log.push(format!(
"STT: partial is_final={} speech_final={} text={:?}",
p.is_final, p.speech_final, p.text
));
if !p.text.trim().is_empty() && (p.speech_final || p.is_final) {
transcript = Some(p.text.clone());
}
}
StreamingSttEvent::Done { text } => {
stt_log.push(format!("STT: done text={:?}", text));
if !text.trim().is_empty() {
transcript = Some(text.clone());
}
break;
}
StreamingSttEvent::Error { message } => {
stt_log.push(format!("STT: error {message}"));
break;
}
}
}
let pcm_bytes = byte_count.load(Ordering::Relaxed);
Ok(VoiceProbeReport {
pcm_bytes,
stt_log,
transcript,
})
}
/// Record mic only (no STT) — quick hardware check.
#[cfg(feature = "audio")]
pub fn run_mic_only_probe(sample_rate: u32, seconds: u32) -> Result<(usize, u32), VoiceError> {
let (pcm, chunks) = crate::audio::capture_pcm_for_duration(sample_rate, seconds)?;
Ok((pcm.len(), chunks))
}
#[cfg(not(feature = "audio"))]
pub async fn run_streaming_probe(_opts: VoiceProbeOptions) -> Result<VoiceProbeReport, VoiceError> {
Err(VoiceError::Config(
"voice probe requires the `audio` feature (cpal)".into(),
))
}
/// Human-readable multi-line report for terminal output.
pub fn format_probe_report(report: &VoiceProbeReport) -> String {
let mut out = String::from("=== xai-grok-voice probe ===\n\n");
out.push_str(&format!(
"Mic capture (streamed)\n pcm_bytes: {}\n",
report.pcm_bytes
));
if report.pcm_bytes == 0 {
out.push_str(" WARNING: no PCM captured — check mic permission / default input device\n");
} else {
let secs_approx = report.pcm_bytes as f64 / (16000.0 * 2.0);
out.push_str(&format!(
" approx duration: {secs_approx:.2}s @ 16kHz mono PCM16\n"
));
}
out.push_str("\nSTT events\n");
if report.stt_log.is_empty() {
out.push_str(" (none)\n");
} else {
for line in &report.stt_log {
out.push_str(&format!(" {line}\n"));
}
}
out.push_str("\nTranscript\n");
match &report.transcript {
Some(t) if !t.trim().is_empty() => out.push_str(&format!(" {t}\n")),
Some(_) => out.push_str(" (empty string)\n"),
None => out.push_str(" (none — STT returned no text)\n"),
}
out
}

View file

@ -0,0 +1,7 @@
//! xAI Speech-to-Text: streaming `wss://api.x.ai/v1/stt`.
mod streaming;
mod types;
pub use streaming::{StreamingSttEvent, StreamingSttSession};
pub use types::{SttServerEvent, SttTranscriptPartial};

View file

@ -0,0 +1,330 @@
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::error::{Error as WsError, ProtocolError};
use tokio_tungstenite::tungstenite::handshake::client::Request as WsRequest;
use tokio_tungstenite::tungstenite::http::HeaderValue;
use url::Url;
use crate::config::VoiceConfig;
use crate::error::VoiceError;
use crate::stt::types::{SttServerEvent, SttTranscriptPartial};
/// Events delivered from an active streaming STT session.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StreamingSttEvent {
Ready,
Partial(SttTranscriptPartial),
Done { text: String },
Error { message: String },
}
/// Streaming STT over `wss://api.x.ai/v1/stt`.
pub struct StreamingSttSession {
audio_tx: Option<mpsc::Sender<Vec<u8>>>,
event_rx: mpsc::Receiver<StreamingSttEvent>,
_writer_task: JoinHandle<()>,
_reader_task: JoinHandle<()>,
}
impl StreamingSttSession {
/// Connect and wait for `transcript.created` before sending audio.
pub async fn connect(config: &VoiceConfig, bearer: &str) -> Result<Self, VoiceError> {
let url = build_stt_ws_url(config)?;
let mut request = url
.as_str()
.into_client_request()
.map_err(|e| VoiceError::WebSocket(format!("request: {e}")))?;
request.headers_mut().insert(
"Authorization",
format!("Bearer {bearer}")
.parse()
.map_err(|e| VoiceError::WebSocket(format!("auth header: {e}")))?,
);
// Request-identity headers so the backend can attribute and meter voice
// usage by client, mirroring what the sampler / imagine request paths
// send. Billing itself follows the `Authorization` bearer (per-user for
// OAuth, BYOK key owner otherwise); these are purely for usage
// attribution. Skipped when empty (e.g. the probe binary / tests) or
// when a value isn't a valid header (never fatal — the connection is
// still fully authorized without them).
insert_optional_header(
&mut request,
"x-grok-client-identifier",
&config.client_identifier,
);
insert_optional_header(&mut request, "User-Agent", &config.user_agent);
let (ws, _) = tokio::time::timeout(
Duration::from_secs(15),
tokio_tungstenite::connect_async(request),
)
.await
.map_err(|_| VoiceError::WebSocket("connect timed out".into()))?
.map_err(|e| VoiceError::WebSocket(format!("connect: {e}")))?;
let (mut ws_write, mut ws_read) = ws.split();
let (audio_tx, mut audio_rx) = mpsc::channel::<Vec<u8>>(64);
let (event_tx, event_rx) = mpsc::channel::<StreamingSttEvent>(64);
let writer_task = tokio::spawn(async move {
while let Some(chunk) = audio_rx.recv().await {
if ws_write.send(Message::Binary(chunk.into())).await.is_err() {
break;
}
}
let _ = ws_write
.send(Message::Text(r#"{"type":"audio.done"}"#.into()))
.await;
});
let reader_task = tokio::spawn(async move {
loop {
match ws_read.next().await {
Some(Ok(Message::Text(text))) => {
let event = match serde_json::from_str::<SttServerEvent>(&text) {
Ok(SttServerEvent::Created {}) => Some(StreamingSttEvent::Ready),
Ok(SttServerEvent::Partial {
text,
is_final,
speech_final,
}) => Some(StreamingSttEvent::Partial(SttTranscriptPartial {
text,
is_final,
speech_final,
})),
Ok(SttServerEvent::Done { text, .. }) => {
Some(StreamingSttEvent::Done { text })
}
Ok(SttServerEvent::Error { message }) => {
Some(StreamingSttEvent::Error { message })
}
Ok(SttServerEvent::Unknown) => None,
Err(e) => Some(StreamingSttEvent::Error {
message: format!("parse error: {e}"),
}),
};
if let Some(ev) = event
&& event_tx.send(ev).await.is_err()
{
break;
}
}
// Non-text frames (Close/Binary/Ping/Pong): ignore and let a
// subsequent `None` terminate the loop. A graceful close is
// treated as a normal end, not an error.
Some(Ok(_)) => continue,
// Transport-level failure: surface it so the pager can render
// and stop listening — except for an abrupt reset, which is
// also what we see when the socket is torn down at the end of
// a turn (incl. our own teardown), so reporting it would just
// produce a spurious "connection lost" toast.
Some(Err(e)) => {
if !is_benign_disconnect(&e) {
let _ = event_tx
.send(StreamingSttEvent::Error {
message: format!("connection lost: {e}"),
})
.await;
}
break;
}
// Stream ended cleanly: normal end of session, no error.
None => break,
}
}
});
let mut session = Self {
audio_tx: Some(audio_tx),
event_rx,
_writer_task: writer_task,
_reader_task: reader_task,
};
session.wait_ready().await?;
Ok(session)
}
async fn wait_ready(&mut self) -> Result<(), VoiceError> {
match tokio::time::timeout(Duration::from_secs(10), self.event_rx.recv()).await {
Ok(Some(StreamingSttEvent::Ready)) => Ok(()),
Ok(Some(StreamingSttEvent::Error { message })) => Err(VoiceError::Stt(message)),
Ok(_) => Err(VoiceError::Stt("unexpected event before ready".into())),
Err(_) => Err(VoiceError::Stt(
"timed out waiting for transcript.created".into(),
)),
}
}
pub async fn send_pcm(&self, pcm_bytes: Vec<u8>) -> Result<(), VoiceError> {
let Some(tx) = &self.audio_tx else {
return Err(VoiceError::Stt("audio input closed".into()));
};
tx.send(pcm_bytes)
.await
.map_err(|_| VoiceError::Stt("audio channel closed".into()))
}
pub async fn recv(&mut self) -> Option<StreamingSttEvent> {
self.event_rx.recv().await
}
/// Stop accepting PCM; the WebSocket task sends `audio.done` when the channel closes.
pub fn finish_audio(&mut self) {
self.audio_tx.take();
}
/// Clone the live audio sender for the capture bridge.
pub fn audio_sender(&self) -> Option<mpsc::Sender<Vec<u8>>> {
self.audio_tx.clone()
}
}
impl Drop for StreamingSttSession {
fn drop(&mut self) {
// Dropping a `JoinHandle` only detaches the task — it does not stop it.
// Abort both halves so an aborted setup (e.g. `connect` returning `Err`
// after `wait_ready` fails, or the caller failing to open the mic after
// a successful connect) tears the socket down immediately instead of
// leaving the writer to emit a stray `audio.done` and the reader to
// linger on an idle connection. On the healthy path both tasks have
// already finished (audio drained, `audio.done` flushed) before drop,
// so these aborts are no-ops.
self._writer_task.abort();
self._reader_task.abort();
}
}
/// Disconnects that aren't worth surfacing to the user: a socket torn down
/// without a closing handshake is the normal result of ending a turn (the
/// client or server just drops the connection), not a real failure.
fn is_benign_disconnect(err: &WsError) -> bool {
matches!(
err,
WsError::ConnectionClosed
| WsError::AlreadyClosed
| WsError::Protocol(ProtocolError::ResetWithoutClosingHandshake)
)
}
/// Insert `name: value` into the handshake request, unless `value` is empty
/// (header omitted) or not a valid header value (skipped with a debug log). A
/// missing identity header never fails the connection — the bearer alone fully
/// authorizes the request; these headers only enrich server-side attribution.
fn insert_optional_header(request: &mut WsRequest, name: &'static str, value: &str) {
if value.is_empty() {
return;
}
match HeaderValue::from_str(value) {
Ok(header_value) => {
request.headers_mut().insert(name, header_value);
}
Err(e) => {
tracing::debug!(
header = name,
"skipping voice STT header (invalid value): {e}"
);
}
}
}
fn build_stt_ws_url(config: &VoiceConfig) -> Result<Url, VoiceError> {
// Resolve `auto` / aliases here so the wire value is always a concrete
// catalog code (the STT API does not accept `auto`, unlike TTS).
let language = crate::language_for_api(&config.language);
Url::parse_with_params(
&config.stt_ws_url()?,
&[
("sample_rate", config.sample_rate.to_string()),
("encoding", "pcm".into()),
("interim_results", config.stt_interim_results.to_string()),
("language", language.to_string()),
("endpointing", config.stt_endpointing_ms.to_string()),
],
)
.map_err(|e| VoiceError::Stt(format!("bad STT URL: {e}")))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stt_url_includes_query_params() {
let cfg = VoiceConfig::default();
let url = build_stt_ws_url(&cfg).unwrap();
let q = url.query().unwrap_or_default();
assert!(q.contains("sample_rate=16000"));
assert!(q.contains("encoding=pcm"));
assert!(q.contains("language=en"), "default language on wire: {q}");
}
#[test]
fn stt_url_resolves_auto_to_concrete_language() {
let cfg = VoiceConfig {
language: "auto".into(),
..VoiceConfig::default()
};
let url = build_stt_ws_url(&cfg).unwrap();
let q = url.query().unwrap_or_default();
assert!(
!q.contains("language=auto"),
"must never send auto to STT API: {q}"
);
assert!(
q.contains("language="),
"resolved language query param missing: {q}"
);
// Resolved value must be a catalog code.
let lang = q
.split('&')
.find_map(|p| p.strip_prefix("language="))
.expect("language param");
assert!(
crate::stt_language_by_code(lang).is_some(),
"resolved language {lang:?} not in STT catalog"
);
}
#[test]
fn stt_url_passes_through_catalog_language() {
let cfg = VoiceConfig {
language: "ja".into(),
..VoiceConfig::default()
};
let url = build_stt_ws_url(&cfg).unwrap();
assert!(url.query().unwrap_or_default().contains("language=ja"));
}
#[test]
fn optional_header_inserted_when_present_skipped_when_empty() {
let mut req = "wss://api.x.ai/v1/stt".into_client_request().unwrap();
insert_optional_header(&mut req, "x-grok-client-identifier", "grok-shell");
insert_optional_header(&mut req, "User-Agent", "");
assert_eq!(
req.headers().get("x-grok-client-identifier").unwrap(),
"grok-shell"
);
assert!(
req.headers().get("user-agent").is_none(),
"empty value must omit the header entirely"
);
}
#[test]
fn optional_header_skips_invalid_value_without_panic() {
let mut req = "wss://api.x.ai/v1/stt".into_client_request().unwrap();
// A control char is not a valid header value; it must be dropped
// silently, never panic or fail the (already-authorized) handshake.
insert_optional_header(&mut req, "User-Agent", "bad\nvalue");
assert!(
req.headers().get("user-agent").is_none(),
"invalid value must omit the header, not panic"
);
}
}

View file

@ -0,0 +1,71 @@
use serde::Deserialize;
/// Parsed server → client STT WebSocket events.
#[derive(Debug, Clone, Deserialize, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum SttServerEvent {
#[serde(rename = "transcript.created")]
Created {},
#[serde(rename = "transcript.partial")]
Partial {
#[serde(default)]
text: String,
#[serde(default)]
is_final: bool,
#[serde(default)]
speech_final: bool,
},
#[serde(rename = "transcript.done")]
Done {
#[serde(default)]
text: String,
#[serde(default)]
duration: Option<f32>,
},
#[serde(rename = "error")]
Error {
#[serde(default)]
message: String,
},
#[serde(other)]
Unknown,
}
/// Normalized partial transcript for the UI.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SttTranscriptPartial {
pub text: String,
pub is_final: bool,
pub speech_final: bool,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_partial_event() {
let raw =
r#"{"type":"transcript.partial","text":"hello","is_final":false,"speech_final":false}"#;
let ev: SttServerEvent = serde_json::from_str(raw).unwrap();
let SttServerEvent::Partial {
text, speech_final, ..
} = ev
else {
panic!("expected partial");
};
assert_eq!(text, "hello");
assert!(!speech_final);
}
#[test]
fn parse_speech_final() {
let raw =
r#"{"type":"transcript.partial","text":"done","is_final":true,"speech_final":true}"#;
let ev: SttServerEvent = serde_json::from_str(raw).unwrap();
let SttServerEvent::Partial { speech_final, .. } = ev else {
panic!("expected partial");
};
assert!(speech_final);
}
}