Synced from monorepo

Changes:
- Detect the herdr multiplexer
- Mark /gboom as non-production code
- Bound peak memory when loading a large session
- Add a subagent lifecycle soak bounding threads, fds, and heap
- Stream inherited replay to bound fork memory
- Copy full plan from plan approval with y
- Stop armed signature verification from deleting the managed-deny smoke policy
- Add source-tagged terminal version telemetry
- Show the UI instantly and fetch models and settings in the background
- Session test helpers
- computer_reason on the ConversationHistoryDone trailer

crates/codegen/xai-grok-test-support/src/env.rs

@@ -6,6 +6,21 @@ use std::process::Command;
 
 use crate::sandbox::TestSandbox;
 
+/// Parse env var `key` into `T`, falling back to `default` when it is unset or
+/// present-but-unparseable (warning in the latter case).
+pub fn env_parse<T: std::str::FromStr>(key: &str, default: T) -> T {
+    let Ok(raw) = std::env::var(key) else {
+        return default;
+    };
+    match raw.parse() {
+        Ok(value) => value,
+        Err(_) => {
+            eprintln!("[test-support] ignoring unparseable {key}={raw:?}; using default");
+            default
+        }
+    }
+}
+
 /// RAII guard for a single environment variable in `#[serial]` tests: snapshots
 /// the prior value on construction, applies the change, then restores the prior
 /// value (or unsets it) on drop — even if an assertion panics. Restoring rather
@@ -110,7 +125,9 @@ pub fn grok_binary() -> PathBuf {
     if let Ok(path) = std::env::var("GROK_BINARY") {
         let p = PathBuf::from(path);
         assert!(p.exists(), "GROK_BINARY does not exist: {}", p.display());
-        return p;
+        // Bazel's GROK_BINARY is runfiles-relative; the harness spawns the child
+        // with a different cwd, so absolutize against the (runfiles-root) cwd now.
+        return std::path::absolute(&p).unwrap_or(p);
     }
 
     if let Ok(path) = std::env::var("CARGO_BIN_EXE_xai-grok-pager") {

crates/codegen/xai-grok-test-support/src/leader.rs

@@ -56,6 +56,8 @@ pub struct Capture {
     chunks: std::sync::Mutex<Vec<String>>,
     notification_count: AtomicU32,
     reconnected_count: AtomicU32,
+    models_update_count: AtomicU32,
+    settings_update_count: AtomicU32,
 }
 
 struct LeaderAcpClient {
@@ -96,10 +98,23 @@ impl acp::Client for LeaderAcpClient {
     }
 
     async fn ext_notification(&self, args: acp::ExtNotification) -> acp::Result<()> {
-        if &*args.method == "x.ai/leader_reconnected" {
-            self.capture
-                .reconnected_count
-                .fetch_add(1, Ordering::SeqCst);
+        match &*args.method {
+            "x.ai/leader_reconnected" => {
+                self.capture
+                    .reconnected_count
+                    .fetch_add(1, Ordering::SeqCst);
+            }
+            "x.ai/models/update" => {
+                self.capture
+                    .models_update_count
+                    .fetch_add(1, Ordering::SeqCst);
+            }
+            "x.ai/settings/update" => {
+                self.capture
+                    .settings_update_count
+                    .fetch_add(1, Ordering::SeqCst);
+            }
+            _ => {}
         }
         Ok(())
     }
@@ -180,12 +195,46 @@ impl LeaderFixture {
         Self::start_with_binary_timeout(binary, server, cwd, sandbox, Duration::from_secs(30)).await
     }
 
+    /// Start a leader pointed at an arbitrary base URL; offline-startup tests
+    /// aim it at an unreachable endpoint to prove it boots from local data.
+    pub async fn start_with_base_url(
+        base_url: &str,
+        cwd: &Path,
+        sandbox: &TestSandbox,
+    ) -> io::Result<Self> {
+        Self::start_with_binary_base_url_timeout(
+            &grok_binary(),
+            base_url,
+            cwd,
+            sandbox,
+            Duration::from_secs(30),
+        )
+        .await
+    }
+
     async fn start_with_binary_timeout(
         binary: &Path,
         server: &MockInferenceServer,
         cwd: &Path,
         sandbox: &TestSandbox,
         readiness_timeout: Duration,
+    ) -> io::Result<Self> {
+        Self::start_with_binary_base_url_timeout(
+            binary,
+            &server.url(),
+            cwd,
+            sandbox,
+            readiness_timeout,
+        )
+        .await
+    }
+
+    async fn start_with_binary_base_url_timeout(
+        binary: &Path,
+        base_url: &str,
+        cwd: &Path,
+        sandbox: &TestSandbox,
+        readiness_timeout: Duration,
     ) -> io::Result<Self> {
         let socket = sandbox.grok_home().join("leader.sock");
         let lock = sandbox.grok_home().join("leader.lock");
@@ -202,11 +251,11 @@ impl LeaderFixture {
         .stdout(std::process::Stdio::null());
         sandbox.apply_to_std_command(&mut cmd);
         cmd.envs(xai_tty_utils::pager_env())
-            .env("GROK_CLI_CHAT_PROXY_BASE_URL", server.url())
-            .env("GROK_XAI_API_BASE_URL", server.url())
-            .env("GROK_MODELS_BASE_URL", server.url())
-            .env("GROK_FEEDBACK_BASE_URL", server.url())
-            .env("GROK_TRACE_UPLOAD_URL", server.url())
+            .env("GROK_CLI_CHAT_PROXY_BASE_URL", base_url)
+            .env("GROK_XAI_API_BASE_URL", base_url)
+            .env("GROK_MODELS_BASE_URL", base_url)
+            .env("GROK_FEEDBACK_BASE_URL", base_url)
+            .env("GROK_TRACE_UPLOAD_URL", base_url)
             .env("XAI_API_KEY", "test-key-for-ci")
             .env("GROK_LEADER_SOCKET", &socket)
             .env("RUST_LOG", "xai_grok_shell=debug");
@@ -262,6 +311,18 @@ impl LeaderFixture {
         server: &MockInferenceServer,
         cwd: &Path,
         sandbox: &TestSandbox,
+    ) -> io::Result<LeaderStdioClient> {
+        self.spawn_client_with_base_url(&server.url(), cwd, sandbox)
+            .await
+    }
+
+    /// Spawn a relay client whose own endpoints point at an arbitrary base URL;
+    /// pairs with [`Self::start_with_base_url`] for a fully offline stack.
+    pub async fn spawn_client_with_base_url(
+        &self,
+        base_url: &str,
+        cwd: &Path,
+        sandbox: &TestSandbox,
     ) -> io::Result<LeaderStdioClient> {
         let binary = self
             .inner
@@ -269,7 +330,7 @@ impl LeaderFixture {
             .unwrap_or_else(|error| error.into_inner())
             .binary
             .clone();
-        self.spawn_client_with_binary(&binary, server, cwd, sandbox)
+        self.spawn_client_with_binary_base_url(&binary, base_url, cwd, sandbox)
             .await
     }
 
@@ -279,6 +340,17 @@ impl LeaderFixture {
         server: &MockInferenceServer,
         cwd: &Path,
         sandbox: &TestSandbox,
+    ) -> io::Result<LeaderStdioClient> {
+        self.spawn_client_with_binary_base_url(binary, &server.url(), cwd, sandbox)
+            .await
+    }
+
+    async fn spawn_client_with_binary_base_url(
+        &self,
+        binary: &Path,
+        base_url: &str,
+        cwd: &Path,
+        sandbox: &TestSandbox,
     ) -> io::Result<LeaderStdioClient> {
         let socket = self
             .inner
@@ -289,7 +361,7 @@ impl LeaderFixture {
         let registration = FixtureClientRegistration::new(&self.inner);
         LeaderStdioClient::spawn_with_binary_and_socket(
             binary,
-            server,
+            base_url,
             cwd,
             sandbox,
             socket,
@@ -550,7 +622,7 @@ fn wait_std_child_bounded(
 impl LeaderStdioClient {
     async fn spawn_with_binary_and_socket(
         binary: &Path,
-        server: &MockInferenceServer,
+        base_url: &str,
         cwd: &Path,
         sandbox: &TestSandbox,
         leader_socket: PathBuf,
@@ -565,11 +637,11 @@ impl LeaderStdioClient {
                 .label("grok leader stdio client")
                 .stdin(TestStdin::Piped)
                 .stdout(TestOutput::Piped)
-                .env("GROK_CLI_CHAT_PROXY_BASE_URL", server.url())
-                .env("GROK_XAI_API_BASE_URL", server.url())
-                .env("GROK_MODELS_BASE_URL", server.url())
-                .env("GROK_FEEDBACK_BASE_URL", server.url())
-                .env("GROK_TRACE_UPLOAD_URL", server.url())
+                .env("GROK_CLI_CHAT_PROXY_BASE_URL", base_url)
+                .env("GROK_XAI_API_BASE_URL", base_url)
+                .env("GROK_MODELS_BASE_URL", base_url)
+                .env("GROK_FEEDBACK_BASE_URL", base_url)
+                .env("GROK_TRACE_UPLOAD_URL", base_url)
                 .env("XAI_API_KEY", "test-key-for-ci")
                 .env("GROK_LEADER_SOCKET", leader_socket)
                 .env("RUST_LOG", "xai_grok_shell=debug"),
@@ -753,6 +825,16 @@ impl LeaderStdioClient {
     pub fn notification_count(&self) -> u32 {
         self.capture.notification_count.load(Ordering::SeqCst)
     }
+
+    /// Count of `x.ai/models/update` notifications received (catalog self-heal).
+    pub fn models_update_count(&self) -> u32 {
+        self.capture.models_update_count.load(Ordering::SeqCst)
+    }
+
+    /// Count of `x.ai/settings/update` notifications received (settings self-heal).
+    pub fn settings_update_count(&self) -> u32 {
+        self.capture.settings_update_count.load(Ordering::SeqCst)
+    }
 }
 
 pub fn leader_lock_path(home: &Path) -> PathBuf {

crates/codegen/xai-grok-test-support/src/lib.rs

@@ -21,6 +21,7 @@
 //! - [`grok_binary`] — Resolve the grok binary path (GROK_BINARY env or cargo_bin)
 //! - [`spawn_counting_server`] — Connection-counting HTTP/1.1 server for wire/pooling tests
 //! - [`uds_proxy::UdsProxy`] — Frame-aware fault-injection proxy for leader IPC sockets (unix)
+//! - [`ResourceSnapshot`] — RSS/threads/fds sampling for soak tests
 /// Multiply a harness timeout by `GROK_TEST_TIMEOUT_SCALE` (positive integer,
 /// default 1). CI lanes on shared runner pools raise it so pool load slows
 /// tests instead of failing them (see the Grok Build merge CI workflow).
@@ -41,6 +42,7 @@ mod inference_override;
 pub mod leader;
 pub mod mock_server;
 pub mod process;
+pub mod resources;
 pub mod sandbox;
 pub mod scripted;
 pub mod sse;
@@ -67,4 +69,5 @@ pub use process::{
     TestOutput, TestOutputSnapshot, TestProcess, TestProcessConfig, TestProcessState,
     TestProcessStderr, TestProcessStdout, TestProcessTermination, TestProcessTree, TestStdin,
 };
+pub use resources::{ResourceGrowth, ResourceSnapshot};
 pub use sandbox::{TestSandbox, TestSandboxBuilder};

crates/codegen/xai-grok-test-support/src/mock_server.rs

@@ -267,6 +267,9 @@ pub struct MockInferenceServer {
     chunk_delay: Arc<std::sync::RwLock<Option<Duration>>>,
     /// Mock `/v1/storage` 401 gate + accepted-upload record.
     storage: Arc<StorageState>,
+    /// When set, `/v1/models` and `/v1/settings` hang forever (never
+    /// respond); see [`Self::set_hang`].
+    hang: Arc<std::sync::atomic::AtomicBool>,
     /// See [`Self::set_user_subscription_tier`].
     user_tier: Arc<std::sync::RwLock<Option<String>>>,
 }
@@ -306,6 +309,7 @@ impl MockInferenceServer {
         let messages_stop_reason = Arc::new(std::sync::RwLock::new("end_turn".to_string()));
         let chunk_delay = Arc::new(std::sync::RwLock::new(None::<Duration>));
         let storage = Arc::new(StorageState::default());
+        let hang = Arc::new(std::sync::atomic::AtomicBool::new(false));
         let user_tier = Arc::new(std::sync::RwLock::new(None::<String>));
         let app = Self::build_router(
             log.clone(),
@@ -317,6 +321,7 @@ impl MockInferenceServer {
             messages_stop_reason.clone(),
             chunk_delay.clone(),
             storage.clone(),
+            hang.clone(),
             user_tier.clone(),
         );
 
@@ -356,6 +361,7 @@ impl MockInferenceServer {
             messages_stop_reason,
             chunk_delay,
             storage,
+            hang,
             user_tier,
         })
     }
@@ -426,6 +432,12 @@ impl MockInferenceServer {
         self.set_settings(json!({ "allow_access": true }));
     }
 
+    /// Make `/v1/models` and `/v1/settings` hang forever, standing in for a
+    /// black-holed backend in non-blocking-startup tests.
+    pub fn set_hang(&self, hang: bool) {
+        self.hang.store(hang, std::sync::atomic::Ordering::Release);
+    }
+
     /// Set the `subscriptionTier` served by `GET /v1/user`. `None`
     /// (default) omits the field, which the shell treats as "no qualifying
     /// subscription" (free tier).
@@ -663,8 +675,11 @@ impl MockInferenceServer {
         messages_stop_reason: Arc<std::sync::RwLock<String>>,
         chunk_delay: Arc<std::sync::RwLock<Option<Duration>>>,
         storage: Arc<StorageState>,
+        hang: Arc<std::sync::atomic::AtomicBool>,
         user_tier: Arc<std::sync::RwLock<Option<String>>>,
     ) -> Router {
+        let hang_models = hang.clone();
+        let hang_settings = hang;
         let log_cc = log.clone();
         let log_rs = log.clone();
         let log_msg = log.clone();
@@ -920,8 +935,12 @@ impl MockInferenceServer {
                     move || {
                         let log = log.clone();
                         let models = models.clone();
+                        let hang = hang_models.clone();
                         async move {
                             log.record("GET", "/v1/models", None, None, Vec::new());
+                            if hang.load(std::sync::atomic::Ordering::Acquire) {
+                                tokio::time::sleep(Duration::from_secs(3600)).await;
+                            }
                             let models_json = models.read().unwrap().clone();
                             Json(json!({
                                 "object": "list",
@@ -935,12 +954,18 @@ impl MockInferenceServer {
                 "/v1/settings",
                 get({
                     let log = log.clone();
+                    let settings = settings.clone();
+                    let hang = hang_settings.clone();
                     move || {
                         let log = log.clone();
                         let settings = settings.clone();
                         let overrides = overrides_settings.clone();
+                        let hang = hang.clone();
                         async move {
                             log.record("GET", "/v1/settings", None, None, Vec::new());
+                            if hang.load(std::sync::atomic::Ordering::Acquire) {
+                                tokio::time::sleep(Duration::from_secs(3600)).await;
+                            }
                             // Scripted one-shots take precedence (FIFO), so a
                             // test can serve a transient payload (e.g. one
                             // stale gated snapshot) and fall back to the

crates/codegen/xai-grok-test-support/src/resources.rs

@@ -0,0 +1,134 @@
+//! Generic OS resource snapshots for soak tests. No shell types: `rss_bytes`
+//! reads `/proc` (Linux) or shells out to `ps` (macOS); the task/fd counters
+//! are Linux-only and return `None` elsewhere.
+
+/// RSS (bytes), live threads, and open fds sampled together. `None` marks a
+/// metric the platform can't report.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct ResourceSnapshot {
+    pub rss: Option<usize>,
+    pub threads: Option<usize>,
+    pub fds: Option<usize>,
+}
+
+/// Saturating per-field growth of one [`ResourceSnapshot`] over an earlier
+/// baseline. A distinct type from a snapshot so a delta can't be mistaken for
+/// an absolute sample. `None` marks a field either side couldn't report.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct ResourceGrowth {
+    pub rss: Option<usize>,
+    pub threads: Option<usize>,
+    pub fds: Option<usize>,
+}
+
+impl ResourceSnapshot {
+    pub fn capture() -> Self {
+        Self {
+            rss: rss_bytes(),
+            threads: thread_count(),
+            fds: fd_count(),
+        }
+    }
+
+    /// RSS only, skipping the thread and fd probes. For hot sampling loops that
+    /// use just `rss`: on Linux this avoids the per-tick `/proc/self/{task,fd}`
+    /// directory scans. The RSS read itself still shells out to `ps` on macOS.
+    pub fn capture_rss() -> Option<usize> {
+        rss_bytes()
+    }
+
+    /// Growth of `self` (after) over `baseline` (before); see [`ResourceGrowth`].
+    pub fn growth_from(&self, baseline: &ResourceSnapshot) -> ResourceGrowth {
+        let delta = |after: Option<usize>, before: Option<usize>| {
+            before.zip(after).map(|(b, a)| a.saturating_sub(b))
+        };
+        ResourceGrowth {
+            rss: delta(self.rss, baseline.rss),
+            threads: delta(self.threads, baseline.threads),
+            fds: delta(self.fds, baseline.fds),
+        }
+    }
+}
+
+fn rss_bytes() -> Option<usize> {
+    #[cfg(target_os = "linux")]
+    {
+        let status = std::fs::read_to_string("/proc/self/status").ok()?;
+        for line in status.lines() {
+            if let Some(val) = line.strip_prefix("VmRSS:") {
+                let kb: usize = val.trim().trim_end_matches(" kB").trim().parse().ok()?;
+                return Some(kb * 1024);
+            }
+        }
+        None
+    }
+
+    #[cfg(target_os = "macos")]
+    {
+        use std::process::Command;
+        let output = Command::new("ps")
+            .args(["-o", "rss=", "-p", &std::process::id().to_string()])
+            .output()
+            .ok()?;
+        let kb: usize = String::from_utf8_lossy(&output.stdout)
+            .trim()
+            .parse()
+            .ok()?;
+        Some(kb * 1024)
+    }
+
+    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+    {
+        None
+    }
+}
+
+fn thread_count() -> Option<usize> {
+    #[cfg(target_os = "linux")]
+    {
+        Some(std::fs::read_dir("/proc/self/task").ok()?.count())
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        None
+    }
+}
+
+/// The read's own transient fd closes with the iterator, so before and after
+/// samples stay symmetric.
+fn fd_count() -> Option<usize> {
+    #[cfg(target_os = "linux")]
+    {
+        Some(std::fs::read_dir("/proc/self/fd").ok()?.count())
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn growth_from_saturates_and_propagates_none() {
+        let before = ResourceSnapshot {
+            rss: Some(100),
+            threads: Some(5),
+            fds: None,
+        };
+        let after = ResourceSnapshot {
+            rss: Some(30),
+            threads: Some(9),
+            fds: Some(3),
+        };
+        let growth = after.growth_from(&before);
+        assert_eq!(growth.rss, Some(0), "a shrink saturates to zero");
+        assert_eq!(growth.threads, Some(4), "growth is the delta");
+        assert_eq!(
+            growth.fds, None,
+            "a missing baseline sample propagates None"
+        );
+    }
+}