grok-build-upstream-mirror/crates/codegen/xai-grok-shell/tests/test_nonblocking_startup.rs

//! Non-blocking startup regression tests. `#[ignore]`; requires pre-built binary.
//!
//! These exercise leader startup through the persistent-leader fixture: the
//! leader must bind its socket and become ready without blocking on the remote
//! `/settings` + `/v1/models` fetch, and must self-heal its catalog once the
//! endpoint recovers.
//!
//! ```bash
//! cargo test -p xai-grok-shell --test test_nonblocking_startup -- --ignored
//! ```

#![cfg(unix)]

mod common;

use std::time::{Duration, Instant};

use xai_grok_test_support::leader::LeaderFixture;
use xai_grok_test_support::*;

async fn poll_until(ceiling: Duration, interval: Duration, condition: impl Fn() -> bool) -> bool {
    let deadline = Instant::now() + ceiling;
    while Instant::now() < deadline {
        if condition() {
            return true;
        }
        tokio::time::sleep(interval).await;
    }
    false
}

/// A hanging proxy must not delay leader readiness: the fixture (which waits for
/// the leader socket) must come up and a session must be created well within the
/// blocking-fetch window.
#[tokio::test]
#[ignore] // requires pre-built binary; run with --ignored
async fn leader_ready_while_proxy_hangs() {
    tokio::task::LocalSet::new()
        .run_until(async {
            let server = MockInferenceServer::start().await.unwrap();
            server.set_hang(true);

            let workdir = git_workdir();
            let sandbox = TestSandbox::new();

            let started = Instant::now();
            let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
                .await
                .expect("leader must become ready while the proxy hangs");
            let mut clients = Vec::new();
            common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
                Box::pin(async move {
                    clients.push(
                        fixture
                            .spawn_client(&server, workdir.workspace(), &sandbox)
                            .await
                            .expect("spawn leader client"),
                    );
                    clients[0].initialize().await;
                    let _session = clients[0].create_session(workdir.workspace()).await;

                    let elapsed = started.elapsed();
                    assert!(
                        elapsed < Duration::from_secs(25),
                        "startup took {elapsed:?} with a hanging proxy; readiness appears \
                         to block on the network fetch\nstderr:\n{}",
                        clients[0].stderr_text(),
                    );
                })
            })
            .await;
        })
        .await;
}

/// The background catalog refresh must re-fetch and push `x.ai/models/update`
/// once a previously-hanging endpoint recovers.
#[tokio::test]
#[ignore] // requires pre-built binary; run with --ignored
async fn catalog_self_heals_after_endpoint_recovers() {
    tokio::task::LocalSet::new()
        .run_until(async {
            let server = MockInferenceServer::start().await.unwrap();
            server.set_hang(true);

            let workdir = git_workdir();
            let sandbox = TestSandbox::new();

            let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
                .await
                .expect("leader must become ready while the proxy hangs");
            let mut clients = Vec::new();
            common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
                Box::pin(async move {
                    clients.push(
                        fixture
                            .spawn_client(&server, workdir.workspace(), &sandbox)
                            .await
                            .expect("spawn leader client"),
                    );
                    clients[0].initialize().await;

                    // Recover the endpoint. The background catalog refresh
                    // (5s-base backoff) re-fetches and pushes `x.ai/models/update`.
                    server.set_hang(false);

                    let healed =
                        poll_until(Duration::from_secs(60), Duration::from_millis(500), || {
                            clients[0].models_update_count() > 0
                        })
                        .await;
                    assert!(
                        healed,
                        "no x.ai/models/update after the endpoint recovered\nstderr:\n{}",
                        clients[0].stderr_text(),
                    );
                })
            })
            .await;
        })
        .await;
}

/// Custom-backend reality: a user points at their own backend that serves
/// `/v1/models` + chat but blocks the cli-chat-proxy `/settings` (404). The
/// leader must boot fast, load its catalog from the served models, and run a
/// real prompt, even though remote settings never arrive.
#[tokio::test]
#[ignore] // requires pre-built binary; run with --ignored
async fn leader_usable_when_settings_blocked_but_models_served() {
    tokio::task::LocalSet::new()
        .run_until(async {
            // Models + chat are served; `set_settings` is never called, so
            // `/v1/settings` 404s, mimicking a blocked proxy settings endpoint.
            let server = MockInferenceServer::start().await.unwrap();

            let workdir = git_workdir();
            let sandbox = TestSandbox::new();

            let started = Instant::now();
            let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
                .await
                .expect("leader must become ready with /settings blocked");
            let mut clients = Vec::new();
            common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
                Box::pin(async move {
                    clients.push(
                        fixture
                            .spawn_client(&server, workdir.workspace(), &sandbox)
                            .await
                            .expect("spawn leader client"),
                    );
                    clients[0].initialize().await;
                    let session = clients[0].create_session(workdir.workspace()).await;

                    // The custom backend serves chat, so a prompt round-trips
                    // despite the blocked settings endpoint.
                    clients[0]
                        .prompt(&session, "ping")
                        .await
                        .expect("prompt against the custom backend");
                    assert!(
                        server.has_chat_completion_request() || server.has_responses_request(),
                        "the prompt must reach the served chat endpoint\nstderr:\n{}",
                        clients[0].stderr_text(),
                    );

                    assert!(
                        started.elapsed() < Duration::from_secs(25),
                        "startup/usage blocked on the unreachable /settings\nstderr:\n{}",
                        clients[0].stderr_text(),
                    );
                    assert_eq!(
                        clients[0].settings_update_count(),
                        0,
                        "no settings update should land while /settings is blocked",
                    );
                })
            })
            .await;
        })
        .await;
}