Synced from monorepo
Changes: - Non-blocking coding-data sharing upsell banner - Consolidate remediation in Doctor - Auto mode defers fail-closed gate asks to the classifier - Coalesce marketplace list fetches - Allow removing a marketplace source by name - Contain hung git marketplace sources (timeouts, non-blocking refresh, unbrick modal) - Label failed workspace RPCs with error_kind - Drop redundant explicit tonic/prost deps from xai-grok-shell - Report real exit codes for completed background shells - Narrow the date-rollover reminder to date-bearing templates - Wire toolOverrides through the session and agent - Security: Bash(git:*) allowlist matches whole command chain by prefix - Split prompt-trigger telemetry and record classifier provenance - Raise connectors-manager timeout to 60s - Auto classifier honors recorded approvals for repeat actions - Apply doctor fixes in the TUI - Auto-mode classifier timeouts prompt instead of silently denying - Scope subagent completion drains to the owning session - Add the toolOverrides wire types - Set client_identifier=grok-agent-sdk - Accept both spellings of the workspace-teleport kill switch - Persist one-shot occurrence journal - Stop turns that poll the exact same tool call 16x in a row - Copy compaction checkpoint files when forking sessions - Auto-focus permission prompt from scrollback - Esc cancels the running turn in non-vim and minimal modes - List Ctrl+Z undo and redo in keyboard shortcuts - Out-of-process macOS mic capture - Show active auth mode on session-info - Install the npm binary under $GROK_HOME - Remove hover/click dead zones between dashboard items - Route startup warnings to doctor - Document [feedback.user] author identity config - Extend bang command timeout - Close combine-queued edit-hold race - Integrate relocation recovery - Expose privacy notice rollout flag - Break harness discovery ref cycle so connections can idle-evict - Shift/Alt+Enter inserts newline when editing a queued prompt - Gate project Claude permissions on folder trust - Echo response.create.event_id on response.created - Toast when session creation fails from disk full - Add shared test process lifecycle - Enable dynamic workflows by default - Add relocation transaction state machine - Add shared test sandbox - Surface auth failures on model-switch compact - Persist durable scheduler expiry - Confirm before removing extensions-modal items - Re-run compact and prompt after login when compact hit expired auth - Recap sends hosted tools under backend search
This commit is contained in:
grokkybara[bot]
parent
3af4d5d398
commit
a5727c5960
482 changed files with 37627 additions and 13402 deletions
|
|
@ -2,15 +2,307 @@
|
|||
|
||||
use xai_grok_shell::sampling::{ApiBackend, Client, SamplerConfig};
|
||||
|
||||
#[cfg(unix)]
|
||||
pub mod leader {
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
|
||||
use futures::FutureExt as _;
|
||||
use xai_grok_test_support::leader::{LeaderFixture, LeaderStdioClient};
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub type TestBody<'a> = Pin<Box<dyn Future<Output = ()> + 'a>>;
|
||||
|
||||
type PanicPayload = Box<dyn std::any::Any + Send>;
|
||||
|
||||
fn finish_body(body_result: Result<(), PanicPayload>, cleanup_error: Option<io::Error>) {
|
||||
match body_result {
|
||||
Ok(()) => {
|
||||
if let Some(error) = cleanup_error {
|
||||
panic!("leader integration cleanup failed: {error}");
|
||||
}
|
||||
}
|
||||
Err(payload) => {
|
||||
if let Some(error) = cleanup_error {
|
||||
eprintln!("leader integration cleanup after panic failed: {error}");
|
||||
}
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait CleanupClient {
|
||||
async fn graceful_close(&mut self) -> io::Result<()>;
|
||||
async fn hard_close(&mut self) -> io::Result<()>;
|
||||
fn contain_failed_cleanup_for_unwind(&mut self);
|
||||
}
|
||||
|
||||
impl CleanupClient for LeaderStdioClient {
|
||||
async fn graceful_close(&mut self) -> io::Result<()> {
|
||||
self.close().await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn hard_close(&mut self) -> io::Result<()> {
|
||||
self.kill_and_close().await.map(|_| ())
|
||||
}
|
||||
|
||||
fn contain_failed_cleanup_for_unwind(&mut self) {
|
||||
LeaderStdioClient::contain_failed_cleanup_for_unwind(self);
|
||||
}
|
||||
}
|
||||
|
||||
trait CleanupFixture {
|
||||
async fn close_fixture(&self) -> io::Result<()>;
|
||||
fn contain_failed_cleanup_for_unwind(&self);
|
||||
}
|
||||
|
||||
impl CleanupFixture for LeaderFixture {
|
||||
async fn close_fixture(&self) -> io::Result<()> {
|
||||
self.close().await
|
||||
}
|
||||
|
||||
fn contain_failed_cleanup_for_unwind(&self) {
|
||||
LeaderFixture::contain_failed_cleanup_for_unwind(self);
|
||||
}
|
||||
}
|
||||
|
||||
struct ClientCleanupOutcome {
|
||||
all_closed: bool,
|
||||
error: Option<io::Error>,
|
||||
}
|
||||
|
||||
async fn close_clients<C: CleanupClient>(clients: &mut Vec<C>) -> ClientCleanupOutcome {
|
||||
let pending = std::mem::take(clients).into_iter();
|
||||
let mut retained = Vec::new();
|
||||
let mut first_error = None;
|
||||
for mut client in pending {
|
||||
match client.graceful_close().await {
|
||||
Ok(()) => {}
|
||||
Err(close_error) => match client.hard_close().await {
|
||||
Ok(()) => {}
|
||||
Err(kill_error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(io::Error::new(
|
||||
close_error.kind(),
|
||||
format!(
|
||||
"leader client close failed: {close_error}; bounded hard cleanup also failed: {kill_error}"
|
||||
),
|
||||
));
|
||||
}
|
||||
retained.push(client);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
*clients = retained;
|
||||
ClientCleanupOutcome {
|
||||
all_closed: clients.is_empty(),
|
||||
error: first_error,
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_owned_processes<C, F>(fixture: &F, clients: &mut Vec<C>) -> Option<io::Error>
|
||||
where
|
||||
C: CleanupClient,
|
||||
F: CleanupFixture,
|
||||
{
|
||||
let cleanup = close_clients(clients).await;
|
||||
let mut cleanup_error = cleanup.error;
|
||||
if !cleanup.all_closed {
|
||||
// This error-only path requests hard kills, then intentionally
|
||||
// leaks concrete owners so panic unwind cannot run blocking Drop.
|
||||
// The leak is bounded by the lifetime of the test process.
|
||||
for client in clients.iter_mut() {
|
||||
client.contain_failed_cleanup_for_unwind();
|
||||
}
|
||||
let retained = std::mem::take(clients);
|
||||
std::mem::forget(retained);
|
||||
fixture.contain_failed_cleanup_for_unwind();
|
||||
return cleanup_error;
|
||||
}
|
||||
if let Err(error) = fixture.close_fixture().await {
|
||||
cleanup_error = Some(match cleanup_error {
|
||||
Some(client_error) => io::Error::new(
|
||||
client_error.kind(),
|
||||
format!("{client_error}; fixture cleanup also failed: {error}"),
|
||||
),
|
||||
None => error,
|
||||
});
|
||||
}
|
||||
cleanup_error
|
||||
}
|
||||
|
||||
/// Run a leader test body, then close only directly-owned stdio clients and
|
||||
/// the concrete initial fixture leader. Detached replacement leaders are
|
||||
/// intentionally outside cleanup ownership; tests that create one remain
|
||||
/// ignored/manual until OS containment or a test-only leader binary exists.
|
||||
#[allow(dead_code)]
|
||||
pub async fn run_with_cleanup<F>(
|
||||
fixture: &LeaderFixture,
|
||||
clients: &mut Vec<LeaderStdioClient>,
|
||||
body: F,
|
||||
) where
|
||||
F: for<'a> FnOnce(&'a LeaderFixture, &'a mut Vec<LeaderStdioClient>) -> TestBody<'a>,
|
||||
{
|
||||
let body_result = std::panic::AssertUnwindSafe(body(fixture, clients))
|
||||
.catch_unwind()
|
||||
.await;
|
||||
let cleanup_error = cleanup_owned_processes(fixture, clients).await;
|
||||
finish_body(body_result, cleanup_error);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct FakeClient {
|
||||
graceful_fails: bool,
|
||||
hard_fails: bool,
|
||||
graceful_calls: Arc<AtomicUsize>,
|
||||
hard_calls: Arc<AtomicUsize>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
containment_calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl CleanupClient for FakeClient {
|
||||
async fn graceful_close(&mut self) -> io::Result<()> {
|
||||
self.graceful_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.graceful_fails {
|
||||
Err(io::Error::other("injected graceful failure"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn hard_close(&mut self) -> io::Result<()> {
|
||||
self.hard_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.hard_fails {
|
||||
Err(io::Error::other("injected hard failure"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn contain_failed_cleanup_for_unwind(&mut self) {
|
||||
self.containment_calls.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FakeClient {
|
||||
fn drop(&mut self) {
|
||||
self.drops.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeFixture {
|
||||
close_calls: AtomicUsize,
|
||||
containment_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl CleanupFixture for FakeFixture {
|
||||
async fn close_fixture(&self) -> io::Result<()> {
|
||||
self.close_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn contain_failed_cleanup_for_unwind(&self) {
|
||||
self.containment_calls.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn double_failed_client_transfers_to_unwind_containment() {
|
||||
let graceful_calls = Arc::new(AtomicUsize::new(0));
|
||||
let hard_calls = Arc::new(AtomicUsize::new(0));
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
let containment_calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut clients = vec![FakeClient {
|
||||
graceful_fails: true,
|
||||
hard_fails: true,
|
||||
graceful_calls: graceful_calls.clone(),
|
||||
hard_calls: hard_calls.clone(),
|
||||
drops: drops.clone(),
|
||||
containment_calls: containment_calls.clone(),
|
||||
}];
|
||||
let fixture = FakeFixture::default();
|
||||
|
||||
let error = cleanup_owned_processes(&fixture, &mut clients)
|
||||
.await
|
||||
.expect("double failure must be reported");
|
||||
|
||||
assert!(error.to_string().contains("injected graceful failure"));
|
||||
assert!(error.to_string().contains("injected hard failure"));
|
||||
assert!(
|
||||
clients.is_empty(),
|
||||
"retained owner must transfer to leaked containment"
|
||||
);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(graceful_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(hard_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(containment_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(fixture.close_calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(fixture.containment_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_owners_drop_before_fixture_close() {
|
||||
let graceful_calls = Arc::new(AtomicUsize::new(0));
|
||||
let hard_calls = Arc::new(AtomicUsize::new(0));
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
let containment_calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut clients = vec![
|
||||
FakeClient {
|
||||
graceful_fails: false,
|
||||
hard_fails: false,
|
||||
graceful_calls: graceful_calls.clone(),
|
||||
hard_calls: hard_calls.clone(),
|
||||
drops: drops.clone(),
|
||||
containment_calls: containment_calls.clone(),
|
||||
},
|
||||
FakeClient {
|
||||
graceful_fails: true,
|
||||
hard_fails: false,
|
||||
graceful_calls: graceful_calls.clone(),
|
||||
hard_calls: hard_calls.clone(),
|
||||
drops: drops.clone(),
|
||||
containment_calls: containment_calls.clone(),
|
||||
},
|
||||
];
|
||||
let fixture = FakeFixture::default();
|
||||
|
||||
let error = cleanup_owned_processes(&fixture, &mut clients).await;
|
||||
|
||||
assert!(
|
||||
error.is_none(),
|
||||
"successful bounded hard cleanup must recover the graceful failure"
|
||||
);
|
||||
assert!(clients.is_empty());
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(graceful_calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(hard_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(containment_calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(fixture.close_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(fixture.containment_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a sampling client configured for a mock server. Shared by the
|
||||
/// integration tests so the ~30-field `SamplerConfig` literal lives in one
|
||||
/// place (`SamplerConfig` has no `Default`).
|
||||
#[allow(dead_code)]
|
||||
pub fn create_test_client(base_url: &str, api_backend: ApiBackend) -> Client {
|
||||
create_test_client_with_extra_headers(base_url, api_backend, &[])
|
||||
}
|
||||
|
||||
/// Like [`create_test_client`] but seeds `SamplerConfig::extra_headers`, so a
|
||||
/// test can assert that session-injected headers reach the wire.
|
||||
#[allow(dead_code)]
|
||||
pub fn create_test_client_with_extra_headers(
|
||||
base_url: &str,
|
||||
api_backend: ApiBackend,
|
||||
|
|
@ -22,6 +314,7 @@ pub fn create_test_client_with_extra_headers(
|
|||
/// The shared mock-server `SamplerConfig`; tests needing a non-default field
|
||||
/// (e.g. `doom_loop_recovery`) mutate the returned value before building the
|
||||
/// client themselves.
|
||||
#[allow(dead_code)]
|
||||
pub fn test_sampler_config(
|
||||
base_url: &str,
|
||||
api_backend: ApiBackend,
|
||||
|
|
|
|||
|
|
@ -17,9 +17,7 @@
|
|||
//! ```bash
|
||||
//! cargo test -p xai-grok-shell --test test_agent_type_invariant -- --ignored
|
||||
//! ```
|
||||
use agent_client_protocol::Agent as _;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
use xai_grok_test_support::*;
|
||||
async fn with_local_set<F, Fut>(f: F)
|
||||
where
|
||||
|
|
@ -70,9 +68,11 @@ async fn test_default_model_uses_grok_build_harness() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "prompt failed: {:?}", result.err());
|
||||
let sys_prompt = server
|
||||
|
|
@ -94,10 +94,10 @@ async fn test_same_type_model_switch_no_rebuild() {
|
|||
with_local_set(|| async {
|
||||
let server = same_type_server().await;
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client
|
||||
.create_session_with_model_timeout(workdir.path(), "model-a")
|
||||
.create_session_with_model_timeout(workdir.workspace(), "model-a")
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "first prompt failed: {:?}", result.err());
|
||||
|
|
@ -128,21 +128,24 @@ async fn test_session_resume_preserves_harness() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
writer.initialize_with_timeout().await;
|
||||
let session_id = writer.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = writer
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let result = writer.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "prompt failed: {:?}", result.err());
|
||||
let original_sys_prompt = server
|
||||
.last_system_prompt()
|
||||
.expect("should have captured system prompt");
|
||||
let shared_home = writer.take_home();
|
||||
invalidate_models_cache(shared_home.path());
|
||||
let shared_sandbox = writer.take_sandbox();
|
||||
invalidate_models_cache(shared_sandbox.home());
|
||||
drop(writer);
|
||||
let reader = GrokStdioClient::spawn_with_home(&server, workdir.path(), shared_home).await;
|
||||
let reader =
|
||||
GrokStdioClient::spawn_with_sandbox(&server, workdir.workspace(), shared_sandbox).await;
|
||||
reader.initialize_with_timeout().await;
|
||||
let _ = reader
|
||||
.load_session_with_timeout(&session_id, workdir.path())
|
||||
.load_session_with_timeout(&session_id, workdir.workspace())
|
||||
.await;
|
||||
let result2 = reader.prompt_with_timeout(&session_id, "say goodbye").await;
|
||||
assert!(
|
||||
|
|
@ -178,15 +181,20 @@ async fn test_session_resume_preserves_harness() {
|
|||
async fn test_model_without_agent_type_defaults_to_grok_build() {
|
||||
with_local_set(|| async {
|
||||
let server = MockInferenceServer::start_with_models(
|
||||
vec![MockModelEntry::new("no-agent-type-model"),],
|
||||
vec![
|
||||
MockModelEntry::new("no-agent-type-model"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client
|
||||
.create_session_with_model_timeout(workdir.path(), "no-agent-type-model")
|
||||
.create_session_with_model_timeout(
|
||||
workdir.workspace(),
|
||||
"no-agent-type-model",
|
||||
)
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "prompt failed: {:?}", result.err());
|
||||
|
|
@ -194,10 +202,10 @@ async fn test_model_without_agent_type_defaults_to_grok_build() {
|
|||
.last_system_prompt()
|
||||
.expect("should have at least one inference request");
|
||||
assert!(
|
||||
sys_prompt.contains("Grok") || sys_prompt.contains("grok"),
|
||||
"model without agent_type should default to grok-build harness\nsystem prompt preview: {}",
|
||||
& sys_prompt[..sys_prompt.len().min(500)]
|
||||
);
|
||||
sys_prompt.contains("Grok") || sys_prompt.contains("grok"),
|
||||
"model without agent_type should default to grok-build harness\nsystem prompt preview: {}",
|
||||
&sys_prompt[..sys_prompt.len().min(500)]
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
@ -210,134 +218,34 @@ async fn test_grok_agent_env_overrides_model_agent_type() {
|
|||
with_local_set(|| async {
|
||||
let server = dual_model_server().await;
|
||||
let workdir = git_workdir();
|
||||
let binary = grok_binary();
|
||||
let home = tempfile::TempDir::new().expect("create temp home");
|
||||
let mut cmd = tokio::process::Command::new(&binary);
|
||||
cmd.args(["agent", "stdio"])
|
||||
.current_dir(workdir.path())
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(
|
||||
&mut cmd,
|
||||
&server.url(),
|
||||
home.path(),
|
||||
);
|
||||
cmd.env("GROK_AGENT", "grok-build");
|
||||
let mut child = cmd.spawn().expect("spawn grok");
|
||||
let outgoing = child.stdin.take().unwrap();
|
||||
let incoming = child.stdout.take().unwrap();
|
||||
use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
|
||||
let outgoing = outgoing.compat_write();
|
||||
let incoming = incoming.compat();
|
||||
let incoming = xai_acp_lib::LineBufferedRead::spawn_local(incoming);
|
||||
use agent_client_protocol as acp;
|
||||
struct NoopClient;
|
||||
#[async_trait::async_trait(?Send)]
|
||||
impl acp::Client for NoopClient {
|
||||
async fn request_permission(
|
||||
&self,
|
||||
args: acp::RequestPermissionRequest,
|
||||
) -> acp::Result<acp::RequestPermissionResponse> {
|
||||
let outcome = args
|
||||
.options
|
||||
.iter()
|
||||
.find(|o| o.kind == acp::PermissionOptionKind::AllowOnce)
|
||||
.or(args.options.first())
|
||||
.map(|o| acp::RequestPermissionOutcome::Selected(
|
||||
acp::SelectedPermissionOutcome::new(o.option_id.clone()),
|
||||
))
|
||||
.unwrap_or(acp::RequestPermissionOutcome::Cancelled);
|
||||
Ok(acp::RequestPermissionResponse::new(outcome))
|
||||
}
|
||||
async fn session_notification(
|
||||
&self,
|
||||
_args: acp::SessionNotification,
|
||||
) -> acp::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let (conn, handle_io) = acp::ClientSideConnection::new(
|
||||
NoopClient,
|
||||
outgoing,
|
||||
incoming,
|
||||
|fut| {
|
||||
tokio::task::spawn_local(fut);
|
||||
},
|
||||
);
|
||||
tokio::task::spawn_local(handle_io);
|
||||
let _init = tokio::time::timeout(
|
||||
Duration::from_secs(20),
|
||||
conn
|
||||
.initialize(
|
||||
acp::InitializeRequest::new(acp::ProtocolVersion::V1)
|
||||
.client_capabilities(
|
||||
acp::ClientCapabilities::new()
|
||||
.fs(acp::FileSystemCapabilities::new())
|
||||
.terminal(false),
|
||||
)
|
||||
.meta(
|
||||
serde_json::json!(
|
||||
{ "startupHints" : { "nonInteractive" : true,
|
||||
"skipGitStatus" : true, "skipProjectLayout" : true },
|
||||
"clientType" : "test-client", "clientVersion" : "0.0.0-test"
|
||||
}
|
||||
)
|
||||
.as_object()
|
||||
.cloned(),
|
||||
),
|
||||
),
|
||||
let sandbox = TestSandbox::builder().mock_url(server.url()).build();
|
||||
let client = GrokStdioClient::spawn_with_sandbox_env_and_args(
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
sandbox,
|
||||
&[("GROK_AGENT", "grok-build")],
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.expect("init timed out")
|
||||
.expect("init failed");
|
||||
conn.authenticate(
|
||||
acp::AuthenticateRequest::new(acp::AuthMethodId::new("xai.api_key"))
|
||||
.meta(
|
||||
serde_json::json!({ "headless" : true }).as_object().cloned(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("auth failed");
|
||||
let session = tokio::time::timeout(
|
||||
Duration::from_secs(20),
|
||||
conn
|
||||
.new_session(
|
||||
acp::NewSessionRequest::new(workdir.path().to_path_buf())
|
||||
.meta(
|
||||
serde_json::json!({ "modelId" : "cursor-model" })
|
||||
.as_object()
|
||||
.cloned(),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("session/new timed out")
|
||||
.expect("session/new failed");
|
||||
let _prompt = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
conn
|
||||
.prompt(
|
||||
acp::PromptRequest::new(
|
||||
session.session_id.clone(),
|
||||
vec![
|
||||
acp::ContentBlock::Text(acp::TextContent::new("say hello"))
|
||||
],
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("prompt timed out")
|
||||
.expect("prompt failed");
|
||||
.await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client
|
||||
.create_session_with_model_timeout(workdir.workspace(), "cursor-model")
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"prompt with GROK_AGENT override failed: {:?}\nstderr:\n{}",
|
||||
result.err(),
|
||||
client.stderr()
|
||||
);
|
||||
let sys_prompt = server
|
||||
.last_system_prompt()
|
||||
.expect("should have inference request");
|
||||
assert!(
|
||||
sys_prompt.contains("Grok") || sys_prompt.contains("grok"),
|
||||
"GROK_AGENT=grok-build should override cursor model's agent_type\nsystem prompt preview: {}",
|
||||
& sys_prompt[..sys_prompt.len().min(500)]
|
||||
);
|
||||
sys_prompt.contains("Grok") || sys_prompt.contains("grok"),
|
||||
"GROK_AGENT=grok-build should override catalog model agent_type\nsystem prompt preview: {}",
|
||||
&sys_prompt[..sys_prompt.len().min(500)]
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,10 +22,12 @@ async fn provider_backed_model_sends_minted_token_on_the_wire() {
|
|||
let server = MockInferenceServer::start()
|
||||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::TempDir::new().unwrap();
|
||||
let mut sandbox = TestSandbox::builder().git().mock_url(server.url()).build();
|
||||
// The baseline already omits the leader socket; keep the test's explicit
|
||||
// fresh-process intent at the typed sandbox layer that survives env_clear().
|
||||
sandbox.remove_env("GROK_LEADER_SOCKET");
|
||||
|
||||
let grok_home = home.path().join(".grok");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
std::fs::create_dir_all(&grok_home).expect("create .grok home");
|
||||
|
||||
let counter = grok_home.join("mint-count");
|
||||
|
|
@ -77,17 +79,14 @@ auth_provider = "gateway"
|
|||
"json",
|
||||
])
|
||||
.arg("--cwd")
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(sandbox.workspace())
|
||||
.current_dir(sandbox.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
// Don't attach to a developer's ambient leader; spawn fresh against the mock.
|
||||
cmd.env_remove("GROK_LEADER_SOCKET");
|
||||
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox_borrowed(cmd, &sandbox).await;
|
||||
assert_headless_success(&result, "auth provider e2e", Some(&server));
|
||||
|
||||
let runs = std::fs::read_to_string(&counter)
|
||||
|
|
@ -142,10 +141,12 @@ async fn undefined_provider_fails_closed_and_never_leaks_session_key() {
|
|||
)
|
||||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::TempDir::new().unwrap();
|
||||
let mut sandbox = TestSandbox::builder().git().mock_url(server.url()).build();
|
||||
// The baseline already omits the leader socket; keep the test's explicit
|
||||
// fresh-process intent at the typed sandbox layer that survives env_clear().
|
||||
sandbox.remove_env("GROK_LEADER_SOCKET");
|
||||
|
||||
let grok_home = home.path().join(".grok");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
std::fs::create_dir_all(&grok_home).expect("create .grok home");
|
||||
|
||||
// Model references `gateway`, but no `[auth_provider.gateway]` table exists.
|
||||
|
|
@ -176,18 +177,16 @@ auth_provider = "gateway"
|
|||
"json",
|
||||
])
|
||||
.arg("--cwd")
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(sandbox.workspace())
|
||||
.current_dir(sandbox.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
cmd.env_remove("GROK_LEADER_SOCKET");
|
||||
|
||||
// The turn is expected to fail (the mock 401s the unauthenticated request);
|
||||
// we assert on the wire, not the exit code.
|
||||
let _ = run_headless_with_cmd(cmd).await;
|
||||
let _ = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
let requests = server.requests();
|
||||
// Non-vacuity: the model was actually exercised.
|
||||
|
|
@ -220,10 +219,12 @@ async fn provider_with_args_and_json_output_sends_minted_token() {
|
|||
let server = MockInferenceServer::start()
|
||||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::TempDir::new().unwrap();
|
||||
let mut sandbox = TestSandbox::builder().git().mock_url(server.url()).build();
|
||||
// The baseline already omits the leader socket; keep the test's explicit
|
||||
// fresh-process intent at the typed sandbox layer that survives env_clear().
|
||||
sandbox.remove_env("GROK_LEADER_SOCKET");
|
||||
|
||||
let grok_home = home.path().join(".grok");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
std::fs::create_dir_all(&grok_home).expect("create .grok home");
|
||||
|
||||
// The helper records the args it was invoked with (proving direct exec, no
|
||||
|
|
@ -279,16 +280,14 @@ auth_provider = "gateway"
|
|||
"json",
|
||||
])
|
||||
.arg("--cwd")
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(sandbox.workspace())
|
||||
.current_dir(sandbox.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
cmd.env_remove("GROK_LEADER_SOCKET");
|
||||
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox_borrowed(cmd, &sandbox).await;
|
||||
assert_headless_success(&result, "auth provider args/json e2e", Some(&server));
|
||||
|
||||
let args = std::fs::read_to_string(&seen_args).expect("helper must have run");
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ async fn test_headless_session_in_git_repo() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.path()).await;
|
||||
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.workspace()).await;
|
||||
|
||||
assert_headless_success(&result, "grok -p in git repo", Some(&server));
|
||||
assert_no_crashes(&result.stderr);
|
||||
|
|
@ -211,7 +211,7 @@ async fn test_headless_tools_allowlist_keeps_enabled_web_tools() {
|
|||
"--tools",
|
||||
"read_file,grep,list_dir,web_search,web_fetch",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
&[("GROK_WEB_FETCH", "1")],
|
||||
)
|
||||
.await;
|
||||
|
|
@ -272,7 +272,7 @@ async fn test_headless_tools_allowlist_does_not_fail_open_for_disabled_web_fetch
|
|||
"--tools",
|
||||
"read_file,web_fetch",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
&[("GROK_WEB_FETCH", "0")],
|
||||
)
|
||||
.await;
|
||||
|
|
@ -302,7 +302,7 @@ async fn test_headless_terminal_only_allowlist_is_foreground_only() {
|
|||
let result = run_headless(
|
||||
&server,
|
||||
&["-p", "say hello", "--yolo", "--tools", "run_terminal_cmd"],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -357,7 +357,7 @@ async fn test_headless_free_usage_exhausted_prints_paywall_message() {
|
|||
}
|
||||
let workdir = git_workdir();
|
||||
|
||||
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.path()).await;
|
||||
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.workspace()).await;
|
||||
|
||||
assert!(
|
||||
!result.timed_out && !result.status.success(),
|
||||
|
|
@ -396,7 +396,7 @@ async fn test_headless_streaming_json_output() {
|
|||
"--output-format",
|
||||
"streaming-json",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -485,7 +485,7 @@ async fn test_headless_json_reports_server_cost() {
|
|||
"--output-format",
|
||||
"json",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -537,7 +537,7 @@ async fn test_headless_json_reports_usage_on_max_turns() {
|
|||
"--output-format",
|
||||
"json",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -563,7 +563,7 @@ async fn test_headless_streaming_json_usage() {
|
|||
"--output-format",
|
||||
"streaming-json",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -602,7 +602,7 @@ async fn headless_json_schema_chat_completions_uses_response_format() {
|
|||
"--max-turns",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -661,7 +661,7 @@ async fn headless_json_schema_responses_uses_text_format() {
|
|||
"--max-turns",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -717,7 +717,7 @@ async fn headless_json_schema_messages_backend_uses_structured_output_tool() {
|
|||
"--max-turns",
|
||||
"2",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -799,7 +799,7 @@ async fn headless_json_schema_messages_validates_text_when_tool_not_called() {
|
|||
"--max-turns",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -846,7 +846,7 @@ async fn headless_json_schema_messages_retries_on_schema_violation() {
|
|||
"--max-turns",
|
||||
"3",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -887,7 +887,7 @@ async fn invalid_json_schema_disables_structured_output_and_surfaces_error() {
|
|||
"--max-turns",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -946,7 +946,7 @@ async fn test_stdio_full_session_lifecycle() {
|
|||
with_local_set(|| async {
|
||||
let server = MockInferenceServer::start().await.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
// Initialize and authenticate
|
||||
let init_resp = client.initialize_with_timeout().await;
|
||||
|
|
@ -956,7 +956,7 @@ async fn test_stdio_full_session_lifecycle() {
|
|||
);
|
||||
|
||||
// Create session (triggers libgit2 init)
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client.create_session_with_timeout(workdir.workspace()).await;
|
||||
assert!(!session_id.0.is_empty(), "session ID should be non-empty");
|
||||
|
||||
// Send prompt — triggers inference to mock server
|
||||
|
|
@ -993,10 +993,12 @@ async fn test_stdio_session_close() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
|
||||
// Session should be alive — session/info returns data with sessionId
|
||||
let info_resp = client
|
||||
|
|
@ -1055,7 +1057,7 @@ async fn test_stdio_prompt_then_immediate_load_session() {
|
|||
with_local_set(|| async {
|
||||
let server = MockInferenceServer::start().await.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
let init_resp = writer.initialize_with_timeout().await;
|
||||
assert!(
|
||||
|
|
@ -1063,7 +1065,7 @@ async fn test_stdio_prompt_then_immediate_load_session() {
|
|||
"agent should return at least one auth method"
|
||||
);
|
||||
|
||||
let session_id = writer.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = writer.create_session_with_timeout(workdir.workspace()).await;
|
||||
let result = writer.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
|
|
@ -1073,13 +1075,18 @@ async fn test_stdio_prompt_then_immediate_load_session() {
|
|||
stderr_tail(&writer.stderr(), 1200)
|
||||
);
|
||||
|
||||
let shared_home = writer.take_home();
|
||||
let shared_sandbox = writer.take_sandbox();
|
||||
drop(writer);
|
||||
|
||||
let reader = GrokStdioClient::spawn_with_home(&server, workdir.path(), shared_home).await;
|
||||
let reader = GrokStdioClient::spawn_with_sandbox(
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
shared_sandbox,
|
||||
)
|
||||
.await;
|
||||
reader.initialize_with_timeout().await;
|
||||
let _ = reader
|
||||
.load_session_with_timeout(&session_id, workdir.path())
|
||||
.load_session_with_timeout(&session_id, workdir.workspace())
|
||||
.await;
|
||||
assert!(
|
||||
reader.notification_count() > 0,
|
||||
|
|
@ -1133,7 +1140,7 @@ async fn test_stdio_xcode_escaped_slash_methods_get_responses() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let mut agent = RawStdioClient::spawn(&server, workdir.path()).await;
|
||||
let mut agent = RawStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
// initialize/authenticate carry no slash (they work from Xcode too), but
|
||||
// ride string UUID ids and minimal capabilities like Xcode's client.
|
||||
|
|
@ -1173,7 +1180,7 @@ async fn test_stdio_xcode_escaped_slash_methods_get_responses() {
|
|||
"jsonrpc": "2.0",
|
||||
"id": new_id,
|
||||
"method": "session/new",
|
||||
"params": { "cwd": workdir.path(), "mcpServers": [] },
|
||||
"params": { "cwd": workdir.workspace(), "mcpServers": [] },
|
||||
}),
|
||||
"session/new",
|
||||
);
|
||||
|
|
@ -1235,57 +1242,34 @@ async fn test_stdio_xcode_escaped_slash_methods_get_responses() {
|
|||
/// Isolated headless run with a custom `~/.grok/`. Clean env (no leaked
|
||||
/// host credentials). Write config files into `grok_dir()` before `run()`.
|
||||
struct ConfigTestHarness {
|
||||
home: tempfile::TempDir,
|
||||
workdir: tempfile::TempDir,
|
||||
env: Vec<(String, String)>,
|
||||
sandbox: TestSandbox,
|
||||
}
|
||||
|
||||
impl ConfigTestHarness {
|
||||
fn new(server: &MockInferenceServer) -> Self {
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
Self {
|
||||
home,
|
||||
workdir: git_workdir(),
|
||||
env: vec![
|
||||
("GROK_CLI_CHAT_PROXY_BASE_URL".into(), server.url()),
|
||||
("GROK_TELEMETRY_ENABLED".into(), "false".into()),
|
||||
("GROK_FEEDBACK_ENABLED".into(), "false".into()),
|
||||
("GROK_TRACE_UPLOAD".into(), "false".into()),
|
||||
("GROK_INSTRUMENTATION".into(), "disabled".into()),
|
||||
("GROK_DISABLE_AUTOUPDATER".into(), "1".into()),
|
||||
],
|
||||
sandbox: TestSandbox::builder().mock_url(server.url()).git().build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn grok_dir(&self) -> std::path::PathBuf {
|
||||
self.home.path().join(".grok")
|
||||
self.sandbox.grok_home().to_path_buf()
|
||||
}
|
||||
|
||||
fn env(&mut self, key: &str, value: &str) -> &mut Self {
|
||||
self.env.push((key.into(), value.into()));
|
||||
self.sandbox.set_env(key, value);
|
||||
self
|
||||
}
|
||||
|
||||
async fn run(&self) -> HeadlessResult {
|
||||
async fn run(self) -> HeadlessResult {
|
||||
let mut cmd = tokio::process::Command::new(grok_binary());
|
||||
cmd.args(["-p", "say hello", "--yolo"])
|
||||
.current_dir(self.workdir.path())
|
||||
.current_dir(self.sandbox.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.env_clear()
|
||||
.env("HOME", self.home.path())
|
||||
// Windows resolves `~` via USERPROFILE, not HOME — pin the grok
|
||||
// home explicitly so the sandbox holds on all platforms (see
|
||||
// `test_env_cmd_tokio`).
|
||||
.env("GROK_HOME", self.grok_dir())
|
||||
.env("PATH", std::env::var("PATH").unwrap_or_default());
|
||||
for (k, v) in &self.env {
|
||||
cmd.env(k, v);
|
||||
}
|
||||
run_headless_with_cmd(cmd).await
|
||||
.kill_on_drop(true);
|
||||
run_headless_in_sandbox(cmd, self.sandbox).await
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1375,7 +1359,7 @@ async fn headless_reasoning_efforts_payload_parses_and_legacy_effort_rides_wire(
|
|||
"--max-turns",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -1493,7 +1477,7 @@ async fn test_headless_timeout_exit_kills_pending_background_task() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let pid_file = workdir.path().join("task_pid.txt");
|
||||
let pid_file = workdir.workspace().join("task_pid.txt");
|
||||
enqueue_background_task_turn(&server, &pid_file);
|
||||
|
||||
let result = run_headless(
|
||||
|
|
@ -1505,7 +1489,7 @@ async fn test_headless_timeout_exit_kills_pending_background_task() {
|
|||
"--background-wait-timeout",
|
||||
"1",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -1534,7 +1518,7 @@ async fn test_headless_no_wait_exit_kills_background_task() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let pid_file = workdir.path().join("task_pid.txt");
|
||||
let pid_file = workdir.workspace().join("task_pid.txt");
|
||||
enqueue_background_task_turn(&server, &pid_file);
|
||||
|
||||
let result = run_headless(
|
||||
|
|
@ -1545,7 +1529,7 @@ async fn test_headless_no_wait_exit_kills_background_task() {
|
|||
"--yolo",
|
||||
"--no-wait-for-background",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -1571,7 +1555,7 @@ async fn test_headless_waits_for_short_background_task_and_exits_clean() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let marker = workdir.path().join("finished.txt");
|
||||
let marker = workdir.workspace().join("finished.txt");
|
||||
let command = format!("/bin/sleep 1 && echo ok > {}", marker.display());
|
||||
let args = serde_json::json!({
|
||||
"command": command,
|
||||
|
|
@ -1610,7 +1594,7 @@ async fn test_headless_waits_for_short_background_task_and_exits_clean() {
|
|||
"--background-wait-timeout",
|
||||
"30",
|
||||
],
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,12 @@ fn debug_cmd(
|
|||
home: &Path,
|
||||
workdir: &Path,
|
||||
extra: &[&str],
|
||||
) -> tokio::process::Command {
|
||||
) -> (tokio::process::Command, TestSandbox) {
|
||||
let mut sandbox = TestSandbox::builder().mock_url(server.url()).build();
|
||||
sandbox
|
||||
.set_env("HOME", home)
|
||||
.set_env("USERPROFILE", home)
|
||||
.set_env("GROK_HOME", home.join(".grok"));
|
||||
let mut cmd = tokio::process::Command::new(grok_binary());
|
||||
cmd.args(["-p", "say hi", "--yolo", "--output-format", "json"])
|
||||
.args(extra)
|
||||
|
|
@ -87,14 +92,8 @@ fn debug_cmd(
|
|||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home);
|
||||
// Pin the home location and drop inherited firehose toggles for determinism.
|
||||
cmd.env("GROK_HOME", home.join(".grok"));
|
||||
cmd.env_remove("GROK_DEBUG_LOG");
|
||||
cmd.env_remove("GROK_LOG_FILE");
|
||||
cmd.env_remove("GROK_LOG_SAMPLING");
|
||||
cmd.env_remove("GROK_HOOKS_LOG");
|
||||
cmd
|
||||
sandbox.apply_to_tokio_command(&mut cmd);
|
||||
(cmd, sandbox)
|
||||
}
|
||||
|
||||
/// Poll up to 50×100ms for the per-session firehose at `path` to become non-empty
|
||||
|
|
@ -142,8 +141,8 @@ async fn debug_flag_enables_firehose_without_crashing() {
|
|||
let workdir = git_workdir();
|
||||
let home = TempDir::new().expect("create temp home");
|
||||
|
||||
let cmd = debug_cmd(&server, home.path(), workdir.path(), &["--debug"]);
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let (cmd, sandbox) = debug_cmd(&server, home.path(), workdir.workspace(), &["--debug"]);
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
assert_headless_success(&result, "grok --debug headless", Some(&server));
|
||||
assert_no_crashes(&result.stderr);
|
||||
|
|
@ -159,8 +158,8 @@ async fn no_debug_flag_writes_no_debug_dir() {
|
|||
let workdir = git_workdir();
|
||||
let home = TempDir::new().expect("create temp home");
|
||||
|
||||
let cmd = debug_cmd(&server, home.path(), workdir.path(), &[]);
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let (cmd, sandbox) = debug_cmd(&server, home.path(), workdir.workspace(), &[]);
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
assert_headless_success(&result, "grok headless (no --debug)", Some(&server));
|
||||
assert!(
|
||||
|
|
@ -182,19 +181,16 @@ async fn agent_session_writes_named_session_file() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = TempDir::new().expect("create temp home");
|
||||
let grok_home = home.path().join(".grok");
|
||||
let grok_home_str = grok_home.to_string_lossy().into_owned();
|
||||
let mut sandbox = TestSandbox::new();
|
||||
sandbox.set_env("GROK_DEBUG_LOG", "1");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
|
||||
let client = GrokStdioClient::spawn_with_home_and_env(
|
||||
&server,
|
||||
workdir.path(),
|
||||
home,
|
||||
&[("GROK_DEBUG_LOG", "1"), ("GROK_HOME", &grok_home_str)],
|
||||
)
|
||||
.await;
|
||||
let client =
|
||||
GrokStdioClient::spawn_with_sandbox(&server, workdir.workspace(), sandbox).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
// New session ids are UUID v7 (filesystem-safe), so the firehose file is
|
||||
// named verbatim `<sessionId>.txt`.
|
||||
let sid = session_id.0.to_string();
|
||||
|
|
@ -231,24 +227,25 @@ async fn debug_flag_master_switch_enables_firehose() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = TempDir::new().expect("create temp home");
|
||||
let grok_home = home.path().join(".grok");
|
||||
let grok_home_str = grok_home.to_string_lossy().into_owned();
|
||||
let sandbox = TestSandbox::new();
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
|
||||
// Drive `grok --debug agent stdio`: the master switch (which runs before
|
||||
// the agent dispatch) must be what enables the firehose — NOT a direct
|
||||
// GROK_DEBUG_LOG env. The spawn helper clears inherited firehose toggles,
|
||||
// so the `--debug` flag is the only thing that can enable logging here.
|
||||
let client = GrokStdioClient::spawn_with_home_env_and_args(
|
||||
// GROK_DEBUG_LOG env. The sandbox baseline excludes inherited firehose
|
||||
// toggles, so the `--debug` flag is the only thing enabling logging here.
|
||||
let client = GrokStdioClient::spawn_with_sandbox_env_and_args(
|
||||
&server,
|
||||
workdir.path(),
|
||||
home,
|
||||
&[("GROK_HOME", &grok_home_str)],
|
||||
workdir.workspace(),
|
||||
sandbox,
|
||||
&[],
|
||||
&["--debug"],
|
||||
)
|
||||
.await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let sid = session_id.0.to_string();
|
||||
let _ = client.prompt_with_timeout(&session_id, "say hi").await;
|
||||
|
||||
|
|
@ -284,13 +281,13 @@ async fn debug_file_flag_writes_single_file_and_bypasses_routing() {
|
|||
let explicit = home.path().join("explicit-firehose.txt");
|
||||
let explicit_str = explicit.to_string_lossy().into_owned();
|
||||
|
||||
let cmd = debug_cmd(
|
||||
let (cmd, sandbox) = debug_cmd(
|
||||
&server,
|
||||
home.path(),
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
&["--debug-file", &explicit_str],
|
||||
);
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
assert_headless_success(&result, "grok --debug-file", Some(&server));
|
||||
assert_no_crashes(&result.stderr);
|
||||
|
|
@ -318,9 +315,9 @@ async fn grok_log_file_explicit_path_is_written() {
|
|||
let home = TempDir::new().expect("create temp home");
|
||||
let custom = home.path().join("custom-log-file.log");
|
||||
|
||||
let mut cmd = debug_cmd(&server, home.path(), workdir.path(), &[]);
|
||||
cmd.env("GROK_LOG_FILE", &custom);
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let (cmd, mut sandbox) = debug_cmd(&server, home.path(), workdir.workspace(), &[]);
|
||||
sandbox.set_env("GROK_LOG_FILE", &custom);
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
assert_headless_success(&result, "grok GROK_LOG_FILE=path", Some(&server));
|
||||
assert_no_crashes(&result.stderr);
|
||||
|
|
|
|||
|
|
@ -537,10 +537,11 @@ async fn headless_config_enables_doom_loop_check_header() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = xai_grok_test_support::git_workdir();
|
||||
let home = tempfile::TempDir::new().unwrap();
|
||||
let sandbox = xai_grok_test_support::TestSandbox::builder()
|
||||
.mock_url(server.url())
|
||||
.build();
|
||||
|
||||
let grok_home = home.path().join(".grok");
|
||||
std::fs::create_dir_all(&grok_home).expect("create .grok home");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
std::fs::write(
|
||||
grok_home.join("config.toml"),
|
||||
"[doom_loop_recovery]\nenabled = true\n",
|
||||
|
|
@ -550,18 +551,14 @@ async fn headless_config_enables_doom_loop_check_header() {
|
|||
let mut cmd = tokio::process::Command::new(xai_grok_test_support::grok_binary());
|
||||
cmd.args(["-p", "say hi", "--yolo", "--output-format", "json"])
|
||||
.arg("--cwd")
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(workdir.workspace())
|
||||
.current_dir(workdir.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
cmd.env("GROK_HOME", grok_home);
|
||||
// Don't attach to a developer's ambient leader; spawn fresh against the mock.
|
||||
cmd.env_remove("GROK_LEADER_SOCKET");
|
||||
|
||||
let result = xai_grok_test_support::run_headless_with_cmd(cmd).await;
|
||||
let result = xai_grok_test_support::run_headless_in_sandbox(cmd, sandbox).await;
|
||||
xai_grok_test_support::assert_headless_success(&result, "doom-loop header e2e", Some(&server));
|
||||
|
||||
let requests = server.requests();
|
||||
|
|
|
|||
|
|
@ -29,10 +29,9 @@ async fn global_models_config_reaches_inference_request() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::TempDir::new().unwrap();
|
||||
let sandbox = TestSandbox::builder().mock_url(server.url()).build();
|
||||
|
||||
let grok_home = home.path().join(".grok");
|
||||
std::fs::create_dir_all(&grok_home).expect("create .grok home");
|
||||
let grok_home = sandbox.grok_home().to_path_buf();
|
||||
std::fs::write(
|
||||
grok_home.join("config.toml"),
|
||||
r#"[models]
|
||||
|
|
@ -50,18 +49,14 @@ stream_tool_calls = true
|
|||
let mut cmd = tokio::process::Command::new(grok_binary());
|
||||
cmd.args(["-p", "say hi", "--yolo", "--output-format", "json"])
|
||||
.arg("--cwd")
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(workdir.workspace())
|
||||
.current_dir(workdir.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
cmd.env("GROK_HOME", grok_home);
|
||||
// Don't attach to a developer's ambient leader; spawn fresh against the mock.
|
||||
cmd.env_remove("GROK_LEADER_SOCKET");
|
||||
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
assert_headless_success(&result, "global models config e2e", Some(&server));
|
||||
|
||||
let requests = server.requests();
|
||||
|
|
|
|||
|
|
@ -19,312 +19,276 @@
|
|||
|
||||
#![cfg(unix)]
|
||||
|
||||
mod common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use agent_client_protocol::{self as acp, Agent as _};
|
||||
|
||||
use xai_grok_test_support::leader::{
|
||||
LeaderStdioClient, leader_log, wait_for_live_leader, wait_for_new_leader,
|
||||
wait_for_replay_notifications,
|
||||
LeaderFixture, leader_log, wait_for_live_leader, wait_for_replay_notifications,
|
||||
};
|
||||
use xai_grok_test_support::*;
|
||||
|
||||
/// THE repro. Kill the shared leader with SIGKILL while two clients are
|
||||
/// connected; both must recover their sessions on the re-elected leader.
|
||||
/// Kill the shared leader while two clients are connected; both must recover.
|
||||
#[tokio::test]
|
||||
#[ignore] // requires pre-built binary; run with --ignored
|
||||
#[ignore = "leader-acceptance: detached replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn test_leader_sigkill_clients_recover_sessions() {
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
|
||||
// ── Phase 1: two clients, one leader, two sessions ────────────
|
||||
let client_a = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client_a.initialize().await;
|
||||
let session_a = client_a.create_session(workdir.path()).await;
|
||||
let r = client_a.prompt(&session_a, "hello from A").await;
|
||||
assert!(
|
||||
r.is_ok(),
|
||||
"pre-crash prompt A failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
r.err(),
|
||||
client_a.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
let client_b = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client_b.initialize().await;
|
||||
let session_b = client_b.create_session(workdir.path()).await;
|
||||
let r = client_b.prompt(&session_b, "hello from B").await;
|
||||
assert!(
|
||||
r.is_ok(),
|
||||
"pre-crash prompt B failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
r.err(),
|
||||
client_b.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("no live leader PID in lock file");
|
||||
assert_ne!(leader_pid, client_a.child.id().unwrap_or(0));
|
||||
assert_ne!(leader_pid, client_b.child.id().unwrap_or(0));
|
||||
.expect("start persistent leader fixture");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn client A"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session_a = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0]
|
||||
.prompt(&session_a, "hello from A")
|
||||
.await
|
||||
.expect("pre-crash prompt A");
|
||||
|
||||
// ── Phase 2: SIGKILL the leader (simulated crash) ─────────────
|
||||
let base_a = client_a.notification_count();
|
||||
let base_b = client_b.notification_count();
|
||||
eprintln!("killing leader pid {leader_pid}");
|
||||
unsafe {
|
||||
libc::kill(leader_pid as i32, libc::SIGKILL);
|
||||
}
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn client B"),
|
||||
);
|
||||
clients[1].initialize().await;
|
||||
let session_b = clients[1].create_session(workdir.workspace()).await;
|
||||
clients[1]
|
||||
.prompt(&session_b, "hello from B")
|
||||
.await
|
||||
.expect("pre-crash prompt B");
|
||||
|
||||
// ── Phase 3: clients must re-elect a leader and reconnect ─────
|
||||
let new_pid = wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no new leader was elected after SIGKILL\n\
|
||||
client A stderr:\n{}\nclient B stderr:\n{}\nleader log:\n{}",
|
||||
client_a.stderr_text(),
|
||||
client_b.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
eprintln!("new leader elected: pid {new_pid}");
|
||||
|
||||
let a_reconnected =
|
||||
wait_for_replay_notifications(&client_a, base_a, Duration::from_secs(60)).await;
|
||||
let b_reconnected =
|
||||
wait_for_replay_notifications(&client_b, base_b, Duration::from_secs(60)).await;
|
||||
eprintln!("replay evidence: A={a_reconnected} B={b_reconnected}");
|
||||
|
||||
// ── Phase 4: prompts on the ORIGINAL session IDs must work ────
|
||||
let res_a = client_a.prompt(&session_a, "after crash A").await;
|
||||
let res_b = client_b.prompt(&session_b, "after crash B").await;
|
||||
|
||||
assert!(
|
||||
res_a.is_ok(),
|
||||
"client A prompt after leader crash failed: {:?}\n\
|
||||
stderr:\n{}\nleader log:\n{}",
|
||||
res_a.err(),
|
||||
client_a.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
assert!(
|
||||
res_b.is_ok(),
|
||||
"client B prompt after leader crash failed: {:?}\n\
|
||||
stderr:\n{}\nleader log:\n{}",
|
||||
res_b.err(),
|
||||
client_b.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
let leader_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("live leader");
|
||||
let base_a = clients[0].notification_count();
|
||||
let base_b = clients[1].notification_count();
|
||||
assert_eq!(
|
||||
fixture
|
||||
.kill_current_concrete_leader()
|
||||
.expect("kill owned leader"),
|
||||
leader_pid
|
||||
);
|
||||
fixture
|
||||
.reap_exited_concrete_leaders()
|
||||
.await
|
||||
.expect("reap crashed concrete leader");
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"no replacement leader\nA:\n{}\nB:\n{}\nleader:\n{}",
|
||||
clients[0].stderr_text(),
|
||||
clients[1].stderr_text(),
|
||||
leader_log(sandbox.home()),
|
||||
)
|
||||
});
|
||||
wait_for_replay_notifications(&clients[0], base_a, Duration::from_secs(60))
|
||||
.await;
|
||||
wait_for_replay_notifications(&clients[1], base_b, Duration::from_secs(60))
|
||||
.await;
|
||||
clients[0]
|
||||
.prompt(&session_a, "after crash A")
|
||||
.await
|
||||
.expect("client A recovery");
|
||||
clients[1]
|
||||
.prompt(&session_b, "after crash B")
|
||||
.await
|
||||
.expect("client B recovery");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Single-client variant: kill -9 the leader, the lone client must re-elect
|
||||
/// and restore. Narrower failure surface than the two-client test.
|
||||
/// Single-client recovery variant.
|
||||
#[tokio::test]
|
||||
#[ignore] // requires pre-built binary; run with --ignored
|
||||
#[ignore = "leader-acceptance: detached replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn test_leader_sigkill_single_client_recovers() {
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
|
||||
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client.initialize().await;
|
||||
let session = client.create_session(workdir.path()).await;
|
||||
client
|
||||
.prompt(&session, "hello")
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("pre-crash prompt failed");
|
||||
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("no live leader PID in lock file");
|
||||
let base = client.notification_count();
|
||||
eprintln!("killing leader pid {leader_pid}");
|
||||
unsafe {
|
||||
libc::kill(leader_pid as i32, libc::SIGKILL);
|
||||
}
|
||||
|
||||
let new_pid = wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no new leader was elected after SIGKILL\nstderr:\n{}\nleader log:\n{}",
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
eprintln!("new leader elected: pid {new_pid}");
|
||||
|
||||
let reconnected =
|
||||
wait_for_replay_notifications(&client, base, Duration::from_secs(60)).await;
|
||||
eprintln!("replay evidence: {reconnected}");
|
||||
|
||||
let res = client.prompt(&session, "after crash").await;
|
||||
assert!(
|
||||
res.is_ok(),
|
||||
"prompt after leader crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
res.err(),
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
.expect("start fixture");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0].prompt(&session, "hello").await.expect("prompt");
|
||||
let leader_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("live leader");
|
||||
let base = clients[0].notification_count();
|
||||
assert_eq!(
|
||||
fixture
|
||||
.kill_current_concrete_leader()
|
||||
.expect("kill owned leader"),
|
||||
leader_pid
|
||||
);
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("replacement leader");
|
||||
wait_for_replay_notifications(&clients[0], base, Duration::from_secs(60)).await;
|
||||
clients[0]
|
||||
.prompt(&session, "after crash")
|
||||
.await
|
||||
.expect("recovered prompt");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// One client driving TWO sessions over a single stdio bridge (the IDE
|
||||
/// shape). After a leader SIGKILL, BOTH sessions must be replayed onto the
|
||||
/// re-elected leader — restoring only the most recent one left the other
|
||||
/// failing with "unknown session id".
|
||||
/// One client must restore both of its sessions after re-election.
|
||||
#[tokio::test]
|
||||
#[ignore] // requires pre-built binary; run with --ignored
|
||||
#[ignore = "leader-acceptance: detached replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn test_leader_sigkill_multi_session_client_recovers_all_sessions() {
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
|
||||
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client.initialize().await;
|
||||
let session_one = client.create_session(workdir.path()).await;
|
||||
client
|
||||
.prompt(&session_one, "hello one")
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("pre-crash prompt on session one failed");
|
||||
let session_two = client.create_session(workdir.path()).await;
|
||||
client
|
||||
.prompt(&session_two, "hello two")
|
||||
.await
|
||||
.expect("pre-crash prompt on session two failed");
|
||||
assert_ne!(session_one.0, session_two.0);
|
||||
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("no live leader PID in lock file");
|
||||
let base = client.notification_count();
|
||||
eprintln!("killing leader pid {leader_pid}");
|
||||
unsafe {
|
||||
libc::kill(leader_pid as i32, libc::SIGKILL);
|
||||
}
|
||||
|
||||
wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no new leader was elected after SIGKILL\nstderr:\n{}\nleader log:\n{}",
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
wait_for_replay_notifications(&client, base, Duration::from_secs(60)).await;
|
||||
|
||||
// BOTH sessions must work on the new leader.
|
||||
let res_one = client.prompt(&session_one, "after crash one").await;
|
||||
let res_two = client.prompt(&session_two, "after crash two").await;
|
||||
assert!(
|
||||
res_one.is_ok(),
|
||||
"session one prompt after crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
res_one.err(),
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
assert!(
|
||||
res_two.is_ok(),
|
||||
"session two prompt after crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
res_two.err(),
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
.expect("start fixture");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session_one = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0]
|
||||
.prompt(&session_one, "hello one")
|
||||
.await
|
||||
.expect("session one prompt");
|
||||
let session_two = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0]
|
||||
.prompt(&session_two, "hello two")
|
||||
.await
|
||||
.expect("session two prompt");
|
||||
let leader_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("live leader");
|
||||
let base = clients[0].notification_count();
|
||||
assert_eq!(
|
||||
fixture
|
||||
.kill_current_concrete_leader()
|
||||
.expect("kill owned leader"),
|
||||
leader_pid
|
||||
);
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("replacement leader");
|
||||
wait_for_replay_notifications(&clients[0], base, Duration::from_secs(60)).await;
|
||||
clients[0]
|
||||
.prompt(&session_one, "after crash one")
|
||||
.await
|
||||
.expect("session one recovery");
|
||||
clients[0]
|
||||
.prompt(&session_two, "after crash two")
|
||||
.await
|
||||
.expect("session two recovery");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Prompt sent DURING the outage (after the bridge noticed the dead leader
|
||||
/// but before the new one is ready). The stdio bridge must hold and deliver
|
||||
/// it once the session is restored — not silently drop it (which left the
|
||||
/// client's request hanging forever).
|
||||
/// A prompt queued during re-election must be delivered after recovery.
|
||||
#[tokio::test]
|
||||
#[ignore] // requires pre-built binary; run with --ignored
|
||||
#[ignore = "leader-acceptance: detached replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn test_prompt_sent_during_outage_is_delivered_after_recovery() {
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
|
||||
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client.initialize().await;
|
||||
let session = client.create_session(workdir.path()).await;
|
||||
client
|
||||
.prompt(&session, "hello")
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("pre-crash prompt failed");
|
||||
.expect("start fixture");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0].prompt(&session, "hello").await.expect("prompt");
|
||||
let leader_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("live leader");
|
||||
assert_eq!(
|
||||
fixture
|
||||
.kill_current_concrete_leader()
|
||||
.expect("kill owned leader"),
|
||||
leader_pid
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
||||
.await
|
||||
.expect("no live leader PID in lock file");
|
||||
eprintln!("killing leader pid {leader_pid}");
|
||||
unsafe {
|
||||
libc::kill(leader_pid as i32, libc::SIGKILL);
|
||||
}
|
||||
// Give the bridge a moment to observe the dead socket (its send
|
||||
// channel closes), then prompt mid-outage: re-election + session
|
||||
// restore are still seconds away.
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
let res = tokio::time::timeout(
|
||||
Duration::from_secs(90),
|
||||
client.conn.prompt(acp::PromptRequest::new(session.clone(), vec![acp::ContentBlock::Text(acp::TextContent::new("sent during outage".to_string()))])),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"prompt sent during outage never completed (dropped by bridge?)\n\
|
||||
stderr:\n{}\nleader log:\n{}",
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
assert!(
|
||||
res.is_ok(),
|
||||
"prompt sent during outage failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
res.err(),
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
// A session-scoped request other than prompt (model switch) must
|
||||
// also survive — same "unknown session id" class.
|
||||
let set_model = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
client.conn.set_session_model(acp::SetSessionModelRequest::new(session.clone(), acp::ModelId::new("test-model"))),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"set_session_model after recovery never completed\nstderr:\n{}\nleader log:\n{}",
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
assert!(
|
||||
set_model.is_ok(),
|
||||
"set_session_model after recovery failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
set_model.err(),
|
||||
client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(90),
|
||||
clients[0].conn.prompt(acp::PromptRequest::new(
|
||||
session.clone(),
|
||||
vec![acp::ContentBlock::Text(acp::TextContent::new(
|
||||
"sent during outage".to_string(),
|
||||
))],
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("outage prompt timeout")
|
||||
.expect("outage prompt failed");
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(leader_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("replacement leader");
|
||||
clients[0]
|
||||
.conn
|
||||
.set_session_model(acp::SetSessionModelRequest::new(
|
||||
session,
|
||||
acp::ModelId::new("test-model"),
|
||||
))
|
||||
.await
|
||||
.expect("set model after recovery");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -582,8 +582,7 @@ async fn test_runtime_profile_start_status_stop_across_clients() {
|
|||
};
|
||||
assert!(matches!(
|
||||
started,
|
||||
ControlPayload::CpuProfileStarted { svg_path, .. }
|
||||
if svg_path == output_path
|
||||
ControlPayload::CpuProfileStarted { svg_path, .. } if svg_path == output_path
|
||||
));
|
||||
|
||||
let status = client_b
|
||||
|
|
@ -603,8 +602,7 @@ if svg_path == output_path
|
|||
svg_path: Some(path),
|
||||
frequency_hz: Some(200),
|
||||
..
|
||||
}
|
||||
if path == output_path
|
||||
} if path == output_path
|
||||
));
|
||||
|
||||
let stopped = client_b
|
||||
|
|
@ -614,8 +612,7 @@ if path == output_path
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
stopped,
|
||||
ControlPayload::CpuProfileStopped { svg_path, .. }
|
||||
if svg_path == output_path
|
||||
ControlPayload::CpuProfileStopped { svg_path, .. } if svg_path == output_path
|
||||
));
|
||||
assert!(output_path.exists());
|
||||
} else {
|
||||
|
|
@ -730,8 +727,7 @@ async fn test_runtime_profile_creates_missing_parent_directory_end_to_end() {
|
|||
};
|
||||
assert!(matches!(
|
||||
started,
|
||||
ControlPayload::CpuProfileStarted { svg_path, .. }
|
||||
if svg_path == nested_output
|
||||
ControlPayload::CpuProfileStarted { svg_path, .. } if svg_path == nested_output
|
||||
));
|
||||
|
||||
let stopped = client
|
||||
|
|
@ -741,8 +737,7 @@ if svg_path == nested_output
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
stopped,
|
||||
ControlPayload::CpuProfileStopped { svg_path, .. }
|
||||
if svg_path == nested_output
|
||||
ControlPayload::CpuProfileStopped { svg_path, .. } if svg_path == nested_output
|
||||
));
|
||||
assert!(nested_output.exists());
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -3,21 +3,15 @@
|
|||
//! cross-version eviction with real processes.
|
||||
//!
|
||||
//! Binaries are resolved per role:
|
||||
//! - `GROK_BINARY_LEADER` — the binary that elects the initial leader
|
||||
//! (typically the latest released stable, e.g. fetched from
|
||||
//! `https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-<ver>-linux-x86_64`).
|
||||
//! - `GROK_BINARY_CLIENT` — the second client (typically a freshly built main).
|
||||
//! - `GROK_BINARY_LEADER` — the binary that elects the initial leader.
|
||||
//! - `GROK_BINARY_CLIENT` — the second client.
|
||||
//!
|
||||
//! All tests are `#[ignore]`d: they need two pre-built binaries and spawn real
|
||||
//! leader subprocesses. On-demand today — no CI lane runs them; invoke with:
|
||||
//!
|
||||
//! ```bash
|
||||
//! GROK_BINARY_LEADER=/path/to/grok-old GROK_BINARY_CLIENT=/path/to/grok-new \
|
||||
//! cargo test -p xai-grok-shell --test test_leader_version_skew -- --ignored --nocapture
|
||||
//! ```
|
||||
//! These ignored tests require two pre-built binaries.
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
mod common;
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
@ -25,14 +19,12 @@ use xai_grok_shell::leader::{
|
|||
ClientCapabilities, ClientMode, ControlCommand, ControlPayload, LeaderClient,
|
||||
};
|
||||
use xai_grok_test_support::leader::{
|
||||
LeaderStdioClient, client_binary, leader_binary, leader_log, pid_alive, read_leader_pid,
|
||||
wait_for_live_leader, wait_for_new_leader, wait_for_replay_notifications,
|
||||
LeaderFixture, client_binary, leader_binary, leader_log, pid_alive, read_leader_pid,
|
||||
wait_for_live_leader, wait_for_replay_notifications,
|
||||
};
|
||||
use xai_grok_test_support::*;
|
||||
|
||||
/// Skew tests are meaningless when both roles resolve to the same binary
|
||||
/// (e.g. a local `--ignored` run without the env vars): the version floor
|
||||
/// never trips. Skip loudly instead of failing.
|
||||
/// Skip when both roles resolve to the same binary; such a run tests no skew.
|
||||
fn skew_binaries() -> Option<(std::path::PathBuf, std::path::PathBuf)> {
|
||||
let old = leader_binary();
|
||||
let new = client_binary();
|
||||
|
|
@ -62,11 +54,9 @@ fn sandbox_unified_log(home: &Path) -> String {
|
|||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// End-to-end version-skew: an old leader is running; a newer client connects,
|
||||
/// evicts it under the version floor, spawns a replacement from its own
|
||||
/// binary, and the old client's session survives via reconnect + reload.
|
||||
/// A new client evicts an old leader and the old session survives replay.
|
||||
#[tokio::test]
|
||||
#[ignore = "two-binary version-skew test; set GROK_BINARY_LEADER/GROK_BINARY_CLIENT and run with --ignored"]
|
||||
#[ignore = "leader-acceptance: version-skew replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn new_client_evicts_old_leader_and_sessions_reload() {
|
||||
let Some((old_bin, new_bin)) = skew_binaries() else {
|
||||
return;
|
||||
|
|
@ -75,83 +65,98 @@ async fn new_client_evicts_old_leader_and_sessions_reload() {
|
|||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture =
|
||||
LeaderFixture::start_with_binary(&old_bin, &server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("start owned version-skew leader");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(
|
||||
&fixture,
|
||||
&mut clients,
|
||||
|fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn old leader client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0]
|
||||
.prompt(&session, "hello from the old world")
|
||||
.await
|
||||
.expect("pre-skew prompt failed");
|
||||
let old_pid =
|
||||
wait_for_live_leader(sandbox.home(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
let base = clients[0].notification_count();
|
||||
|
||||
// Old binary elects the leader and completes a turn.
|
||||
let old_client = LeaderStdioClient::spawn_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn new leader client"),
|
||||
);
|
||||
clients[1].initialize().await;
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"no replacement leader after version-floor eviction\nold stderr:\n{}\nnew stderr:\n{}\nleader log:\n{}",
|
||||
clients[0].stderr_text(),
|
||||
clients[1].stderr_text(),
|
||||
leader_log(sandbox.home()),
|
||||
)
|
||||
});
|
||||
assert_ne!(_new_pid, old_pid);
|
||||
assert!(
|
||||
wait_for_pid_death(old_pid, Duration::from_secs(30)).await,
|
||||
"old leader pid {old_pid} still alive after eviction\nleader log:\n{}",
|
||||
leader_log(sandbox.home()),
|
||||
);
|
||||
|
||||
wait_for_replay_notifications(
|
||||
&clients[0],
|
||||
base,
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await;
|
||||
let response = clients[0].prompt(&session, "after the eviction").await;
|
||||
assert!(
|
||||
response.is_ok(),
|
||||
"old client prompt after eviction failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
response.err(),
|
||||
clients[0].stderr_text(),
|
||||
leader_log(sandbox.home()),
|
||||
);
|
||||
let new_session = clients[1].create_session(workdir.workspace()).await;
|
||||
clients[1]
|
||||
.prompt(&new_session, "hello from the new world")
|
||||
.await
|
||||
.expect("new client prompt failed");
|
||||
})
|
||||
},
|
||||
)
|
||||
.await;
|
||||
old_client.initialize().await;
|
||||
let session = old_client.create_session(workdir.path()).await;
|
||||
old_client
|
||||
.prompt(&session, "hello from the old world")
|
||||
.await
|
||||
.expect("pre-skew prompt failed");
|
||||
let old_pid = wait_for_live_leader(home.path(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
let base = old_client.notification_count();
|
||||
|
||||
// New binary connects: version floor → evict → respawn.
|
||||
let new_client = LeaderStdioClient::spawn_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
.await;
|
||||
new_client.initialize().await;
|
||||
|
||||
let new_pid = wait_for_new_leader(home.path(), old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no replacement leader after version-floor eviction\n\
|
||||
old client stderr:\n{}\nnew client stderr:\n{}\nleader log:\n{}",
|
||||
old_client.stderr_text(),
|
||||
new_client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
)
|
||||
});
|
||||
assert_ne!(new_pid, old_pid);
|
||||
|
||||
// The evicted leader must actually exit within the evict grace
|
||||
// (EVICT_WAIT_TIMEOUT is 8s; force-kill covers overruns).
|
||||
assert!(
|
||||
wait_for_pid_death(old_pid, Duration::from_secs(30)).await,
|
||||
"old leader pid {old_pid} still alive after eviction\nleader log:\n{}",
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
// The old client reconnects and its original session still works.
|
||||
wait_for_replay_notifications(&old_client, base, Duration::from_secs(60)).await;
|
||||
let res = old_client.prompt(&session, "after the eviction").await;
|
||||
assert!(
|
||||
res.is_ok(),
|
||||
"old client prompt after eviction failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
||||
res.err(),
|
||||
old_client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
// And the new client works against the leader it spawned.
|
||||
let new_session = new_client.create_session(workdir.path()).await;
|
||||
new_client
|
||||
.prompt(&new_session, "hello from the new world")
|
||||
.await
|
||||
.expect("new client prompt failed");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// New leader + old client: the older client adopts the newer leader (the
|
||||
/// floor is directional — never downgrade), keeps functioning through
|
||||
/// serde-default compat, and the leader records the version mismatch.
|
||||
/// An old client adopts a directly-owned new leader without triggering a downgrade.
|
||||
#[tokio::test]
|
||||
#[ignore = "two-binary version-skew test; set GROK_BINARY_LEADER/GROK_BINARY_CLIENT and run with --ignored"]
|
||||
async fn old_client_adopts_new_leader_and_still_functions() {
|
||||
|
|
@ -162,166 +167,181 @@ async fn old_client_adopts_new_leader_and_still_functions() {
|
|||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture =
|
||||
LeaderFixture::start_with_binary(&new_bin, &server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("start owned new leader");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn new leader client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let leader_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live new leader");
|
||||
|
||||
// NEW binary elects the leader first.
|
||||
let new_client = LeaderStdioClient::spawn_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn old leader client"),
|
||||
);
|
||||
clients[1].initialize().await;
|
||||
assert_eq!(
|
||||
read_leader_pid(sandbox.home()),
|
||||
Some(leader_pid),
|
||||
"an older client must never evict a newer leader"
|
||||
);
|
||||
let session = clients[1].create_session(workdir.workspace()).await;
|
||||
clients[1]
|
||||
.prompt(&session, "old client on new leader")
|
||||
.await
|
||||
.expect("old client prompt on new leader failed");
|
||||
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut saw_mismatch = false;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
if leader_log(sandbox.home()).contains("Version mismatch") {
|
||||
saw_mismatch = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
assert!(
|
||||
saw_mismatch,
|
||||
"leader never logged the version mismatch\nleader log:\n{}",
|
||||
leader_log(sandbox.home()),
|
||||
);
|
||||
})
|
||||
})
|
||||
.await;
|
||||
new_client.initialize().await;
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live new leader");
|
||||
|
||||
// OLD binary connects: must adopt (no downgrade eviction).
|
||||
let old_client = LeaderStdioClient::spawn_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
.await;
|
||||
old_client.initialize().await;
|
||||
assert_eq!(
|
||||
read_leader_pid(home.path()),
|
||||
Some(leader_pid),
|
||||
"an older client must never evict a newer leader"
|
||||
);
|
||||
|
||||
// Old client functions across the skew: session + prompt succeed,
|
||||
// exercising serde-default wire compat in anger.
|
||||
let session = old_client.create_session(workdir.path()).await;
|
||||
old_client
|
||||
.prompt(&session, "old client on new leader")
|
||||
.await
|
||||
.expect("old client prompt on new leader failed");
|
||||
|
||||
// The leader records the client/leader version mismatch (the
|
||||
// x.ai/leader/version_mismatch notification's server-side warn).
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut saw_mismatch = false;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
if leader_log(home.path()).contains("Version mismatch") {
|
||||
saw_mismatch = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
assert!(
|
||||
saw_mismatch,
|
||||
"leader never logged the version mismatch\nleader log:\n{}",
|
||||
leader_log(home.path()),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// `grok update`'s relaunch signal against a REAL old leader: connect,
|
||||
/// require `relaunch_v1`, send `RelaunchForUpdate`, and the leader exits so
|
||||
/// the surviving client re-elects. Mirrors the private
|
||||
/// `signal_leaders_to_relaunch` in `xai-grok-pager-bin/src/main.rs` (which is
|
||||
/// bin-private, so the per-leader body is replicated here).
|
||||
/// Update relaunch exits the current leader and elects another current binary.
|
||||
#[tokio::test]
|
||||
#[ignore = "two-binary version-skew test; set GROK_BINARY_LEADER/GROK_BINARY_CLIENT and run with --ignored"]
|
||||
#[ignore = "leader-acceptance: version-skew replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn relaunch_for_update_drives_real_old_leader_to_exit() {
|
||||
let Some((old_bin, _new_bin)) = skew_binaries() else {
|
||||
let Some((old_bin, new_bin)) = skew_binaries() else {
|
||||
return;
|
||||
};
|
||||
tokio::task::LocalSet::new()
|
||||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture =
|
||||
LeaderFixture::start_with_binary(&old_bin, &server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("start owned version-skew leader");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn old leader client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let session = clients[0].create_session(workdir.workspace()).await;
|
||||
clients[0]
|
||||
.prompt(&session, "before relaunch")
|
||||
.await
|
||||
.expect("pre-relaunch prompt failed");
|
||||
let old_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
|
||||
let old_client = LeaderStdioClient::spawn_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
.await;
|
||||
old_client.initialize().await;
|
||||
let session = old_client.create_session(workdir.path()).await;
|
||||
old_client
|
||||
.prompt(&session, "before relaunch")
|
||||
.await
|
||||
.expect("pre-relaunch prompt failed");
|
||||
let old_pid = wait_for_live_leader(home.path(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
let base = old_client.notification_count();
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn new leader client"),
|
||||
);
|
||||
clients[1].initialize().await;
|
||||
let current_pid = fixture
|
||||
.wait_for_new_leader(old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("current client must replace old leader");
|
||||
clients[0]
|
||||
.close()
|
||||
.await
|
||||
.expect("close old client before relaunch");
|
||||
|
||||
// The update-signal body, against the sandboxed socket.
|
||||
let control = LeaderClient::connect(
|
||||
home.path().join(".grok").join("leader.sock"),
|
||||
"grok-pager-update",
|
||||
ClientMode::Stdio,
|
||||
ClientCapabilities::default(),
|
||||
)
|
||||
.await
|
||||
.expect("control connect to old leader failed");
|
||||
|
||||
if !control.registration().supports_relaunch() {
|
||||
// Pre-relaunch_v1 releases degrade to the manual-restart
|
||||
// message; nothing to drive here.
|
||||
eprintln!(
|
||||
"SKIP: old leader {:?} does not advertise relaunch_v1",
|
||||
control.registration().leader_binary_version
|
||||
);
|
||||
control.cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
let ack = control
|
||||
.send_control(ControlCommand::RelaunchForUpdate {
|
||||
to_version: "999.0.0".to_string(),
|
||||
})
|
||||
.await;
|
||||
control.cancel();
|
||||
match ack {
|
||||
Ok(Ok(ControlPayload::Relaunching { .. })) => {}
|
||||
// The leader may exit before the ack flushes — acceptable.
|
||||
Err(_) => {}
|
||||
other => panic!("unexpected RelaunchForUpdate reply: {other:?}"),
|
||||
}
|
||||
|
||||
assert!(
|
||||
wait_for_pid_death(old_pid, Duration::from_secs(30)).await,
|
||||
"old leader pid {old_pid} did not exit after accepting relaunch\nleader log:\n{}",
|
||||
leader_log(home.path()),
|
||||
);
|
||||
|
||||
// The surviving client re-elects and restores its session.
|
||||
wait_for_new_leader(home.path(), old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no re-elected leader after relaunch\nstderr:\n{}\nleader log:\n{}",
|
||||
old_client.stderr_text(),
|
||||
leader_log(home.path()),
|
||||
let control = LeaderClient::connect(
|
||||
sandbox.home().join(".grok").join("leader.sock"),
|
||||
"grok-pager-update",
|
||||
ClientMode::Stdio,
|
||||
ClientCapabilities::default(),
|
||||
)
|
||||
});
|
||||
wait_for_replay_notifications(&old_client, base, Duration::from_secs(60)).await;
|
||||
old_client
|
||||
.prompt(&session, "after relaunch")
|
||||
.await
|
||||
.expect("prompt after relaunch failed");
|
||||
.await
|
||||
.expect("control connect to current leader failed");
|
||||
if !control.registration().supports_relaunch() {
|
||||
eprintln!(
|
||||
"SKIP: current leader {:?} does not advertise relaunch_v1",
|
||||
control.registration().leader_binary_version
|
||||
);
|
||||
control.cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
let ack = control
|
||||
.send_control(ControlCommand::RelaunchForUpdate {
|
||||
to_version: "999.0.0".to_string(),
|
||||
})
|
||||
.await;
|
||||
control.cancel();
|
||||
match ack {
|
||||
Ok(Ok(ControlPayload::Relaunching { .. })) | Err(_) => {}
|
||||
other => panic!("unexpected RelaunchForUpdate reply: {other:?}"),
|
||||
}
|
||||
assert!(
|
||||
wait_for_pid_death(current_pid, Duration::from_secs(30)).await,
|
||||
"leader pid {current_pid} did not exit after relaunch\nleader log:\n{}",
|
||||
leader_log(sandbox.home()),
|
||||
);
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(current_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("no re-elected leader after relaunch");
|
||||
})
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Single-ownership after eviction: exactly one leader remains (old pid dead,
|
||||
/// lock names the live replacement), the eviction is attributable in the
|
||||
/// sandbox unified log, and no second writer touched `auth.json` during the
|
||||
/// swap (API-key auth here, so any write would be a regression).
|
||||
/// Eviction leaves one leader and does not race auth-file ownership.
|
||||
#[tokio::test]
|
||||
#[ignore = "two-binary version-skew test; set GROK_BINARY_LEADER/GROK_BINARY_CLIENT and run with --ignored"]
|
||||
#[ignore = "leader-acceptance: version-skew replacement cleanup needs OS containment or a test-only leader binary"]
|
||||
async fn eviction_leaves_single_leader_and_single_auth_owner() {
|
||||
let Some((old_bin, new_bin)) = skew_binaries() else {
|
||||
return;
|
||||
|
|
@ -330,79 +350,84 @@ async fn eviction_leaves_single_leader_and_single_auth_owner() {
|
|||
.run_until(async {
|
||||
let server = MockInferenceServer::start().await.unwrap();
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture =
|
||||
LeaderFixture::start_with_binary(&old_bin, &server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("start owned version-skew leader");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(&fixture, &mut clients, |fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn old leader client"),
|
||||
);
|
||||
clients[0].initialize().await;
|
||||
let old_pid = wait_for_live_leader(sandbox.home(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
let auth_path = sandbox.home().join(".grok").join("auth.json");
|
||||
let auth_before = std::fs::metadata(&auth_path)
|
||||
.ok()
|
||||
.and_then(|metadata| metadata.modified().ok());
|
||||
|
||||
let old_client = LeaderStdioClient::spawn_with_binary(
|
||||
&old_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.workspace(),
|
||||
&sandbox,
|
||||
)
|
||||
.await
|
||||
.expect("spawn new leader client"),
|
||||
);
|
||||
clients[1].initialize().await;
|
||||
let _new_pid = fixture
|
||||
.wait_for_new_leader(old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("no replacement leader after eviction");
|
||||
assert!(
|
||||
wait_for_pid_death(old_pid, Duration::from_secs(30)).await,
|
||||
"evicted leader must exit"
|
||||
);
|
||||
assert!(pid_alive(_new_pid), "replacement leader must stay alive");
|
||||
assert_eq!(read_leader_pid(sandbox.home()), Some(_new_pid));
|
||||
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut attributed = false;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
let log = sandbox_unified_log(sandbox.home());
|
||||
if log.contains("leader.evict.vacate_requested")
|
||||
|| log.contains("leader.spawn.replacement")
|
||||
{
|
||||
attributed = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
assert!(
|
||||
attributed,
|
||||
"eviction must be attributable in unified.jsonl\nlog:\n{}",
|
||||
sandbox_unified_log(sandbox.home()),
|
||||
);
|
||||
let auth_after = std::fs::metadata(&auth_path)
|
||||
.ok()
|
||||
.and_then(|metadata| metadata.modified().ok());
|
||||
assert_eq!(
|
||||
auth_before, auth_after,
|
||||
"auth.json must not be written during an eviction swap"
|
||||
);
|
||||
})
|
||||
})
|
||||
.await;
|
||||
old_client.initialize().await;
|
||||
let old_pid = wait_for_live_leader(home.path(), Duration::from_secs(10))
|
||||
.await
|
||||
.expect("no live old leader");
|
||||
|
||||
let auth_path = home.path().join(".grok").join("auth.json");
|
||||
let auth_before = std::fs::metadata(&auth_path)
|
||||
.ok()
|
||||
.and_then(|m| m.modified().ok());
|
||||
|
||||
let new_client = LeaderStdioClient::spawn_with_binary(
|
||||
&new_bin,
|
||||
&server,
|
||||
workdir.path(),
|
||||
home.path(),
|
||||
)
|
||||
.await;
|
||||
new_client.initialize().await;
|
||||
|
||||
let new_pid = wait_for_new_leader(home.path(), old_pid, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("no replacement leader after eviction");
|
||||
assert!(
|
||||
wait_for_pid_death(old_pid, Duration::from_secs(30)).await,
|
||||
"evicted leader must exit"
|
||||
);
|
||||
assert!(pid_alive(new_pid), "replacement leader must stay alive");
|
||||
assert_eq!(
|
||||
read_leader_pid(home.path()),
|
||||
Some(new_pid),
|
||||
"the lock file must name exactly the surviving leader"
|
||||
);
|
||||
|
||||
// Attribution: the evicting client recorded the vacate/replace in
|
||||
// the sandbox unified log.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut attributed = false;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
let log = sandbox_unified_log(home.path());
|
||||
if log.contains("leader.evict.vacate_requested")
|
||||
|| log.contains("leader.spawn.replacement")
|
||||
{
|
||||
attributed = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
assert!(
|
||||
attributed,
|
||||
"eviction must be attributable in unified.jsonl\nlog:\n{}",
|
||||
sandbox_unified_log(home.path()),
|
||||
);
|
||||
|
||||
// API-key sandbox: neither leader generation may write auth.json
|
||||
// during the swap (single auth ownership; a concurrent refresher
|
||||
// in the dying leader would show up as a write here).
|
||||
let auth_after = std::fs::metadata(&auth_path)
|
||||
.ok()
|
||||
.and_then(|m| m.modified().ok());
|
||||
assert_eq!(
|
||||
auth_before, auth_after,
|
||||
"auth.json must not be written during an eviction swap"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@
|
|||
|
||||
use std::future::Future;
|
||||
|
||||
#[cfg(unix)]
|
||||
mod common;
|
||||
|
||||
use agent_client_protocol as acp;
|
||||
use xai_grok_test_support::*;
|
||||
|
||||
|
|
@ -71,11 +74,11 @@ async fn test_refusal_turn_completes_with_single_messages_request() {
|
|||
with_local_set(|| async {
|
||||
let server = refusal_messages_server().await;
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client
|
||||
.create_session_with_model_timeout(workdir.path(), "messages-compatible-model")
|
||||
.create_session_with_model_timeout(workdir.workspace(), "messages-compatible-model")
|
||||
.await;
|
||||
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
|
|
@ -122,10 +125,10 @@ mod leader {
|
|||
|
||||
use agent_client_protocol as acp;
|
||||
|
||||
use xai_grok_test_support::leader::{LeaderStdioClient, wait_for_live_leader};
|
||||
use xai_grok_test_support::leader::{LeaderFixture, wait_for_live_leader};
|
||||
use xai_grok_test_support::*;
|
||||
|
||||
use super::{refusal_messages_server, turn_messages_request_count, with_local_set};
|
||||
use super::{common, refusal_messages_server, turn_messages_request_count, with_local_set};
|
||||
|
||||
/// Leader-mode variant of the regression: the refusal-terminated turn
|
||||
/// must complete cleanly (single request, prompt response delivered)
|
||||
|
|
@ -136,60 +139,81 @@ mod leader {
|
|||
with_local_set(|| async {
|
||||
let server = refusal_messages_server().await;
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(home.path().join(".grok")).unwrap();
|
||||
|
||||
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
||||
client.initialize().await;
|
||||
let session_id = client
|
||||
.create_session_with_model(workdir.path(), "messages-compatible-model")
|
||||
.await;
|
||||
|
||||
let result = client.prompt(&session_id, "say hello").await;
|
||||
|
||||
// Prove the session is leader-hosted: a live leader process,
|
||||
// distinct from the client subprocess, holds the lock.
|
||||
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
||||
let sandbox = TestSandbox::new();
|
||||
let fixture = LeaderFixture::start(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no live leader PID in lock file — turn did not run under the leader\nstderr:\n{}",
|
||||
client.stderr_text()
|
||||
)
|
||||
});
|
||||
assert_ne!(
|
||||
Some(leader_pid),
|
||||
client.child.id(),
|
||||
"leader must be a separate process from the stdio client"
|
||||
);
|
||||
let response = result.unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"leader-hosted refusal turn must complete, got error: {e:?}\nrequest log:\n{}\nstderr:\n{}",
|
||||
server.request_log_summary(),
|
||||
client.stderr_text()
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
response.stop_reason,
|
||||
acp::StopReason::EndTurn,
|
||||
"refusal must end the turn cleanly under the leader"
|
||||
);
|
||||
assert!(
|
||||
client.captured_text().contains("Echo:"),
|
||||
"streamed response text must reach the client through the leader, got: {:?}",
|
||||
client.captured_text()
|
||||
);
|
||||
assert_eq!(
|
||||
turn_messages_request_count(&server),
|
||||
1,
|
||||
"exactly one turn request to /v1/messages (no retry storm)\nrequest log:\n{}",
|
||||
server.request_log_summary()
|
||||
);
|
||||
assert!(
|
||||
server.messages_request_count() <= 2,
|
||||
"at most turn + title-generation requests\nrequest log:\n{}",
|
||||
server.request_log_summary()
|
||||
);
|
||||
.expect("start persistent leader fixture");
|
||||
let mut clients = Vec::new();
|
||||
common::leader::run_with_cleanup(
|
||||
&fixture,
|
||||
&mut clients,
|
||||
|fixture, clients| {
|
||||
Box::pin(async move {
|
||||
clients.push(
|
||||
fixture
|
||||
.spawn_client(&server, workdir.workspace(), &sandbox)
|
||||
.await
|
||||
.expect("spawn leader client"),
|
||||
);
|
||||
let client = &clients[0];
|
||||
client.initialize().await;
|
||||
let session_id = client
|
||||
.create_session_with_model(
|
||||
workdir.workspace(),
|
||||
"messages-compatible-model",
|
||||
)
|
||||
.await;
|
||||
|
||||
let result = client.prompt(&session_id, "say hello").await;
|
||||
|
||||
// Prove the session is leader-hosted: a live leader process,
|
||||
// distinct from the client subprocess, holds the lock.
|
||||
let leader_pid =
|
||||
wait_for_live_leader(sandbox.home(), Duration::from_secs(5))
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"no live leader PID in lock file — turn did not run under the leader\nstderr:\n{}",
|
||||
client.stderr_text()
|
||||
)
|
||||
});
|
||||
assert_ne!(
|
||||
Some(leader_pid),
|
||||
client.child_pid(),
|
||||
"leader must be a separate process from the stdio client"
|
||||
);
|
||||
let response = result.unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"leader-hosted refusal turn must complete, got error: {e:?}\nrequest log:\n{}\nstderr:\n{}",
|
||||
server.request_log_summary(),
|
||||
client.stderr_text()
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
response.stop_reason,
|
||||
acp::StopReason::EndTurn,
|
||||
"refusal must end the turn cleanly under the leader"
|
||||
);
|
||||
assert!(
|
||||
client.captured_text().contains("Echo:"),
|
||||
"streamed response text must reach the client through the leader, got: {:?}",
|
||||
client.captured_text()
|
||||
);
|
||||
assert_eq!(
|
||||
turn_messages_request_count(&server),
|
||||
1,
|
||||
"exactly one turn request to /v1/messages (no retry storm)\nrequest log:\n{}",
|
||||
server.request_log_summary()
|
||||
);
|
||||
assert!(
|
||||
server.messages_request_count() <= 2,
|
||||
"at most turn + title-generation requests\nrequest log:\n{}",
|
||||
server.request_log_summary()
|
||||
);
|
||||
})
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ async fn new_session(conn: &acp::ClientSideConnection, cwd: &std::path::Path) ->
|
|||
RPC_TIMEOUT,
|
||||
conn.new_session(
|
||||
acp::NewSessionRequest::new(cwd.to_path_buf())
|
||||
.meta(json!({ "modelId" : "test-model" }).as_object().cloned()),
|
||||
.meta(json!({ "modelId": "test-model" }).as_object().cloned()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
|
@ -126,7 +126,7 @@ async fn close_session(conn: &acp::ClientSideConnection, session_id: &acp::Sessi
|
|||
let resp = ext_method(
|
||||
conn,
|
||||
"x.ai/session/close",
|
||||
json!({ "sessionId" : session_id.0.as_ref() }),
|
||||
json!({ "sessionId": session_id.0.as_ref() }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
|
|
@ -183,12 +183,15 @@ async fn connect_and_auth() -> acp::ClientSideConnection {
|
|||
.terminal(false),
|
||||
)
|
||||
.meta(
|
||||
json!(
|
||||
{ "startupHints" : { "nonInteractive" : true,
|
||||
"skipGitStatus" : true, "skipProjectLayout" : true, },
|
||||
"clientType" : "registry-churn-test", "clientVersion" :
|
||||
"0.0-test", }
|
||||
)
|
||||
json!({
|
||||
"startupHints": {
|
||||
"nonInteractive": true,
|
||||
"skipGitStatus": true,
|
||||
"skipProjectLayout": true,
|
||||
},
|
||||
"clientType": "registry-churn-test",
|
||||
"clientVersion": "0.0-test",
|
||||
})
|
||||
.as_object()
|
||||
.cloned(),
|
||||
),
|
||||
|
|
@ -206,7 +209,7 @@ async fn connect_and_auth() -> acp::ClientSideConnection {
|
|||
RPC_TIMEOUT,
|
||||
client_conn.authenticate(
|
||||
acp::AuthenticateRequest::new(method.id().clone())
|
||||
.meta(json!({ "headless" : true }).as_object().cloned()),
|
||||
.meta(json!({ "headless": true }).as_object().cloned()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
//! cargo test -p xai-grok-shell --test test_stop_hook_e2e -- --ignored
|
||||
//! ```
|
||||
|
||||
use xai_grok_test_support::env::test_env_cmd_tokio;
|
||||
use xai_grok_test_support::*;
|
||||
|
||||
/// Everything a test needs to assert on after a headless run with a Stop hook.
|
||||
|
|
@ -12,8 +11,6 @@ struct StopHookRun {
|
|||
result: HeadlessResult,
|
||||
server: MockInferenceServer,
|
||||
state_dir: tempfile::TempDir,
|
||||
_home: tempfile::TempDir,
|
||||
_workdir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl StopHookRun {
|
||||
|
|
@ -44,15 +41,14 @@ impl StopHookRun {
|
|||
/// Runs the built binary headless with a global Stop hook whose script body is
|
||||
/// `respond`. `$n` holds the 1-based invocation number when `respond` runs.
|
||||
async fn run_with_stop_hook(respond: &str) -> StopHookRun {
|
||||
let home = tempfile::TempDir::new().expect("create temp home");
|
||||
let state_dir = tempfile::TempDir::new().expect("create state dir");
|
||||
let workdir = git_workdir();
|
||||
let server = MockInferenceServer::start()
|
||||
.await
|
||||
.expect("start mock server");
|
||||
let sandbox = TestSandbox::builder().mock_url(server.url()).git().build();
|
||||
|
||||
let state = state_dir.path().display();
|
||||
let script_path = home.path().join("stop_hook.sh");
|
||||
let script_path = sandbox.home().join("stop_hook.sh");
|
||||
// Only turn-end gate fires (`reason: "end_turn"`) are counted and
|
||||
// responded to, so a session-end Stop fire (`channel_closed`/`shutdown`)
|
||||
// can never skew the counts these tests assert on.
|
||||
|
|
@ -71,7 +67,7 @@ async fn run_with_stop_hook(respond: &str) -> StopHookRun {
|
|||
)
|
||||
.expect("write hook script");
|
||||
|
||||
let hooks_dir = home.path().join(".grok").join("hooks");
|
||||
let hooks_dir = sandbox.grok_home().join("hooks");
|
||||
std::fs::create_dir_all(&hooks_dir).expect("create hooks dir");
|
||||
std::fs::write(
|
||||
hooks_dir.join("stop.json"),
|
||||
|
|
@ -92,20 +88,17 @@ async fn run_with_stop_hook(respond: &str) -> StopHookRun {
|
|||
|
||||
let mut cmd = tokio::process::Command::new(grok_binary());
|
||||
cmd.args(["-p", "say hello", "--yolo"])
|
||||
.current_dir(workdir.path())
|
||||
.current_dir(sandbox.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
|
||||
StopHookRun {
|
||||
result,
|
||||
server,
|
||||
state_dir,
|
||||
_home: home,
|
||||
_workdir: workdir,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -57,17 +57,18 @@ async fn resume_reconciles_orphaned_running_subagent() {
|
|||
let workdir = git_workdir();
|
||||
|
||||
// Phase 1: create a real session, then take its home so we can seed it.
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let mut writer = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
writer.initialize_with_timeout().await;
|
||||
let session_id = writer.create_session_with_timeout(workdir.path()).await;
|
||||
let shared_home = writer.take_home();
|
||||
let session_id = writer
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let shared_sandbox = writer.take_sandbox();
|
||||
drop(writer);
|
||||
|
||||
// Simulate a crash: inject a subagent meta left `running` on disk (no
|
||||
// terminal write, no SubagentFinished) — exactly what a dead process
|
||||
// leaves behind.
|
||||
// GrokStdioClient sets HOME=<temp>; the binary uses <HOME>/.grok as GROK_HOME.
|
||||
let grok_home = shared_home.path().join(".grok");
|
||||
let grok_home = shared_sandbox.grok_home().to_path_buf();
|
||||
let session_dir = locate_session_dir(&grok_home, session_id.0.as_ref());
|
||||
let sub_id = "sa-orphan";
|
||||
let meta_path = session_dir.join("subagents").join(sub_id).join("meta.json");
|
||||
|
|
@ -89,10 +90,11 @@ async fn resume_reconciles_orphaned_running_subagent() {
|
|||
.unwrap();
|
||||
|
||||
// Phase 2: resume in a fresh process. `load_session` runs the reconcile.
|
||||
let reader = GrokStdioClient::spawn_with_home(&server, workdir.path(), shared_home).await;
|
||||
let reader =
|
||||
GrokStdioClient::spawn_with_sandbox(&server, workdir.workspace(), shared_sandbox).await;
|
||||
reader.initialize_with_timeout().await;
|
||||
let _ = reader
|
||||
.load_session_with_timeout(&session_id, workdir.path())
|
||||
.load_session_with_timeout(&session_id, workdir.workspace())
|
||||
.await;
|
||||
|
||||
// The orphan's on-disk meta must now be terminal (cancelled), not running.
|
||||
|
|
|
|||
|
|
@ -59,9 +59,8 @@ async fn test_fresh_session_persists_reasoning_effort() {
|
|||
// Configure the mock catalog's model with an explicit effort via the
|
||||
// user config override (the same path a remote settings catalog entry or
|
||||
// `--effort` would populate).
|
||||
let home = tempfile::TempDir::new().expect("create temp home");
|
||||
let grok_dir = home.path().join(".grok");
|
||||
std::fs::create_dir_all(&grok_dir).expect("create .grok dir");
|
||||
let sandbox = TestSandbox::new();
|
||||
let grok_dir = sandbox.grok_home();
|
||||
std::fs::write(
|
||||
grok_dir.join("config.toml"),
|
||||
r#"
|
||||
|
|
@ -72,13 +71,16 @@ reasoning_effort = "high"
|
|||
)
|
||||
.expect("write config.toml");
|
||||
|
||||
let client = GrokStdioClient::spawn_with_home(&server, workdir.path(), home).await;
|
||||
let client =
|
||||
GrokStdioClient::spawn_with_sandbox(&server, workdir.workspace(), sandbox).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "prompt failed: {:?}", result.err());
|
||||
|
||||
let summary = read_summary(client.home_path(), &session_id.0);
|
||||
let summary = read_summary(client.sandbox().home(), &session_id.0);
|
||||
assert_eq!(
|
||||
summary.get("reasoning_effort").and_then(|v| v.as_str()),
|
||||
Some("high"),
|
||||
|
|
@ -98,14 +100,16 @@ async fn test_fresh_session_without_effort_omits_field() {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
||||
let client = GrokStdioClient::spawn(&server, workdir.workspace()).await;
|
||||
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
||||
assert!(result.is_ok(), "prompt failed: {:?}", result.err());
|
||||
|
||||
let summary = read_summary(client.home_path(), &session_id.0);
|
||||
let summary = read_summary(client.sandbox().home(), &session_id.0);
|
||||
assert_eq!(
|
||||
summary.get("reasoning_effort"),
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -247,17 +247,19 @@ async fn headless_session_refreshes_trusted_local_plugin_and_writes_session_json
|
|||
"json",
|
||||
"--cwd",
|
||||
])
|
||||
.arg(workdir.path())
|
||||
.current_dir(workdir.path())
|
||||
.arg(workdir.workspace())
|
||||
.current_dir(workdir.workspace())
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
xai_grok_test_support::env::test_env_cmd_tokio(&mut cmd, &server.url(), &home);
|
||||
cmd.env("HOME", &home);
|
||||
cmd.env("GROK_HOME", &grok_home);
|
||||
let mut sandbox = TestSandbox::builder().mock_url(server.url()).build();
|
||||
sandbox
|
||||
.set_env("HOME", &home)
|
||||
.set_env("USERPROFILE", &home)
|
||||
.set_env("GROK_HOME", &grok_home);
|
||||
|
||||
let result = run_headless_with_cmd(cmd).await;
|
||||
let result = run_headless_in_sandbox(cmd, sandbox).await;
|
||||
assert_headless_success(
|
||||
&result,
|
||||
"headless session with trusted local plugin refresh",
|
||||
|
|
@ -281,10 +283,10 @@ async fn headless_session_refreshes_trusted_local_plugin_and_writes_session_json
|
|||
enabled: vec!["demo-plugin".to_string()],
|
||||
};
|
||||
let plugin_registry = SharedPluginRegistryHandle::new(None, Vec::new())
|
||||
.build_for_cwd(workdir.path(), &config, &[], true)
|
||||
.build_for_cwd(workdir.workspace(), &config, &[], true)
|
||||
.expect("registry built from refreshed snapshot");
|
||||
let agents = xai_grok_agent::discovery::all_subagents_with_plugins(
|
||||
workdir.path(),
|
||||
workdir.workspace(),
|
||||
&HashMap::new(),
|
||||
Some(plugin_registry.as_ref()),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -99,12 +99,15 @@ async fn run_scenario(env: &[(&str, &str)]) -> String {
|
|||
.await
|
||||
.expect("start mock server");
|
||||
let workdir = git_workdir();
|
||||
let home = tempfile::TempDir::new().expect("create temp home");
|
||||
seed_fixtures(home.path(), workdir.path());
|
||||
let mut sandbox = TestSandbox::new();
|
||||
seed_fixtures(sandbox.home(), workdir.workspace());
|
||||
sandbox.extend_env(env.iter().copied());
|
||||
|
||||
let client = GrokStdioClient::spawn_with_home_and_env(&server, workdir.path(), home, env).await;
|
||||
let client = GrokStdioClient::spawn_with_sandbox(&server, workdir.workspace(), sandbox).await;
|
||||
client.initialize_with_timeout().await;
|
||||
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
||||
let session_id = client
|
||||
.create_session_with_timeout(workdir.workspace())
|
||||
.await;
|
||||
let _ = client.prompt_with_timeout(&session_id, "hello").await;
|
||||
|
||||
let bodies: Vec<String> = server
|
||||
|
|
|
|||
Loading…
Reference in a new issue