Publish harness and TUI open-source

initial sync from the monorepo

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/campaign_leader_mode_remote_dismiss_on_model_pick.rs

@@ -0,0 +1,140 @@
+// Per-test-case module for the `leader_pty_e2e` integration test crate.
+#[allow(unused_imports)]
+use super::common::*;
+
+/// **Leader mode: a `/model` pick in the TUI dismisses a remote campaign.**
+///
+/// The dismiss chokepoint (`persist_user_choice`) runs in the **TUI process**,
+/// but in leader mode no in-process agent ever seeds the TUI's remote campaign
+/// cache — only `app::run`'s own seed makes a remote campaign visible to
+/// `resolve_dismissable_campaigns`. Without that seed this test times out in
+/// the dismiss phase: the pick persists but no dismissal is recorded, and the
+/// leader re-nudges every new session over the user's explicit choice.
+///
+/// The TUI's settings prefetch is deliberately 2s-capped, so on a loaded
+/// runner a spawn can miss the fetch (unseeded cache — the documented
+/// transient leader-mode divergence). The test retries with fresh TUI spawns
+/// (same leader) until a pick lands the dismissal, then proves it sticks.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[ignore = "PTY e2e; run with cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1"]
+async fn campaign_leader_mode_remote_dismiss_on_model_pick() {
+    const CONFIG_MODEL: &str = "config-model";
+    const CAMPAIGN_MODEL: &str = "campaign-model";
+    const CAMPAIGN_ID: &str = "e2e-leader-remote-nudge";
+
+    let content = ContentController::start_with_models(vec![
+        MockModel::new(CONFIG_MODEL),
+        MockModel::new(CAMPAIGN_MODEL),
+    ])
+    .await
+    .expect("start content with two models");
+
+    // Serve the campaign from the settings endpoint (restating `allow_access`,
+    // which the preset otherwise provides).
+    content.server().set_settings(json!({
+        "allow_access": true,
+        "campaigns": [
+            { "id": CAMPAIGN_ID, "models": { "default": CAMPAIGN_MODEL } }
+        ]
+    }));
+
+    // Seed config.toml with the user's own default model; a fixed leader
+    // socket under the shared GROK_HOME so every spawn elects/attaches to the
+    // same leader (mirrors `LeaderCluster`).
+    let grok_home = content.home().join(".grok");
+    std::fs::create_dir_all(&grok_home).expect("create GROK_HOME");
+    std::fs::write(
+        grok_home.join("config.toml"),
+        format!("[models]\ndefault = \"{CONFIG_MODEL}\"\n"),
+    )
+    .expect("write config.toml");
+    let socket = grok_home.join("leader-e2e.sock");
+    let socket = socket.to_str().expect("socket path is utf-8").to_owned();
+
+    // Session (OAuth) auth, not the harness's default XAI_API_KEY: the
+    // settings fetch requires `auth_manager.auth()` — in ApiKey/BYOK mode the
+    // pager never requests `/v1/settings`, so a remote campaign would be
+    // structurally unreachable (see `spawn_polling_session`'s doc).
+    seed_fake_oauth(&content, "pty-campaign-leader");
+    let binary = pager_binary().expect("resolve pager binary");
+    let env = oauth_env_for_pager(&content);
+    let spawn = || -> PtyHarness {
+        let env_refs: Vec<(&str, &str)> =
+            env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
+        PtyHarness::new(
+            &binary,
+            DEFAULT_ROWS,
+            DEFAULT_COLS,
+            &["--leader", "--leader-socket", &socket],
+            &env_refs,
+        )
+        .expect("spawn leader-mode pager")
+    };
+    let state_path = grok_home.join("campaigns_state.json");
+    let dismissed = |state_path: &std::path::Path| {
+        std::fs::read_to_string(state_path)
+            .map(|s| s.contains(CAMPAIGN_ID))
+            .unwrap_or(false)
+    };
+
+    // ── Phase 1+2: nudge on a new session; a pick records the dismissal in
+    // the TUI process. Retries fresh TUI spawns (same leader) so a missed
+    // 2s prefetch window on a loaded runner can't wedge the test.
+    let mut recorded = false;
+    'attempts: for attempt in 0..3 {
+        let mut h = spawn();
+        h.wait_for_text(WELCOME_SCREEN_SENTINEL, LEADER_TIMEOUT)
+            .unwrap_or_else(|_| {
+                panic!(
+                    "leader-mode welcome never rendered (attempt {attempt})\nscreen:\n{}",
+                    h.screen_contents()
+                )
+            });
+        if !wait_for_model_via_new_sessions(&mut h, CAMPAIGN_MODEL, Duration::from_secs(60)) {
+            // Campaign never applied on this spawn; try a fresh TUI.
+            h.quit().expect("clean quit");
+            continue;
+        }
+        h.inject_keys(format!("/model {CONFIG_MODEL}\r").as_bytes())
+            .expect("pick model");
+        let deadline = Instant::now() + Duration::from_secs(20);
+        while Instant::now() < deadline {
+            h.update(Duration::from_millis(200));
+            if dismissed(&state_path) {
+                recorded = true;
+                h.quit().expect("clean quit");
+                break 'attempts;
+            }
+        }
+        // The regression under test: pick persisted but dismissal missing.
+        // With the app::run seed present this only happens when the prefetch
+        // missed on this spawn; retry once more before declaring failure.
+        h.quit().expect("clean quit");
+    }
+    assert!(
+        recorded,
+        "leader-mode TUI must record the remote campaign dismissal in {state_path:?}"
+    );
+
+    // ── Phase 3: the dismissal is durable and the pick is persisted. The
+    // user's choice must be in config.toml (campaign value never laundered
+    // in), and the dismissed id on disk is what every future resolution —
+    // leader or not — filters on (`dismissed_id_is_dropped_from_override`
+    // pins the filter; the sibling remote-settings e2e pins the full
+    // no-re-nudge reboot in-process). A fresh same-leader-socket client is
+    // deliberately not asserted on-screen here: reattach paint timing is the
+    // one flaky piece and adds no coverage over the disk + sibling asserts.
+    let config = std::fs::read_to_string(grok_home.join("config.toml")).expect("read config.toml");
+    assert!(
+        config.contains(&format!("default = \"{CONFIG_MODEL}\"")),
+        "the user's pick must be persisted to config.toml:\n{config}"
+    );
+    assert!(
+        !config.contains(CAMPAIGN_MODEL),
+        "the campaign value must never be written to config.toml:\n{config}"
+    );
+    assert!(
+        dismissed(&state_path),
+        "the dismissal must survive on disk after the client exits"
+    );
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/common.rs

@@ -0,0 +1,44 @@
+//! Shared constants for the leader-mode PTY e2e tests.
+//!
+//! Drive/seed helpers live in `xai_grok_pager_pty_harness::flows` (one
+//! canonical copy shared with `pty_e2e`); only suite-local constants stay
+//! here.
+
+pub(crate) use serde_json::json;
+pub(crate) use std::time::{Duration, Instant};
+pub(crate) use xai_grok_pager_pty_harness::{
+    ContentController, LeaderCluster, MockModel, PtyHarness, inference_request_count, keys,
+    oauth_env_for_pager, pager_binary, seed_fake_oauth, submit_turn, wait_for_labels_absent,
+    wait_for_model_via_new_sessions,
+};
+
+/// Default PTY size used by every e2e test (same as `pty_e2e`).
+pub(crate) const DEFAULT_ROWS: u16 = 50;
+
+pub(crate) const DEFAULT_COLS: u16 = 120;
+
+/// Substring we wait for on the welcome screen (matches the menu label).
+pub(crate) const WELCOME_SCREEN_SENTINEL: &str = "Quit";
+
+/// Prompt sent to the agent in content-driven tests.
+pub(crate) const PROMPT: &str = "go";
+
+/// Response sentinel the mock server streams back.
+pub(crate) const MOCK_RESPONSE_SENTINEL: &str = "MOCKRESPONSE";
+
+/// Cold leader-client bring-up budget. History: 60s → 120s → 240s while these
+/// cases ran interleaved with the full `pty_e2e` suite (each leader case
+/// spawns multiple full pager processes; suite-wide contention pushed cold
+/// bring-up past two minutes). Now that the leader cases run in their own
+/// serialized target the budget can come back down after a green soak week —
+/// tracked in the leader test-infra plan.
+pub(crate) const LEADER_TIMEOUT: Duration = Duration::from_secs(240);
+
+/// Streamed-turn deadline in leader mode (same contention rationale).
+pub(crate) const STREAM_TIMEOUT: Duration = Duration::from_secs(120);
+
+/// Sentinel for leader-test turn `n`, short enough to never wrap at 120 cols
+/// (wrapping would break the exactly-once occurrence counts).
+pub(crate) fn turn_sentinel(n: u8) -> String {
+    format!("{MOCK_RESPONSE_SENTINEL}_T{n}")
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/leader_n_clients_shared_session.rs

@@ -0,0 +1,109 @@
+// Per-test-case module for the `leader_pty_e2e` integration test crate.
+#[allow(unused_imports)]
+use super::common::*;
+
+/// 23. **Leader mode — N clients share one session.**
+/// Scales the multi-client fan-out to N = 1 driver + `VIEWERS` viewers (bump
+/// `VIEWERS` to widen it). The driver spawns the leader and runs a turn; every
+/// viewer attaches through the SAME leader and must replay that turn exactly
+/// once (duplicated replay history and an empty/stuck pane both fail); a later
+/// turn driven from the driver must stream live into ALL viewers exactly once;
+/// and the leader + viewers must survive the spawning client's exit.
+/// Deliberately NOT a superset of `leader_two_clients_shared_session`: the
+/// driver/viewer role-flip (a turn driven from a viewer back to the driver)
+/// and the multi-turn scrollback exactly-once integrity stay unique to that
+/// test, so it must not be deleted as redundant.
+#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
+#[ignore = "PTY e2e; run with cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1"]
+async fn leader_n_clients_shared_session() {
+    // N = 1 driver + VIEWERS viewers; bump to scale the live fan-out. Keep N
+    // small: worker_threads above and the per-viewer survival pump below are
+    // sized for it — raise them together if you scale VIEWERS up.
+    const VIEWERS: usize = 2;
+
+    let cluster = LeaderCluster::start(DEFAULT_ROWS, DEFAULT_COLS)
+        .await
+        .expect("start cluster");
+    cluster
+        .content()
+        .set_response(format!("{} first turn payload.", turn_sentinel(1)));
+
+    // The driver spawns the leader and runs turn 1.
+    let mut a = cluster.spawn_leader(&[]).expect("spawn driver");
+    a.wait_for_text(WELCOME_SCREEN_SENTINEL, LEADER_TIMEOUT)
+        .expect("driver welcome");
+    a.inject_keys(format!("{PROMPT}\r").as_bytes())
+        .expect("driver submit turn 1");
+    a.wait_for_text(&turn_sentinel(1), STREAM_TIMEOUT)
+        .expect("driver turn 1");
+
+    // Every viewer attaches through the shared leader and must replay the
+    // driver's transcript exactly once (duplicated replay or an empty pane
+    // both fail).
+    let mut viewers: Vec<PtyHarness> = Vec::new();
+    for i in 0..VIEWERS {
+        let mut v = cluster
+            .attach(&[])
+            .unwrap_or_else(|e| panic!("spawn viewer {i}: {e}"));
+        v.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+            .unwrap_or_else(|e| panic!("viewer {i} replayed driver's transcript: {e}"));
+        // Settle the PTY before counting: wait_for_text returns on first match,
+        // so a duplicate replay arriving in a later batch would slip past an
+        // immediate count. A pump can only reveal a duplicate, never hide one.
+        v.update(Duration::from_millis(500));
+        let screen = v.screen_contents();
+        assert_eq!(
+            screen.matches(&turn_sentinel(1)).count(),
+            1,
+            "turn 1 must appear in viewer {i} exactly once (duplicated replay?)\nscreen:\n{screen}"
+        );
+        viewers.push(v);
+    }
+
+    // Turn 2 driven from the driver streams live into EVERY viewer (fan-out).
+    cluster
+        .content()
+        .set_response(format!("{} second turn payload.", turn_sentinel(2)));
+    a.inject_keys(b"again\r").expect("driver submit turn 2");
+    a.wait_for_text(&turn_sentinel(2), STREAM_TIMEOUT)
+        .expect("driver turn 2");
+    for (i, v) in viewers.iter_mut().enumerate() {
+        v.wait_for_text(&turn_sentinel(2), STREAM_TIMEOUT)
+            .unwrap_or_else(|e| panic!("viewer {i} received driver's live turn: {e}"));
+        // Same settle-then-count guard as the replay check, now for the LIVE
+        // stream: a duplicated fan-out frame must fail too.
+        v.update(Duration::from_millis(500));
+        let screen = v.screen_contents();
+        assert_eq!(
+            screen.matches(&turn_sentinel(2)).count(),
+            1,
+            "turn 2 must appear in viewer {i} exactly once (duplicated live stream?)\nscreen:\n{screen}"
+        );
+    }
+
+    // The spawning client's exit must not take the leader (or the viewers)
+    // down: each viewer keeps its transcript and stays attached.
+    drop(a);
+    for (i, v) in viewers.iter_mut().enumerate() {
+        v.update(Duration::from_secs(3));
+        assert!(
+            v.is_running(),
+            "viewer {i} exited after the driver quit\nscreen:\n{}",
+            v.screen_contents()
+        );
+        assert!(
+            !v.contains_text("panicked"),
+            "viewer {i} rendered a panic\nscreen:\n{}",
+            v.screen_contents()
+        );
+        assert!(
+            v.contains_text(&turn_sentinel(2)),
+            "viewer {i} lost its transcript after the driver quit\nscreen:\n{}",
+            v.screen_contents()
+        );
+    }
+
+    for mut v in viewers {
+        v.quit().expect("quit viewer");
+    }
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/leader_reattach_cancellation_roundtrips_durable_log.rs

@@ -0,0 +1,128 @@
+// Per-test-case module for the `leader_pty_e2e` integration test crate.
+#[allow(unused_imports)]
+use super::common::*;
+
+/// 25. **Leader reattach — cancellation round-trips through the durable log.**
+/// A turn driven on leader client A is Ctrl+C-cancelled mid-stream; the leader
+/// must persist a `turn_completed` terminal with `stop_reason == cancelled`
+/// (the producer fail-before). A FRESH client must replay the cancelled
+/// transcript through the same leader and land clean — running, no panic, and
+/// not stranded on the "Waiting"/"Cancelling" spinners — and must still hold
+/// that transcript after A (the original driver) exits. A keep-alive viewer
+/// holds the leader up across A's exit (the leader stops with its last client).
+///
+/// C attaches *before* A is dropped: `PtyHarness` Drop SIGKILLs the child, and
+/// under full-suite contention a cold `--resume` handshake racing that teardown
+/// flakes with an empty screen for the whole `LEADER_TIMEOUT` (the observed
+/// "C replayed the cancelled transcript" timeout). Replaying while A is still
+/// up, then proving C survives A's exit, covers the durable-log + multi-client
+/// survival invariants without that race.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "PTY e2e; run with cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1"]
+async fn leader_reattach_cancellation_roundtrips_durable_log() {
+    let cluster = LeaderCluster::start(DEFAULT_ROWS, DEFAULT_COLS)
+        .await
+        .expect("start leader cluster");
+    // Paced enough to cancel mid-stream, short enough that the heavy
+    // multi-client cancel drain does not dominate suite-wide contention.
+    let long_response = format!(
+        "{} {}",
+        turn_sentinel(1),
+        "more streamed filler to hold the turn open. ".repeat(40)
+    );
+    cluster.content().set_response(long_response);
+    cluster
+        .content()
+        .set_chunk_delay(Some(Duration::from_millis(40)));
+
+    let mut a = cluster.spawn_leader(&[]).expect("spawn leader client A");
+    a.wait_for_text(WELCOME_SCREEN_SENTINEL, LEADER_TIMEOUT)
+        .expect("A welcome");
+    a.inject_keys(format!("{PROMPT}\r").as_bytes())
+        .expect("A submit turn");
+    // Wait until the turn is clearly streaming (sentinel visible); this also
+    // closes the leader's rewind window so cancel is not confused with rewind.
+    a.wait_for_text(&turn_sentinel(1), STREAM_TIMEOUT)
+        .expect("A turn streaming");
+
+    // Ctrl+C on an empty prompt cancels while streaming (Esc no longer cancels).
+    a.inject_keys(keys::CTRL_C).expect("A press ctrl+c");
+    a.update(Duration::from_millis(200));
+    // Generous budget: the heavy multi-client leader cluster drains the paced
+    // cancel slower than the single-client path, so match the test's other
+    // waits (LEADER/STREAM_TIMEOUT) rather than the single-client 15s.
+    a.wait_for_text("Turn cancelled by user", STREAM_TIMEOUT)
+        .expect("A turn cancelled marker");
+
+    // Producer fail-before: the cancel must have persisted a durable terminal
+    // carrying the cancelled stop reason.
+    let rec = cluster
+        .wait_for_turn_completed(STREAM_TIMEOUT)
+        .expect("turn_completed persisted to updates.jsonl");
+    assert_eq!(
+        rec["stop_reason"], "cancelled",
+        "cancelled turn must record stop_reason=cancelled, got {rec}"
+    );
+
+    // Keep-alive viewer attaches AFTER the cancel so the leader survives A's
+    // exit; waiting for it to replay proves it is attached to the same session.
+    let mut keep = cluster.attach(&[]).expect("spawn keep-alive viewer");
+    keep.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+        .expect("keep-alive replayed the cancelled transcript");
+
+    // Reattach must replay from the durable log, not re-drive a turn: the mock
+    // must see no new inference request while C catches up.
+    let inference_before_reattach = inference_request_count(cluster.content());
+
+    // Fresh reattach while A is still up (see module comment), then prove the
+    // original driver's exit does not take C (or the transcript) down.
+    let mut c = cluster.attach(&[]).expect("spawn fresh reattach client C");
+    c.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+        .expect("C replayed the cancelled transcript");
+
+    drop(a);
+    // Leader processes A's disconnect; brief settle then re-check C still has
+    // the durable replay (not a fixed long sleep that would mask a hang).
+    c.update(Duration::from_millis(500));
+    assert!(
+        c.is_running(),
+        "C exited after A quit\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert!(
+        c.contains_text(&turn_sentinel(1)),
+        "C lost the cancelled transcript after A quit\nscreen:\n{}",
+        c.screen_contents()
+    );
+
+    // Same fidelity caveat as the completion case: a fresh reattach lands Idle
+    // either way, so absent spinners are regression guards, not fail-befores.
+    // Bare substrings (not the full `…`-suffixed labels) are simple stable
+    // matches for the spinner labels. Bound the replay wait rather than a fixed
+    // settle.
+    wait_for_labels_absent(&mut c, &["Waiting", "Cancelling"], Duration::from_secs(5));
+
+    assert!(
+        !c.contains_text("panicked"),
+        "C rendered a panic\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert!(
+        !c.contains_text("Waiting"),
+        "C is stranded on the active-turn spinner\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert!(
+        !c.contains_text("Cancelling"),
+        "C is stranded on the cancelling spinner\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert_eq!(
+        inference_request_count(cluster.content()),
+        inference_before_reattach,
+        "reattach must replay from the durable log, not re-drive a turn (no new inference request)"
+    );
+
+    c.quit().expect("quit C");
+    keep.quit().expect("quit keep-alive viewer");
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/leader_reattach_completion_roundtrips_durable_log.rs

@@ -0,0 +1,103 @@
+// Per-test-case module for the `leader_pty_e2e` integration test crate.
+#[allow(unused_imports)]
+use super::common::*;
+
+/// 24. **Leader reattach — completion round-trips through the durable log.**
+/// A turn driven on the leader-electing client A completes; the leader must
+/// persist a replayable `turn_completed` terminal — the producer fail-before:
+/// without the producer the record simply would not exist. A FRESH client that
+/// re-attaches after A exits must then replay the completed transcript exactly
+/// once through the same leader and land clean: running, no panic, and not
+/// stranded on the active-turn "Waiting" spinner. A keep-alive viewer holds
+/// the leader up across A's exit (the leader stops with its last client).
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "PTY e2e; run with cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1"]
+async fn leader_reattach_completion_roundtrips_durable_log() {
+    let cluster = LeaderCluster::start(DEFAULT_ROWS, DEFAULT_COLS)
+        .await
+        .expect("start leader cluster");
+    cluster
+        .content()
+        .set_response(format!("{} completed turn payload.", turn_sentinel(1)));
+
+    // A elects the leader and drives one turn to completion.
+    let mut a = cluster.spawn_leader(&[]).expect("spawn leader client A");
+    a.wait_for_text(WELCOME_SCREEN_SENTINEL, LEADER_TIMEOUT)
+        .expect("A welcome");
+    a.inject_keys(format!("{PROMPT}\r").as_bytes())
+        .expect("A submit turn");
+    a.wait_for_text(&turn_sentinel(1), STREAM_TIMEOUT)
+        .expect("A turn rendered");
+
+    // Attach the keep-alive viewer AFTER the turn so the leader survives A's
+    // exit; waiting for it to replay the transcript proves it is attached.
+    let mut keep = cluster.attach(&[]).expect("spawn keep-alive viewer");
+    keep.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+        .expect("keep-alive replayed A's transcript");
+
+    // Producer fail-before: the completed turn must have persisted a durable,
+    // replayable terminal carrying the real stop reason.
+    let rec = cluster
+        .wait_for_turn_completed(STREAM_TIMEOUT)
+        .expect("turn_completed persisted to updates.jsonl");
+    assert_eq!(
+        rec["stop_reason"], "end_turn",
+        "completed turn must record stop_reason=end_turn, got {rec}"
+    );
+    assert!(
+        rec["prompt_id"].as_str().is_some_and(|s| !s.is_empty()),
+        "turn_completed must carry a non-empty prompt_id, got {rec}"
+    );
+
+    // Reattach must replay from the durable log, not re-drive a turn: the mock
+    // must see no new inference request while C catches up.
+    let inference_before_reattach = inference_request_count(cluster.content());
+
+    // Fresh reattach AFTER A exits: it replays the completed transcript via the
+    // durable rail through the surviving leader.
+    drop(a);
+    let mut c = cluster.attach(&[]).expect("spawn fresh reattach client C");
+    c.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+        .expect("C replayed the completed transcript");
+
+    // A finished turn clears the leader's prompt slot, so a fresh reattach lands
+    // Idle regardless of the consumer guard; absent "Waiting" is thus a
+    // regression guard, not a fail-before. The bare substring (not the full
+    // `…`-suffixed label) is a simple stable match for the spinner label, and
+    // nothing else renders "Waiting" in a settled reattached session. Bound the
+    // replay wait rather than a fixed settle that could flake under load.
+    wait_for_labels_absent(&mut c, &["Waiting"], Duration::from_secs(5));
+
+    assert!(
+        c.is_running(),
+        "C exited unexpectedly\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert!(
+        !c.contains_text("panicked"),
+        "C rendered a panic\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert!(
+        !c.contains_text("Waiting"),
+        "C is stranded on the active-turn spinner\nscreen:\n{}",
+        c.screen_contents()
+    );
+    assert_eq!(
+        inference_request_count(cluster.content()),
+        inference_before_reattach,
+        "reattach must replay from the durable log, not re-drive a turn (no new inference request)"
+    );
+    // Pump once more so a late or duplicate replay batch would be painted before
+    // the exactly-once count below — this test's headline guard.
+    c.update(Duration::from_millis(500));
+    let screen = c.screen_contents();
+    assert_eq!(
+        screen.matches(&turn_sentinel(1)).count(),
+        1,
+        "C must replay the turn exactly once (duplicated replay?)\nscreen:\n{screen}"
+    );
+
+    c.quit().expect("quit C");
+    keep.quit().expect("quit keep-alive viewer");
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/leader_two_clients_shared_session.rs

@@ -0,0 +1,146 @@
+// Per-test-case module for the `leader_pty_e2e` integration test crate.
+#[allow(unused_imports)]
+use super::common::*;
+
+/// 12. **Leader mode — two clients share one session.**
+/// The two-terminal flow: pager A starts with `--leader` on an isolated
+/// socket (spawning the leader), runs a turn; pager B starts with the same
+/// socket + `--resume` and attaches to A's session through the SAME leader.
+/// Discriminating for the multi-client replay surface: B must render A's
+/// transcript exactly once (duplicated replay history and an empty/stuck
+/// pane both fail), later turns must stream live into BOTH panes regardless
+/// of which client drives, and the leader + viewer must survive the
+/// spawning client's exit.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "PTY e2e; run with cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1"]
+async fn leader_two_clients_shared_session() {
+    // One shared leader (shared HOME/GROK_HOME hold the sessions AND the
+    // explicit leader socket), so B attaches to the leader A spawned instead
+    // of the machine's default one.
+    let cluster = LeaderCluster::start(DEFAULT_ROWS, DEFAULT_COLS)
+        .await
+        .expect("start cluster");
+    cluster
+        .content()
+        .set_response(format!("{} first turn payload.", turn_sentinel(1)));
+
+    let mut a = cluster.spawn_leader(&[]).expect("spawn pager A");
+    a.wait_for_text(WELCOME_SCREEN_SENTINEL, LEADER_TIMEOUT)
+        .expect("A welcome");
+
+    a.inject_keys(format!("{PROMPT}\r").as_bytes())
+        .expect("A submit turn 1");
+    a.wait_for_text(&turn_sentinel(1), STREAM_TIMEOUT)
+        .expect("A turn 1");
+
+    // B attaches to A's session (most recent in the shared cwd) via the
+    // shared leader and must replay A's transcript.
+    let mut b = cluster.attach(&[]).expect("spawn pager B");
+    b.wait_for_text(&turn_sentinel(1), LEADER_TIMEOUT)
+        .expect("B replayed A's transcript");
+
+    let b_screen = b.screen_contents();
+    assert_eq!(
+        b_screen.matches(&turn_sentinel(1)).count(),
+        1,
+        "A's turn must appear in B exactly once (duplicated replay?)\nB screen:\n{b_screen}"
+    );
+
+    // Turn 2 driven from A streams live into the attached viewer B.
+    cluster
+        .content()
+        .set_response(format!("{} second turn payload.", turn_sentinel(2)));
+    submit_turn(&mut a, "again", &turn_sentinel(2), STREAM_TIMEOUT);
+    b.wait_for_text(&turn_sentinel(2), STREAM_TIMEOUT)
+        .expect("B received A's live turn");
+
+    // Turn 3 driven from B reaches A (driver/viewer flip).
+    cluster
+        .content()
+        .set_response(format!("{} third turn payload.", turn_sentinel(3)));
+    submit_turn(&mut b, "more", &turn_sentinel(3), STREAM_TIMEOUT);
+    a.wait_for_text(&turn_sentinel(3), STREAM_TIMEOUT)
+        .expect("A received B's live turn");
+
+    // Each submitted prompt top-anchors its turn, scrolling earlier turns
+    // above the viewport. Grow the viewport and wheel-scroll to the top
+    // (keyboard scroll keys are captured by the focused input box) so the
+    // whole 3-turn transcript is on screen, then count every sentinel
+    // exactly once — a duplicated replay or a dropped turn both fail here,
+    // on either pane.
+    //
+    // Non-dev builds no longer metronome on tracing_rx; scroll finalize runs
+    // via has_active_stream ticks and cadence-suppressed wheels must still
+    // schedule ticks (handle_input returns Changed). Bazel/linux amd64 is
+    // slower than macOS local: keep retrying wheel bursts until every turn
+    // sentinel is visible (or time out), not a single fixed burst budget.
+    fn wheel_scroll_to_top(h: &mut PtyHarness) {
+        for burst in 0..4 {
+            for _ in 0..50 {
+                let _ = h.inject_keys(b"\x1b[<64;40;10M");
+            }
+            h.update(Duration::from_millis(350 + burst * 120));
+        }
+        h.update(Duration::from_millis(500));
+    }
+
+    fn all_turns_once(screen: &str) -> bool {
+        (1..=3).all(|turn| screen.matches(&turn_sentinel(turn)).count() == 1)
+    }
+
+    for (name, h) in [("A", &mut a), ("B", &mut b)] {
+        h.resize(200, DEFAULT_COLS).expect("grow viewport");
+        h.update(Duration::from_millis(600));
+
+        let deadline = std::time::Instant::now() + Duration::from_secs(20);
+        let mut screen = h.screen_contents();
+        while !all_turns_once(&screen) && std::time::Instant::now() < deadline {
+            wheel_scroll_to_top(h);
+            // Home/goto-top via Esc then wheel again if turn 1 still missing.
+            if screen.matches(&turn_sentinel(1)).count() == 0 {
+                let _ = h.inject_keys(keys::ESC);
+                h.update(Duration::from_millis(200));
+                let _ = h.inject_keys(keys::ESC);
+                h.update(Duration::from_millis(200));
+                wheel_scroll_to_top(h);
+            }
+            screen = h.screen_contents();
+        }
+
+        assert!(
+            h.is_running(),
+            "pager {name} exited\nscreen:\n{}",
+            h.screen_contents()
+        );
+        assert!(
+            !h.contains_text("panicked"),
+            "pager {name} rendered a panic\nscreen:\n{}",
+            h.screen_contents()
+        );
+        let screen = h.screen_contents();
+        for turn in 1..=3 {
+            assert_eq!(
+                screen.matches(&turn_sentinel(turn)).count(),
+                1,
+                "turn {turn} must appear in {name} exactly once\nscreen:\n{screen}"
+            );
+        }
+    }
+
+    // The spawning client's exit must not take the leader (or B) down: B
+    // keeps its transcript and stays attached.
+    drop(a);
+    b.update(Duration::from_secs(3));
+    assert!(
+        b.is_running(),
+        "B exited after A quit\nscreen:\n{}",
+        b.screen_contents()
+    );
+    assert!(
+        b.contains_text(&turn_sentinel(3)),
+        "B lost its transcript after A quit\nscreen:\n{}",
+        b.screen_contents()
+    );
+
+    b.quit().expect("quit B");
+}

crates/codegen/xai-grok-pager/tests/leader_pty_e2e/mod.rs

@@ -0,0 +1,24 @@
+//! Leader-mode PTY e2e tests, split out of the shared `pty_e2e` target.
+//!
+//! These cases spawn multi-process leader clusters (each test boots 2-3 full
+//! pager processes plus a leader subprocess), so running them interleaved
+//! with the ~45-test `pty_e2e` suite was the root of the 60→240s
+//! `LEADER_TIMEOUT` flake history. As their own `[[test]]` target they get
+//! their own Bazel test action (serialized from the main PTY pool) and can be
+//! invoked in isolation:
+//!
+//! ```bash
+//! cargo test -p xai-grok-pager --test leader_pty_e2e -- --ignored --test-threads=1 --nocapture
+//! ```
+//!
+//! Binary resolution and harness plumbing are identical to `pty_e2e` (see
+//! that target's `mod.rs`); the shared helpers these tests need live in this
+//! directory's `common.rs`.
+
+mod common;
+
+mod campaign_leader_mode_remote_dismiss_on_model_pick;
+mod leader_n_clients_shared_session;
+mod leader_reattach_cancellation_roundtrips_durable_log;
+mod leader_reattach_completion_roundtrips_durable_log;
+mod leader_two_clients_shared_session;