grok-build-upstream-mirror/crates/codegen/xai-grok-update/tests/test_concurrent_convergence.rs

//! End-to-end tests for the lock-free concurrent-updater convergence model
//! (the "double download" fix): updaters key staleness off the on-disk
//! install, so a binary another process already installed is never
//! downloaded again — and the accepted same-instant residual race is
//! genuinely harmless thanks to per-attempt download temp names.
//!
//! Production has three independent downloader paths that can race around a
//! release:
//!
//! 1. TUI startup: `check_update_background` spawns a detached `grok update`
//!    (the Ctrl+U path now adopts this child instead of spawning a second).
//! 2. Explicit `grok update` (incl. the Ctrl+U fallback when there is no
//!    live child).
//! 3. Leader mode: the hourly checker runs `ensure_latest_on_disk`
//!    in-process.
//!
//! Two layers are exercised here:
//!
//! - **Convergence** (`ensure_latest_on_disk`, `run_update`): a sequential
//!   updater finds the target already on disk and skips the download. The
//!   artifact server / fake `gh` count downloads so the skip is asserted,
//!   not assumed.
//! - **Race integrity** (`install_internal_from_base` run concurrently): the
//!   same-instant race is accepted as rare; these tests pin the property
//!   that makes it acceptable — concurrent installs (same or *different*
//!   versions) never corrupt the active binary. Before the per-attempt
//!   temp-name fix, every `0.1.x` download shared one `grok-0.1.tmp`
//!   (`with_extension("tmp")` eats everything after the last dot), so racer
//!   A could atomically rename racer B's half-written file into place.

#![cfg(unix)]

mod common;

use std::os::unix::fs::PermissionsExt;
use std::path::Path;

use serial_test::serial;

use common::artifact_server::ArtifactServer;
use common::{
    FakeBinGuard, can_exec_shell_scripts, host_platform, make_update_config, reset_home,
    set_test_version, small_good_artifact, test_home,
};
use xai_grok_update::auto_update::{ensure_latest_on_disk, install_internal_from_base, run_update};
use xai_grok_update::version::installed_on_disk_version;

/// Assert the active `~/.grok/bin/grok` resolves to the expected versioned
/// binary, actually runs, and has exactly the expected content (the content
/// check is what catches a cross-racer temp-file corruption).
fn assert_active_binary(home: &Path, version: &str, platform: &str, expected_content: &[u8]) {
    let link = home.join("bin").join("grok");
    assert!(link.is_symlink(), "grok must be a symlink");
    let resolved = dunce::canonicalize(&link)
        .unwrap_or_else(|e| panic!("active grok symlink does not resolve: {e}"));
    assert_eq!(
        resolved.file_name().unwrap().to_string_lossy(),
        format!("grok-{version}-{platform}"),
        "active grok must be the expected version"
    );
    assert_eq!(
        std::fs::read(&resolved).unwrap(),
        expected_content,
        "active binary content must be exactly the served artifact (no \
         partial/interleaved writes from a racing updater)"
    );
    let ran_ok = std::process::Command::new(&resolved)
        .arg("--version")
        .stdin(std::process::Stdio::null())
        .stdout(std::process::Stdio::null())
        .stderr(std::process::Stdio::null())
        .status()
        .map(|s| s.success())
        .unwrap_or(false);
    assert!(ran_ok, "active grok must pass the smoke-test");
}

/// Lay down a managed-install layout in the test GROK_HOME:
/// `bin/grok -> ../downloads/grok-<version>-<platform>` (what
/// `install_internal_from_base` produces).
fn fake_managed_install(version: &str) {
    let home = test_home();
    let downloads = home.join("downloads");
    let bin = home.join("bin");
    std::fs::create_dir_all(&downloads).unwrap();
    std::fs::create_dir_all(&bin).unwrap();
    let name = format!("grok-{version}-{}", host_platform());
    std::fs::write(downloads.join(&name), small_good_artifact()).unwrap();
    std::fs::set_permissions(
        downloads.join(&name),
        std::fs::Permissions::from_mode(0o755),
    )
    .unwrap();
    std::os::unix::fs::symlink(
        std::path::Path::new("../downloads").join(&name),
        bin.join("grok"),
    )
    .unwrap();
}

/// Fake `gh` that logs argv to `<dir>/gh-args.log`, answers
/// `release list --exclude-pre-releases` from `<dir>/gh-stable-only-stdout`,
/// and for `release download ... --output <path>` writes a smoke-passing
/// artifact to the output path.
fn fake_gh_serving_releases(dir: &std::path::Path) -> String {
    let dq = format!("'{}'", dir.to_string_lossy().replace('\'', "'\\''"));
    format!(
        r#"#!/bin/sh
echo "$@" >> {dq}/gh-args.log
case "$*" in
  *"release list"*)
    if [ -f {dq}/gh-stable-only-stdout ]; then cat {dq}/gh-stable-only-stdout; fi
    ;;
  *"release download"*)
    out=""
    prev=""
    for a in "$@"; do
      if [ "$prev" = "--output" ]; then out="$a"; fi
      prev="$a"
    done
    if [ -n "$out" ]; then
      printf '#!/bin/sh\nexit 0\n' > "$out"
      chmod +x "$out"
    fi
    ;;
esac
exit 0
"#
    )
}

/// Count `release download` invocations in the fake gh's argv log.
fn gh_download_count(g: &FakeBinGuard) -> usize {
    g.args_log()
        .iter()
        .filter(|l| l.contains("release download"))
        .count()
}

fn setup_gh_release(running_version: &str) -> FakeBinGuard {
    let _ = test_home();
    reset_home();
    set_test_version(running_version);
    // SAFETY: serial_test ensures no race; reset_home clears this between tests.
    unsafe { std::env::set_var("GROK_INSTALLER", "gh-release") };
    FakeBinGuard::install("gh", fake_gh_serving_releases)
}

// ─────────────────────────────────────────────────────────────────────────────
// Convergence: ensure_latest_on_disk downloads once, then every subsequent
// pass (the leader's hourly re-entry) converges without re-downloading.
// This is the e2e companion to the decision-level tests in
// test_downgrade_matrix.rs — it asserts on actual download invocations.
// ─────────────────────────────────────────────────────────────────────────────

#[tokio::test]
#[serial]
async fn ensure_latest_downloads_once_then_converges_without_redownload() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let g = setup_gh_release("0.2.5");
    g.set_stable_only_stdout("v0.2.7\n");
    let cfg = make_update_config("stable");

    // Pass 1: disk is empty → downloads and installs.
    let first = ensure_latest_on_disk(&cfg).await.unwrap();
    assert_eq!(first.installed.as_deref(), Some("0.2.7"));
    assert!(first.relaunch_needed, "running 0.2.5 < disk 0.2.7");
    assert_eq!(gh_download_count(&g), 1, "first pass downloads");
    assert_eq!(installed_on_disk_version().as_deref(), Some("0.2.7"));

    // Pass 2 (the pre-fix hourly re-download): disk already current →
    // no download, but the stale running process still gets the relaunch
    // signal.
    let second = ensure_latest_on_disk(&cfg).await.unwrap();
    assert_eq!(second.installed, None, "second pass must not re-download");
    assert!(second.relaunch_needed, "still running 0.2.5 < disk 0.2.7");
    assert_eq!(
        gh_download_count(&g),
        1,
        "hourly re-entry must not download again"
    );
}

// ─────────────────────────────────────────────────────────────────────────────
// Convergence: explicit `grok update` (the Ctrl+U fallback path) finds the
// binary another process already installed and skips the download — while
// still returning the target version so stale leaders get signalled.
// ─────────────────────────────────────────────────────────────────────────────

#[tokio::test]
#[serial]
async fn run_update_skips_download_when_disk_already_current() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let g = setup_gh_release("0.2.5");
    g.set_stable_only_stdout("v0.2.7\n");
    // Another process (TUI background download) already installed 0.2.7.
    fake_managed_install("0.2.7");
    let mut cfg = make_update_config("stable");

    let result = run_update(false, None, None, &mut cfg).await.unwrap();

    assert_eq!(
        result.as_deref(),
        Some("0.2.7"),
        "run_update must still report the on-disk target so the caller \
         signals stale leaders to relaunch"
    );
    assert_eq!(
        gh_download_count(&g),
        0,
        "a binary someone else installed must not be downloaded again"
    );
}

#[tokio::test]
#[serial]
async fn run_update_force_still_redownloads_when_disk_current() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let g = setup_gh_release("0.2.7");
    g.set_stable_only_stdout("v0.2.7\n");
    fake_managed_install("0.2.7");
    let mut cfg = make_update_config("stable");

    let result = run_update(true, None, None, &mut cfg).await.unwrap();

    assert_eq!(result.as_deref(), Some("0.2.7"));
    assert_eq!(
        gh_download_count(&g),
        1,
        "--force must bypass the disk-current skip and reinstall"
    );
}

// ─────────────────────────────────────────────────────────────────────────────
// Installer gating: the disk-version probe must only be trusted for
// installers that actually maintain the managed `~/.grok/bin/grok` symlink
// (internal, gh-release). For npm, a symlink left over from a previous
// internal install LIES about the npm install's version — and in the worst
// direction (leftover "newer" than the registry) it would silently suppress
// npm updates forever.
// ─────────────────────────────────────────────────────────────────────────────

fn setup_npm(running_version: &str) -> FakeBinGuard {
    let _ = test_home();
    reset_home();
    set_test_version(running_version);
    // SAFETY: serial_test ensures no race; reset_home clears this between tests.
    unsafe { std::env::set_var("GROK_INSTALLER", "npm") };
    FakeBinGuard::install_npm()
}

#[tokio::test]
#[serial]
async fn npm_update_not_suppressed_by_leftover_newer_internal_symlink() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let g = setup_npm("0.2.5");
    g.set_stdout("\"0.2.7\"\n");
    // Leftover symlink from a previous internal install, claiming to be
    // NEWER than the npm registry. It says nothing about the npm-managed
    // global install and must be ignored for npm staleness decisions.
    fake_managed_install("0.2.9");
    let mut cfg = make_update_config("stable");

    let result = run_update(false, None, None, &mut cfg).await.unwrap();

    assert_eq!(
        result.as_deref(),
        Some("0.2.7"),
        "npm update must proceed despite the lying leftover symlink"
    );
    assert!(
        g.args_log().iter().any(|l| l.contains("i -g")),
        "npm install must actually run: {:?}",
        g.args_log()
    );
}

#[tokio::test]
#[serial]
async fn ensure_latest_npm_ignores_leftover_internal_symlink() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let g = setup_npm("0.2.5");
    g.set_stdout("\"0.2.7\"\n");
    fake_managed_install("0.2.9");
    let cfg = make_update_config("stable");

    let outcome = ensure_latest_on_disk(&cfg).await.unwrap();

    assert_eq!(
        outcome.installed.as_deref(),
        Some("0.2.7"),
        "npm leader pass must install despite the lying leftover symlink"
    );
    assert!(
        outcome.relaunch_needed,
        "running 0.2.5 < freshly installed 0.2.7"
    );
    assert!(
        g.args_log().iter().any(|l| l.contains("i -g")),
        "npm install must actually run: {:?}",
        g.args_log()
    );
}

#[tokio::test]
#[serial]
async fn disk_probe_preserves_prerelease_versions() {
    let _ = test_home();
    reset_home();
    // An alpha install must read back as the full pre-release version —
    // truncating to "0.1.220" would mask the alpha → stable update.
    fake_managed_install("0.1.220-alpha.4");
    assert_eq!(
        installed_on_disk_version().as_deref(),
        Some("0.1.220-alpha.4")
    );
}

#[tokio::test]
#[serial]
async fn disk_probe_rejects_dangling_symlink() {
    // If the symlink survives but its target binary was deleted (manual
    // ~/.grok/downloads cleanup), the probe must report None — otherwise
    // every updater would claim "already up to date" forever while no
    // runnable binary exists, and nothing would ever repair the install.
    let home = test_home();
    reset_home();
    let platform = host_platform();
    fake_managed_install("0.2.7");
    assert_eq!(installed_on_disk_version().as_deref(), Some("0.2.7"));

    std::fs::remove_file(
        home.join("downloads")
            .join(format!("grok-0.2.7-{platform}")),
    )
    .unwrap();

    assert_eq!(
        installed_on_disk_version(),
        None,
        "a dangling symlink must not report an installed version"
    );
}

#[tokio::test]
#[serial]
async fn ensure_latest_repairs_dangling_symlink_by_downloading() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    // Dangling symlink + stale running process: the probe returns None, so
    // the decision falls back to the running version and the download runs,
    // repairing the install instead of wedging on "already up to date".
    let g = setup_gh_release("0.2.5");
    g.set_stable_only_stdout("v0.2.7\n");
    let home = test_home();
    let platform = host_platform();
    fake_managed_install("0.2.7");
    std::fs::remove_file(
        home.join("downloads")
            .join(format!("grok-0.2.7-{platform}")),
    )
    .unwrap();
    let cfg = make_update_config("stable");

    let outcome = ensure_latest_on_disk(&cfg).await.unwrap();

    assert_eq!(
        outcome.installed.as_deref(),
        Some("0.2.7"),
        "dangling symlink must be repaired by an actual download"
    );
    assert_eq!(gh_download_count(&g), 1);
    assert_eq!(
        installed_on_disk_version().as_deref(),
        Some("0.2.7"),
        "probe healthy again after the repair install"
    );
}

// ─────────────────────────────────────────────────────────────────────────────
// Race integrity: the accepted same-instant race must stay harmless. Two (or
// three) installers running concurrently — even for DIFFERENT versions —
// must never leave a corrupt active binary. Pre-fix, all 0.1.x downloads
// shared one `grok-0.1.tmp`, so a concurrent racer could atomically rename a
// half-written file into place.
// ─────────────────────────────────────────────────────────────────────────────

async fn run_concurrent_installs(
    server: &ArtifactServer,
    versions: &[&str],
) -> Vec<anyhow::Result<()>> {
    let base = server.uri();
    let mut tasks = Vec::new();
    for version in versions {
        let base = base.clone();
        let version = version.to_string();
        tasks.push(tokio::spawn(async move {
            let cfg = make_update_config("stable");
            install_internal_from_base(Some(&version), &cfg, &base).await
        }));
    }
    let mut results = Vec::new();
    for t in tasks {
        results.push(t.await.expect("install task must not panic"));
    }
    results
}

#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_same_version_installs_leave_valid_active_binary() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let home = test_home();
    reset_home();
    let platform = host_platform();
    let artifact = small_good_artifact();
    let server = ArtifactServer::start(artifact.clone());
    // Hold responses open so the racers genuinely overlap mid-download.
    server.set_slow(true);

    let results = run_concurrent_installs(&server, &["0.1.181", "0.1.181", "0.1.181"]).await;
    for r in results {
        r.expect("every racing install must succeed (atomic swap, last writer wins)");
    }

    // Lock-free model: concurrent racers may each download (accepted waste);
    // the invariant is integrity, not the count.
    assert!(server.request_count() >= 1);
    assert_active_binary(home, "0.1.181", &platform, &artifact);
}

#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_different_version_installs_do_not_corrupt_each_other() {
    if !can_exec_shell_scripts() {
        eprintln!("skipping: shell scripts cannot execute in this sandbox");
        return;
    }
    let home = test_home();
    reset_home();
    let platform = host_platform();
    let artifact = small_good_artifact();
    let server = ArtifactServer::start(artifact.clone());
    server.set_slow(true);

    // Pre-fix, BOTH of these wrote to downloads/grok-0.1.tmp concurrently
    // (with_extension("tmp") truncates at the last dot), so one racer could
    // rename the other's partial file into its own versioned path.
    let results = run_concurrent_installs(&server, &["0.1.181", "0.1.182"]).await;
    for r in results {
        r.expect("both racing installs must succeed");
    }

    // Both versioned binaries must exist with full, uncorrupted content.
    for version in ["0.1.181", "0.1.182"] {
        let path = home
            .join("downloads")
            .join(format!("grok-{version}-{platform}"));
        assert_eq!(
            std::fs::read(&path).unwrap(),
            artifact,
            "binary {version} must contain exactly the served artifact"
        );
    }

    // The active symlink points at whichever racer swapped last; it must
    // resolve and run regardless.
    let resolved = dunce::canonicalize(home.join("bin").join("grok")).unwrap();
    assert_eq!(std::fs::read(&resolved).unwrap(), artifact);
    let name = resolved.file_name().unwrap().to_string_lossy().to_string();
    assert!(
        !name.contains(".tmp"),
        "active grok must never be a temp file: {name}"
    );

    // No stray shared temp file left behind (the pre-fix collision name).
    assert!(
        !home.join("downloads").join("grok-0.1.tmp").exists(),
        "the pre-fix shared temp name must not exist"
    );
}