bingshuo/grok-build-upstream-mirror
Watch
1
0
Fork
You've already forked grok-build-upstream-mirror
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Synced from monorepo

Browse source 
Synced from monorepo

Changes:
- Refresh tool search when the managed MCP catalog is re-fetched
- Prevent duplicate leader process spawn and startup hang from stale leaders
- Document marketplaces, plugins, and organization controls
- Stamp session ID on image generation direct-to-API requests
- Fix auto mode blocked documentation
- Auto mode considers recent user intent
- Expose deploy archive, taken-down, limit, and in-progress reasons on the chat API
- Fail-closed auth refresh contract for shell clients
- Emit a chat-supplied per-session turn index in turn hooks
- Show bash mode chrome in minimal mode
- Add metrics for true-noop and stationarity stops
- Include voice interim text on prompt submit
- Silently end turn on true-noop thrash
- Quiet copy toast when clipboard delivery is confirmed
- Fix session fork truncating at the wrong prompt in rewound sessions
- Make the idle "still running" watcher cue clickable to open the tasks pane
- Default web search model to grok-4.5
- Let plugin subagents inherit parent MCP servers
- Gate no-op end-turn reminder on system reminders
- Add gateway bridge lifecycle telemetry
- Allow editing finalized text while voice is open
- Relocate token carrier to turn-commit events and plumb per-turn origin context
- Raise workflow scratch quotas and make failed runs resumable
- Workflows overlay: auto-progress phases, live agent status, and drop budget meter

Source-Revision: 9b8d35b46d959c042ea9aa31cbbebbd1f0c5c527
This commit is contained in:
grokkybara[bot] 2026-07-24 16:59:42 +00:00
commit 6e38642082
103 changed files with 4964 additions and 1261 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/codegen/xai-grok-shell/src/auth/credential_provider.rs
View file

@ -43,7 +43,7 @@ impl HttpAuth for ShellAuthCredentialProvider {
fn apply(&self, builder: RequestBuilder, base_url: &str) -> RequestBuilder {
let mut creds = self.static_credentials.clone();
if creds.deployment_key.is_none()
&& let Some(auth) = self.auth_manager.current_or_expired()
&& let Some(auth) = self.auth_manager.current_wire_valid()
{
creds.user_token = Some(auth.key);
}
@ -60,12 +60,12 @@ impl AuthCredentialProvider for ShellAuthCredentialProvider {
..Default::default()
};
}
let auth = self.auth_manager.current_or_expired();
let user_id = auth.as_ref().map(|a| a.user_id.clone());
let team_id = auth.as_ref().and_then(|a| a.team_id.clone());
let organization_id = auth.as_ref().and_then(|a| a.organization_id.clone());
let api_key_id = api_key_id_for(auth.as_ref());
let token = auth.map(|a| a.key);
let identity = self.auth_manager.current_or_expired();
let user_id = identity.as_ref().map(|a| a.user_id.clone());
let team_id = identity.as_ref().and_then(|a| a.team_id.clone());
let organization_id = identity.as_ref().and_then(|a| a.organization_id.clone());
let api_key_id = api_key_id_for(identity.as_ref());
let token = self.auth_manager.current_wire_valid().map(|a| a.key);
CredentialSnapshot {
token,
user_id,

6
crates/codegen/xai-grok-shell/src/auth/error.rs
View file

@ -141,4 +141,10 @@ impl AuthError {
pub(crate) fn permanent(reason: RefreshTokenFailedReason) -> Self {
AuthError::Refresh(RefreshTokenError::Permanent(reason.into()))
}
/// Retryable refresh failure (network, 5xx, sleep/dark-wake defer, etc.).
/// Permanent failures, NotLoggedIn, and policy rejects are not transient.
pub(crate) fn is_transient(&self) -> bool {
matches!(self, AuthError::Refresh(RefreshTokenError::Transient(_)))
}
}

5
crates/codegen/xai-grok-shell/src/auth/flow.rs
View file

@ -701,9 +701,8 @@ pub(crate) async fn try_ensure_session_noninteractive(
let grok_home = grok_home::grok_home();
let auth_manager = Arc::new(AuthManager::new(&grok_home, grok_com_config.clone()));
// A refresh failure leaves the session on disk (credentials are retained;
// the verdict gates re-attempts). Return it so consumers self-recover on
// 401, rather than disabling the relay for the leader's lifetime.
// Transient refresh failure: credentials remain (usable on 401 recovery).
// Permanent failure already discarded them.
if let Some(expired) = expired_refreshable_session(&auth_manager) {
return Some(expired);
}

211
crates/codegen/xai-grok-shell/src/auth/manager.rs
View file

@ -478,6 +478,9 @@ impl AuthManager {
);
if scope == self.scope {
self.clear_inner();
// Intentional logout/scope removal: drop sticky permanent so the
// next state is NotLoggedIn, not a retained invalid_grant verdict.
*self.permanent_failure.write() = None;
}
Ok(())
}
@ -499,9 +502,9 @@ impl AuthManager {
}
}
/// Drop the in-memory auth. The sticky permanent-failure verdict is scoped
/// to a credential key, so an empty cache reads through as "no failure"
/// without explicit clearing.
/// Drop the in-memory auth. Sticky `RefreshTokenRejected` still
/// short-circuits with no live credential until a wire-valid login;
/// non-sticky verdicts read absent once their scoped key is gone.
fn clear_inner(&self) {
*self.inner.write() = None;
}
@ -588,8 +591,11 @@ impl AuthManager {
}
/// Drop the in-memory credentials, loudly. Logs the discard (with `reason`)
/// before routing through [`clear_inner`] so the cached permanent_failure
/// (if any) goes with them. Centralizes the "credentials gone" telemetry.
/// before routing through [`clear_inner`]. Also clears a sticky permanent
/// verdict so force-reload / disk-anomaly paths surface `NotLoggedIn`
/// rather than a retained `invalid_grant`. Permanent discard after a live
/// IdP rejection uses [`clear_inner`] alone so the sticky short-circuit
/// survives until login.
fn drop_in_memory_credentials(&self, reason: &str) {
if let Some(d) = self.current_or_expired() {
xai_grok_telemetry::unified_log::warn(
@ -605,6 +611,7 @@ impl AuthManager {
);
}
self.clear_inner();
*self.permanent_failure.write() = None;
}
// ── Read methods ─────────────────────────────────────────────────
@ -851,6 +858,7 @@ impl AuthManager {
// current session work with fresh credentials while the user fixes the
// filesystem (e.g. read-only disk). Without this, a disk failure leaves
// the stale/dead token in memory and the user is completely stuck.
*self.permanent_failure.write() = None;
self.with_inner_write(|inner| *inner = Some(auth.clone()));
// Fire-and-forget enrichment. Off the critical path -- a slow
@ -908,6 +916,7 @@ impl AuthManager {
),
}
// Always update in-memory, even if disk write failed (see update()).
*self.permanent_failure.write() = None;
self.with_inner_write(|inner| *inner = Some(auth.clone()));
write_result?;
Ok(auth)
@ -962,13 +971,19 @@ impl AuthManager {
}
/// Hot-swap credentials (called by config watcher). Does NOT write to disk.
/// Clears a sticky permanent verdict only when the new bearer is wire-valid
/// (login / sibling adopt). Hard-expired swaps keep the sticky short-circuit
/// so a dead RT is not re-tried until a real login.
pub(crate) fn hot_swap(&self, new_auth: GrokAuth) {
if !self.is_token_hard_expired(&new_auth) {
*self.permanent_failure.write() = None;
}
self.with_inner_write(|inner| *inner = Some(new_auth));
}
/// Clear in-memory credentials. Does NOT touch disk, and does NOT clear the
/// permanent-failure verdict: that is credential-scoped and self-invalidates
/// on the next lookup once the credential it targets is gone.
/// Clear in-memory credentials. Does NOT touch disk. Sticky
/// `RefreshTokenRejected` remains until wire-valid login; other verdicts
/// are key-scoped and drop out once their credential is gone.
pub(crate) fn clear_in_memory(&self) {
self.clear_inner();
}
@ -1053,23 +1068,23 @@ impl AuthManager {
/// disk (disk RT differs from in-memory RT). Used by `refresh_chain`
/// to demote a `PermanentFailure` to transient so the sibling's
/// fresher token can be tried on the next attempt.
///
/// Requires an in-memory RT: empty `inner` means the disk credential is
/// the only candidate (not a multi-process rotation). Does **not**
/// require a non-expired disk AT — a sibling may still hold a usable RT
/// while its AT is buffer/hard-expired.
fn sibling_has_different_refresh_token(&self) -> bool {
let disk_auth = self.read_disk_auth();
let Some(ref disk) = disk_auth else {
return false;
};
// Expired AT = dead sibling, not a live one. Disk may have
// diverged from memory due to failed writes (e.g. disk full)
// while both RTs are revoked.
if self.is_token_expired(disk) {
return false;
}
let disk_rt = disk.refresh_token.as_deref();
let Some(disk_rt) = disk_rt else {
let Some(disk_rt) = disk.refresh_token.as_deref() else {
return false;
};
let mem_rt = self.expired_auth().and_then(|a| a.refresh_token);
mem_rt.as_deref() != Some(disk_rt)
let Some(mem_rt) = self.current_or_expired().and_then(|a| a.refresh_token) else {
return false;
};
mem_rt.as_str() != disk_rt
}
/// Re-read `auth.json` from disk without updating in-memory state.
@ -1348,13 +1363,18 @@ impl AuthManager {
{
Ok(auth) => Ok(auth),
Err(e) => {
// Grace: the early-invalidation buffer is OUR
// conservative estimate, not the IdP's actual
// expiry. If the cached token is still wire-valid
// ([`Self::is_token_hard_expired`]), return it so a
// transient IdP blip during the buffer window
// is invisible to the user.
if let Some(auth) = snapshot
// Grace for still wire-valid ATs on transient failures
// and retain-path permanents (ClientRejected / Other).
// RefreshTokenRejected discards AT+RT — never re-serve
// that snapshot even when it is only soft-expired.
let deny_grace = matches!(
&e,
AuthError::Refresh(crate::auth::RefreshTokenError::Permanent(pe))
if pe.reason
== crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected
);
if !deny_grace
&& let Some(auth) = snapshot
&& !self.is_token_hard_expired(&auth)
{
tracing::debug!(
@ -1804,16 +1824,67 @@ impl AuthManager {
) {
return Ok(refreshed);
}
if self.sibling_has_different_refresh_token() {
tracing::info!("auth: sibling-rotation detected; demoting to transient");
return Err(AuthError::transient(format!("sibling-rotation: {error}")));
}
// No clear: the verdict (+ TTL) gates re-attempts; the dead
// bearer is dropped only on explicit logout. Key on the
// credential the refresher actually sent (`tried_key`), falling
// back to our own resolution when the authority has no key.
// Client contract: only genuine IdP RT rejection discards.
// Escalated `Other` / `ClientRejected` retain credentials.
// When mem and disk RTs diverge, clear only the side that was
// actually tried so an untried successor RT (e.g. mem after a
// disk-persist failure) is not wiped by a disk-first invalid_grant.
let failed_reason = error.reason;
if let Some(key) = tried_key.or(attempted_key) {
let is_rtr = failed_reason
== crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected;
if is_rtr {
let mem = self.current_or_expired();
let disk = self.read_disk_auth();
// Unattributed + diverging RTs: demote without recording so
// the next attempt can try the other side (no sticky lockout).
if tried_key.is_none() && self.sibling_has_different_refresh_token() {
tracing::info!("auth: sibling-rotation detected; demoting to transient");
return Err(AuthError::transient(format!(
"sibling-rotation: {failed_reason:?}"
)));
}
let (clear_mem, clear_disk) = match (tried_key.as_ref(), &mem, &disk) {
(Some(tk), m, d) => {
let mem_match = m.as_ref().is_some_and(|a| a.key == *tk);
let disk_match = d.as_ref().is_some_and(|a| a.key == *tk);
if mem_match || disk_match {
(mem_match, disk_match)
} else {
(true, true)
}
}
(None, _, _) => (true, true),
};
if let Some(key) = tried_key.or(attempted_key) {
self.record_permanent_failure(key, error);
}
let mut disk_mutation = "unchanged";
if clear_disk {
disk_mutation = match self.write_scope_removal(&self.scope) {
Ok(m) => m.label(),
Err(e) => {
tracing::warn!(
error = %e,
"auth: failed to clear credentials after permanent refresh failure"
);
"write_failed"
}
};
}
if clear_mem {
self.clear_inner();
}
xai_grok_telemetry::unified_log::warn(
"auth: cleared credentials after permanent refresh failure",
None,
Some(serde_json::json!({
"reason": format!("{failed_reason:?}"),
"disk_mutation": disk_mutation,
"cleared_mem": clear_mem,
"cleared_disk": clear_disk,
})),
);
} else if let Some(key) = tried_key.or(attempted_key) {
self.record_permanent_failure(key, error);
}
Err(AuthError::permanent(failed_reason))
@ -1905,6 +1976,16 @@ impl AuthManager {
/// the common no-verdict case returns before any disk I/O; only a stored
/// verdict triggers [`Self::attempted_verdict_key`]'s disk read.
///
/// After a permanent failure **discards** credentials, sticky reasons
/// (`RefreshTokenRejected`) still short-circuit with no live credential so
/// concurrent callers cannot re-hit the IdP with a dead RT. Login
/// (`hot_swap` / `update`) and logout clear the verdict.
///
/// Sticky applies only to the **same** rejected key or to **no** live
/// credential (post-discard). A different attempted key (sibling RT/AT
/// on disk) must be allowed to refresh — otherwise a hard-expired sibling
/// AT strands a process that could still refresh a live RT.
///
/// TTL expiry is judged on *both* clocks (see [`GateRaise`]): the monotonic
/// clock pauses during a system suspend, so a wall-clock arm is required
/// for the TTL to elapse across sleep. Without it, a recoverable failure
@ -1929,8 +2010,14 @@ impl AuthManager {
// would attempt. Guard dropped above so `inner` isn't co-held.
// Deliberately `ServerRejected` (the widest resolution) regardless of
// the caller's reason, so the read never misses a stored verdict.
(self.attempted_verdict_key(RefreshReason::ServerRejected)? == token_key)
.then(|| AuthError::permanent(reason))
match self.attempted_verdict_key(RefreshReason::ServerRejected) {
Some(k) if k == token_key => Some(AuthError::permanent(reason)),
// Different credential key: never sticky-block a sibling RT.
Some(_) => None,
// No live credential after discard: sticky short-circuit until login.
None if reason.is_sticky() => Some(AuthError::permanent(reason)),
None => None,
}
}
/// `true` iff [`Self::permanent_failure`] has a non-expired entry. Lets
@ -1990,17 +2077,51 @@ impl AuthManager {
crate::auth::recovery::UnauthorizedRecovery::new(self.clone(), rejected, source)
}
/// One-shot 401 recovery off the live bearer, snapshotted once so the
/// rejected key and KPI attribution describe one credential.
/// 401 recovery off the live bearer. Snapshots the rejected credential once
/// for KPI attribution. On **transient** refresh failure (network, 5xx,
/// sleep/dark-wake defer, lock timeout) retries with backoff before giving
/// up. Permanent failures and NotLoggedIn stop immediately.
///
/// After a successful recovery the **caller** retries the original request
/// (turn-level may resubmit more than once; API resubmit is separate from
/// refresh retries).
pub(crate) async fn try_recover_unauthorized(
self: &Arc<Self>,
source: crate::auth::recovery::RecoverySource,
) -> bool {
/// Bounded refresh attempts for non-permanent failures. Kept strictly
/// below OidcRefresher's consecutive-transient escalation threshold so
/// one 401 recovery cannot alone escalate a network blip to permanent
/// `Other`.
const MAX_TRANSIENT_ATTEMPTS: u32 = 2;
let cached = self.with_inner_read(|inner| inner.cloned());
self.unauthorized_recovery(cached, source)
.next()
.await
.is_ok()
let mut delay = StdDuration::from_millis(500);
for attempt in 0..MAX_TRANSIENT_ATTEMPTS {
match self
.unauthorized_recovery(cached.clone(), source)
.next()
.await
{
Ok(_) => return true,
Err(e) if e.is_transient() && attempt + 1 < MAX_TRANSIENT_ATTEMPTS => {
xai_grok_telemetry::unified_log::warn(
"auth recovery: transient failure, retrying",
None,
Some(serde_json::json!({
"attempt": attempt + 1,
"max_attempts": MAX_TRANSIENT_ATTEMPTS,
"delay_ms": delay.as_millis() as u64,
"error": format!("{e}"),
})),
);
tokio::time::sleep(delay).await;
delay = (delay.saturating_mul(2)).min(StdDuration::from_secs(4));
}
Err(_) => return false,
}
}
false
}
pub(crate) fn record_manual_auth(
@ -2335,6 +2456,12 @@ impl AuthManager {
let key = key.map(|k| k.trim().to_string()).filter(|k| !k.is_empty());
*self.process_static_api_key.write() = key;
}
/// Static/BYOK key for export paths (e.g. desktop `getBearerToken`). Never a
/// session JWT; respects kill-switch and preferred-method pin.
pub(crate) fn static_api_key_for_export(&self) -> Option<String> {
resolve_static_api_key(self)
}
}
fn non_empty_key(key: Option<String>) -> Option<String> {

335
crates/codegen/xai-grok-shell/src/auth/manager_tests.rs
View file

@ -729,25 +729,6 @@ fn record_permanent_failure(
auth_manager.record_permanent_failure(key, reason.into());
}
/// Permanent-failure refresher that reports a specific `tried_key` (the
/// credential it claims to have sent to the IdP), letting tests assert the
/// verdict is keyed on the actually-tried credential.
struct TriedKeyFailRefresher {
tried_key: String,
call_count: Arc<AtomicU32>,
}
#[async_trait::async_trait]
impl TokenRefresher for TriedKeyFailRefresher {
async fn refresh(&self, _reason: RefreshReason) -> crate::auth::refresh::RefreshOutcome {
self.call_count.fetch_add(1, Ordering::SeqCst);
crate::auth::refresh::RefreshOutcome::permanent(
crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected,
Some(self.tried_key.clone()),
)
}
}
/// With `inner == None` but a dead refresh-token on disk, the refresher still
/// exchanges that disk RT. The verdict must be keyed on the
/// credential actually tried (the disk RT), so repeated reactive refreshes
@ -791,10 +772,10 @@ async fn storm_cap_engages_with_empty_inner_and_dead_disk_refresh_token() {
}
/// Record/check consistency: in-mem and disk are DIFFERENT stale credentials.
/// The refresher resolves & sends the DISK refresh token, so the verdict must be
/// keyed on THAT — proven by swapping the in-mem bearer afterward and confirming
/// the verdict still caps the storm (a verdict mis-keyed to the in-mem bearer
/// would read absent after the swap and re-hit the IdP). The `tried_key == None`
/// The refresher reports `tried_key = disk`; with a retain-path permanent
/// (`ClientRejected`) credentials stay, so the verdict stays scoped to disk.
/// Swapping the in-mem bearer must not re-open the IdP (a verdict mis-keyed to
/// the in-mem bearer would read absent after the swap). The `tried_key == None`
/// fallback (external-binary flow → `attempted_verdict_key`) is covered by
/// `storm_cap_engages_with_empty_inner_and_dead_disk_refresh_token`.
#[tokio::test]
@ -813,7 +794,7 @@ async fn verdict_not_keyed_on_in_mem_bearer() {
..GrokAuth::test_default()
});
// disk: a DIFFERENT stale credential K_disk (expired, with RT) — what the
// refresher resolves first.
// refresher claims to have tried.
let disk = GrokAuth {
key: "disk-stale".into(),
auth_mode: AuthMode::Oidc,
@ -826,7 +807,23 @@ async fn verdict_not_keyed_on_in_mem_bearer() {
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
let calls = Arc::new(AtomicU32::new(0));
mgr.set_refresher(Arc::new(TriedKeyFailRefresher {
struct TriedKeyClientRejected {
tried_key: String,
call_count: Arc<AtomicU32>,
}
#[async_trait::async_trait]
impl TokenRefresher for TriedKeyClientRejected {
async fn refresh(&self, _reason: RefreshReason) -> crate::auth::refresh::RefreshOutcome {
self.call_count.fetch_add(1, Ordering::SeqCst);
// ClientRejected retains credentials (unlike RefreshTokenRejected),
// so the disk-scoped verdict remains the storm cap after mem swap.
crate::auth::refresh::RefreshOutcome::permanent(
crate::auth::error::RefreshTokenFailedReason::ClientRejected,
Some(self.tried_key.clone()),
)
}
}
mgr.set_refresher(Arc::new(TriedKeyClientRejected {
tried_key: "disk-stale".into(),
call_count: calls.clone(),
}));
@ -839,6 +836,10 @@ async fn verdict_not_keyed_on_in_mem_bearer() {
1,
"first call hits the IdP once"
);
assert!(
mgr.read_disk_auth().is_some(),
"ClientRejected must retain the disk credential the verdict is keyed on",
);
// Swap the in-mem bearer to yet another stale key: a verdict mis-keyed to
// the old in-mem bearer would now read absent.
@ -1197,8 +1198,8 @@ async fn proactive_refresh_backs_off_on_permanent_failure() {
failure is recorded, got {after_failure} calls"
);
assert!(
mgr.permanent_failure().is_some(),
"permanent failure must be cached after invalid_grant",
mgr.current_or_expired().is_none(),
"permanent refresh failure must clear credentials",
);
// The proactive (background) loop must never emit the manual_auth KPI:
// a background failure is not a user-facing forced re-login.
@ -1322,10 +1323,10 @@ async fn reactive_401_recovery_produces_fresh_token_end_to_end() {
// refresh_chain permanent-failure short-circuit via recovery is tested
// in recovery::tests::refresh_authority_short_circuits_on_cached_permanent_failure.
/// Different disk RT with expired AT: PermanentFailure is recorded
/// (not demoted to transient), stopping the retry loop.
/// Different disk RT with expired AT: demote to transient so a sibling's
/// still-usable RT is not wiped by permanent clear.
#[tokio::test]
async fn refresh_chain_records_permanent_failure_when_disk_rt_differs_but_at_expired() {
async fn refresh_chain_demotes_when_disk_rt_differs_even_if_at_expired() {
let dir = tempfile::tempdir().unwrap();
let cfg = GrokComConfig::default();
let scope = cfg.auth_scope();
@ -1355,7 +1356,7 @@ async fn refresh_chain_records_permanent_failure_when_disk_rt_differs_but_at_exp
..GrokAuth::test_default()
};
let mut store = AuthStore::new();
store.insert(scope, sibling);
store.insert(scope, sibling.clone());
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
struct FailingRefresher;
@ -1374,28 +1375,254 @@ async fn refresh_chain_records_permanent_failure_when_disk_rt_differs_but_at_exp
mgr.set_refresher(Arc::new(FailingRefresher));
let err = mgr.auth().await.unwrap_err();
// An expired disk AT means the sibling is dead too — the failure is
// permanent (not demoted to transient). Credentials are retained; the
// scoped verdict is cached and stops the retry storm.
assert!(
matches!(err, AuthError::Refresh(RefreshTokenError::Transient(_))),
"disk RT mismatch must demote even when sibling AT is expired, got: {err:?}",
);
assert_eq!(
mgr.read_disk_auth().and_then(|a| a.refresh_token),
Some("rt-new".into()),
"sibling RT on disk must not be wiped when AT is only expired",
);
assert!(
mgr.permanent_failure().is_none(),
"demotion must not record a sticky permanent verdict",
);
}
/// Disk-first invalid_grant must not wipe an untried in-memory successor RT
/// (mem-ahead-of-disk after a failed persist of a successful rotation).
#[tokio::test]
async fn permanent_rtr_clears_only_the_tried_side_when_rts_diverge() {
let dir = tempfile::tempdir().unwrap();
let cfg = GrokComConfig::default();
let scope = cfg.auth_scope();
let mgr = Arc::new(AuthManager::new(dir.path(), cfg));
// Mem: successor RT after a successful refresh whose disk write failed.
mgr.hot_swap(GrokAuth {
key: "mem-successor".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-new".into()),
expires_at: Some(Utc::now() - Duration::hours(1)),
oidc_issuer: Some("https://issuer.example".into()),
oidc_client_id: Some("client-1".into()),
..GrokAuth::test_default()
});
// Disk: revoked predecessor RT (disk-first resolve will try this).
let disk = GrokAuth {
key: "disk-predecessor".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-old".into()),
expires_at: Some(Utc::now() - Duration::hours(1)),
oidc_issuer: Some("https://issuer.example".into()),
oidc_client_id: Some("client-1".into()),
..GrokAuth::test_default()
};
let mut store = AuthStore::new();
store.insert(scope, disk);
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
let calls = Arc::new(AtomicU32::new(0));
struct TriedDiskRtr(Arc<AtomicU32>);
#[async_trait::async_trait]
impl TokenRefresher for TriedDiskRtr {
async fn refresh(
&self,
_reason: crate::auth::manager::RefreshReason,
) -> crate::auth::refresh::RefreshOutcome {
self.0.fetch_add(1, Ordering::SeqCst);
crate::auth::refresh::RefreshOutcome::permanent(
crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected,
Some("disk-predecessor".into()),
)
}
}
mgr.set_refresher(Arc::new(TriedDiskRtr(calls.clone())));
let err = mgr
.refresh_chain(TokenType::OidcSession, RefreshReason::ServerRejected)
.await
.unwrap_err();
assert!(
matches!(err, AuthError::Refresh(RefreshTokenError::Permanent(_))),
"must surface a permanent failure when disk AT is expired, got: {err:?}",
"must surface permanent for the tried disk RT, got: {err:?}",
);
assert!(
mgr.permanent_failure().is_some(),
"verdict must be cached (scoped to the retained credential)",
mgr.read_disk_auth().is_none(),
"rejected disk predecessor must be cleared",
);
assert_eq!(
mgr.current_or_expired().and_then(|a| a.refresh_token),
Some("rt-new".into()),
"untried in-memory successor RT must not be wiped",
);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
/// Retain-path permanent (ClientRejected) still graces a soft-expired wire-valid AT.
#[tokio::test]
async fn client_rejected_graces_soft_expired_access_token() {
let dir = tempfile::tempdir().unwrap();
let mgr = Arc::new(AuthManager::new(dir.path(), GrokComConfig::default()));
// Inside the early-invalidation buffer but still hard-valid.
mgr.hot_swap(GrokAuth {
key: "buffered-at".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt".into()),
expires_at: Some(Utc::now() + Duration::seconds(30)),
oidc_issuer: Some("https://issuer.example".into()),
oidc_client_id: Some("client-1".into()),
..GrokAuth::test_default()
});
struct AlwaysClientRejected;
#[async_trait::async_trait]
impl TokenRefresher for AlwaysClientRejected {
async fn refresh(
&self,
_reason: crate::auth::manager::RefreshReason,
) -> crate::auth::refresh::RefreshOutcome {
crate::auth::refresh::RefreshOutcome::permanent(
crate::auth::error::RefreshTokenFailedReason::ClientRejected,
Some("buffered-at".into()),
)
}
}
mgr.set_refresher(Arc::new(AlwaysClientRejected));
let auth = mgr
.auth()
.await
.expect("retain-path permanent must grace wire-valid AT");
assert_eq!(auth.key, "buffered-at");
assert!(
mgr.current_or_expired().is_some(),
"ClientRejected must retain credentials",
);
}
/// Escalated permanent `Other` retains AT+RT (only RefreshTokenRejected discards).
#[tokio::test]
async fn permanent_other_retains_credentials() {
let dir = tempfile::tempdir().unwrap();
let mgr = Arc::new(AuthManager::new(dir.path(), GrokComConfig::default()));
let session = GrokAuth {
key: "live-key".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-still-valid".into()),
expires_at: Some(Utc::now() - Duration::hours(1)),
oidc_issuer: Some("https://issuer.example".into()),
oidc_client_id: Some("client-1".into()),
..GrokAuth::test_default()
};
mgr.hot_swap(session.clone());
// Persist so disk clear would be observable.
let mut store = AuthStore::new();
store.insert(GrokComConfig::default().auth_scope(), session);
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
struct OtherPermanent;
#[async_trait::async_trait]
impl crate::auth::refresh::TokenRefresher for OtherPermanent {
async fn refresh(
&self,
_reason: crate::auth::manager::RefreshReason,
) -> crate::auth::refresh::RefreshOutcome {
crate::auth::refresh::RefreshOutcome::permanent(
crate::auth::error::RefreshTokenFailedReason::Other,
Some("live-key".into()),
)
}
}
mgr.set_refresher(Arc::new(OtherPermanent));
let err = mgr.auth().await.unwrap_err();
assert!(
matches!(err, AuthError::Refresh(RefreshTokenError::Permanent(_))),
"escalated Other must still surface permanent, got: {err:?}",
);
// No-clear invariant: a refresh failure must NOT delete auth.json (a future
// regression that re-adds disk-clear-on-invalid_grant would fail here).
assert!(
mgr.read_disk_auth().is_some(),
"invalid_grant must not delete auth.json (no auto-clear)",
"Other must not clear disk credentials",
);
// Second attempt short-circuits on the cached verdict — no extra IdP call.
assert!(matches!(
mgr.auth().await.unwrap_err(),
AuthError::Refresh(RefreshTokenError::Permanent(_))
));
assert_eq!(
mgr.current_or_expired().and_then(|a| a.refresh_token),
Some("rt-still-valid".into()),
"Other must retain in-memory RT",
);
}
/// Sticky permanent must not block a different credential key (sibling RT).
#[tokio::test]
async fn sticky_permanent_allows_refresh_when_attempted_key_differs() {
let dir = tempfile::tempdir().unwrap();
let cfg = GrokComConfig::default();
let scope = cfg.auth_scope();
let mgr = Arc::new(AuthManager::new(dir.path(), cfg));
mgr.hot_swap(GrokAuth {
key: "dead-key".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-dead".into()),
expires_at: Some(Utc::now() - Duration::hours(1)),
..GrokAuth::test_default()
});
record_permanent_failure(
&mgr,
crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected,
);
assert!(mgr.permanent_failure().is_some());
// Sibling writes a different key + RT (AT hard-expired, RT may still work).
let sibling = GrokAuth {
key: "sibling-key".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-sibling".into()),
expires_at: Some(Utc::now() - Duration::minutes(30)),
oidc_issuer: Some("https://issuer.example".into()),
oidc_client_id: Some("client-1".into()),
..GrokAuth::test_default()
};
let mut store = AuthStore::new();
store.insert(scope, sibling.clone());
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
// Load sibling into memory without clearing sticky via wire-valid hot_swap.
mgr.with_inner_write(|inner| *inner = Some(sibling));
assert!(
mgr.permanent_failure().is_none(),
"sticky verdict must not apply to a different credential key",
);
let calls = Arc::new(AtomicU32::new(0));
struct CountingOk(Arc<AtomicU32>);
#[async_trait::async_trait]
impl crate::auth::refresh::TokenRefresher for CountingOk {
async fn refresh(
&self,
_reason: crate::auth::manager::RefreshReason,
) -> crate::auth::refresh::RefreshOutcome {
self.0.fetch_add(1, Ordering::SeqCst);
crate::auth::refresh::RefreshOutcome::Success(Box::new(GrokAuth {
key: "fresh-from-sibling-rt".into(),
auth_mode: AuthMode::Oidc,
refresh_token: Some("rt-sibling".into()),
expires_at: Some(Utc::now() + Duration::hours(1)),
..GrokAuth::test_default()
}))
}
}
mgr.set_refresher(Arc::new(CountingOk(calls.clone())));
let auth = mgr
.auth()
.await
.expect("sibling key must reach refresh_chain");
assert_eq!(auth.key, "fresh-from-sibling-rt");
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
/// Different disk RT with valid AT: adopt the sibling's token directly.
@ -1514,10 +1741,15 @@ async fn permanent_failure_reads_absent_after_clear_so_auth_reports_not_logged_i
crate::auth::error::RefreshTokenFailedReason::RefreshTokenRejected,
);
mgr.clear_in_memory();
// clear_in_memory drops the credential but keeps a sticky permanent
// verdict so a just-revoked RT is not re-tried until login.
let err = mgr.auth().await.unwrap_err();
assert!(
matches!(err, AuthError::NotLoggedIn),
"auth() after hot_swap_clear() must report NotLoggedIn, got: {err:?}",
matches!(
err,
AuthError::Refresh(RefreshTokenError::Permanent(_)) | AuthError::NotLoggedIn
),
"auth() after clear_in_memory must not re-hit a dead RT, got: {err:?}",
);
}
@ -2629,9 +2861,10 @@ async fn update_recovers_from_whitespace_only_auth_json() {
// -- sibling_has_different_refresh_token ----------------------------------
/// Expired disk AT with different RT is not a live sibling.
/// Expired disk AT with different RT is still treated as a sibling RT
/// (may still be refreshable; must not be wiped by permanent clear).
#[tokio::test]
async fn sibling_different_rt_with_expired_at_is_not_treated_as_live() {
async fn sibling_different_rt_with_expired_at_is_still_sibling() {
let dir = tempfile::tempdir().unwrap();
let cfg = GrokComConfig::default();
let mgr = Arc::new(AuthManager::new(dir.path(), cfg.clone()));
@ -2659,8 +2892,8 @@ async fn sibling_different_rt_with_expired_at_is_not_treated_as_live() {
write_auth_json(&dir.path().join("auth.json"), &store).unwrap();
assert!(
!mgr.sibling_has_different_refresh_token(),
"expired disk token must not be treated as a live sibling"
mgr.sibling_has_different_refresh_token(),
"different disk RT must demote even when the sibling AT is expired"
);
}

11
crates/codegen/xai-grok-shell/src/auth/refresh/auth_backend_contract_tests.rs
View file

@ -319,9 +319,11 @@ async fn auth_backend_contract_transient_failures_escalate_to_non_sticky_permane
auth_manager.hot_swap(expired_oidc(&base_url));
// One refresher instance: it owns the consecutive-failure counter.
// Budget is above try_recover_unauthorized's per-recovery attempts so a
// single 401 recovery cannot alone escalate; exhaust the full budget here.
let refresher = OidcRefresher::new(auth_manager.clone());
let mut outcomes = Vec::new();
for _ in 0..3 {
for _ in 0..5 {
outcomes.push(refresher.refresh(RefreshReason::ServerRejected).await);
}
@ -330,7 +332,12 @@ async fn auth_backend_contract_transient_failures_escalate_to_non_sticky_permane
"first blip is transient, not a lockout: {:?}",
outcomes[0],
);
match &outcomes[2] {
assert!(
matches!(outcomes[3], RefreshOutcome::TransientFailure { .. }),
"4th blip still under escalation budget: {:?}",
outcomes[3],
);
match &outcomes[4] {
RefreshOutcome::PermanentFailure { error, .. } => {
assert_eq!(
error.reason,

5
crates/codegen/xai-grok-shell/src/auth/refresh/mod.rs
View file

@ -94,8 +94,9 @@ pub(crate) enum RefreshOutcome {
Success(Box<GrokAuth>),
/// Terminal failure (e.g. invalid_grant), or a transient escalated to
/// `Other` after repeated blips. Caller records a verdict scoped to the
/// rejected credential and retains it (`RefreshTokenRejected` is sticky,
/// the rest age out past the TTL).
/// rejected credential. `refresh_chain` discards AT+RT only for
/// `RefreshTokenRejected` (sticky until login); `ClientRejected` / `Other`
/// retain credentials and age out past the TTL.
PermanentFailure {
error: crate::auth::error::RefreshTokenFailedError,
/// Key of the credential the refresher actually sent to the IdP, so

5
crates/codegen/xai-grok-shell/src/auth/refresh/oidc_refresher.rs
View file

@ -13,8 +13,9 @@ use crate::auth::manager::AuthManager;
/// Escalate to `PermanentFailure` after this many consecutive transient
/// failures (then `PERMANENT_FAILURE_TTL` allows recovery). OIDC tolerates more
/// blips than `ExternalBinaryRefresher` (1) since network refreshes flake more
/// than a local binary.
const MAX_CONSECUTIVE_TRANSIENT_FAILURES: u32 = 3;
/// than a local binary. Kept above `try_recover_unauthorized`'s per-recovery
/// attempt budget so one 401 recovery cannot alone escalate.
const MAX_CONSECUTIVE_TRANSIENT_FAILURES: u32 = 5;
/// Consecutive transient-failure budget, scoped to the credential it accrued
/// against. Held under one lock so the credential check, reset, and increment

83
crates/codegen/xai-grok-shell/src/auth/refresh/oidc_refresher_tests.rs
View file

@ -301,26 +301,29 @@ async fn oidc_refresher_e2e_near_expiry_idp_rejects_refresh() {
};
mgr.hot_swap(near_expiry);
// auth() dispatches to refresh_chain -> OidcRefresher -> invalid_grant.
// Because the token is still within real expires_at (3 min from now),
// the grace path returns the cached token as a fallback.
// Permanent invalid_grant discards AT+RT (no grace re-serve of pre-refresh
// snapshot). Grace remains for *transient* refresh failures only.
mgr.set_refresher(std::sync::Arc::new(OidcRefresher::new(mgr.clone())));
let refreshed = mgr.auth().await;
let err = mgr.auth().await.unwrap_err();
assert!(
refreshed.is_ok(),
"grace path should return the cached token while within real expires_at"
matches!(
err,
crate::auth::AuthError::Refresh(crate::auth::RefreshTokenError::Permanent(_))
),
"permanent invalid_grant must not grace-serve the pre-refresh AT, got: {err:?}",
);
assert!(
mgr.current_or_expired().is_none(),
"permanent invalid_grant must clear credentials",
);
assert_eq!(refreshed.unwrap().key, "about-to-expire-token");
server.abort();
}
/// On `invalid_client` (client_id rotated, soft-deleted, or disabled), the
/// credential is retained and a permanent-failure verdict cached. Verdict + TTL
/// stop the retry loop; the bearer drops only on explicit logout, so a
/// transient client-rotation blip self-heals without a fleet re-login.
/// On `invalid_client` (client_id rotated, soft-deleted, or disabled) with a
/// hard-expired AT, permanent failure retains AT+RT (only invalid_grant discards).
#[tokio::test]
async fn oidc_refresher_e2e_invalid_client_caches_verdict_and_retains_credentials() {
async fn oidc_refresher_e2e_invalid_client_retains_credentials() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base_url = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
let base_for_discovery = base_url.clone();
@ -371,34 +374,25 @@ async fn oidc_refresher_e2e_invalid_client_caches_verdict_and_retains_credential
mgr.hot_swap(expired);
mgr.set_refresher(std::sync::Arc::new(OidcRefresher::new(mgr.clone())));
let refreshed = mgr.auth().await.ok();
let err = mgr.auth().await.unwrap_err();
assert!(
refreshed.is_none(),
"refresh should fail when client is unknown"
matches!(
err,
crate::auth::AuthError::Refresh(crate::auth::RefreshTokenError::Permanent(_))
),
"refresh should fail permanently when client is unknown, got: {err:?}",
);
assert_eq!(
mgr.current_or_expired()
.and_then(|a| a.refresh_token)
.as_deref(),
Some("rt-valid"),
"invalid_client must retain RT for TTL-gated retry after client rotation",
);
// Credential retained (not cleared) — the bearer may be fine; the client
// credential isn't.
assert!(
mgr.expired_auth().is_some(),
"credentials must be retained after invalid_client",
mgr.read_disk_auth().is_some() || mgr.current_or_expired().is_some(),
"invalid_client must not clear credentials",
);
// The verdict is cached, scoped to the retained credential, and carries
// the non-sticky `ClientRejected` reason (so it ages out, not stuck-forever).
match mgr.permanent_failure() {
Some(crate::auth::AuthError::Refresh(crate::auth::RefreshTokenError::Permanent(e))) => {
assert_eq!(
e.reason,
crate::auth::RefreshTokenFailedReason::ClientRejected,
"invalid_client must map to ClientRejected",
);
assert!(
!e.reason.is_sticky(),
"ClientRejected must age out past the TTL, not stick forever",
);
}
other => panic!("invalid_client must cache a permanent-failure verdict, got {other:?}"),
}
server.abort();
}
@ -990,22 +984,18 @@ async fn refresher_disk_retry_invalid_client_with_different_client_id_preserves_
other => panic!("expected PermanentFailure, got: {other:?}"),
}
// Credential retained; the cached verdict (scoped to it) stops the storm.
// Disk-retry already tried the sibling RT and got invalid_client —
// permanent is recorded, but ClientRejected retains credentials.
assert!(
mgr.current_or_expired().is_some(),
"credential must be retained on permanent failure"
);
assert!(
mgr.permanent_failure().is_some(),
"verdict must be cached to stop the retry storm"
mgr.current_or_expired().is_some() || mgr.read_disk_auth().is_some(),
"invalid_client permanent must retain credentials (only invalid_grant discards)"
);
assert_eq!(attempts.load(Ordering::SeqCst), 2, "no recursion");
server.abort();
}
/// Both RTs revoked: retry is strictly one-shot (no third call);
/// refresh_chain's disk-RT-differs guard preserves disk creds.
/// Both RTs revoked: retry is strictly one-shot (no third call).
#[tokio::test]
async fn refresher_disk_retry_is_one_shot() {
use std::sync::atomic::{AtomicU32, Ordering};
@ -1060,7 +1050,8 @@ async fn refresher_disk_retry_is_one_shot() {
"exactly two IdP calls — disk-token retry must NOT recurse"
);
// Disk auth must still be present (the refresher never clears).
// This test calls the refresher directly (not refresh_chain); disk is
// unchanged here — refresh_chain is responsible for permanent clear.
assert!(
mgr.read_disk_auth().is_some(),
"refresher must not touch disk; clearing is refresh_chain's responsibility"