Synced from monorepo
Synced from monorepo Changes: - Release a shell session's resources in one drop - Make the tools blocking-wait cap client-configurable and self-describing - Recognize API "exceeds budget" errors as context overflow - Retry /btw on model overload - Carry running background tasks and subagents across compaction - Require round-trip time for SDK liveness checks - Background-subagent completion reminders with a selectable delivery surface - Make a PTY shell reap itself until it reaches the registry - Recover the OS error code from a TLS-phase connection reset - Consume the attached-client signal and report why idle is withheld - Treat `.grok/sandbox.toml` edits as protected so auto mode prompts before writing - Surface history/search in the Ctrl+. cheatsheet and keep it working in history view - Delete sessions from the dashboard and welcome list - Release a session's activity record when the session ends - Stop charging auth-retry budget for fail-closed 401s; reset it across suspends - Scope skills watches on project vendor roots - Make [stop] cancel in-flight compaction - Make the leader soak measure the leader, not its harness Source-Revision: 8d69c91f02bcacf01e98d5aebbf2f92547c45738
This commit is contained in:
parent
dd04f397b1
commit
a422116582
165 changed files with 15161 additions and 1969 deletions
|
|
@ -18,7 +18,7 @@ use tokio_util::sync::CancellationToken;
|
|||
use tracing::Instrument;
|
||||
|
||||
use xai_grok_sampling_types::{
|
||||
ConversationRequest, ConversationResponse, EmptyResponseContext, SamplingError,
|
||||
ConversationRequest, ConversationResponse, EmptyResponseContext, SamplingError, SentCredential,
|
||||
error::Result as SamplingResult,
|
||||
};
|
||||
|
||||
|
|
@ -692,7 +692,10 @@ fn synthesize_from_info(info: &SamplingErrorInfo) -> SamplingError {
|
|||
.find_map(|tok| tok.strip_suffix('s').and_then(|n| n.parse::<u64>().ok()))
|
||||
.unwrap_or(0),
|
||||
},
|
||||
SamplingErrorKind::Auth => SamplingError::Auth(info.message.clone()),
|
||||
SamplingErrorKind::Auth => SamplingError::Auth {
|
||||
message: info.message.clone(),
|
||||
credential: info.credential,
|
||||
},
|
||||
// Must stay Serialization: EventStreamError is retryable, and a
|
||||
// response-parse failure is deterministic on retry. `info.message`
|
||||
// is the variant's rendered Display, so rebuild via the constructor
|
||||
|
|
@ -826,6 +829,7 @@ fn handle_cancellation(
|
|||
empty_response_context: None,
|
||||
doom_loop_triggers: None,
|
||||
doom_loop_aborted_at_chunk: None,
|
||||
credential: SentCredential::Unknown,
|
||||
};
|
||||
let _ = event_tx.send(SamplingEvent::Failed {
|
||||
request_id: request_id.clone(),
|
||||
|
|
@ -833,7 +837,7 @@ fn handle_cancellation(
|
|||
});
|
||||
send_completion(
|
||||
completion_tx,
|
||||
Err(SamplingError::Auth("request cancelled".to_string())),
|
||||
Err(SamplingError::auth_unknown("request cancelled")),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -863,6 +867,7 @@ mod tests {
|
|||
empty_response_context: None,
|
||||
doom_loop_triggers: None,
|
||||
doom_loop_aborted_at_chunk: None,
|
||||
credential: SentCredential::Unknown,
|
||||
};
|
||||
let err = synthesize_from_info(&info);
|
||||
match err {
|
||||
|
|
@ -883,6 +888,7 @@ mod tests {
|
|||
empty_response_context: None,
|
||||
doom_loop_triggers: None,
|
||||
doom_loop_aborted_at_chunk: None,
|
||||
credential: SentCredential::Unknown,
|
||||
};
|
||||
let err = synthesize_from_info(&info);
|
||||
match err {
|
||||
|
|
@ -908,6 +914,7 @@ mod tests {
|
|||
empty_response_context: None,
|
||||
doom_loop_triggers: None,
|
||||
doom_loop_aborted_at_chunk: None,
|
||||
credential: SentCredential::Unknown,
|
||||
};
|
||||
let err = synthesize_from_info(&info);
|
||||
match err {
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ use xai_grok_sampling_types::error::{try_parse_stream_error, user_facing_api_err
|
|||
use xai_grok_sampling_types::{
|
||||
ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, ConversationRequest,
|
||||
ConversationResponse, CreateResponseWrapper, DOOM_LOOP_CHECK_HEADER, MessagesRequestWrapper,
|
||||
ResponseModelMetadata, Result, SamplingError, build_messages_request, is_check_event, messages,
|
||||
rs,
|
||||
ResponseModelMetadata, Result, SamplingError, SentCredential, build_messages_request,
|
||||
is_check_event, messages, rs,
|
||||
};
|
||||
|
||||
use crate::attribution::bearer_tail_fragment;
|
||||
|
|
@ -488,6 +488,28 @@ pub fn user_agent_string_for(origin: &OriginClientInfo) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// A request builder coupled to the credential state it was built with, so
|
||||
/// a 401 arm cannot classify from anything but the build-time capture. The
|
||||
/// wire default (`SentCredential::Unknown`, which charges the retry budget)
|
||||
/// stays the fail-closed one; only an explicit `sent_bearer: None` — a send
|
||||
/// the builder provably stamped no credential onto — reaches the uncharged
|
||||
/// lane via [`auth_rejected`].
|
||||
struct SentRequest {
|
||||
builder: reqwest::RequestBuilder,
|
||||
/// Tail fragment of the credential in the built headers (`None` = no
|
||||
/// credential header at all).
|
||||
sent_bearer: Option<String>,
|
||||
}
|
||||
|
||||
/// The one way a 401 becomes a `SamplingError::Auth` with a wire-derived
|
||||
/// credential classification: from the fragment its [`SentRequest`] captured.
|
||||
fn auth_rejected(message: String, sent_bearer: Option<&str>) -> SamplingError {
|
||||
SamplingError::Auth {
|
||||
message,
|
||||
credential: SentCredential::from_sent_fragment(sent_bearer),
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SamplingClient
|
||||
// =============================================================================
|
||||
|
|
@ -510,9 +532,8 @@ impl SamplingClient {
|
|||
api_key = %api_key,
|
||||
"Invalid api_key: cannot be converted to a valid HTTP header"
|
||||
);
|
||||
SamplingError::Auth(
|
||||
"Invalid api_key: cannot be converted to a valid HTTP header"
|
||||
.to_string(),
|
||||
SamplingError::auth_unknown(
|
||||
"Invalid api_key: cannot be converted to a valid HTTP header",
|
||||
)
|
||||
})?;
|
||||
headers.insert(HeaderName::from_static("x-api-key"), header_value);
|
||||
|
|
@ -524,9 +545,8 @@ impl SamplingClient {
|
|||
api_key = %api_key,
|
||||
"Invalid api_key: cannot be converted to a valid HTTP Authorization header"
|
||||
);
|
||||
SamplingError::Auth(
|
||||
"Invalid api_key: cannot be converted to a valid HTTP Authorization header"
|
||||
.to_string(),
|
||||
SamplingError::auth_unknown(
|
||||
"Invalid api_key: cannot be converted to a valid HTTP Authorization header",
|
||||
)
|
||||
})?;
|
||||
headers.insert(AUTHORIZATION, header_value);
|
||||
|
|
@ -658,7 +678,7 @@ impl SamplingClient {
|
|||
self.defaults.api_backend.clone()
|
||||
}
|
||||
|
||||
/// POST with default headers, returning the builder plus the tail
|
||||
/// POST with default headers, returning the builder coupled to the tail
|
||||
/// fragment of the credential actually placed in its headers (`None` =
|
||||
/// no credential) — captured at build time because a record-time
|
||||
/// re-read races with the recovery a 401 triggers.
|
||||
|
|
@ -666,7 +686,7 @@ impl SamplingClient {
|
|||
/// A wired bearer_resolver is the sole auth source: a missing live
|
||||
/// bearer strips default Authorization / x-api-key so a hard-expired
|
||||
/// seed key cannot ride on the wire.
|
||||
fn post(&self, url: impl reqwest::IntoUrl) -> (reqwest::RequestBuilder, Option<String>) {
|
||||
fn post(&self, url: impl reqwest::IntoUrl) -> SentRequest {
|
||||
let mut headers = self.default_headers.clone();
|
||||
if let Some(resolver) = &self.bearer_resolver {
|
||||
headers.remove(AUTHORIZATION);
|
||||
|
|
@ -713,7 +733,10 @@ impl SamplingClient {
|
|||
if let Some(injector) = &self.header_injector {
|
||||
injector.inject(&mut headers);
|
||||
}
|
||||
(self.http.post(url).headers(headers), sent_bearer)
|
||||
SentRequest {
|
||||
builder: self.http.post(url).headers(headers),
|
||||
sent_bearer,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tail fragment of the credential in `headers` — `x-api-key`
|
||||
|
|
@ -856,9 +879,10 @@ impl SamplingClient {
|
|||
sent_bearer,
|
||||
);
|
||||
let server_message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401): {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401): {server_message}"),
|
||||
sent_bearer,
|
||||
));
|
||||
}
|
||||
let message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
return Err(SamplingError::Api {
|
||||
|
|
@ -911,7 +935,10 @@ impl SamplingClient {
|
|||
deployment_id: payload.x_grok_deployment_id.as_deref(),
|
||||
user_id: payload.x_grok_user_id.as_deref(),
|
||||
};
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("chat/completions"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("chat/completions"));
|
||||
let http_request = grok_headers.apply(builder).json(&payload);
|
||||
|
||||
let response = http_request.send().await.map_err(|e| {
|
||||
|
|
@ -968,7 +995,10 @@ impl SamplingClient {
|
|||
deployment_id: payload.x_grok_deployment_id.as_deref(),
|
||||
user_id: payload.x_grok_user_id.as_deref(),
|
||||
};
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("chat/completions"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("chat/completions"));
|
||||
let http_request = grok_headers
|
||||
.apply(builder)
|
||||
.header(ACCEPT, HeaderValue::from_static("text/event-stream"))
|
||||
|
|
@ -1009,9 +1039,10 @@ impl SamplingClient {
|
|||
let endpoint = self.endpoint("chat/completions");
|
||||
let body = response.bytes().await.unwrap_or_default();
|
||||
let server_message = user_facing_api_error_message(status, body.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401) from {endpoint}: {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401) from {endpoint}: {server_message}"),
|
||||
sent_bearer.as_deref(),
|
||||
));
|
||||
}
|
||||
|
||||
let bytes = response.bytes().await?;
|
||||
|
|
@ -1183,7 +1214,10 @@ impl SamplingClient {
|
|||
// it in post-serialize. This is the last surviving piece of the
|
||||
// old raw_output machinery.
|
||||
xai_grok_sampling_types::patch_reasoning_text_types(&mut request_body);
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("responses"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("responses"));
|
||||
let http_request = grok_headers.apply(builder).json(&request_body);
|
||||
|
||||
let response = http_request.send().await.map_err(|e| {
|
||||
|
|
@ -1205,9 +1239,10 @@ impl SamplingClient {
|
|||
);
|
||||
let endpoint = self.endpoint("responses");
|
||||
let server_message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401) from {endpoint}: {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401) from {endpoint}: {server_message}"),
|
||||
sent_bearer.as_deref(),
|
||||
));
|
||||
}
|
||||
|
||||
let message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
|
|
@ -1327,7 +1362,10 @@ impl SamplingClient {
|
|||
.defaults
|
||||
.doom_loop_recovery
|
||||
.map(crate::doom_loop::DoomLoopSignalCollector::new);
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("responses"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("responses"));
|
||||
let mut http_request = grok_headers
|
||||
.apply(builder)
|
||||
.header(ACCEPT, HeaderValue::from_static("text/event-stream"));
|
||||
|
|
@ -1369,9 +1407,10 @@ impl SamplingClient {
|
|||
let endpoint = self.endpoint("responses");
|
||||
let body = response.bytes().await.unwrap_or_default();
|
||||
let server_message = user_facing_api_error_message(status, body.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401) from {endpoint}: {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401) from {endpoint}: {server_message}"),
|
||||
sent_bearer.as_deref(),
|
||||
));
|
||||
}
|
||||
let model_metadata = extract_model_metadata(response.headers());
|
||||
let retry_after_secs = extract_retry_after(response.headers());
|
||||
|
|
@ -1528,7 +1567,10 @@ impl SamplingClient {
|
|||
deployment_id: request.x_grok_deployment_id.as_deref(),
|
||||
user_id: request.x_grok_user_id.as_deref(),
|
||||
};
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("messages"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("messages"));
|
||||
let http_request = grok_headers.apply(builder).json(&request.inner);
|
||||
|
||||
let response = http_request.send().await.map_err(|e| {
|
||||
|
|
@ -1550,9 +1592,10 @@ impl SamplingClient {
|
|||
);
|
||||
let endpoint = self.endpoint("messages");
|
||||
let server_message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401) from {endpoint}: {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401) from {endpoint}: {server_message}"),
|
||||
sent_bearer.as_deref(),
|
||||
));
|
||||
}
|
||||
|
||||
let message = user_facing_api_error_message(status, bytes.as_ref());
|
||||
|
|
@ -1637,7 +1680,10 @@ impl SamplingClient {
|
|||
deployment_id: request.x_grok_deployment_id.as_deref(),
|
||||
user_id: request.x_grok_user_id.as_deref(),
|
||||
};
|
||||
let (builder, sent_bearer) = self.post(self.endpoint("messages"));
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer,
|
||||
} = self.post(self.endpoint("messages"));
|
||||
let http_request = grok_headers
|
||||
.apply(builder)
|
||||
.header(ACCEPT, HeaderValue::from_static("text/event-stream"))
|
||||
|
|
@ -1675,9 +1721,10 @@ impl SamplingClient {
|
|||
let endpoint = self.endpoint("messages");
|
||||
let body = response.bytes().await.unwrap_or_default();
|
||||
let server_message = user_facing_api_error_message(status, body.as_ref());
|
||||
return Err(SamplingError::Auth(format!(
|
||||
"Unauthorized (401) from {endpoint}: {server_message}"
|
||||
)));
|
||||
return Err(auth_rejected(
|
||||
format!("Unauthorized (401) from {endpoint}: {server_message}"),
|
||||
sent_bearer.as_deref(),
|
||||
));
|
||||
}
|
||||
let model_metadata = extract_model_metadata(response.headers());
|
||||
let retry_after_secs = extract_retry_after(response.headers());
|
||||
|
|
@ -2323,7 +2370,7 @@ mod tests {
|
|||
let mut config = minimal_config();
|
||||
config.header_injector = Some(std::sync::Arc::new(TestInjector));
|
||||
let client = SamplingClient::new(config).expect("build");
|
||||
let (builder, _sent) = client.post("http://localhost/test");
|
||||
let SentRequest { builder, .. } = client.post("http://localhost/test");
|
||||
let req = builder.build().expect("build request");
|
||||
assert!(
|
||||
req.headers().contains_key("traceparent"),
|
||||
|
|
@ -2403,7 +2450,10 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (_builder, bearer) = client.post("https://example.test/v1/chat/completions");
|
||||
let SentRequest {
|
||||
sent_bearer: bearer,
|
||||
..
|
||||
} = client.post("https://example.test/v1/chat/completions");
|
||||
assert_eq!(bearer.as_deref(), Some("r-1234567890"));
|
||||
assert_eq!(
|
||||
bearer.as_deref().map(str::len),
|
||||
|
|
@ -2422,7 +2472,10 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (_builder, bearer) = client.post("https://example.test/v1/messages");
|
||||
let SentRequest {
|
||||
sent_bearer: bearer,
|
||||
..
|
||||
} = client.post("https://example.test/v1/messages");
|
||||
assert_eq!(bearer.as_deref(), Some("c-key-abc123"));
|
||||
assert_eq!(
|
||||
bearer.as_deref().map(str::len),
|
||||
|
|
@ -2439,7 +2492,10 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (_builder, bearer) = client.post("https://example.test/v1/chat/completions");
|
||||
let SentRequest {
|
||||
sent_bearer: bearer,
|
||||
..
|
||||
} = client.post("https://example.test/v1/chat/completions");
|
||||
assert!(bearer.is_none());
|
||||
}
|
||||
|
||||
|
|
@ -2468,7 +2524,10 @@ mod tests {
|
|||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
|
||||
let (_builder, sent_at_build) = client.post("https://example.test/v1/responses");
|
||||
let SentRequest {
|
||||
sent_bearer: sent_at_build,
|
||||
..
|
||||
} = client.post("https://example.test/v1/responses");
|
||||
// The 401 kicks recovery; the resolver rotates before the callback runs.
|
||||
*resolver.0.lock().unwrap() = "fresh-token-newtail99".to_string();
|
||||
|
||||
|
|
@ -2496,7 +2555,7 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (builder, _sent) = client.post("https://example.test/v1/messages");
|
||||
let SentRequest { builder, .. } = client.post("https://example.test/v1/messages");
|
||||
let request = builder.build().expect("request should build");
|
||||
let auth = request
|
||||
.headers()
|
||||
|
|
@ -2522,7 +2581,7 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (builder, _sent) = client.post("https://example.test/v1/responses");
|
||||
let SentRequest { builder, .. } = client.post("https://example.test/v1/responses");
|
||||
let request = builder.build().expect("request should build");
|
||||
let auth_count = request.headers().get_all(AUTHORIZATION).iter().count();
|
||||
assert_eq!(
|
||||
|
|
@ -2548,7 +2607,7 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (builder, _sent) = client.post("https://example.test/v1/messages");
|
||||
let SentRequest { builder, .. } = client.post("https://example.test/v1/messages");
|
||||
let request = builder.build().expect("request should build");
|
||||
let api_key = request
|
||||
.headers()
|
||||
|
|
@ -2572,7 +2631,8 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (_builder, sent_bearer) = client.post("https://example.test/v1/chat/completions");
|
||||
let SentRequest { sent_bearer, .. } =
|
||||
client.post("https://example.test/v1/chat/completions");
|
||||
client.record_401_attribution(
|
||||
crate::attribution::SamplingConsumer::ChatCompletionsStream,
|
||||
sent_bearer.as_deref(),
|
||||
|
|
@ -2636,7 +2696,10 @@ mod tests {
|
|||
..minimal_config()
|
||||
};
|
||||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
let (builder, sent) = client.post("https://example.test/v1/responses");
|
||||
let SentRequest {
|
||||
builder,
|
||||
sent_bearer: sent,
|
||||
} = client.post("https://example.test/v1/responses");
|
||||
let request = builder.body("").build().expect("request should build");
|
||||
assert_eq!(sent, None, "capture must agree: nothing was sent");
|
||||
assert!(
|
||||
|
|
@ -2670,7 +2733,7 @@ mod tests {
|
|||
let client = SamplingClient::new(cfg).expect("client should build");
|
||||
|
||||
// Build a request to inspect the final headers.
|
||||
let (builder, _sent) = client.post("https://example.test/v1/responses");
|
||||
let SentRequest { builder, .. } = client.post("https://example.test/v1/responses");
|
||||
let request = builder.body("").build().expect("request should build");
|
||||
|
||||
let auth_values: Vec<_> = request.headers().get_all(AUTHORIZATION).iter().collect();
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
|
|||
|
||||
use xai_grok_sampling_types::{
|
||||
ConversationResponse, EmptyResponseContext, ResponseModelMetadata, SamplingError,
|
||||
SentCredential,
|
||||
};
|
||||
|
||||
use crate::metrics::InferenceLatencyStats;
|
||||
|
|
@ -168,6 +169,11 @@ pub struct SamplingErrorInfo {
|
|||
/// Telemetry only; `None` for terminal-response detections.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub doom_loop_aborted_at_chunk: Option<u64>,
|
||||
/// Meaningful only when `kind == Auth`: whether the rejected request
|
||||
/// actually carried a credential on the wire. Defaults to `Unknown`
|
||||
/// (charge-the-budget behavior) for payloads from older peers.
|
||||
#[serde(default, skip_serializing_if = "SentCredential::is_unknown")]
|
||||
pub credential: SentCredential,
|
||||
}
|
||||
|
||||
/// Coarse-grained classification of a sampling failure.
|
||||
|
|
@ -217,7 +223,7 @@ impl From<&SamplingError> for SamplingErrorInfo {
|
|||
let message = err.to_string();
|
||||
|
||||
let (kind, status_code, retry_after_secs, model_metadata) = match err {
|
||||
SamplingError::Auth(_) => (SamplingErrorKind::Auth, None, None, None),
|
||||
SamplingError::Auth { .. } => (SamplingErrorKind::Auth, None, None, None),
|
||||
SamplingError::InvalidConfiguration(_) => (SamplingErrorKind::Api, None, None, None),
|
||||
SamplingError::Http(_) => (SamplingErrorKind::Http, None, None, None),
|
||||
SamplingError::Serialization(_) => (SamplingErrorKind::Serialization, None, None, None),
|
||||
|
|
@ -264,6 +270,10 @@ impl From<&SamplingError> for SamplingErrorInfo {
|
|||
} => (Some(triggers.clone()), *aborted_at_chunk),
|
||||
_ => (None, None),
|
||||
};
|
||||
let credential = match err {
|
||||
SamplingError::Auth { credential, .. } => *credential,
|
||||
_ => SentCredential::Unknown,
|
||||
};
|
||||
|
||||
Self {
|
||||
kind,
|
||||
|
|
@ -275,6 +285,7 @@ impl From<&SamplingError> for SamplingErrorInfo {
|
|||
empty_response_context,
|
||||
doom_loop_triggers,
|
||||
doom_loop_aborted_at_chunk,
|
||||
credential,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -286,7 +297,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn auth_variant_classified_as_auth() {
|
||||
let err = SamplingError::Auth("bad token".into());
|
||||
let err = SamplingError::auth_unknown("bad token");
|
||||
let info = SamplingErrorInfo::from(&err);
|
||||
assert_eq!(info.kind, SamplingErrorKind::Auth);
|
||||
assert_eq!(info.status_code, None);
|
||||
|
|
@ -296,6 +307,18 @@ mod tests {
|
|||
assert!(info.message.contains("bad token"));
|
||||
}
|
||||
|
||||
/// A payload from a peer that predates `credential` must still parse,
|
||||
/// defaulting to `Unknown` (charge-the-budget behavior).
|
||||
#[test]
|
||||
fn info_without_credential_field_deserializes_to_unknown() {
|
||||
let info: SamplingErrorInfo = serde_json::from_str(
|
||||
r#"{"kind":"Auth","status_code":401,"message":"x","is_retryable":false,
|
||||
"retry_after_secs":null,"model_metadata":null}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(info.credential, SentCredential::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_configuration_classified_as_api() {
|
||||
let err = SamplingError::InvalidConfiguration("missing model");
|
||||
|
|
|
|||
|
|
@ -149,8 +149,8 @@ impl SamplerHandle {
|
|||
request_id: cancel_id,
|
||||
});
|
||||
completion_rx.await.unwrap_or_else(|_| {
|
||||
Err(SamplingError::Auth(
|
||||
"sampler actor dropped before completion".to_string(),
|
||||
Err(SamplingError::auth_unknown(
|
||||
"sampler actor dropped before completion",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -173,26 +173,20 @@ pub fn classify_error(
|
|||
return RetryDecision::RetryWithImageStrip;
|
||||
}
|
||||
|
||||
// Server explicitly said don't retry (x-should-retry: false).
|
||||
// Trust the server — it knows if the error is request-content-caused
|
||||
// (e.g. malformed tool call in conversation history) vs transient.
|
||||
//
|
||||
// x-should-retry: true is intentionally NOT handled here — we only
|
||||
// use the header to suppress retries (false), not to force them
|
||||
// (true). Forcing retries on non-retryable status codes could
|
||||
// amplify failures. true falls through to existing status-code logic.
|
||||
// Shared retry vetoes (`SamplingError::is_retry_vetoed`, also used by
|
||||
// one-shot callers like /btw):
|
||||
// - x-should-retry: false — trust the server, it knows if the error is
|
||||
// request-content-caused (e.g. malformed tool call in history) vs
|
||||
// transient. x-should-retry: true is intentionally NOT handled — the
|
||||
// header only suppresses retries; forcing them on non-retryable
|
||||
// statuses could amplify failures.
|
||||
// - Context-window / size overflow — deterministic, re-sending the same
|
||||
// (or larger) payload always fails, whatever status the backend used.
|
||||
//
|
||||
// Checked AFTER image-strip guards: image stripping changes the
|
||||
// request payload, so a server "don't retry" on the original
|
||||
// request doesn't apply to the stripped request.
|
||||
if let Some(false) = err.should_retry_header() {
|
||||
return RetryDecision::Fatal(clone_error(err));
|
||||
}
|
||||
|
||||
// Context-window / size overflow is deterministic — re-sending the same (or
|
||||
// larger) payload always fails — so never retry it, whatever status the backend
|
||||
// used (in-stream `ResponseError`→500, HTTP 400/500, OpenAI/Anthropic variants).
|
||||
if err.is_context_length_error() {
|
||||
if err.is_retry_vetoed() {
|
||||
return RetryDecision::Fatal(clone_error(err));
|
||||
}
|
||||
|
||||
|
|
@ -263,10 +257,10 @@ pub fn format_sampling_error(err: &SamplingError, retry_count: Option<u32>) -> S
|
|||
};
|
||||
|
||||
match err {
|
||||
SamplingError::Auth(msg) => {
|
||||
SamplingError::Auth { message, .. } => {
|
||||
format!(
|
||||
"{}Authentication failed: {}. Please check your API key configuration.",
|
||||
retry_prefix, msg
|
||||
retry_prefix, message
|
||||
)
|
||||
}
|
||||
SamplingError::InvalidConfiguration(msg) => {
|
||||
|
|
@ -389,7 +383,13 @@ pub fn format_sampling_error(err: &SamplingError, retry_count: Option<u32>) -> S
|
|||
/// retry budget re-generating a response that fails the same way.
|
||||
pub(crate) fn clone_error(err: &SamplingError) -> SamplingError {
|
||||
match err {
|
||||
SamplingError::Auth(msg) => SamplingError::Auth(msg.clone()),
|
||||
SamplingError::Auth {
|
||||
message,
|
||||
credential,
|
||||
} => SamplingError::Auth {
|
||||
message: message.clone(),
|
||||
credential: *credential,
|
||||
},
|
||||
SamplingError::InvalidConfiguration(msg) => SamplingError::InvalidConfiguration(msg),
|
||||
SamplingError::Http(e) => {
|
||||
// reqwest::Error is not Clone; preserve the rendered message
|
||||
|
|
@ -520,9 +520,9 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn classify_auth_error_emits_to_session() {
|
||||
let err = SamplingError::Auth("bad token".into());
|
||||
let err = SamplingError::auth_unknown("bad token");
|
||||
match classify_error(&err, 0, 5, RATE_LIMIT_RETRY_THRESHOLD) {
|
||||
RetryDecision::EmitToSession(SamplingError::Auth(_)) => {}
|
||||
RetryDecision::EmitToSession(SamplingError::Auth { .. }) => {}
|
||||
other => panic!("expected EmitToSession(Auth), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
|
@ -763,14 +763,14 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn format_includes_retry_prefix_when_count_present() {
|
||||
let err = SamplingError::Auth("bad".into());
|
||||
let err = SamplingError::auth_unknown("bad");
|
||||
let s = format_sampling_error(&err, Some(3));
|
||||
assert!(s.starts_with("Request failed after 3 retries."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_omits_retry_prefix_when_count_absent() {
|
||||
let err = SamplingError::Auth("bad".into());
|
||||
let err = SamplingError::auth_unknown("bad");
|
||||
let s = format_sampling_error(&err, None);
|
||||
assert!(!s.starts_with("Request failed after"));
|
||||
assert!(s.starts_with("Authentication failed:"));
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ pub async fn collect_response(
|
|||
empty_response_context: None,
|
||||
doom_loop_triggers: None,
|
||||
doom_loop_aborted_at_chunk: None,
|
||||
credential: xai_grok_sampling_types::SentCredential::Unknown,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue