Synced from monorepo
Synced from monorepo Changes: - Workspace server: surface preview-proxy metrics through the hub metric pump - Shell: reclaim a session’s retained state in one entry - Shell: reclaim a session’s resident state in one entry - Pager: withhold key event types from Alacritty builds that double keys - Tools: cancel a session’s subagents when it closes - Pager: keep the whole plan in scrollback and separate reasoning from output in minimal mode - Pager: probe terminal version over DA2 and include it with feedback - SuperGrok Plus: identity, CLI, and analytics tier surfaces - Shell: inherit the session process scope into subagents - Pager: build @-file-search matcher lazily on first use - Tools: fix description and output contradictions in tool definitions - Workspace: degrade @-file-search instead of aborting on thread exhaustion - Tools: reap a session’s LSP servers when it closes - Tools: fix contradictions and defects in tool descriptions, schemas, and harness pools - MCP: reap stdio MCP children on session close - Shell: reuse spawn-time skill discovery for session telemetry - Tools: stop leaking shell-wrapper positional params into sourced scripts (fixes activate_conda under persistent/static shell) - Shell: self-heal corrupt session-search SQLite cache - Workspace: cap workspace-server tokio workers on many-core hosts - Shell: reap a session’s child processes when it closes - Crash handler: capture SIGABRT so panic-aborts leave crash reports - CLI chat proxy: team-scoped Grok Code managed-config admin routes - MCP: add CLI enable/disable for MCP servers - Shell: cap tokio worker threads for startup thread demand - Workspace: harden git_commit and add git_sync_base operation - Circuit breaker: add feature-gated gRPC retry policy Source-Revision: 2a818575225183d8ca915f5632a09b8067b5156a
This commit is contained in:
parent
02d9359435
commit
5da6962e4a
192 changed files with 10337 additions and 3421 deletions
116
crates/common/xai-circuit-breaker/src/grpc.rs
Normal file
116
crates/common/xai-circuit-breaker/src/grpc.rs
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
//! [`GrpcRetryPolicy`] — classifies a `tonic::Code` into a [`Disposition`], the
|
||||
//! gRPC analogue of [`crate::RetryPolicy`]. Behind the `grpc` feature.
|
||||
|
||||
use crate::retry_policy::Disposition;
|
||||
use tonic::Code;
|
||||
|
||||
/// Maps a gRPC [`Code`] to a [`Disposition`].
|
||||
pub struct GrpcRetryPolicy {
|
||||
retryable: &'static [Code],
|
||||
}
|
||||
|
||||
impl GrpcRetryPolicy {
|
||||
/// Retry only transient connection errors (`Unavailable`, `Unknown`);
|
||||
/// excluding `Internal`/`DeadlineExceeded` avoids amplifying a sick peer.
|
||||
pub const DEFAULT: Self = Self::new(&[Code::Unavailable, Code::Unknown]);
|
||||
|
||||
/// Permissive preset: also retry `Internal` and `DeadlineExceeded`.
|
||||
pub const PERMISSIVE: Self = Self::new(&[
|
||||
Code::Unavailable,
|
||||
Code::Unknown,
|
||||
Code::Internal,
|
||||
Code::DeadlineExceeded,
|
||||
]);
|
||||
|
||||
/// Construct from an explicit retryable-code set.
|
||||
pub const fn new(retryable: &'static [Code]) -> Self {
|
||||
Self { retryable }
|
||||
}
|
||||
|
||||
/// Classify `code`. Returns `None` for `Code::Ok` (success, not an error).
|
||||
pub fn classify(&self, code: Code) -> Option<Disposition> {
|
||||
match code {
|
||||
Code::Ok => None,
|
||||
c if self.is_retryable(c) => Some(Disposition::Retryable),
|
||||
_ => Some(Disposition::Terminal),
|
||||
}
|
||||
}
|
||||
|
||||
/// `true` iff `code` is in the retryable set.
|
||||
pub fn is_retryable(&self, code: Code) -> bool {
|
||||
self.retryable.contains(&code)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GrpcRetryPolicy {
|
||||
fn default() -> Self {
|
||||
Self::DEFAULT
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_retries_transient_codes() {
|
||||
for c in [Code::Unavailable, Code::Unknown] {
|
||||
assert!(GrpcRetryPolicy::DEFAULT.is_retryable(c));
|
||||
assert_eq!(
|
||||
GrpcRetryPolicy::DEFAULT.classify(c),
|
||||
Some(Disposition::Retryable)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_excludes_internal_and_deadline_exceeded() {
|
||||
for c in [Code::Internal, Code::DeadlineExceeded] {
|
||||
assert!(!GrpcRetryPolicy::DEFAULT.is_retryable(c));
|
||||
assert_eq!(
|
||||
GrpcRetryPolicy::DEFAULT.classify(c),
|
||||
Some(Disposition::Terminal)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_terminal_for_permanent_codes() {
|
||||
for c in [
|
||||
Code::NotFound,
|
||||
Code::PermissionDenied,
|
||||
Code::InvalidArgument,
|
||||
Code::AlreadyExists,
|
||||
Code::Unauthenticated,
|
||||
] {
|
||||
assert_eq!(
|
||||
GrpcRetryPolicy::DEFAULT.classify(c),
|
||||
Some(Disposition::Terminal)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_classifies_as_none() {
|
||||
assert_eq!(GrpcRetryPolicy::DEFAULT.classify(Code::Ok), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permissive_also_retries_internal_and_deadline() {
|
||||
for c in [
|
||||
Code::Unavailable,
|
||||
Code::Unknown,
|
||||
Code::Internal,
|
||||
Code::DeadlineExceeded,
|
||||
] {
|
||||
assert!(GrpcRetryPolicy::PERMISSIVE.is_retryable(c));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_set_is_respected() {
|
||||
let policy = GrpcRetryPolicy::new(&[Code::ResourceExhausted]);
|
||||
assert!(policy.is_retryable(Code::ResourceExhausted));
|
||||
assert!(!policy.is_retryable(Code::Unavailable));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +1,20 @@
|
|||
//! Shared HTTP circuit breaker.
|
||||
//! Shared circuit breaker.
|
||||
//!
|
||||
//! Sliding-window-with-min-samples algorithm: the breaker trips when
|
||||
//! `sample_count >= min_samples AND error_rate >= error_rate_threshold`
|
||||
//! over the live window. Server- and client-side consumers run the same
|
||||
//! state machine and pick a preset via [`BreakerConfig::server`] or
|
||||
//! [`BreakerConfig::client`].
|
||||
//!
|
||||
//! The breaker is protocol-agnostic (it operates on [`Outcome`]); classification
|
||||
//! helpers exist for HTTP ([`RetryPolicy`]) and gRPC ([`GrpcRetryPolicy`], `grpc`
|
||||
//! feature).
|
||||
|
||||
mod breaker;
|
||||
mod clock;
|
||||
mod config;
|
||||
#[cfg(feature = "grpc")]
|
||||
mod grpc;
|
||||
mod observer;
|
||||
mod registry;
|
||||
mod retry_policy;
|
||||
|
|
@ -20,6 +26,8 @@ pub use breaker::CircuitBreaker;
|
|||
pub use clock::MockClock;
|
||||
pub use clock::{Clock, SystemClock};
|
||||
pub use config::{BreakerConfig, default_failure_codes, parse_failure_codes};
|
||||
#[cfg(feature = "grpc")]
|
||||
pub use grpc::GrpcRetryPolicy;
|
||||
pub use observer::{NoopObserver, Observer};
|
||||
pub use registry::CircuitBreakerRegistry;
|
||||
pub use retry_policy::{Disposition, RetryPolicy};
|
||||
|
|
|
|||
Loading…
Reference in a new issue