Publish harness and TUI open-source

initial sync from the monorepo

crates/common/xai-circuit-breaker/src/breaker.rs

@@ -0,0 +1,365 @@
+//! The [`CircuitBreaker`] state machine: sliding-window-with-min-samples
+//! algorithm with three states (`Closed`, `Open`, `HalfOpen`) and an
+//! atomic-mirror lock-free fast-path for `is_open()`.
+
+use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, OnceLock};
+use std::time::{Duration, Instant};
+
+use crate::clock::{Clock, SystemClock};
+use crate::config::BreakerConfig;
+use crate::observer::{NoopObserver, Observer};
+use crate::state::{BreakerOpen, BreakerState, Outcome};
+use crate::window::SlidingWindow;
+
+static NOOP_OBSERVER: NoopObserver = NoopObserver;
+
+/// Cheaply-clonable handle around a shared [`CircuitBreakerInner`].
+#[derive(Clone)]
+pub struct CircuitBreaker {
+    inner: Arc<CircuitBreakerInner>,
+}
+
+pub(crate) struct CircuitBreakerInner {
+    config: BreakerConfig,
+    state: AtomicU8,
+    /// Monotonic baseline captured at construction; `opened_at_millis`
+    /// stores millisecond offsets from this instant, avoiding NTP
+    /// drift issues and letting `MockClock` drive cool-down windows.
+    baseline: Instant,
+    opened_at_millis: AtomicU64,
+    half_open_probes: AtomicUsize,
+    /// When the most recent half-open probe slot was claimed
+    /// (millisecond offset from `baseline`). A probe whose owner never
+    /// reaches `record()` — e.g. its future is dropped on caller
+    /// cancellation — would otherwise hold its slot forever and strand
+    /// the breaker in `HalfOpen`, shedding all traffic with no path
+    /// back to `Closed`. `try_half_open_probe` treats a claim older
+    /// than `open_duration` as abandoned and lets one caller reclaim
+    /// it, so a lost probe delays recovery by at most one cool-down.
+    probe_claimed_at_millis: AtomicU64,
+    /// Lock-free mirror of `state == Open`. Written after the
+    /// authoritative `state` store with `Release`; read with
+    /// `Relaxed` from the `is_open()` hot path.
+    is_open_fast: AtomicBool,
+    window: Mutex<SlidingWindow>,
+    clock: Arc<dyn Clock>,
+    /// Install-once-on-shared-inner so `with_observer` keeps working
+    /// after a clone (the registry hands out clones).
+    observer: OnceLock<Arc<dyn Observer>>,
+}
+
+impl CircuitBreaker {
+    /// Construct a breaker with the [`SystemClock`] and a no-op observer.
+    pub fn new(config: BreakerConfig) -> Self {
+        Self::with_clock(config, Arc::new(SystemClock))
+    }
+
+    /// Construct a breaker with an injected clock (used by tests to
+    /// drive cool-down windows deterministically).
+    pub fn with_clock(mut config: BreakerConfig, clock: Arc<dyn Clock>) -> Self {
+        config.half_open_max_probes = config.half_open_max_probes.max(1);
+        let baseline = clock.now();
+        Self {
+            inner: Arc::new(CircuitBreakerInner {
+                config,
+                state: AtomicU8::new(BreakerState::Closed as u8),
+                baseline,
+                opened_at_millis: AtomicU64::new(0),
+                half_open_probes: AtomicUsize::new(0),
+                probe_claimed_at_millis: AtomicU64::new(0),
+                is_open_fast: AtomicBool::new(false),
+                window: Mutex::new(SlidingWindow::new()),
+                clock,
+                observer: OnceLock::new(),
+            }),
+        }
+    }
+
+    /// Install an [`Observer`] that receives telemetry callbacks.
+    /// First install wins (`OnceLock`); safe after clone.
+    pub fn with_observer(self, observer: Arc<dyn Observer>) -> Self {
+        let _ = self.inner.observer.set(observer);
+        self
+    }
+
+    fn observer(&self) -> &dyn Observer {
+        self.inner
+            .observer
+            .get()
+            .map(|a| a.as_ref() as &dyn Observer)
+            .unwrap_or(&NOOP_OBSERVER)
+    }
+
+    /// Consult the breaker before issuing a request. Returns `Ok` if
+    /// the request may proceed, `Err(BreakerOpen)` if the breaker is
+    /// currently shedding traffic.
+    pub fn check(&self) -> Result<(), BreakerOpen> {
+        if !self.inner.config.enabled {
+            return Ok(());
+        }
+        match self.state() {
+            BreakerState::Closed => Ok(()),
+            BreakerState::Open => self.check_open(),
+            BreakerState::HalfOpen => self.try_half_open_probe(),
+        }
+    }
+
+    /// Record the outcome of a request.
+    pub fn record(&self, outcome: Outcome) {
+        if !self.inner.config.enabled {
+            return;
+        }
+        let is_failure = matches!(outcome, Outcome::Failure);
+        let now = self.inner.clock.now();
+        let prev_state = self.state();
+
+        match prev_state {
+            BreakerState::Closed => {
+                let should_trip = {
+                    let mut window = self.lock_window();
+                    window.push(is_failure, now);
+                    window.evict(self.inner.config.window_duration, now);
+                    window.sample_count() >= self.inner.config.min_samples
+                        && window.error_rate() >= self.inner.config.error_rate_threshold
+                };
+                if should_trip {
+                    self.trip(prev_state, "trip");
+                }
+            }
+            BreakerState::HalfOpen => {
+                if is_failure {
+                    self.trip(prev_state, "probe_failure");
+                } else {
+                    self.close(prev_state, "probe_success");
+                }
+            }
+            BreakerState::Open => {
+                let mut window = self.lock_window();
+                window.push(is_failure, now);
+                window.evict(self.inner.config.window_duration, now);
+            }
+        }
+
+        let new_state = self.state();
+        self.observer().on_outcome(outcome, new_state);
+    }
+
+    /// Current authoritative [`BreakerState`].
+    pub fn state(&self) -> BreakerState {
+        BreakerState::from_u8(self.inner.state.load(Ordering::Acquire))
+    }
+
+    /// Lock-free "is the breaker currently open?" check (`Relaxed`
+    /// load of the `is_open_fast` mirror).
+    pub fn is_open(&self) -> bool {
+        self.inner.is_open_fast.load(Ordering::Relaxed)
+    }
+
+    /// Failure rate over the live sliding window (`0.0` for an empty
+    /// window). Evicts samples older than `window_duration` against
+    /// the breaker's clock before computing the rate so reads stay
+    /// time-window-accurate even when no `record()` fired recently.
+    pub fn error_rate(&self) -> f64 {
+        let now = self.inner.clock.now();
+        let mut window = self.lock_window();
+        window.evict(self.inner.config.window_duration, now);
+        window.error_rate()
+    }
+
+    /// `true` if `status` is in the configured failure code set.
+    pub fn is_failure_status(&self, status: u16) -> bool {
+        self.inner.config.is_failure_status(status)
+    }
+
+    /// Force-transition to `HalfOpen` for tests (bypasses the
+    /// open-duration timer).
+    #[cfg(any(test, feature = "test-hooks"))]
+    pub fn force_half_open(&self) {
+        let prev = self.state();
+        self.inner
+            .state
+            .store(BreakerState::HalfOpen as u8, Ordering::Release);
+        self.inner.is_open_fast.store(false, Ordering::Release);
+        self.inner.half_open_probes.store(0, Ordering::Release);
+        if prev != BreakerState::HalfOpen {
+            self.observer()
+                .on_state_change(prev, BreakerState::HalfOpen, "force_half_open");
+        }
+    }
+
+    fn lock_window(&self) -> std::sync::MutexGuard<'_, SlidingWindow> {
+        self.inner.window.lock().unwrap_or_else(|e| e.into_inner())
+    }
+
+    fn elapsed_millis(&self) -> u64 {
+        self.inner
+            .clock
+            .now()
+            .saturating_duration_since(self.inner.baseline)
+            .as_millis() as u64
+    }
+
+    fn check_open(&self) -> Result<(), BreakerOpen> {
+        let opened = self.inner.opened_at_millis.load(Ordering::Acquire);
+        let now = self.elapsed_millis();
+        let elapsed = Duration::from_millis(now.saturating_sub(opened));
+
+        if elapsed >= self.inner.config.open_duration {
+            if self.cas_state(BreakerState::Open, BreakerState::HalfOpen) {
+                self.inner.is_open_fast.store(false, Ordering::Release);
+                // Do NOT reset `half_open_probes` here. It is already 0:
+                // `trip()` zeroes it on entry to `Open` and nothing
+                // increments it while `Open`. Resetting after the CAS
+                // publishes `HalfOpen` races a loser thread that observes
+                // `HalfOpen` and claims a probe slot in the gap, which the
+                // reset would then clear — admitting two probes instead of
+                // one.
+                self.observer().on_state_change(
+                    BreakerState::Open,
+                    BreakerState::HalfOpen,
+                    "open_elapsed",
+                );
+                // Route through the shared probe-accounting path so
+                // the loser of the CAS race and the winner agree on
+                // the counter.
+                return self.try_half_open_probe();
+            }
+            // Lost CAS race — re-evaluate.
+            match self.state() {
+                BreakerState::Closed => return Ok(()),
+                BreakerState::HalfOpen => return self.try_half_open_probe(),
+                BreakerState::Open => {
+                    let opened = self.inner.opened_at_millis.load(Ordering::Acquire);
+                    let elapsed =
+                        Duration::from_millis(self.elapsed_millis().saturating_sub(opened));
+                    return Err(BreakerOpen {
+                        retry_after: self.inner.config.open_duration.saturating_sub(elapsed),
+                    });
+                }
+            }
+        }
+
+        Err(BreakerOpen {
+            retry_after: self.inner.config.open_duration.saturating_sub(elapsed),
+        })
+    }
+
+    fn trip(&self, prev: BreakerState, reason: &'static str) {
+        self.inner
+            .state
+            .store(BreakerState::Open as u8, Ordering::Release);
+        self.inner
+            .opened_at_millis
+            .store(self.elapsed_millis(), Ordering::Release);
+        self.inner.half_open_probes.store(0, Ordering::Release);
+        // Mirror after the authoritative state store.
+        self.inner.is_open_fast.store(true, Ordering::Release);
+        if prev != BreakerState::Open {
+            self.observer()
+                .on_state_change(prev, BreakerState::Open, reason);
+        }
+    }
+
+    fn close(&self, prev: BreakerState, reason: &'static str) {
+        self.inner
+            .state
+            .store(BreakerState::Closed as u8, Ordering::Release);
+        self.lock_window().clear();
+        self.inner.half_open_probes.store(0, Ordering::Release);
+        self.inner.is_open_fast.store(false, Ordering::Release);
+        if prev != BreakerState::Closed {
+            self.observer()
+                .on_state_change(prev, BreakerState::Closed, reason);
+        }
+    }
+
+    fn try_half_open_probe(&self) -> Result<(), BreakerOpen> {
+        let now = self.elapsed_millis();
+        let prev = self.inner.half_open_probes.fetch_add(1, Ordering::AcqRel);
+        if prev < self.inner.config.half_open_max_probes {
+            self.inner
+                .probe_claimed_at_millis
+                .store(now, Ordering::Release);
+            self.observer().on_probe_admission(true);
+            return Ok(());
+        }
+        self.inner.half_open_probes.fetch_sub(1, Ordering::AcqRel);
+
+        // All probe slots are claimed. A claim is only released via
+        // `record()`; if a probe's owner was cancelled before recording
+        // (its future dropped mid-flight), the slot would be held forever
+        // and the breaker could never leave `HalfOpen`. Treat a claim
+        // older than `open_duration` as abandoned and let exactly one
+        // caller (the CAS winner) take it over. A slow-but-alive probe
+        // that outlives the lease may briefly coexist with its
+        // replacement; both outcomes are recorded, same as running with
+        // an extra probe slot.
+        let lease_millis = self.inner.config.open_duration.as_millis() as u64;
+        let claimed = self.inner.probe_claimed_at_millis.load(Ordering::Acquire);
+        if now.saturating_sub(claimed) >= lease_millis
+            && self
+                .inner
+                .probe_claimed_at_millis
+                .compare_exchange(claimed, now, Ordering::AcqRel, Ordering::Acquire)
+                .is_ok()
+        {
+            self.observer().on_probe_admission(true);
+            return Ok(());
+        }
+
+        self.observer().on_probe_admission(false);
+        // Slot-exhausted rejection: callers that map this to HTTP
+        // `Retry-After` shouldn't advertise the full open-duration
+        // cool-down; advertise a small fixed backoff (capped to
+        // `open_duration`).
+        const HALF_OPEN_PROBE_BACKOFF: Duration = Duration::from_millis(50);
+        Err(BreakerOpen {
+            retry_after: HALF_OPEN_PROBE_BACKOFF.min(self.inner.config.open_duration),
+        })
+    }
+
+    fn cas_state(&self, from: BreakerState, to: BreakerState) -> bool {
+        self.inner
+            .state
+            .compare_exchange(from as u8, to as u8, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+    }
+}
+
+impl std::fmt::Debug for CircuitBreaker {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CircuitBreaker")
+            .field("state", &self.state())
+            .field("error_rate", &self.error_rate())
+            .finish()
+    }
+}
+
+#[cfg(test)]
+#[path = "breaker_tests/support.rs"]
+mod support;
+
+#[cfg(test)]
+#[path = "breaker_tests/state_machine.rs"]
+mod state_machine;
+
+#[cfg(test)]
+#[path = "breaker_tests/half_open.rs"]
+mod half_open;
+
+#[cfg(test)]
+#[path = "breaker_tests/parity.rs"]
+mod parity;
+
+#[cfg(test)]
+#[path = "breaker_tests/observer.rs"]
+mod observer_tests;
+
+#[cfg(test)]
+#[path = "breaker_tests/breaker_size.rs"]
+mod breaker_size;
+
+#[cfg(test)]
+#[path = "breaker_tests/concurrent.rs"]
+mod concurrent;

crates/common/xai-circuit-breaker/src/breaker_tests/breaker_size.rs

@@ -0,0 +1,45 @@
+//! Crate-contract tests: handle size and `with_observer` after clone.
+
+use std::sync::Arc;
+use std::sync::Mutex as StdMutex;
+
+use super::super::*;
+
+/// Pins the 8-byte handle size that the `clippy::large_enum_variant`
+/// fix depends on.
+#[test]
+fn handle_is_pointer_sized() {
+    assert_eq!(
+        std::mem::size_of::<CircuitBreaker>(),
+        std::mem::size_of::<usize>()
+    );
+}
+
+/// `with_observer` must work even after the handle has already
+/// been cloned — the registry hands out clones, so this is the
+/// realistic call shape.
+#[test]
+fn with_observer_works_after_clone() {
+    #[derive(Default)]
+    struct Counting {
+        transitions: StdMutex<usize>,
+    }
+    impl Observer for Counting {
+        fn on_state_change(&self, _: BreakerState, _: BreakerState, _: &str) {
+            *self.transitions.lock().unwrap_or_else(|e| e.into_inner()) += 1;
+        }
+    }
+
+    let cb = CircuitBreaker::new(BreakerConfig::client());
+    let cloned = cb.clone();
+    let obs = Arc::new(Counting::default());
+    let _ = cloned.with_observer(obs.clone());
+
+    // The observer was installed on the SHARED inner, so both
+    // `cb` and `cloned` see it.
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert_eq!(*obs.transitions.lock().unwrap(), 1);
+}

crates/common/xai-circuit-breaker/src/breaker_tests/concurrent.rs

@@ -0,0 +1,86 @@
+//! Concurrent-access stress tests.
+
+use std::sync::Arc;
+use std::thread;
+
+use super::super::*;
+use super::support::fast_config;
+
+#[test]
+fn concurrent_check_and_record_no_panic() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 5;
+        c.open_duration = std::time::Duration::from_millis(10);
+        c.half_open_max_probes = 2;
+    }));
+
+    let handles: Vec<_> = (0..8)
+        .map(|i| {
+            let cb = cb.clone();
+            thread::spawn(move || {
+                for j in 0..200 {
+                    let _ = cb.check();
+                    let outcome = if (i + j) % 3 == 0 {
+                        Outcome::Failure
+                    } else {
+                        Outcome::Success
+                    };
+                    cb.record(outcome);
+                }
+            })
+        })
+        .collect();
+
+    for h in handles {
+        h.join().unwrap();
+    }
+
+    // State and error_rate must be readable without panic
+    let _state = cb.state();
+    let _rate = cb.error_rate();
+}
+
+/// 100 threads × 100 `record(Failure)` calls. The breaker must remain
+/// readable, its sliding window must stay bounded at
+/// `MAX_WINDOW_ENTRIES` (10k), and `error_rate()` must read as a
+/// finite f64 (no NaN from divide-by-zero or counter corruption).
+#[test]
+fn concurrent_record_does_not_panic_or_corrupt_window() {
+    let cb = Arc::new(CircuitBreaker::new(BreakerConfig {
+        // Keep the breaker closed throughout so every record() goes
+        // through the `Closed` branch that mutates the window.
+        enabled: true,
+        min_samples: usize::MAX,
+        error_rate_threshold: 2.0,
+        ..BreakerConfig::server()
+    }));
+    let handles: Vec<_> = (0..100)
+        .map(|_| {
+            let cb = cb.clone();
+            thread::spawn(move || {
+                for _ in 0..100 {
+                    cb.record(Outcome::Failure);
+                }
+            })
+        })
+        .collect();
+    for h in handles {
+        h.join().unwrap();
+    }
+
+    // Breaker must still report a valid state.
+    assert!(matches!(
+        cb.state(),
+        BreakerState::Closed | BreakerState::Open | BreakerState::HalfOpen
+    ));
+    // min_samples = usize::MAX and threshold = 2.0 keep us Closed.
+    assert_eq!(cb.state(), BreakerState::Closed);
+    // 10,000 failures into an unbounded-rate-threshold breaker must
+    // produce a finite error_rate — no NaN from a corrupted counter.
+    let rate = cb.error_rate();
+    assert!(rate.is_finite(), "error_rate must be finite, got {rate}");
+    assert!(
+        (0.0..=1.0).contains(&rate),
+        "error_rate out of range: {rate}"
+    );
+}

crates/common/xai-circuit-breaker/src/breaker_tests/half_open.rs

@@ -0,0 +1,172 @@
+//! Half-open probe limiting, `half_open_max_probes = 0` clamping,
+//! abandoned-probe lease reclaim, and CAS-loss recovery on the
+//! Open → HalfOpen transition.
+
+use std::sync::Arc;
+use std::thread;
+use std::time::Duration;
+
+use super::super::*;
+use super::support::{breaker_with_mock, fast_config};
+
+#[test]
+fn half_open_limits_concurrent_probes() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+        c.half_open_max_probes = 2;
+    }));
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+
+    assert!(cb.check().is_ok());
+
+    // Third exceeds max_probes
+    assert!(cb.check().is_err());
+}
+
+#[test]
+fn max_probes_clamped_to_at_least_one() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig {
+        half_open_max_probes: 0,
+        min_samples: 1,
+        open_duration: Duration::from_millis(50),
+        ..Default::default()
+    });
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+
+    // Even with max_probes=0 in config, clamped to 1 so one probe gets through
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+    // Second is rejected
+    assert!(cb.check().is_err());
+}
+
+#[test]
+fn breaker_half_open_serialises_concurrent_probes() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig {
+        half_open_max_probes: 1,
+        ..BreakerConfig::client()
+    });
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    clock.advance(Duration::from_secs(61));
+
+    // First check claims the only probe slot.
+    assert!(cb.check().is_ok());
+    // Subsequent checks must short-circuit until the probe
+    // resolves and the breaker transitions.
+    for _ in 0..10 {
+        assert!(cb.check().is_err());
+    }
+}
+
+/// A probe whose owner never records (its future was dropped on caller
+/// cancellation) must not strand the breaker in `HalfOpen` forever:
+/// once the claim is older than `open_duration`, one caller reclaims
+/// the slot and recovery proceeds.
+#[test]
+fn abandoned_probe_slot_reclaimed_after_lease_expiry() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+        c.half_open_max_probes = 1;
+    }));
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+
+    // Claim the only probe slot, then abandon it: no record() ever fires.
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+    // While the lease is live, the slot stays claimed.
+    assert!(cb.check().is_err());
+
+    // Once the lease (open_duration) expires, the claim is treated as
+    // abandoned: exactly one caller takes the slot over.
+    clock.advance(Duration::from_millis(50));
+    assert!(
+        cb.check().is_ok(),
+        "expired probe lease must be reclaimable"
+    );
+    assert!(cb.check().is_err(), "only one takeover per expired lease");
+
+    // The takeover probe's outcome drives the state machine as usual.
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+/// The reclaim path must also handle repeated abandonment: each expired
+/// lease admits exactly one replacement probe.
+#[test]
+fn repeatedly_abandoned_probes_keep_recovery_alive() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+        c.half_open_max_probes = 1;
+    }));
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+
+    for round in 0..3 {
+        assert!(cb.check().is_ok(), "round {round}: probe must be admitted");
+        assert!(cb.check().is_err(), "round {round}: second probe rejected");
+        // Abandon the probe and let its lease expire.
+        clock.advance(Duration::from_millis(50));
+    }
+
+    // A probe that finally records still closes the breaker.
+    assert!(cb.check().is_ok());
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+/// Race two threads attempting the Open → HalfOpen CAS. Only one
+/// should win the CAS; the loser must observe `HalfOpen` and
+/// take the same probe-counting path so the half_open_probes
+/// counter is consistent.
+#[test]
+fn cas_loss_recovery_with_mock_clock() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig {
+        half_open_max_probes: 1,
+        ..fast_config(|c| {
+            c.min_samples = 1;
+            c.open_duration = Duration::from_millis(50);
+        })
+    });
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    clock.advance(Duration::from_millis(70));
+
+    // Spawn many threads simultaneously. Only one probe slot;
+    // exactly one Ok overall.
+    let cb_arc = Arc::new(cb);
+    let barrier = Arc::new(std::sync::Barrier::new(16));
+    let handles: Vec<_> = (0..16)
+        .map(|_| {
+            let cb = cb_arc.clone();
+            let barrier = barrier.clone();
+            thread::spawn(move || {
+                barrier.wait();
+                cb.check().is_ok()
+            })
+        })
+        .collect();
+    let oks: usize = handles
+        .into_iter()
+        .map(|h| h.join().unwrap() as usize)
+        .sum();
+    assert_eq!(oks, 1, "exactly one thread should claim the probe slot");
+    assert_eq!(cb_arc.state(), BreakerState::HalfOpen);
+}

crates/common/xai-circuit-breaker/src/breaker_tests/observer.rs

@@ -0,0 +1,160 @@
+//! Observer-invocation invariants: fires-once-per-transition,
+//! post-transition state visible to the observer, and `is_open()`
+//! Release-ordering after `record()`.
+
+use std::sync::Arc;
+use std::sync::Barrier;
+use std::sync::Mutex as StdMutex;
+use std::thread;
+use std::time::{Duration, Instant};
+
+use super::super::*;
+use crate::clock::MockClock;
+
+/// Uses a `Mutex<Vec<_>>` recording observer to assert exactly one
+/// warn on open and one info on close, rather than an in-breaker
+/// `warn_count` counter.
+#[test]
+fn observer_emits_exactly_one_open_and_one_close_transition() {
+    #[derive(Default)]
+    struct RecordingObserver {
+        transitions: StdMutex<Vec<(BreakerState, BreakerState)>>,
+    }
+    impl Observer for RecordingObserver {
+        fn on_state_change(&self, old: BreakerState, new: BreakerState, _reason: &str) {
+            self.transitions
+                .lock()
+                .unwrap_or_else(|e| e.into_inner())
+                .push((old, new));
+        }
+    }
+
+    let obs = Arc::new(RecordingObserver::default());
+    let clock = Arc::new(MockClock::new());
+    let cb = CircuitBreaker::with_clock(BreakerConfig::client(), clock.clone())
+        .with_observer(obs.clone());
+
+    // Cross the threshold many times in the open state -- no
+    // additional Closed->Open transitions should be reported.
+    for _ in 0..50 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    // Close via probe.
+    clock.advance(Duration::from_secs(61));
+    assert!(cb.check().is_ok());
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+
+    let transitions = obs.transitions.lock().unwrap();
+    let to_open = transitions
+        .iter()
+        .filter(|(_, to)| *to == BreakerState::Open)
+        .count();
+    let to_closed = transitions
+        .iter()
+        .filter(|(from, to)| *from == BreakerState::HalfOpen && *to == BreakerState::Closed)
+        .count();
+    assert_eq!(to_open, 1, "exactly one open transition");
+    assert_eq!(to_closed, 1, "exactly one close-via-probe transition");
+}
+
+/// Observer's `on_state_change` is called AFTER the inner state
+/// has transitioned. We assert this by having the observer call
+/// `breaker.state()` and compare against the `new` argument.
+#[test]
+fn observer_sees_post_transition_state() {
+    struct StateProbingObserver {
+        cb: StdMutex<Option<CircuitBreaker>>,
+        mismatches: StdMutex<Vec<(BreakerState, BreakerState)>>,
+    }
+    impl Observer for StateProbingObserver {
+        fn on_state_change(&self, _old: BreakerState, new: BreakerState, _reason: &str) {
+            let cb_guard = self.cb.lock().unwrap_or_else(|e| e.into_inner());
+            if let Some(cb) = cb_guard.as_ref() {
+                let observed = cb.state();
+                if observed != new {
+                    self.mismatches
+                        .lock()
+                        .unwrap_or_else(|e| e.into_inner())
+                        .push((new, observed));
+                }
+            }
+        }
+    }
+    let obs = Arc::new(StateProbingObserver {
+        cb: StdMutex::new(None),
+        mismatches: StdMutex::new(Vec::new()),
+    });
+    let cb = CircuitBreaker::new(BreakerConfig::client()).with_observer(obs.clone());
+    *obs.cb.lock().unwrap() = Some(cb.clone());
+
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    let mismatches = obs.mismatches.lock().unwrap();
+    assert!(
+        mismatches.is_empty(),
+        "observer observed pre-transition states: {mismatches:?}"
+    );
+}
+
+/// `is_open()` must reflect the post-transition state on a separate
+/// reader thread within a bounded spin. This exercises the
+/// `is_open_fast` `AtomicBool` mirror's cross-thread Release/Acquire
+/// visibility: a `Relaxed` store on the writer side would still allow
+/// this test to pass under x86's TSO, but a regression that drops the
+/// `state` Release store before the mirror would let the reader
+/// observe `is_open() == true` *before* `state() == Open` is visible,
+/// which the post-spin invariants assert against.
+#[test]
+fn is_open_visible_to_reader_thread_after_trip() {
+    let cb = Arc::new(CircuitBreaker::new(BreakerConfig::client()));
+    let barrier = Arc::new(Barrier::new(2));
+
+    let writer = {
+        let cb = cb.clone();
+        let barrier = barrier.clone();
+        thread::spawn(move || {
+            barrier.wait();
+            for _ in 0..5 {
+                cb.record(Outcome::Failure);
+            }
+        })
+    };
+
+    let reader = {
+        let cb = cb.clone();
+        let barrier = barrier.clone();
+        thread::spawn(move || {
+            barrier.wait();
+            let deadline = Instant::now() + Duration::from_secs(5);
+            while !cb.is_open() {
+                assert!(
+                    Instant::now() < deadline,
+                    "reader thread never observed is_open() == true \
+                     within the timeout — possible Release-mirror regression"
+                );
+                std::hint::spin_loop();
+            }
+            // Mirror saw the trip; the authoritative `state` Acquire
+            // load must also reflect Open (or HalfOpen on a racing
+            // open-elapsed CAS, which can't happen here — no clock
+            // advance).
+            cb.state()
+        })
+    };
+
+    writer.join().unwrap();
+    let observed_state = reader.join().unwrap();
+    assert_eq!(
+        observed_state,
+        BreakerState::Open,
+        "reader's state() must agree with the is_open() mirror"
+    );
+    assert!(cb.is_open());
+    assert_eq!(cb.state(), BreakerState::Open);
+}

crates/common/xai-circuit-breaker/src/breaker_tests/parity.rs

@@ -0,0 +1,131 @@
+//! Parity tests for the `server` and `client` presets, including a
+//! sustained high-401-rate failure pattern.
+
+use std::time::Duration;
+
+use super::super::*;
+use super::support::breaker_with_mock;
+
+#[test]
+fn client_preset_trips_on_5x_401s() {
+    // With the sliding-window algorithm, 5 × 401 against `client()`
+    // gives sample_count=5 >= min_samples=5 and rate=1.0 >= 0.5.
+    let cb = CircuitBreaker::new(BreakerConfig::client());
+    for _ in 0..4 {
+        cb.record(Outcome::Failure);
+        assert_eq!(cb.state(), BreakerState::Closed);
+    }
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!(cb.is_open());
+}
+
+#[test]
+fn breaker_half_open_after_cool_down_success() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig::client());
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    clock.advance(Duration::from_secs(61));
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+    assert!(!cb.is_open());
+}
+
+#[test]
+fn breaker_half_open_after_cool_down_failure_reopens() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig::client());
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    clock.advance(Duration::from_secs(61));
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!(cb.is_open());
+}
+
+#[test]
+fn parity_server_trips_on_sustained_500s() {
+    let cb = CircuitBreaker::new(BreakerConfig::server());
+    for _ in 0..10 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+}
+
+#[test]
+fn parity_server_does_not_trip_below_min_samples() {
+    let cb = CircuitBreaker::new(BreakerConfig::server());
+    for _ in 0..9 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+#[test]
+fn parity_server_does_not_trip_below_threshold() {
+    let cb = CircuitBreaker::new(BreakerConfig::server());
+    // 5 failures and 6 successes interleaved (lead with the
+    // successes so the partial rate never crosses 0.5 once
+    // min_samples is reached): SSSSSS FFFFF → 11 samples,
+    // rate = 5/11 ≈ 0.4545.
+    for _ in 0..6 {
+        cb.record(Outcome::Success);
+    }
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Closed);
+    assert!(cb.error_rate() < 0.5);
+}
+
+#[test]
+fn parity_server_half_open_probe_then_close() {
+    let (cb, clock) = breaker_with_mock(BreakerConfig::server());
+    for _ in 0..10 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    clock.advance(BreakerConfig::server().open_duration + Duration::from_millis(1));
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+#[test]
+fn parity_client_trips_on_fresh_session_5x_401() {
+    let cb = CircuitBreaker::new(BreakerConfig::client());
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!((cb.error_rate() - 1.0).abs() < f64::EPSILON);
+}
+
+#[test]
+fn client_preset_trips_on_interleaved_success_pattern() {
+    // [401×4, 200, 401×5] = 10 samples, 9 failures, rate = 0.9.
+    let cb = CircuitBreaker::new(BreakerConfig::client());
+    for _ in 0..4 {
+        cb.record(Outcome::Failure);
+    }
+    cb.record(Outcome::Success);
+    for _ in 0..5 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!((cb.error_rate() - 0.9).abs() < 1e-9);
+}

crates/common/xai-circuit-breaker/src/breaker_tests/state_machine.rs

@@ -0,0 +1,359 @@
+//! Closed → Open → HalfOpen → Closed transitions plus threshold,
+//! min-samples, window-eviction, and disabled-breaker behaviour.
+
+use std::time::Duration;
+
+use super::super::*;
+use super::support::{breaker_with_mock, fast_config};
+
+// -- State transitions ----------------------------------------------------
+
+#[test]
+fn closed_to_open_on_high_error_rate() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 2;
+        c.error_rate_threshold = 0.5;
+    }));
+
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Closed);
+
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+}
+
+#[test]
+fn trips_at_exact_threshold() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 2;
+        c.error_rate_threshold = 0.5;
+    }));
+
+    cb.record(Outcome::Success);
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+}
+
+#[test]
+fn does_not_trip_below_threshold() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 3;
+        c.error_rate_threshold = 0.5;
+    }));
+
+    cb.record(Outcome::Success);
+    cb.record(Outcome::Failure);
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+#[test]
+fn does_not_trip_below_min_samples() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 5;
+        c.error_rate_threshold = 0.5;
+    }));
+
+    for _ in 0..4 {
+        cb.record(Outcome::Failure);
+    }
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+#[test]
+fn open_to_half_open_after_duration() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+    }));
+
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!(cb.check().is_err());
+
+    clock.advance(Duration::from_millis(70));
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+}
+
+#[test]
+fn half_open_to_closed_on_probe_success() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+    }));
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+    cb.check().unwrap();
+
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+#[test]
+fn half_open_to_open_on_probe_failure() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(50);
+    }));
+
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(70));
+    cb.check().unwrap();
+
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+}
+
+// -- Window eviction ------------------------------------------------------
+
+#[test]
+fn old_samples_evicted_from_window() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 2;
+        c.window_duration = Duration::from_millis(100);
+        c.error_rate_threshold = 0.5;
+        c.open_duration = Duration::from_millis(50);
+    }));
+
+    cb.record(Outcome::Failure);
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+
+    // Recover: Open -> HalfOpen -> Closed
+    clock.advance(Duration::from_millis(70));
+    cb.check().unwrap();
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+
+    // Record one failure, then wait for it to fall outside the window
+    cb.record(Outcome::Failure);
+    clock.advance(Duration::from_millis(120));
+
+    // New success triggers eviction of the old failure
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+    assert!(cb.error_rate() < 0.01);
+}
+
+// -- Disabled breaker -----------------------------------------------------
+
+#[test]
+fn disabled_breaker_always_allows() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 1;
+        c.enabled = false;
+    }));
+
+    cb.record(Outcome::Failure);
+    cb.record(Outcome::Failure);
+    assert!(cb.check().is_ok());
+}
+
+#[test]
+fn disabled_breaker_does_not_accumulate() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 1;
+        c.enabled = false;
+    }));
+
+    for _ in 0..100 {
+        cb.record(Outcome::Failure);
+    }
+    // Window should be empty since record() is a no-op when disabled
+    assert!(cb.error_rate().abs() < f64::EPSILON);
+    assert_eq!(cb.state(), BreakerState::Closed);
+}
+
+// -- Failure code matching ------------------------------------------------
+
+#[test]
+fn is_failure_status_matches_configured_codes() {
+    let cb = CircuitBreaker::new(BreakerConfig::default());
+    for code in [429, 500, 502, 503, 504] {
+        assert!(cb.is_failure_status(code));
+    }
+    for code in [200, 201, 301, 400, 404, 501] {
+        assert!(!cb.is_failure_status(code));
+    }
+}
+
+#[test]
+fn is_failure_status_with_custom_codes() {
+    let cb = CircuitBreaker::new(BreakerConfig {
+        failure_codes: [500, 503].into_iter().collect(),
+        ..Default::default()
+    });
+    assert!(cb.is_failure_status(500));
+    assert!(cb.is_failure_status(503));
+    assert!(!cb.is_failure_status(429));
+    assert!(!cb.is_failure_status(502));
+}
+
+// The four `parse_failure_codes_*` and four `from_lookup_*` tests live
+// alongside `BreakerConfig` in `config.rs`. We add stub aliases here so
+// the named-test set is complete in this file too.
+
+#[test]
+fn parse_failure_codes_basic() {
+    use crate::config::parse_failure_codes;
+    use std::collections::HashSet;
+    assert_eq!(
+        parse_failure_codes("429,500,502,503,504"),
+        [429, 500, 502, 503, 504]
+            .into_iter()
+            .collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_with_whitespace() {
+    use crate::config::parse_failure_codes;
+    use std::collections::HashSet;
+    assert_eq!(
+        parse_failure_codes(" 429 , 500 , 502 "),
+        [429, 500, 502].into_iter().collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_ignores_invalid() {
+    use crate::config::parse_failure_codes;
+    use std::collections::HashSet;
+    assert_eq!(
+        parse_failure_codes("429,abc,500,,999999"),
+        [429, 500].into_iter().collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_empty_returns_empty_set() {
+    use crate::config::parse_failure_codes;
+    assert!(parse_failure_codes("").is_empty());
+}
+
+#[test]
+fn from_lookup_returns_defaults_when_no_vars_set() {
+    let config = BreakerConfig::from_lookup_with_prefix("CB_", |_| None);
+    assert_eq!(config.window_duration, Duration::from_secs(60));
+    assert_eq!(config.min_samples, 10);
+    assert!((config.error_rate_threshold - 0.5).abs() < f64::EPSILON);
+    assert_eq!(config.open_duration, Duration::from_secs(10));
+    assert_eq!(config.half_open_max_probes, 1);
+    assert!(config.enabled);
+}
+
+#[test]
+fn from_lookup_applies_overrides() {
+    let config = BreakerConfig::from_lookup_with_prefix("CB_", |key| match key {
+        "CB_WINDOW_SECS" => Some("120".into()),
+        "CB_MIN_SAMPLES" => Some("20".into()),
+        "CB_ERROR_RATE_THRESHOLD" => Some("0.8".into()),
+        "CB_OPEN_DURATION_SECS" => Some("30".into()),
+        "CB_HALF_OPEN_MAX_PROBES" => Some("3".into()),
+        "CB_FAILURE_CODES" => Some("500,503".into()),
+        "CB_ENABLED" => Some("false".into()),
+        _ => None,
+    });
+    assert_eq!(config.window_duration, Duration::from_secs(120));
+    assert_eq!(config.min_samples, 20);
+    assert!((config.error_rate_threshold - 0.8).abs() < f64::EPSILON);
+    assert_eq!(config.open_duration, Duration::from_secs(30));
+    assert_eq!(config.half_open_max_probes, 3);
+    assert!(!config.enabled);
+}
+
+#[test]
+fn from_lookup_uses_defaults_for_unparseable_values() {
+    let config = BreakerConfig::from_lookup_with_prefix("CB_", |key| match key {
+        "CB_MIN_SAMPLES" => Some("not_a_number".into()),
+        "CB_ERROR_RATE_THRESHOLD" => Some("abc".into()),
+        "CB_HALF_OPEN_MAX_PROBES" => Some("".into()),
+        _ => None,
+    });
+    assert_eq!(config.min_samples, 10);
+    assert!((config.error_rate_threshold - 0.5).abs() < f64::EPSILON);
+    assert_eq!(config.half_open_max_probes, 1);
+}
+
+#[test]
+fn from_lookup_empty_failure_codes_uses_defaults() {
+    let config = BreakerConfig::from_lookup_with_prefix("CB_", |key| match key {
+        "CB_FAILURE_CODES" => Some("".into()),
+        _ => None,
+    });
+    assert_eq!(config.failure_codes, crate::config::default_failure_codes());
+}
+
+// -- Error rate -----------------------------------------------------------
+
+#[test]
+fn error_rate_reflects_window_contents() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 100;
+    }));
+
+    cb.record(Outcome::Success);
+    cb.record(Outcome::Success);
+    cb.record(Outcome::Failure);
+    assert!((cb.error_rate() - 1.0 / 3.0).abs() < 0.01);
+}
+
+#[test]
+fn error_rate_zero_on_empty_window() {
+    let cb = CircuitBreaker::new(fast_config(|_| {}));
+    assert!(cb.error_rate().abs() < f64::EPSILON);
+}
+
+// -- BreakerOpen ----------------------------------------------------------
+
+#[test]
+fn breaker_open_reports_retry_after() {
+    let cb = CircuitBreaker::new(fast_config(|c| {
+        c.min_samples = 1;
+        c.open_duration = Duration::from_millis(200);
+    }));
+
+    cb.record(Outcome::Failure);
+    let err = cb.check().unwrap_err();
+    assert!(err.retry_after <= Duration::from_millis(200));
+    assert!(err.retry_after > Duration::from_millis(100));
+}
+
+#[test]
+fn breaker_open_display() {
+    let err = BreakerOpen {
+        retry_after: Duration::from_millis(5300),
+    };
+    assert_eq!(err.to_string(), "circuit breaker open; retry after 5.3s");
+}
+
+// -- Full cycle -----------------------------------------------------------
+
+#[test]
+fn full_cycle_closed_open_half_open_closed() {
+    let (cb, clock) = breaker_with_mock(fast_config(|c| {
+        c.min_samples = 2;
+        c.error_rate_threshold = 0.5;
+        c.open_duration = Duration::from_millis(50);
+    }));
+
+    cb.record(Outcome::Failure);
+    cb.record(Outcome::Failure);
+    assert_eq!(cb.state(), BreakerState::Open);
+    assert!(cb.check().is_err());
+
+    clock.advance(Duration::from_millis(70));
+    assert!(cb.check().is_ok());
+    assert_eq!(cb.state(), BreakerState::HalfOpen);
+
+    cb.record(Outcome::Success);
+    assert_eq!(cb.state(), BreakerState::Closed);
+    assert!(cb.check().is_ok());
+
+    for _ in 0..5 {
+        cb.record(Outcome::Success);
+    }
+    assert_eq!(cb.state(), BreakerState::Closed);
+}

crates/common/xai-circuit-breaker/src/breaker_tests/support.rs

@@ -0,0 +1,33 @@
+//! Shared test helpers for the `breaker_tests` sub-modules.
+//!
+//! Items are `pub(super)` so sibling test modules
+//! (`breaker_tests::state_machine`, `breaker_tests::half_open`, …) can
+//! reach them. The `#[cfg(test)] #[path = "breaker_tests/support.rs"]
+//! mod support;` declaration in `breaker.rs` makes `super` resolve to
+//! the parent `breaker` module — sibling sub-modules then import via
+//! `use super::support::*`. A future flatten-the-`#[path]` refactor
+//! would silently break those imports, hence this note.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use super::super::CircuitBreaker;
+use crate::clock::MockClock;
+use crate::config::BreakerConfig;
+
+pub(super) fn fast_config(f: impl FnOnce(&mut BreakerConfig)) -> BreakerConfig {
+    let mut c = BreakerConfig {
+        min_samples: 2,
+        open_duration: Duration::from_millis(50),
+        window_duration: Duration::from_millis(200),
+        ..Default::default()
+    };
+    f(&mut c);
+    c
+}
+
+pub(super) fn breaker_with_mock(config: BreakerConfig) -> (CircuitBreaker, Arc<MockClock>) {
+    let clock = Arc::new(MockClock::new());
+    let cb = CircuitBreaker::with_clock(config, clock.clone());
+    (cb, clock)
+}

crates/common/xai-circuit-breaker/src/clock.rs

@@ -0,0 +1,63 @@
+//! Time source abstraction used by [`crate::CircuitBreaker`].
+//!
+//! Production uses [`SystemClock`]. Tests construct a [`MockClock`]
+//! (gated on `cfg(test)` and the `test-hooks` feature) to drive
+//! open-duration windows deterministically.
+
+#[cfg(any(test, feature = "test-hooks"))]
+use std::sync::Mutex;
+#[cfg(any(test, feature = "test-hooks"))]
+use std::time::Duration;
+use std::time::Instant;
+
+/// Monotonic time source.
+pub trait Clock: Send + Sync + 'static {
+    fn now(&self) -> Instant;
+}
+
+/// `Instant::now()`-backed clock.
+#[derive(Debug, Default)]
+pub struct SystemClock;
+
+impl Clock for SystemClock {
+    fn now(&self) -> Instant {
+        Instant::now()
+    }
+}
+
+/// Controllable clock: starts at construction time and only advances
+/// via [`Self::advance`].
+#[cfg(any(test, feature = "test-hooks"))]
+#[derive(Debug)]
+pub struct MockClock {
+    now: Mutex<Instant>,
+}
+
+#[cfg(any(test, feature = "test-hooks"))]
+impl MockClock {
+    pub fn new() -> Self {
+        Self {
+            now: Mutex::new(Instant::now()),
+        }
+    }
+
+    /// Advance the mock clock by `d`. Panics on overflow.
+    pub fn advance(&self, d: Duration) {
+        let mut g = self.now.lock().unwrap_or_else(|e| e.into_inner());
+        *g = g.checked_add(d).expect("MockClock overflow");
+    }
+}
+
+#[cfg(any(test, feature = "test-hooks"))]
+impl Default for MockClock {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(any(test, feature = "test-hooks"))]
+impl Clock for MockClock {
+    fn now(&self) -> Instant {
+        *self.now.lock().unwrap_or_else(|e| e.into_inner())
+    }
+}

crates/common/xai-circuit-breaker/src/config.rs

@@ -0,0 +1,140 @@
+//! [`BreakerConfig`] — tuning knobs for [`crate::CircuitBreaker`].
+//!
+//! Two named presets:
+//! - [`BreakerConfig::server`] — defaults suited to a shared server-side
+//!   breaker (stricter trip threshold, short cool-down).
+//! - [`BreakerConfig::client`] — defaults suited to client-side breakers
+//!   keyed per endpoint or tenant (fewer samples, longer cool-down).
+//!
+//! [`BreakerConfig::from_env`] reads `CB_*` env vars.
+
+use std::collections::HashSet;
+use std::time::Duration;
+
+const DEFAULT_FAILURE_CODES: &[u16] = &[429, 500, 502, 503, 504];
+
+#[derive(Debug, Clone)]
+pub struct BreakerConfig {
+    pub window_duration: Duration,
+    pub min_samples: usize,
+    pub error_rate_threshold: f64,
+    pub open_duration: Duration,
+    pub half_open_max_probes: usize,
+    pub failure_codes: HashSet<u16>,
+    pub enabled: bool,
+}
+
+impl Default for BreakerConfig {
+    fn default() -> Self {
+        Self::server()
+    }
+}
+
+impl BreakerConfig {
+    /// Server preset (`min_samples=10`, `error_rate=0.5`, 60s window,
+    /// 10s open duration, failure codes `[429,500,502,503,504]`).
+    pub fn server() -> Self {
+        Self {
+            window_duration: Duration::from_secs(60),
+            min_samples: 10,
+            error_rate_threshold: 0.5,
+            open_duration: Duration::from_secs(10),
+            half_open_max_probes: 1,
+            failure_codes: default_failure_codes(),
+            enabled: true,
+        }
+    }
+
+    /// Client preset (`min_samples=5`, `error_rate=0.5`, 60s window,
+    /// 60s open duration, failure codes `[401]`).
+    pub fn client() -> Self {
+        Self {
+            window_duration: Duration::from_secs(60),
+            min_samples: 5,
+            error_rate_threshold: 0.5,
+            open_duration: Duration::from_secs(60),
+            half_open_max_probes: 1,
+            failure_codes: [401].into_iter().collect(),
+            enabled: true,
+        }
+    }
+
+    /// Load knobs from `CB_*` environment variables.
+    pub fn from_env() -> Self {
+        Self::from_env_with_prefix("CB_")
+    }
+
+    /// Load knobs from `<prefix>...` environment variables.
+    pub fn from_env_with_prefix(prefix: &str) -> Self {
+        Self::from_lookup_with_prefix(prefix, |key| std::env::var(key).ok())
+    }
+
+    pub(crate) fn from_lookup_with_prefix(
+        prefix: &str,
+        get: impl Fn(&str) -> Option<String>,
+    ) -> Self {
+        let key = |k: &str| format!("{prefix}{k}");
+        let failure_codes = match get(&key("FAILURE_CODES")) {
+            Some(raw) => {
+                let codes = parse_failure_codes(&raw);
+                if codes.is_empty() {
+                    log::warn!(
+                        "{}FAILURE_CODES={raw:?} produced no valid codes, using defaults",
+                        prefix
+                    );
+                    default_failure_codes()
+                } else {
+                    codes
+                }
+            }
+            None => default_failure_codes(),
+        };
+
+        Self {
+            window_duration: Duration::from_secs(lookup_or(&get, &key("WINDOW_SECS"), 60)),
+            min_samples: lookup_or(&get, &key("MIN_SAMPLES"), 10),
+            error_rate_threshold: lookup_or(&get, &key("ERROR_RATE_THRESHOLD"), 0.5),
+            open_duration: Duration::from_secs(lookup_or(&get, &key("OPEN_DURATION_SECS"), 10)),
+            half_open_max_probes: lookup_or(&get, &key("HALF_OPEN_MAX_PROBES"), 1usize).max(1),
+            failure_codes,
+            enabled: lookup_or(&get, &key("ENABLED"), true),
+        }
+    }
+
+    /// `true` if `status` is in the configured failure code set.
+    pub fn is_failure_status(&self, status: u16) -> bool {
+        self.failure_codes.contains(&status)
+    }
+}
+
+fn lookup_or<T: std::str::FromStr>(
+    get: &impl Fn(&str) -> Option<String>,
+    key: &str,
+    default: T,
+) -> T {
+    match get(key) {
+        Some(v) => match v.parse() {
+            Ok(parsed) => parsed,
+            Err(_) => {
+                log::warn!("env {key}={v:?} failed to parse, using default");
+                default
+            }
+        },
+        None => default,
+    }
+}
+
+/// Default set of HTTP failure codes (`429`, `500`, `502`, `503`, `504`).
+pub fn default_failure_codes() -> HashSet<u16> {
+    DEFAULT_FAILURE_CODES.iter().copied().collect()
+}
+
+/// Parse a comma-separated list of status codes; invalid entries are
+/// silently dropped.
+pub fn parse_failure_codes(s: &str) -> HashSet<u16> {
+    s.split(',').filter_map(|c| c.trim().parse().ok()).collect()
+}
+
+#[cfg(test)]
+#[path = "config_tests.rs"]
+mod tests;

crates/common/xai-circuit-breaker/src/config_tests.rs

@@ -0,0 +1,135 @@
+//! Tests for [`crate::config`].
+
+use super::*;
+use std::collections::HashSet;
+
+fn from_lookup(get: impl Fn(&str) -> Option<String>) -> BreakerConfig {
+    BreakerConfig::from_lookup_with_prefix("CB_", get)
+}
+
+// -- Failure code matching ------------------------------------------------
+
+#[test]
+fn is_failure_status_matches_configured_codes() {
+    let config = BreakerConfig::default();
+
+    for code in [429, 500, 502, 503, 504] {
+        assert!(
+            config.is_failure_status(code),
+            "expected {code} to be failure"
+        );
+    }
+    for code in [200, 201, 301, 400, 404, 501] {
+        assert!(
+            !config.is_failure_status(code),
+            "expected {code} to NOT be failure"
+        );
+    }
+}
+
+#[test]
+fn is_failure_status_with_custom_codes() {
+    let config = BreakerConfig {
+        failure_codes: [500, 503].into_iter().collect(),
+        ..Default::default()
+    };
+    assert!(config.is_failure_status(500));
+    assert!(config.is_failure_status(503));
+    assert!(!config.is_failure_status(429));
+    assert!(!config.is_failure_status(502));
+}
+
+// -- parse_failure_codes --------------------------------------------------
+
+#[test]
+fn parse_failure_codes_basic() {
+    assert_eq!(
+        parse_failure_codes("429,500,502,503,504"),
+        [429, 500, 502, 503, 504]
+            .into_iter()
+            .collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_with_whitespace() {
+    assert_eq!(
+        parse_failure_codes(" 429 , 500 , 502 "),
+        [429, 500, 502].into_iter().collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_ignores_invalid() {
+    assert_eq!(
+        parse_failure_codes("429,abc,500,,999999"),
+        [429, 500].into_iter().collect::<HashSet<_>>()
+    );
+}
+
+#[test]
+fn parse_failure_codes_empty_returns_empty_set() {
+    assert!(parse_failure_codes("").is_empty());
+}
+
+// -- from_lookup ----------------------------------------------------------
+
+#[test]
+fn from_lookup_returns_defaults_when_no_vars_set() {
+    let config = from_lookup(|_| None);
+    assert_eq!(config.window_duration, Duration::from_secs(60));
+    assert_eq!(config.min_samples, 10);
+    assert!((config.error_rate_threshold - 0.5).abs() < f64::EPSILON);
+    assert_eq!(config.open_duration, Duration::from_secs(10));
+    assert_eq!(config.half_open_max_probes, 1);
+    assert_eq!(config.failure_codes, default_failure_codes());
+    assert!(config.enabled);
+}
+
+#[test]
+fn from_lookup_applies_overrides() {
+    let config = from_lookup(|key| match key {
+        "CB_WINDOW_SECS" => Some("120".into()),
+        "CB_MIN_SAMPLES" => Some("20".into()),
+        "CB_ERROR_RATE_THRESHOLD" => Some("0.8".into()),
+        "CB_OPEN_DURATION_SECS" => Some("30".into()),
+        "CB_HALF_OPEN_MAX_PROBES" => Some("3".into()),
+        "CB_FAILURE_CODES" => Some("500,503".into()),
+        "CB_ENABLED" => Some("false".into()),
+        _ => None,
+    });
+
+    assert_eq!(config.window_duration, Duration::from_secs(120));
+    assert_eq!(config.min_samples, 20);
+    assert!((config.error_rate_threshold - 0.8).abs() < f64::EPSILON);
+    assert_eq!(config.open_duration, Duration::from_secs(30));
+    assert_eq!(config.half_open_max_probes, 3);
+    assert_eq!(
+        config.failure_codes,
+        [500, 503].into_iter().collect::<HashSet<_>>()
+    );
+    assert!(!config.enabled);
+}
+
+#[test]
+fn from_lookup_uses_defaults_for_unparseable_values() {
+    let config = from_lookup(|key| match key {
+        "CB_MIN_SAMPLES" => Some("not_a_number".into()),
+        "CB_ERROR_RATE_THRESHOLD" => Some("abc".into()),
+        "CB_HALF_OPEN_MAX_PROBES" => Some("".into()),
+        _ => None,
+    });
+
+    assert_eq!(config.min_samples, 10);
+    assert!((config.error_rate_threshold - 0.5).abs() < f64::EPSILON);
+    assert_eq!(config.half_open_max_probes, 1);
+}
+
+#[test]
+fn from_lookup_empty_failure_codes_uses_defaults() {
+    let config = from_lookup(|key| match key {
+        "CB_FAILURE_CODES" => Some("".into()),
+        _ => None,
+    });
+    assert_eq!(config.failure_codes, default_failure_codes());
+}

crates/common/xai-circuit-breaker/src/lib.rs

@@ -0,0 +1,26 @@
+//! Shared HTTP circuit breaker.
+//!
+//! Sliding-window-with-min-samples algorithm: the breaker trips when
+//! `sample_count >= min_samples AND error_rate >= error_rate_threshold`
+//! over the live window. Server- and client-side consumers run the same
+//! state machine and pick a preset via [`BreakerConfig::server`] or
+//! [`BreakerConfig::client`].
+
+mod breaker;
+mod clock;
+mod config;
+mod observer;
+mod registry;
+mod retry_policy;
+mod state;
+mod window;
+
+pub use breaker::CircuitBreaker;
+#[cfg(any(test, feature = "test-hooks"))]
+pub use clock::MockClock;
+pub use clock::{Clock, SystemClock};
+pub use config::{BreakerConfig, default_failure_codes, parse_failure_codes};
+pub use observer::{NoopObserver, Observer};
+pub use registry::CircuitBreakerRegistry;
+pub use retry_policy::{Disposition, RetryPolicy};
+pub use state::{BreakerOpen, BreakerState, Outcome};

crates/common/xai-circuit-breaker/src/observer.rs

@@ -0,0 +1,35 @@
+//! Telemetry hook trait for [`crate::CircuitBreaker`].
+//!
+//! Observer methods are invoked **outside** the breaker's internal
+//! locks and **after** any state transition is visible via
+//! [`crate::CircuitBreaker::state`]. Observer impls must not block or
+//! perform unbounded I/O — they sit on every `record()` / `check()`
+//! hot path. Short non-contended locks (e.g. a Prometheus per-label
+//! mutex) are fine.
+
+use crate::state::{BreakerState, Outcome};
+
+/// Telemetry hooks. Default implementations are no-ops; consumers
+/// implement only the methods they care about.
+pub trait Observer: Send + Sync {
+    /// Called when the breaker transitions between states. `reason`
+    /// is a short stable string (e.g. `"trip"`, `"probe_success"`,
+    /// `"probe_failure"`, `"open_elapsed"`).
+    fn on_state_change(&self, _old: BreakerState, _new: BreakerState, _reason: &str) {}
+
+    /// Called from `check()` when the breaker is `HalfOpen` and a
+    /// caller attempts to claim a probe slot. `allowed = false` means
+    /// `half_open_max_probes` was already in flight.
+    fn on_probe_admission(&self, _allowed: bool) {}
+
+    /// Called from `record()` after the sample is added to the window
+    /// and any resulting state transition has landed. `status` is the
+    /// post-transition state.
+    fn on_outcome(&self, _outcome: Outcome, _status: BreakerState) {}
+}
+
+/// No-op observer used by [`crate::CircuitBreaker::new`].
+#[derive(Debug, Default)]
+pub struct NoopObserver;
+
+impl Observer for NoopObserver {}

crates/common/xai-circuit-breaker/src/registry.rs

@@ -0,0 +1,69 @@
+//! Per-key registry of [`CircuitBreaker`] instances (one per upstream
+//! endpoint, one per tenant, etc.).
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use crate::breaker::CircuitBreaker;
+use crate::config::BreakerConfig;
+
+pub struct CircuitBreakerRegistry {
+    config: BreakerConfig,
+    breakers: Mutex<HashMap<String, Arc<CircuitBreaker>>>,
+}
+
+impl CircuitBreakerRegistry {
+    pub fn new(config: BreakerConfig) -> Self {
+        Self {
+            config,
+            breakers: Mutex::new(HashMap::new()),
+        }
+    }
+
+    /// Returns `None` if the registry's config has `enabled = false`;
+    /// otherwise returns (and lazily creates) the breaker for `key`.
+    pub fn get(&self, key: &str) -> Option<Arc<CircuitBreaker>> {
+        if !self.config.enabled {
+            return None;
+        }
+        let mut breakers = self.breakers.lock().unwrap_or_else(|e| e.into_inner());
+        if let Some(cb) = breakers.get(key) {
+            return Some(Arc::clone(cb));
+        }
+        let cb = Arc::new(CircuitBreaker::new(self.config.clone()));
+        let ret = Arc::clone(&cb);
+        breakers.insert(key.to_owned(), cb);
+        Some(ret)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn registry_returns_none_when_disabled() {
+        let cfg = BreakerConfig {
+            enabled: false,
+            ..Default::default()
+        };
+        let reg = CircuitBreakerRegistry::new(cfg);
+        assert!(reg.get("endpoint-a").is_none());
+    }
+
+    #[test]
+    fn registry_returns_same_breaker_for_same_key() {
+        let reg = CircuitBreakerRegistry::new(BreakerConfig::default());
+        let a = reg.get("endpoint-a").unwrap();
+        let b = reg.get("endpoint-a").unwrap();
+        assert!(Arc::ptr_eq(&a, &b));
+    }
+
+    #[test]
+    fn registry_returns_distinct_breakers_for_distinct_keys() {
+        let reg = CircuitBreakerRegistry::new(BreakerConfig::default());
+        let a = reg.get("endpoint-a").unwrap();
+        let b = reg.get("endpoint-b").unwrap();
+        assert!(!Arc::ptr_eq(&a, &b));
+    }
+}

crates/common/xai-circuit-breaker/src/retry_policy.rs

@@ -0,0 +1,104 @@
+//! [`RetryPolicy`] — maps a non-2xx HTTP status code to a [`Disposition`],
+//! consolidating the scattered "what should I do with this response" logic.
+//!
+//! Two named presets:
+//! - [`RetryPolicy::server`] — server-side preset: retry on 429 or any 5xx;
+//!   all other non-2xx are terminal.
+//! - [`RetryPolicy::client_storage`] — client upload/storage preset:
+//!   400/403/404 terminal-drop, 401 auth-refresh-once, everything else retried.
+
+/// What a caller should do with a non-2xx HTTP response, by status code.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Disposition {
+    /// transient: retry with backoff (5xx, 429, etc.)
+    Retryable,
+    /// refresh credentials once, then give up (e.g. 401)
+    AuthRefresh,
+    /// permanent: drop immediately, never retry (e.g. 400/403/404)
+    Terminal,
+}
+
+/// Maps an HTTP status code to a [`Disposition`].
+pub struct RetryPolicy {
+    retryable: &'static [u16],
+    auth_refresh: &'static [u16],
+    terminal: &'static [u16],
+    default: Disposition,
+}
+
+impl RetryPolicy {
+    /// Classify `status`. Returns `None` for 2xx (success, not an error).
+    pub fn classify(&self, status: u16) -> Option<Disposition> {
+        if (200..300).contains(&status) {
+            return None;
+        }
+        if self.auth_refresh.contains(&status) {
+            return Some(Disposition::AuthRefresh);
+        }
+        if self.terminal.contains(&status) {
+            return Some(Disposition::Terminal);
+        }
+        if self.retryable.contains(&status) || (500..600).contains(&status) {
+            return Some(Disposition::Retryable);
+        }
+        Some(self.default)
+    }
+
+    /// `true` iff `status` classifies as `Retryable`. This is what an HTTP
+    /// server emits in an `x-should-retry` header.
+    pub fn should_retry(&self, status: u16) -> bool {
+        matches!(self.classify(status), Some(Disposition::Retryable))
+    }
+
+    /// Server preset: 429 and any 5xx are retryable, everything else is
+    /// terminal.
+    pub const fn server() -> Self {
+        Self {
+            retryable: &[429],
+            auth_refresh: &[],
+            terminal: &[],
+            default: Disposition::Terminal,
+        }
+    }
+
+    /// Client storage/upload preset: 400/403/404 terminal-drop, 401
+    /// auth-refresh-once, everything else (429, 5xx, unlisted 4xx) retried.
+    pub const fn client_storage() -> Self {
+        Self {
+            retryable: &[],
+            auth_refresh: &[401],
+            terminal: &[400, 403, 404],
+            default: Disposition::Retryable,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn server_should_retry() {
+        let policy = RetryPolicy::server();
+        for code in [429, 500, 502, 503, 504, 501, 520] {
+            assert!(policy.should_retry(code), "expected {code} to retry");
+        }
+        for code in [400, 401, 403, 404, 200] {
+            assert!(!policy.should_retry(code), "expected {code} to NOT retry");
+        }
+        assert_eq!(policy.classify(200), None);
+    }
+
+    #[test]
+    fn client_storage_classify() {
+        let policy = RetryPolicy::client_storage();
+        for code in [400, 403, 404] {
+            assert_eq!(policy.classify(code), Some(Disposition::Terminal));
+        }
+        assert_eq!(policy.classify(401), Some(Disposition::AuthRefresh));
+        for code in [429, 500, 503, 409, 422] {
+            assert_eq!(policy.classify(code), Some(Disposition::Retryable));
+        }
+        assert_eq!(policy.classify(200), None);
+    }
+}

crates/common/xai-circuit-breaker/src/state.rs

@@ -0,0 +1,68 @@
+//! Breaker state types: state enum, outcome enum, and the `BreakerOpen`
+//! error returned by [`crate::CircuitBreaker::check`] when the breaker is
+//! refusing traffic.
+
+use std::time::Duration;
+
+/// Tri-state circuit-breaker status.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum BreakerState {
+    Closed = 0,
+    Open = 1,
+    HalfOpen = 2,
+}
+
+impl BreakerState {
+    pub(crate) fn from_u8(v: u8) -> Self {
+        match v {
+            0 => Self::Closed,
+            1 => Self::Open,
+            2 => Self::HalfOpen,
+            invalid => {
+                debug_assert!(false, "invalid BreakerState: {invalid}");
+                Self::Closed
+            }
+        }
+    }
+}
+
+/// Outcome of a wire request fed back to the breaker via
+/// [`crate::CircuitBreaker::record`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Outcome {
+    Success,
+    Failure,
+}
+
+/// Returned by [`crate::CircuitBreaker::check`] when the breaker is open
+/// or has already exhausted its half-open probe slots.
+#[derive(Debug)]
+pub struct BreakerOpen {
+    pub retry_after: Duration,
+}
+
+impl std::fmt::Display for BreakerOpen {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "circuit breaker open; retry after {:.1}s",
+            self.retry_after.as_secs_f64()
+        )
+    }
+}
+
+impl std::error::Error for BreakerOpen {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn breaker_open_display() {
+        let err = BreakerOpen {
+            retry_after: Duration::from_millis(5300),
+        };
+        assert_eq!(err.to_string(), "circuit breaker open; retry after 5.3s");
+    }
+}

crates/common/xai-circuit-breaker/src/window.rs

@@ -0,0 +1,76 @@
+//! Bounded sliding window over `(timestamp, is_failure)` samples.
+
+use std::collections::VecDeque;
+use std::time::{Duration, Instant};
+
+/// Safety cap on sliding window entries to bound memory under sustained
+/// high load (e.g. 10K req/s * 60s window would otherwise reach 600K
+/// entries).
+pub(crate) const MAX_WINDOW_ENTRIES: usize = 10_000;
+
+pub(crate) struct SlidingWindow {
+    entries: VecDeque<(Instant, bool)>,
+    /// Incremental count of `is_failure = true` entries currently in
+    /// `entries`. Maintained on push/pop so `error_rate()` is O(1)
+    /// instead of O(n) — avoids a per-request hot-path scan under
+    /// the breaker mutex.
+    failures: usize,
+}
+
+impl SlidingWindow {
+    pub(crate) fn new() -> Self {
+        Self {
+            entries: VecDeque::new(),
+            failures: 0,
+        }
+    }
+
+    pub(crate) fn push(&mut self, is_failure: bool, at: Instant) {
+        if self.entries.len() >= MAX_WINDOW_ENTRIES
+            && let Some((_, was_failure)) = self.entries.pop_front()
+            && was_failure
+        {
+            self.failures -= 1;
+        }
+        self.entries.push_back((at, is_failure));
+        if is_failure {
+            self.failures += 1;
+        }
+    }
+
+    pub(crate) fn evict(&mut self, window: Duration, now: Instant) {
+        let Some(cutoff) = now.checked_sub(window) else {
+            return;
+        };
+        while let Some(&(ts, was_failure)) = self.entries.front() {
+            if ts < cutoff {
+                self.entries.pop_front();
+                if was_failure {
+                    self.failures -= 1;
+                }
+            } else {
+                break;
+            }
+        }
+    }
+
+    pub(crate) fn error_rate(&self) -> f64 {
+        if self.entries.is_empty() {
+            return 0.0;
+        }
+        self.failures as f64 / self.entries.len() as f64
+    }
+
+    pub(crate) fn sample_count(&self) -> usize {
+        self.entries.len()
+    }
+
+    pub(crate) fn clear(&mut self) {
+        self.entries.clear();
+        self.failures = 0;
+    }
+}
+
+#[cfg(test)]
+#[path = "window_tests.rs"]
+mod tests;

crates/common/xai-circuit-breaker/src/window_tests.rs

@@ -0,0 +1,109 @@
+//! Tests for [`crate::window::SlidingWindow`].
+
+use super::*;
+
+#[test]
+fn push_evict_and_error_rate() {
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+    w.push(true, base);
+    w.push(false, base + Duration::from_millis(10));
+    w.push(true, base + Duration::from_millis(20));
+
+    assert_eq!(w.sample_count(), 3);
+    assert!((w.error_rate() - (2.0 / 3.0)).abs() < 1e-9);
+
+    // Evict everything older than 5ms relative to base + 20ms.
+    w.evict(Duration::from_millis(5), base + Duration::from_millis(20));
+    assert_eq!(w.sample_count(), 1);
+}
+
+#[test]
+fn empty_error_rate_is_zero() {
+    let w = SlidingWindow::new();
+    assert_eq!(w.error_rate(), 0.0);
+}
+
+#[test]
+fn push_respects_max_entries_cap() {
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+    for i in 0..(MAX_WINDOW_ENTRIES + 5) {
+        w.push(true, base + Duration::from_nanos(i as u64));
+    }
+    assert_eq!(w.sample_count(), MAX_WINDOW_ENTRIES);
+}
+
+#[test]
+fn failure_count_stays_consistent_under_cap_eviction() {
+    // Push enough failures to overflow the cap and confirm
+    // error_rate() (which is O(1) via the cached failures
+    // counter) still reads 1.0 after entries are dropped from
+    // the front.
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+    for i in 0..(MAX_WINDOW_ENTRIES + 100) {
+        w.push(true, base + Duration::from_nanos(i as u64));
+    }
+    assert_eq!(w.sample_count(), MAX_WINDOW_ENTRIES);
+    assert!((w.error_rate() - 1.0).abs() < f64::EPSILON);
+}
+
+#[test]
+fn failure_count_decrements_on_time_eviction() {
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+    w.push(true, base);
+    w.push(false, base + Duration::from_millis(10));
+    w.push(true, base + Duration::from_millis(20));
+    assert!((w.error_rate() - (2.0 / 3.0)).abs() < 1e-9);
+
+    // Evict the first two entries (the leading true and false).
+    // Remaining is one true → error_rate = 1.0.
+    w.evict(Duration::from_millis(5), base + Duration::from_millis(20));
+    assert_eq!(w.sample_count(), 1);
+    assert!((w.error_rate() - 1.0).abs() < f64::EPSILON);
+}
+
+#[test]
+fn clear_resets_failure_count() {
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+    w.push(true, base);
+    w.push(true, base + Duration::from_millis(1));
+    w.clear();
+    // After clear, pushing one success must read error_rate 0.0;
+    // a stale failures counter would read 2/1 instead.
+    w.push(false, base + Duration::from_millis(2));
+    assert!(w.error_rate().abs() < f64::EPSILON);
+}
+
+/// Push past the cap, then advance time past the window duration and
+/// push more — eviction must continue to read the correct cached
+/// failures count even when the deque is at the cap.
+#[test]
+fn cap_then_time_eviction_keeps_failure_count_consistent() {
+    let mut w = SlidingWindow::new();
+    let base = Instant::now();
+
+    // Fill the deque to the cap with failures.
+    for i in 0..MAX_WINDOW_ENTRIES {
+        w.push(true, base + Duration::from_micros(i as u64));
+    }
+    assert_eq!(w.sample_count(), MAX_WINDOW_ENTRIES);
+    assert!((w.error_rate() - 1.0).abs() < f64::EPSILON);
+
+    // Move past the window and evict — every existing sample falls
+    // out, cached failures counter must reach zero.
+    let way_later = base + Duration::from_secs(3600);
+    w.evict(Duration::from_secs(1), way_later);
+    assert_eq!(w.sample_count(), 0);
+    assert!(w.error_rate().abs() < f64::EPSILON);
+
+    // New samples after a full eviction must continue to read
+    // consistently (regression on a stale `failures` field).
+    w.push(false, way_later);
+    w.push(true, way_later + Duration::from_micros(1));
+    assert_eq!(w.sample_count(), 2);
+    assert!((w.error_rate() - 0.5).abs() < f64::EPSILON);
+}