02d9359435
Synced from monorepo Changes: - Workspace task snapshots only list incomplete backgrounded tasks - Quiet auth, LSP, and config warnings in the shell - Fix observability attributes for warm store errors, restore setup, remote tools, and preview denials - Fail closed when soak metrics are missing - Run plan-mode exit last in mixed tool batches - Allow /loop to store prompts that can terminate the loop - Make subagent maximum nesting depth configurable - Security: apply sandbox profile to the leader process that executes tools Source-Revision: 1adcd1f477870e4a97bacbd6be78c8a3bfbac46d
928 lines
29 KiB
Rust
928 lines
29 KiB
Rust
//! Subagent lifecycle soak: churn spawn/run/completion/eviction and assert
|
|
//! threads, fds, and heap/RSS reach steady state. A stub `ChildRunner` drives
|
|
//! the real coordinator/transport.
|
|
//!
|
|
//! SUBAGENT_SOAK_CYCLES=20000 cargo test -p xai-grok-tools \
|
|
//! [--features dhat-heap] --test test_subagent_soak -- --ignored --nocapture
|
|
|
|
#![cfg(unix)]
|
|
|
|
#[cfg(feature = "dhat-heap")]
|
|
#[global_allocator]
|
|
static DHAT_ALLOC: dhat::Alloc = dhat::Alloc;
|
|
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use serde::ser::SerializeMap;
|
|
use serde::{Serialize, Serializer};
|
|
use strum::{EnumCount, IntoEnumIterator};
|
|
use tokio_util::sync::CancellationToken;
|
|
|
|
use xai_grok_test_support::env::env_parse;
|
|
use xai_grok_test_support::resources::{ResourceGrowth, ResourceSnapshot};
|
|
use xai_grok_tools::implementations::grok_build::task::backend::{ChannelBackend, SubagentBackend};
|
|
use xai_grok_tools::implementations::grok_build::task::coordinator::{
|
|
ChildCompletion, ChildControl, ChildRunOutput, ChildRunRequest, ChildRunner, CoordinatorConfig,
|
|
LocalBoxFuture, MAX_COMPLETED_ENTRIES, StartedChild, SubagentCoordinator, SubagentProgress,
|
|
};
|
|
use xai_grok_tools::implementations::grok_build::task::types::{
|
|
SubagentDescribeOutcome, SubagentOwner, SubagentRegistryCounts, SubagentRequest,
|
|
SubagentResult, SubagentValidateTypeOutcome,
|
|
};
|
|
|
|
const PARENT_SESSION_ID: &str = "subagent-soak-parent";
|
|
|
|
#[derive(Clone, Copy, strum::EnumCount, strum::EnumIter)]
|
|
enum Metric {
|
|
Rss,
|
|
Threads,
|
|
Fds,
|
|
}
|
|
|
|
impl Metric {
|
|
fn label(self) -> &'static str {
|
|
match self {
|
|
Metric::Rss => "rss",
|
|
Metric::Threads => "threads",
|
|
Metric::Fds => "fds",
|
|
}
|
|
}
|
|
|
|
/// RSS reports raw bytes, so its key names the unit.
|
|
fn summary_key(self) -> &'static str {
|
|
match self {
|
|
Metric::Rss => "rss_bytes",
|
|
Metric::Threads => "threads",
|
|
Metric::Fds => "fds",
|
|
}
|
|
}
|
|
|
|
fn unit(self) -> Option<&'static str> {
|
|
match self {
|
|
Metric::Rss => Some("MiB"),
|
|
Metric::Threads | Metric::Fds => None,
|
|
}
|
|
}
|
|
|
|
fn budget(self, bounds: &Bounds) -> f64 {
|
|
match self {
|
|
Metric::Rss => bounds.max_rss_growth_mib as f64,
|
|
Metric::Threads => bounds.max_thread_growth as f64,
|
|
Metric::Fds => bounds.max_fd_growth as f64,
|
|
}
|
|
}
|
|
|
|
/// RSS growth samples are bytes; convert to MiB for the budget comparison.
|
|
fn growth_in_budget_unit(self, raw: usize) -> f64 {
|
|
match self {
|
|
Metric::Rss => bytes_to_mib(raw),
|
|
Metric::Threads | Metric::Fds => raw as f64,
|
|
}
|
|
}
|
|
|
|
/// RSS is sampled on every unix; thread and fd counts are Linux-only.
|
|
fn expected_on_this_platform(self) -> bool {
|
|
match self {
|
|
Metric::Rss => true,
|
|
Metric::Threads | Metric::Fds => cfg!(target_os = "linux"),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Reads a metric's field from a snapshot or a growth delta so serialization and
|
|
/// the gates share one projection instead of repeating it.
|
|
trait MetricValue {
|
|
fn value_of(&self, metric: Metric) -> Option<usize>;
|
|
}
|
|
|
|
impl MetricValue for ResourceSnapshot {
|
|
fn value_of(&self, metric: Metric) -> Option<usize> {
|
|
// Destructure so a new resource field is a compile error here, not a
|
|
// silently dropped metric.
|
|
let ResourceSnapshot { rss, threads, fds } = *self;
|
|
match metric {
|
|
Metric::Rss => rss,
|
|
Metric::Threads => threads,
|
|
Metric::Fds => fds,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl MetricValue for ResourceGrowth {
|
|
fn value_of(&self, metric: Metric) -> Option<usize> {
|
|
let ResourceGrowth { rss, threads, fds } = *self;
|
|
match metric {
|
|
Metric::Rss => rss,
|
|
Metric::Threads => threads,
|
|
Metric::Fds => fds,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn serialize_metrics<T: MetricValue, S: Serializer>(
|
|
value: &T,
|
|
serializer: S,
|
|
) -> Result<S::Ok, S::Error> {
|
|
let mut map = serializer.serialize_map(Some(Metric::COUNT))?;
|
|
for metric in Metric::iter() {
|
|
map.serialize_entry(metric.summary_key(), &value.value_of(metric))?;
|
|
}
|
|
map.end()
|
|
}
|
|
|
|
fn bytes_to_mib(bytes: usize) -> f64 {
|
|
bytes as f64 / (1024.0 * 1024.0)
|
|
}
|
|
|
|
#[cfg_attr(not(feature = "dhat-heap"), allow(dead_code))]
|
|
#[derive(Clone, Copy, Serialize)]
|
|
struct HeapSample {
|
|
blocks: i64,
|
|
bytes: i64,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Serialize)]
|
|
struct HeapMetrics {
|
|
before: HeapSample,
|
|
after: HeapSample,
|
|
blocks_per_cycle: f64,
|
|
bytes_per_cycle: f64,
|
|
}
|
|
|
|
impl HeapMetrics {
|
|
fn new(before: HeapSample, after: HeapSample, cycles: u64) -> Self {
|
|
// `SUBAGENT_SOAK_CYCLES=0` would otherwise divide by zero and feed
|
|
// NaN/inf into the leak gates.
|
|
let cycles = cycles.max(1) as f64;
|
|
Self {
|
|
before,
|
|
after,
|
|
blocks_per_cycle: (after.blocks - before.blocks) as f64 / cycles,
|
|
bytes_per_cycle: (after.bytes - before.bytes) as f64 / cycles,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct Bounds {
|
|
#[serde(rename = "warmup_cycles")]
|
|
warmup: u64,
|
|
#[serde(rename = "measured_cycles")]
|
|
measure: u64,
|
|
concurrency: u64,
|
|
max_thread_growth: u64,
|
|
max_fd_growth: u64,
|
|
max_rss_growth_mib: u64,
|
|
max_blocks_per_cycle: f64,
|
|
max_bytes_per_cycle: f64,
|
|
}
|
|
|
|
impl Bounds {
|
|
fn from_env() -> Self {
|
|
Self {
|
|
// Default warmup to the completed-entry cap so the ring is saturated
|
|
// and the measured window observes steady-state eviction rather than
|
|
// one-time cache fill.
|
|
warmup: env_parse("SUBAGENT_SOAK_WARMUP", MAX_COMPLETED_ENTRIES as u64),
|
|
measure: env_parse("SUBAGENT_SOAK_CYCLES", 512u64),
|
|
concurrency: env_parse("SUBAGENT_SOAK_CONCURRENCY", 16u64),
|
|
// RSS is looser than threads and fds to absorb allocator noise.
|
|
max_thread_growth: env_parse("SUBAGENT_SOAK_MAX_THREAD_GROWTH", 8u64),
|
|
max_fd_growth: env_parse("SUBAGENT_SOAK_MAX_FD_GROWTH", 16u64),
|
|
max_rss_growth_mib: env_parse("SUBAGENT_SOAK_MAX_RSS_GROWTH_MIB", 256u64),
|
|
max_blocks_per_cycle: env_parse("SUBAGENT_SOAK_MAX_BLOCKS_PER_CYCLE", 2.0f64),
|
|
max_bytes_per_cycle: env_parse("SUBAGENT_SOAK_MAX_BYTES_PER_CYCLE", 4096.0f64),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct Measurement {
|
|
#[serde(serialize_with = "serialize_metrics")]
|
|
before: ResourceSnapshot,
|
|
#[serde(serialize_with = "serialize_metrics")]
|
|
after: ResourceSnapshot,
|
|
#[serde(serialize_with = "serialize_metrics")]
|
|
growth: ResourceGrowth,
|
|
#[serde(serialize_with = "serialize_counts")]
|
|
counts: SubagentRegistryCounts,
|
|
heap: Option<HeapMetrics>,
|
|
quiesced: bool,
|
|
}
|
|
|
|
fn serialize_counts<S: Serializer>(
|
|
counts: &SubagentRegistryCounts,
|
|
serializer: S,
|
|
) -> Result<S::Ok, S::Error> {
|
|
// Exhaustive destructure so a new count field is a compile error here, not a
|
|
// silently dropped summary key.
|
|
let SubagentRegistryCounts {
|
|
pending,
|
|
active,
|
|
completed,
|
|
} = counts;
|
|
let entries = [
|
|
("pending", pending),
|
|
("active", active),
|
|
("completed", completed),
|
|
];
|
|
let mut map = serializer.serialize_map(Some(entries.len()))?;
|
|
for (key, value) in entries {
|
|
map.serialize_entry(key, value)?;
|
|
}
|
|
map.end()
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct Summary<'a> {
|
|
#[serde(flatten)]
|
|
bounds: &'a Bounds,
|
|
#[serde(flatten)]
|
|
measurement: &'a Measurement,
|
|
}
|
|
|
|
fn heap_capture() -> Option<HeapSample> {
|
|
#[cfg(feature = "dhat-heap")]
|
|
{
|
|
let stats = dhat::HeapStats::get();
|
|
Some(HeapSample {
|
|
blocks: stats.curr_blocks as i64,
|
|
bytes: stats.curr_bytes as i64,
|
|
})
|
|
}
|
|
#[cfg(not(feature = "dhat-heap"))]
|
|
{
|
|
None
|
|
}
|
|
}
|
|
|
|
async fn quiesce(backend: &ChannelBackend) -> bool {
|
|
const MAX_POLLS: usize = 200;
|
|
const SLEEP: Duration = Duration::from_millis(5);
|
|
for _ in 0..MAX_POLLS {
|
|
let counts = backend.registry_counts().await;
|
|
if counts.pending == 0 && counts.active == 0 {
|
|
return true;
|
|
}
|
|
tokio::time::sleep(SLEEP).await;
|
|
}
|
|
let counts = backend.registry_counts().await;
|
|
eprintln!(
|
|
"[soak] quiesce budget expired with pending={} active={}; snapshot may be noisy",
|
|
counts.pending, counts.active
|
|
);
|
|
false
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct SoakControl {
|
|
cancellation: CancellationToken,
|
|
}
|
|
|
|
impl ChildControl for SoakControl {
|
|
type ProgressFuture = std::future::Ready<SubagentProgress>;
|
|
|
|
fn progress(&self) -> Self::ProgressFuture {
|
|
std::future::ready(SubagentProgress::default())
|
|
}
|
|
|
|
fn cancel(&self) {
|
|
self.cancellation.cancel();
|
|
}
|
|
}
|
|
|
|
struct SoakRunner {
|
|
gate: Arc<tokio::sync::Semaphore>,
|
|
}
|
|
|
|
impl ChildRunner for SoakRunner {
|
|
type Control = SoakControl;
|
|
type CompletionData = ();
|
|
type RunFuture = LocalBoxFuture<ChildRunOutput<()>>;
|
|
type ValidateFuture = LocalBoxFuture<SubagentValidateTypeOutcome>;
|
|
type DescribeFuture = LocalBoxFuture<SubagentDescribeOutcome>;
|
|
|
|
fn run(&self, run: ChildRunRequest<Self::Control>) -> Self::RunFuture {
|
|
let gate = self.gate.clone();
|
|
Box::pin(async move {
|
|
let ChildRunRequest {
|
|
request,
|
|
cancellation,
|
|
reporter,
|
|
} = run;
|
|
let promoted = reporter
|
|
.started(StartedChild {
|
|
child_session_id: request.id.clone(),
|
|
persona: None,
|
|
resumed_from: request.resume_from.clone(),
|
|
child_cwd: request.cwd.clone().unwrap_or_default(),
|
|
worktree_path: None,
|
|
effective_model_id: "soak-model".to_owned(),
|
|
definition_background: false,
|
|
control: SoakControl {
|
|
cancellation: cancellation.clone(),
|
|
},
|
|
})
|
|
.await;
|
|
if !promoted || cancellation.is_cancelled() {
|
|
return ChildRunOutput {
|
|
result: SubagentResult {
|
|
success: false,
|
|
cancelled: true,
|
|
error: Some("cancelled before start".to_owned()),
|
|
subagent_id: request.id.clone(),
|
|
child_session_id: request.id,
|
|
..Default::default()
|
|
},
|
|
completion_data: (),
|
|
snapshot_ref: None,
|
|
};
|
|
}
|
|
if request.id.starts_with("conc-") {
|
|
// Hold in `active` until the concurrent phase releases the gate.
|
|
let _ = gate.acquire().await;
|
|
}
|
|
ChildRunOutput {
|
|
result: SubagentResult {
|
|
success: true,
|
|
output: Arc::from("soak child output"),
|
|
subagent_id: request.id.clone(),
|
|
child_session_id: request.id,
|
|
tool_calls: 1,
|
|
turns: 1,
|
|
..Default::default()
|
|
},
|
|
completion_data: (),
|
|
snapshot_ref: None,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn validate_type(&self, _subagent_type: String, _parent: String) -> Self::ValidateFuture {
|
|
Box::pin(std::future::ready(SubagentValidateTypeOutcome::Ok))
|
|
}
|
|
|
|
fn describe_type(
|
|
&self,
|
|
_subagent_type: String,
|
|
_harness_agent_type: Option<String>,
|
|
_parent: String,
|
|
) -> Self::DescribeFuture {
|
|
Box::pin(std::future::ready(SubagentDescribeOutcome::Unavailable))
|
|
}
|
|
|
|
fn on_completed(&self, _completion: ChildCompletion<Self::CompletionData>) {}
|
|
}
|
|
|
|
fn soak_request(id: String, background: bool) -> SubagentRequest {
|
|
SubagentRequest {
|
|
id,
|
|
prompt: "soak work".to_owned(),
|
|
description: "soak child".to_owned(),
|
|
subagent_type: "explore".to_owned(),
|
|
parent_session_id: PARENT_SESSION_ID.to_owned(),
|
|
parent_prompt_id: Some("soak-prompt".to_owned()),
|
|
resume_from: None,
|
|
cwd: None,
|
|
runtime_overrides: Default::default(),
|
|
run_in_background: background,
|
|
surface_completion: true,
|
|
await_to_completion: false,
|
|
fork_context: false,
|
|
owner: SubagentOwner::Task,
|
|
cancel_token: CancellationToken::new(),
|
|
}
|
|
}
|
|
|
|
async fn run_cycle(backend: &ChannelBackend, i: u64) {
|
|
let fg = backend
|
|
.spawn(soak_request(format!("fg-{i}"), false))
|
|
.await
|
|
.expect("foreground spawn round-trips through the coordinator");
|
|
assert!(fg.success, "cycle {i}: foreground child must complete");
|
|
|
|
let bg_id = format!("bg-{i}");
|
|
let bg = backend
|
|
.spawn(soak_request(bg_id.clone(), true))
|
|
.await
|
|
.expect("background spawn round-trips through the coordinator");
|
|
assert!(bg.success, "cycle {i}: background child must complete");
|
|
|
|
let blocking = true;
|
|
let timeout_ms = Some(5_000);
|
|
let snapshot = backend.query(&bg_id, blocking, timeout_ms).await;
|
|
assert!(
|
|
snapshot.is_some(),
|
|
"cycle {i}: completed subagent must be queryable"
|
|
);
|
|
}
|
|
|
|
async fn await_concurrency(backend: &ChannelBackend, n: u64) -> bool {
|
|
const MAX_POLLS: usize = 400;
|
|
const SLEEP: Duration = Duration::from_millis(5);
|
|
for _ in 0..MAX_POLLS {
|
|
if backend.registry_counts().await.active as u64 >= n {
|
|
return true;
|
|
}
|
|
tokio::time::sleep(SLEEP).await;
|
|
}
|
|
false
|
|
}
|
|
|
|
async fn concurrent_phase(backend: &ChannelBackend, gate: &tokio::sync::Semaphore, n: u64) {
|
|
let handles: Vec<_> = (0..n)
|
|
.map(|k| {
|
|
let backend = backend.clone();
|
|
tokio::task::spawn_local(async move {
|
|
backend.spawn(soak_request(format!("conc-{k}"), true)).await
|
|
})
|
|
})
|
|
.collect();
|
|
|
|
let reached = await_concurrency(backend, n).await;
|
|
|
|
// Release then join before asserting, so no child is left blocked on failure.
|
|
gate.add_permits(n as usize);
|
|
for h in handles {
|
|
let result = h.await.expect("concurrent spawn task");
|
|
assert!(
|
|
result.expect("concurrent spawn round-trips").success,
|
|
"concurrent child must complete"
|
|
);
|
|
}
|
|
assert!(reached, "expected {n} concurrently active children");
|
|
}
|
|
|
|
async fn warmup(backend: &ChannelBackend, cycles: u64) -> bool {
|
|
for i in 0..cycles {
|
|
run_cycle(backend, i).await;
|
|
}
|
|
quiesce(backend).await
|
|
}
|
|
|
|
async fn measure(
|
|
backend: &ChannelBackend,
|
|
bounds: &Bounds,
|
|
baseline_quiesced: bool,
|
|
) -> Measurement {
|
|
let heap_before = heap_capture();
|
|
let before = ResourceSnapshot::capture();
|
|
|
|
// Continue ids past the warmup window so measured cycles use fresh entries
|
|
// and keep exercising eviction instead of colliding with warmup ids.
|
|
for i in bounds.warmup..(bounds.warmup + bounds.measure) {
|
|
run_cycle(backend, i).await;
|
|
}
|
|
// A baseline that never drained already poisons `before`, so skip the
|
|
// measured-window drain and report the window as not quiesced.
|
|
let quiesced = baseline_quiesced && quiesce(backend).await;
|
|
|
|
let heap_after = heap_capture();
|
|
let after = ResourceSnapshot::capture();
|
|
let counts = backend.registry_counts().await;
|
|
|
|
Measurement {
|
|
before,
|
|
after,
|
|
growth: after.growth_from(&before),
|
|
counts,
|
|
heap: heap_before
|
|
.zip(heap_after)
|
|
.map(|(before, after)| HeapMetrics::new(before, after, bounds.measure)),
|
|
quiesced,
|
|
}
|
|
}
|
|
|
|
/// Takes `expected` as a parameter so the skip arm is testable on any platform.
|
|
fn metric_failure(
|
|
metric: Metric,
|
|
value: Option<usize>,
|
|
expected: bool,
|
|
bounds: &Bounds,
|
|
) -> Option<String> {
|
|
let Some(raw) = value else {
|
|
return expected.then(|| {
|
|
format!(
|
|
"{}: growth sample unavailable; the soak cannot bound it",
|
|
metric.label()
|
|
)
|
|
});
|
|
};
|
|
let growth = metric.growth_in_budget_unit(raw);
|
|
let budget = metric.budget(bounds);
|
|
(growth > budget).then(|| {
|
|
let unit = metric.unit().map(|u| format!(" {u}")).unwrap_or_default();
|
|
format!(
|
|
"{}: grew {growth:.1}{unit} over the soak (bound {budget:.1}{unit})",
|
|
metric.label()
|
|
)
|
|
})
|
|
}
|
|
|
|
fn check_bounds(bounds: &Bounds, m: &Measurement) -> Vec<String> {
|
|
// Drain first: a non-quiesced window has nonzero counts and noisy growth, so
|
|
// report the quiesce failure alone; the gates below only mean anything once
|
|
// drained.
|
|
if !m.quiesced {
|
|
return vec![
|
|
"quiesce budget expired before the measured window drained; soak result is unreliable"
|
|
.to_owned(),
|
|
];
|
|
}
|
|
|
|
let mut failures = Vec::new();
|
|
if m.counts.pending != 0 {
|
|
failures.push(format!(
|
|
"no subagent may remain pending, saw {}",
|
|
m.counts.pending
|
|
));
|
|
}
|
|
if m.counts.active != 0 {
|
|
failures.push(format!(
|
|
"no subagent may remain active, saw {}",
|
|
m.counts.active
|
|
));
|
|
}
|
|
if m.counts.completed > MAX_COMPLETED_ENTRIES {
|
|
failures.push(format!(
|
|
"completed retention must stay bounded by its cap, saw {}",
|
|
m.counts.completed
|
|
));
|
|
}
|
|
|
|
for metric in Metric::iter() {
|
|
let expected = metric.expected_on_this_platform();
|
|
if let Some(f) = metric_failure(metric, m.growth.value_of(metric), expected, bounds) {
|
|
failures.push(f);
|
|
}
|
|
}
|
|
|
|
if let Some(h) = m.heap {
|
|
let measure = bounds.measure;
|
|
if h.blocks_per_cycle > bounds.max_blocks_per_cycle {
|
|
failures.push(format!(
|
|
"block-count leak: {:.3} blocks/cycle retained ({} over {measure} cycles) \
|
|
exceeds the {} gate",
|
|
h.blocks_per_cycle,
|
|
h.after.blocks - h.before.blocks,
|
|
bounds.max_blocks_per_cycle
|
|
));
|
|
}
|
|
if h.bytes_per_cycle > bounds.max_bytes_per_cycle {
|
|
failures.push(format!(
|
|
"byte leak: {:.1} bytes/cycle retained ({} over {measure} cycles) \
|
|
exceeds the {} gate",
|
|
h.bytes_per_cycle,
|
|
h.after.bytes - h.before.bytes,
|
|
bounds.max_bytes_per_cycle
|
|
));
|
|
}
|
|
}
|
|
|
|
failures
|
|
}
|
|
|
|
fn assert_bounds(bounds: &Bounds, m: &Measurement) {
|
|
let failures = check_bounds(bounds, m);
|
|
assert!(
|
|
failures.is_empty(),
|
|
"subagent soak bounds violated:\n - {}",
|
|
failures.join("\n - ")
|
|
);
|
|
}
|
|
|
|
/// Keep this the only test in the binary that creates a `dhat::Profiler`.
|
|
#[tokio::test(flavor = "current_thread")]
|
|
#[ignore = "subagent soak; run with --ignored (SUBAGENT_SOAK_CYCLES bounds the measured window)"]
|
|
async fn subagent_lifecycle_soak_bounds_threads_fds_and_heap() {
|
|
#[cfg(feature = "dhat-heap")]
|
|
let _profiler = dhat::Profiler::builder().testing().build();
|
|
|
|
let bounds = Bounds::from_env();
|
|
|
|
let local = tokio::task::LocalSet::new();
|
|
local
|
|
.run_until(async move {
|
|
let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let config = CoordinatorConfig {
|
|
foreground_budget: Duration::from_secs(600),
|
|
..CoordinatorConfig::default()
|
|
};
|
|
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
|
tokio::task::spawn_local(
|
|
SubagentCoordinator::new(command_rx, SoakRunner { gate: gate.clone() }, config)
|
|
.run(),
|
|
);
|
|
let backend = ChannelBackend::new(command_tx);
|
|
|
|
let warmup_quiesced = warmup(&backend, bounds.warmup).await;
|
|
// Drain the concurrent phase into the baseline; a failed drain marks
|
|
// the window unreliable.
|
|
concurrent_phase(&backend, &gate, bounds.concurrency).await;
|
|
let baseline_quiesced = warmup_quiesced && quiesce(&backend).await;
|
|
let measurement = measure(&backend, &bounds, baseline_quiesced).await;
|
|
|
|
let summary = Summary {
|
|
bounds: &bounds,
|
|
measurement: &measurement,
|
|
};
|
|
eprintln!(
|
|
"SUBAGENT_SOAK_SUMMARY {}",
|
|
serde_json::to_string(&summary).expect("summary serializes")
|
|
);
|
|
|
|
assert_bounds(&bounds, &measurement);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
mod tests {
|
|
use super::*;
|
|
use pretty_assertions::assert_eq;
|
|
|
|
#[test]
|
|
fn value_of_reads_the_matching_slot_of_snapshot_and_growth() {
|
|
let snapshot = ResourceSnapshot {
|
|
rss: Some(11),
|
|
threads: Some(22),
|
|
fds: Some(33),
|
|
};
|
|
assert_eq!(snapshot.value_of(Metric::Rss), Some(11));
|
|
assert_eq!(snapshot.value_of(Metric::Threads), Some(22));
|
|
assert_eq!(snapshot.value_of(Metric::Fds), Some(33));
|
|
|
|
let growth = ResourceGrowth {
|
|
rss: Some(1),
|
|
threads: None,
|
|
fds: Some(3),
|
|
};
|
|
assert_eq!(growth.value_of(Metric::Rss), Some(1));
|
|
assert_eq!(growth.value_of(Metric::Threads), None);
|
|
assert_eq!(growth.value_of(Metric::Fds), Some(3));
|
|
}
|
|
|
|
#[test]
|
|
fn serialize_metrics_keys_match_summary_keys_in_order() {
|
|
#[derive(Serialize)]
|
|
struct Wrap(#[serde(serialize_with = "serialize_metrics")] ResourceSnapshot);
|
|
let snapshot = ResourceSnapshot {
|
|
rss: Some(1),
|
|
threads: None,
|
|
fds: Some(3),
|
|
};
|
|
let json = serde_json::to_string(&Wrap(snapshot)).expect("snapshot serializes");
|
|
assert_eq!(json, r#"{"rss_bytes":1,"threads":null,"fds":3}"#);
|
|
}
|
|
|
|
#[test]
|
|
fn bytes_to_mib_divides_by_1024_squared() {
|
|
assert_eq!(bytes_to_mib(0), 0.0);
|
|
assert_eq!(bytes_to_mib(1024 * 1024), 1.0);
|
|
assert_eq!(bytes_to_mib(3 * 1024 * 1024), 3.0);
|
|
}
|
|
|
|
#[test]
|
|
fn growth_in_budget_unit_scales_only_rss() {
|
|
assert_eq!(Metric::Rss.growth_in_budget_unit(2 * 1024 * 1024), 2.0);
|
|
assert_eq!(Metric::Threads.growth_in_budget_unit(7), 7.0);
|
|
assert_eq!(Metric::Fds.growth_in_budget_unit(7), 7.0);
|
|
}
|
|
|
|
#[test]
|
|
fn budget_reads_per_metric_bound() {
|
|
let bounds = Bounds {
|
|
warmup: 0,
|
|
measure: 0,
|
|
concurrency: 0,
|
|
max_thread_growth: 3,
|
|
max_fd_growth: 5,
|
|
max_rss_growth_mib: 7,
|
|
max_blocks_per_cycle: 1.0,
|
|
max_bytes_per_cycle: 2.0,
|
|
};
|
|
assert_eq!(Metric::Rss.budget(&bounds), 7.0);
|
|
assert_eq!(Metric::Threads.budget(&bounds), 3.0);
|
|
assert_eq!(Metric::Fds.budget(&bounds), 5.0);
|
|
}
|
|
|
|
#[test]
|
|
fn heap_metrics_clamps_zero_cycles() {
|
|
let before = HeapSample {
|
|
blocks: 10,
|
|
bytes: 100,
|
|
};
|
|
let after = HeapSample {
|
|
blocks: 20,
|
|
bytes: 400,
|
|
};
|
|
let heap = HeapMetrics::new(before, after, 0);
|
|
assert!(heap.blocks_per_cycle.is_finite());
|
|
assert!(heap.bytes_per_cycle.is_finite());
|
|
assert_eq!(heap.blocks_per_cycle, 10.0);
|
|
assert_eq!(heap.bytes_per_cycle, 300.0);
|
|
}
|
|
|
|
fn generous_bounds() -> Bounds {
|
|
Bounds {
|
|
warmup: 0,
|
|
measure: 4,
|
|
concurrency: 4,
|
|
max_thread_growth: 100,
|
|
max_fd_growth: 100,
|
|
max_rss_growth_mib: 100,
|
|
max_blocks_per_cycle: 10.0,
|
|
max_bytes_per_cycle: 10_000.0,
|
|
}
|
|
}
|
|
|
|
/// Zero growth that reads as measured, unlike `ResourceGrowth::default()`.
|
|
fn zero_growth() -> ResourceGrowth {
|
|
ResourceGrowth {
|
|
rss: Some(0),
|
|
threads: Some(0),
|
|
fds: Some(0),
|
|
}
|
|
}
|
|
|
|
fn drained(growth: ResourceGrowth, heap: Option<HeapMetrics>) -> Measurement {
|
|
Measurement {
|
|
before: ResourceSnapshot::default(),
|
|
after: ResourceSnapshot::default(),
|
|
growth,
|
|
counts: SubagentRegistryCounts {
|
|
pending: 0,
|
|
active: 0,
|
|
completed: 0,
|
|
},
|
|
heap,
|
|
quiesced: true,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_passes_a_clean_drained_window() {
|
|
let m = drained(zero_growth(), None);
|
|
assert!(check_bounds(&generous_bounds(), &m).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_fails_when_an_expected_metric_is_unavailable() {
|
|
let growth = ResourceGrowth {
|
|
rss: None,
|
|
threads: Some(0),
|
|
fds: Some(0),
|
|
};
|
|
let failures = check_bounds(&generous_bounds(), &drained(growth, None));
|
|
assert!(
|
|
failures
|
|
.iter()
|
|
.any(|f| f.starts_with("rss:") && f.contains("unavailable")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn metric_failure_covers_expected_missing_unexpected_missing_and_budget() {
|
|
let b = generous_bounds();
|
|
assert!(metric_failure(Metric::Threads, None, false, &b).is_none());
|
|
assert!(
|
|
metric_failure(Metric::Rss, None, true, &b)
|
|
.unwrap()
|
|
.contains("unavailable")
|
|
);
|
|
assert!(metric_failure(Metric::Fds, Some(0), true, &b).is_none());
|
|
assert!(
|
|
metric_failure(Metric::Rss, Some(500 * 1024 * 1024), true, &b)
|
|
.unwrap()
|
|
.starts_with("rss:")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_reports_non_quiesce_first_and_alone() {
|
|
let mut m = drained(zero_growth(), None);
|
|
m.quiesced = false;
|
|
m.counts.pending = 3;
|
|
let failures = check_bounds(&generous_bounds(), &m);
|
|
assert_eq!(failures.len(), 1);
|
|
assert!(failures[0].contains("quiesce"));
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_over_budget_growth() {
|
|
let growth = ResourceGrowth {
|
|
rss: Some(200 * 1024 * 1024),
|
|
threads: Some(0),
|
|
fds: Some(0),
|
|
};
|
|
let failures = check_bounds(&generous_bounds(), &drained(growth, None));
|
|
assert!(
|
|
failures.iter().any(|f| f.starts_with("rss:")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_treats_the_budget_as_an_inclusive_max() {
|
|
let growth = ResourceGrowth {
|
|
rss: Some(100 * 1024 * 1024),
|
|
threads: Some(100),
|
|
fds: Some(100),
|
|
};
|
|
assert!(check_bounds(&generous_bounds(), &drained(growth, None)).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_nonzero_counts_and_heap_leak() {
|
|
let mut m = drained(
|
|
zero_growth(),
|
|
Some(HeapMetrics {
|
|
before: HeapSample {
|
|
blocks: 0,
|
|
bytes: 0,
|
|
},
|
|
after: HeapSample {
|
|
blocks: 0,
|
|
bytes: 0,
|
|
},
|
|
blocks_per_cycle: 0.0,
|
|
bytes_per_cycle: 1_000_000.0,
|
|
}),
|
|
);
|
|
m.counts.active = 2;
|
|
let failures = check_bounds(&generous_bounds(), &m);
|
|
assert!(
|
|
failures.iter().any(|f| f.contains("active")),
|
|
"{failures:?}"
|
|
);
|
|
assert!(
|
|
failures.iter().any(|f| f.contains("byte leak")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_pending_while_quiesced() {
|
|
let mut m = drained(zero_growth(), None);
|
|
m.counts.pending = 3;
|
|
let failures = check_bounds(&generous_bounds(), &m);
|
|
assert!(
|
|
failures.iter().any(|f| f.contains("pending")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_completed_over_cap() {
|
|
let mut m = drained(zero_growth(), None);
|
|
m.counts.completed = MAX_COMPLETED_ENTRIES + 1;
|
|
let failures = check_bounds(&generous_bounds(), &m);
|
|
assert!(
|
|
failures.iter().any(|f| f.contains("completed retention")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_thread_and_fd_over_budget() {
|
|
let growth = ResourceGrowth {
|
|
rss: Some(0),
|
|
threads: Some(200),
|
|
fds: Some(200),
|
|
};
|
|
let failures = check_bounds(&generous_bounds(), &drained(growth, None));
|
|
assert!(
|
|
failures.iter().any(|f| f.starts_with("threads:")),
|
|
"{failures:?}"
|
|
);
|
|
assert!(
|
|
failures.iter().any(|f| f.starts_with("fds:")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn check_bounds_flags_block_count_leak() {
|
|
let m = drained(
|
|
zero_growth(),
|
|
Some(HeapMetrics {
|
|
before: HeapSample {
|
|
blocks: 0,
|
|
bytes: 0,
|
|
},
|
|
after: HeapSample {
|
|
blocks: 0,
|
|
bytes: 0,
|
|
},
|
|
blocks_per_cycle: 50.0,
|
|
bytes_per_cycle: 0.0,
|
|
}),
|
|
);
|
|
let failures = check_bounds(&generous_bounds(), &m);
|
|
assert!(
|
|
failures.iter().any(|f| f.contains("block-count leak")),
|
|
"{failures:?}"
|
|
);
|
|
}
|
|
}
|