Synced from monorepo
Synced from monorepo Changes: - Workspace task snapshots only list incomplete backgrounded tasks - Quiet auth, LSP, and config warnings in the shell - Fix observability attributes for warm store errors, restore setup, remote tools, and preview denials - Fail closed when soak metrics are missing - Run plan-mode exit last in mixed tool batches - Allow /loop to store prompts that can terminate the loop - Make subagent maximum nesting depth configurable - Security: apply sandbox profile to the leader process that executes tools Source-Revision: 1adcd1f477870e4a97bacbd6be78c8a3bfbac46d
This commit is contained in:
parent
b41c75a578
commit
02d9359435
96 changed files with 2346 additions and 351 deletions
|
|
@ -198,6 +198,33 @@ impl DisconnectCause {
|
|||
_ => None,
|
||||
}
|
||||
}
|
||||
/// Bounded classification of transport error detail for metrics. Collapses
|
||||
/// free-form OS/tungstenite messages into a small allowlist so reconnect
|
||||
/// storms can be attributed without high-cardinality labels.
|
||||
fn detail_class(&self) -> Option<&'static str> {
|
||||
let detail = self.detail()?;
|
||||
Some(classify_transport_detail(detail))
|
||||
}
|
||||
}
|
||||
/// Map a transport error detail string to a bounded class label.
|
||||
fn classify_transport_detail(detail: &str) -> &'static str {
|
||||
let d = detail.to_ascii_lowercase();
|
||||
if d.contains("connection reset") || d.contains("econnreset") || d.contains("reset by peer") {
|
||||
"connection_reset"
|
||||
} else if d.contains("broken pipe") || d.contains("epipe") {
|
||||
"broken_pipe"
|
||||
} else if d.contains("unexpected eof")
|
||||
|| d.contains("connection closed")
|
||||
|| d.contains("connection aborted without closing")
|
||||
{
|
||||
"unexpected_eof"
|
||||
} else if d.contains("timed out") || d.contains("timeout") || d.contains("etimedout") {
|
||||
"timeout"
|
||||
} else if d.contains("connection aborted") || d.contains("econnaborted") {
|
||||
"connection_aborted"
|
||||
} else {
|
||||
"other"
|
||||
}
|
||||
}
|
||||
struct OutageInfo {
|
||||
cause: DisconnectCause,
|
||||
|
|
@ -1458,6 +1485,9 @@ async fn reconnect_and_replay(
|
|||
"server reconnect succeeded"
|
||||
);
|
||||
crate::metrics::reconnect_cause(outage.cause.label());
|
||||
if let Some(detail_class) = outage.cause.detail_class() {
|
||||
crate::metrics::disconnect_detail_class(outage.cause.label(), detail_class);
|
||||
}
|
||||
crate::metrics::reconnect_gap_observe(silent_gap_ms as f64 / 1_000.0);
|
||||
*inner.connection_id.lock().await = Some(ack.connection_id.clone());
|
||||
*inner.hello_capabilities.write() = std::mem::take(&mut ack.capabilities);
|
||||
|
|
@ -1682,6 +1712,29 @@ mod tests {
|
|||
assert_eq!(DisconnectCause::Forced.label(), "forced");
|
||||
}
|
||||
#[test]
|
||||
fn classify_transport_detail_is_bounded() {
|
||||
assert_eq!(
|
||||
classify_transport_detail("Connection reset by peer (os error 104)"),
|
||||
"connection_reset"
|
||||
);
|
||||
assert_eq!(classify_transport_detail("Broken pipe"), "broken_pipe");
|
||||
assert_eq!(
|
||||
classify_transport_detail("Unexpected EOF"),
|
||||
"unexpected_eof"
|
||||
);
|
||||
assert_eq!(classify_transport_detail("operation timed out"), "timeout");
|
||||
assert_eq!(
|
||||
classify_transport_detail("Connection aborted"),
|
||||
"connection_aborted"
|
||||
);
|
||||
assert_eq!(classify_transport_detail("something novel"), "other");
|
||||
assert_eq!(
|
||||
DisconnectCause::ReadError("ECONNRESET".to_owned()).detail_class(),
|
||||
Some("connection_reset")
|
||||
);
|
||||
assert!(DisconnectCause::Eof.detail_class().is_none());
|
||||
}
|
||||
#[test]
|
||||
fn conn_health_snapshot_without_clock_skew_reports_zero_jump() {
|
||||
let health = ConnHealth::new();
|
||||
health.record_inbound();
|
||||
|
|
|
|||
|
|
@ -68,6 +68,17 @@ mod inner {
|
|||
.expect("computer_hub_client_reconnects_by_cause_total must register once")
|
||||
});
|
||||
|
||||
static DISCONNECT_DETAIL_CLASS_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
|
||||
register_int_counter_vec!(
|
||||
"computer_hub_client_disconnect_detail_class_total",
|
||||
"Disconnects with a transport error detail, by cause (transport_read_error |\
|
||||
transport_write_error) and bounded detail_class (connection_reset | \
|
||||
broken_pipe | unexpected_eof | timeout | connection_aborted | other).",
|
||||
&["cause", "detail_class"]
|
||||
)
|
||||
.expect("computer_hub_client_disconnect_detail_class_total must register once")
|
||||
});
|
||||
|
||||
static RECONNECT_GAP_SECONDS: LazyLock<Histogram> = LazyLock::new(|| {
|
||||
register_histogram!(
|
||||
"computer_hub_client_reconnect_gap_seconds",
|
||||
|
|
@ -340,6 +351,12 @@ mod inner {
|
|||
RECONNECTS_BY_CAUSE_TOTAL.with_label_values(&[cause]).inc();
|
||||
}
|
||||
|
||||
pub(crate) fn disconnect_detail_class(cause: &str, detail_class: &str) {
|
||||
DISCONNECT_DETAIL_CLASS_TOTAL
|
||||
.with_label_values(&[cause, detail_class])
|
||||
.inc();
|
||||
}
|
||||
|
||||
pub(crate) fn reconnect_gap_observe(secs: f64) {
|
||||
RECONNECT_GAP_SECONDS.observe(secs);
|
||||
}
|
||||
|
|
@ -551,6 +568,7 @@ mod inner {
|
|||
pub(crate) fn reconnect_failed(_reason: &str) {}
|
||||
pub(crate) fn reconnect_duration_observe(_secs: f64) {}
|
||||
pub(crate) fn reconnect_cause(_cause: &str) {}
|
||||
pub(crate) fn disconnect_detail_class(_cause: &str, _detail_class: &str) {}
|
||||
pub(crate) fn reconnect_gap_observe(_secs: f64) {}
|
||||
pub(crate) fn call_dispatch_observe(_secs: f64) {}
|
||||
pub(crate) fn demux_inbox_depth_set(_depth: i64) {}
|
||||
|
|
@ -598,6 +616,7 @@ pub(crate) use inner::cancel_hook_received;
|
|||
pub(crate) use inner::cancel_no_target;
|
||||
pub(crate) use inner::cancel_pending_tombstoned;
|
||||
pub(crate) use inner::demux_inbox_depth_set;
|
||||
pub(crate) use inner::disconnect_detail_class;
|
||||
pub(crate) use inner::early_notif_buffered;
|
||||
pub(crate) use inner::heartbeat_pong_dropped;
|
||||
pub(crate) use inner::hook_send;
|
||||
|
|
|
|||
Loading…
Reference in a new issue