Publish harness and TUI open-source

initial sync from the monorepo
This commit is contained in:
grokkybara[bot] 2026-07-16 06:46:02 +01:00
commit c68e39f604
2734 changed files with 1437016 additions and 0 deletions

View file

@ -0,0 +1,76 @@
[package]
license = "Apache-2.0"
name = "xai-grok-mcp"
version = "0.1.0"
edition.workspace = true
description = "MCP integration crate. Quarantines rmcp + reqwest 0.13 (rmcp 2.1 requires reqwest >= 0.13.2 while the rest of the workspace uses reqwest 0.12) and owns the MCP credential store and OAuth flow orchestrator."
authors = ["xAI"]
[dependencies]
rmcp = { version = "2.1", features = [
"auth",
"client",
"transport-async-rw",
"transport-streamable-http-client-reqwest",
"reqwest",
] }
xai-grok-version = { workspace = true }
# reqwest 0.13 feature set for MCP transports. Notably:
# - `blocking` is for build-script style use (not actually used here today, but
# kept for future MCP transports that may need it).
# - `query`/`form` opt-in features stay off; rmcp transports don't use them.
reqwest = { version = "0.13", default-features = false, features = [
"blocking",
"json",
"rustls",
"stream",
] }
# Deps for credentials.rs + oauth.rs.
async-trait = { workspace = true }
axum = { workspace = true }
oauth2 = { workspace = true }
parking_lot = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }
webbrowser = { workspace = true }
xai-grok-config = { workspace = true }
# For mcp_http_client.rs (rmcp trait signatures). Must match rmcp 2.1's own
# versions or the trait types won't unify.
http = { workspace = true }
sse-stream = "0.2"
# Deps for servers.rs.
agent-client-protocol = { workspace = true }
xai-file-utils = { path = "../xai-file-utils" }
futures = { workspace = true }
regex = { workspace = true }
which = { workspace = true }
# Atomic descriptor writes (NamedTempFile + persist) in materialize_descriptors.
tempfile = { workspace = true }
xai-grok-tools = { workspace = true }
xai-grok-telemetry = { workspace = true }
xai-grok-workspace-types = { workspace = true }
xai-tool-protocol = { workspace = true }
xai-tool-runtime = { workspace = true }
xai-tool-types = { workspace = true }
[target.'cfg(unix)'.dependencies]
libc = { workspace = true }
[dev-dependencies]
# Used in servers.rs test `test_same_raw_name_different_servers_no_local_registry_collision`
# to verify MCP tools register into a `LocalRegistry` without collision.
xai-computer-hub-sdk = { workspace = true }
# `test-util` enables `tokio::time::pause` / `advance`, required by
# the deterministic timing tests in `liveness.rs`.
tokio = { workspace = true, features = ["test-util"] }
[lints]
workspace = true

View file

@ -0,0 +1,735 @@
//! rmcp transport bridge over the ACP reverse channel.
//!
//! In-process SDK MCP servers (the official `grok-agent-sdk`'s `@tool` /
//! `create_sdk_mcp_server`) run in the SDK-host process, not behind a socket. The
//! agent reaches them by sending each MCP JSON-RPC message to the client as a
//! reverse `x.ai/mcp/sdk_call` request and feeding the response back. This module
//! adapts that request/response channel into an rmcp transport so an in-process
//! server reuses the same `RunningService` / tool-dispatch path as HTTP/stdio
//! servers for tool calls.
//!
//! Half-duplex (v1 limitation): the bridge carries ONLY client→server requests and
//! their responses. Server→client traffic is NOT bridged — neither notifications
//! (`notifications/*`) nor server-initiated requests such as
//! `sampling/createMessage`, `roots/list`, or elicitation are delivered. Tools that
//! depend on those features will not work over this transport yet. The duplex
//! plumbing below exists to decouple slow tool calls (one task per request), not to
//! deliver a second message direction.
//!
//! The invoker is abstract ([`AcpReverseInvoker`]) so this crate stays free of the
//! ACP gateway types; the host (shell) supplies an impl backed by its gateway.
use std::sync::Arc;
use std::time::Duration;
use rmcp::service::RoleClient;
use rmcp::transport::async_rw::AsyncRwTransport;
use serde_json::Value;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, DuplexStream};
/// Sends one MCP JSON-RPC message to an in-process server over the ACP reverse
/// channel (`x.ai/mcp/sdk_call`) and returns its JSON-RPC response. The `Err` string is
/// surfaced as a JSON-RPC error to the waiting rmcp request (fail-closed: a missing
/// tool server is a real error, unlike a hook gate).
///
/// `timeout` bounds the single round trip so a missing or hung client fails this
/// reverse call instead of stalling the agent's tool loop forever. It carries the
/// resolved per-server tool timeout (the same `tool_timeout_ms` the HTTP path uses),
/// threaded in from the bridge so zero-IPC and loopback share one tool budget.
#[async_trait::async_trait]
pub trait AcpReverseInvoker: Send + Sync + 'static {
async fn invoke(
&self,
server_id: &str,
message: Value,
timeout: Duration,
) -> Result<Value, String>;
}
/// rmcp transport for an in-process server reached over ACP reverse-RPC.
pub type AcpBridgeTransport = AsyncRwTransport<RoleClient, DuplexStream, DuplexStream>;
/// Duplex buffer for the bridge. MCP messages are small; this only needs to hold
/// one in-flight message comfortably.
const BRIDGE_BUF: usize = 256 * 1024;
/// Bounded capacity for the server→client response channel. The only producers are
/// the in-flight invoke tasks (one per outstanding rmcp request, and rmcp bounds its
/// own in-flight concurrency), so this small buffer gives backpressure/defensiveness
/// without ever realistically blocking a producer.
const RESPONSE_CHANNEL_CAP: usize = 128;
/// JSON-RPC "Internal error" code, used for every error this bridge synthesizes.
const INTERNAL_ERROR_CODE: i64 = -32603;
/// Build an rmcp transport that bridges to an in-process MCP server via `invoker`.
///
/// Spawns a pump that forwards each client→server message as a reverse
/// `x.ai/mcp/sdk_call` and writes the server→client response back. The pump exits when
/// rmcp drops its half of the duplex (service shutdown), so it never leaks.
///
/// `invoke_timeout` is the resolved per-server tool timeout; it bounds every reverse
/// round trip so the zero-IPC path honors the same budget as loopback/HTTP.
pub fn acp_bridge_transport(
server_id: String,
invoker: Arc<dyn AcpReverseInvoker>,
invoke_timeout: Duration,
) -> AcpBridgeTransport {
let (agent_read, pump_write) = tokio::io::duplex(BRIDGE_BUF); // server -> client
let (pump_read, agent_write) = tokio::io::duplex(BRIDGE_BUF); // client -> server
tokio::spawn(pump(
server_id,
invoker,
invoke_timeout,
pump_read,
pump_write,
));
AsyncRwTransport::new(agent_read, agent_write)
}
/// Forward newline-delimited JSON-RPC between rmcp and the reverse channel.
///
/// Each client→server request is invoked in its own task so a slow tool can't block
/// later requests to the same server (JSON-RPC correlates by `id`, not order). All
/// responses funnel through one writer task so their bytes never interleave on the
/// duplex.
async fn pump(
server_id: String,
invoker: Arc<dyn AcpReverseInvoker>,
invoke_timeout: Duration,
client_to_server: DuplexStream,
server_to_client: DuplexStream,
) {
let (responses_tx, responses_rx) = tokio::sync::mpsc::channel::<String>(RESPONSE_CHANNEL_CAP);
let writer = write_responses(server_to_client, responses_rx);
let reader = read_requests(
server_id,
invoker,
invoke_timeout,
client_to_server,
responses_tx,
);
// `writer` then drains and exits once `reader` returns and closes the channel.
tokio::join!(reader, writer);
}
/// Read each client→server line and dispatch its request on a fresh task.
///
/// The spawned tasks live in a [`tokio::task::JoinSet`] owned by this function rather
/// than as detached `tokio::spawn`s, so when this function returns (EOF = teardown)
/// the set is dropped and every still-running invoke is aborted promptly instead of
/// being left to run out its timeout. Finished tasks are reaped (non-blockingly)
/// after each read so the set can't grow unbounded over a long-lived session.
///
/// IMPORTANT: `read_line` is NOT cancellation-safe, so it must never be raced in a
/// `select!`. A client→server message can arrive across multiple `fill_buf` chunks
/// (e.g. a tool call whose JSON args exceed the read buffer); if another `select!`
/// branch (such as reaping a finished invoke) fired while a `read_line` was pending,
/// the partially-consumed bytes would be dropped on the next `line.clear()`,
/// desyncing the JSON-RPC stream and hanging that request to its tool-level timeout.
/// We therefore read each line to completion FIRST, then reap finished invokes with a
/// synchronous, non-cancelling `try_join_next` drain.
async fn read_requests(
server_id: String,
invoker: Arc<dyn AcpReverseInvoker>,
invoke_timeout: Duration,
client_to_server: DuplexStream,
responses_tx: tokio::sync::mpsc::Sender<String>,
) {
let mut reader = BufReader::new(client_to_server);
let mut line = String::new();
let mut invokes: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
loop {
line.clear();
match reader.read_line(&mut line).await {
Ok(0) | Err(_) => break, // rmcp closed its end
Ok(_) => {}
}
// Reap finished invokes so the set stays bounded.
while invokes.try_join_next().is_some() {}
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let message: Value = match serde_json::from_str(trimmed) {
Ok(value) => value,
Err(err) => {
tracing::warn!(%err, "acp mcp bridge: dropping unparseable client message");
continue;
}
};
// An id-less message is a notification (no response). The SDK peer rejects reverse
// `x.ai/mcp/sdk_call`s without a JSON-RPC id, so id-less messages (e.g. rmcp's
// `notifications/initialized` on every handshake) are logged and discarded locally
// rather than spawning a doomed round-trip. Safe only because the SDK `Server` is
// lenient about never receiving `initialized` (a documented v1 limit).
let Some(id) = message.get("id").filter(|id| !id.is_null()).cloned() else {
tracing::debug!(
%message,
"acp mcp bridge: discarding id-less notification (half-duplex v1)"
);
continue;
};
let invoker = invoker.clone();
let server_id = server_id.clone();
let responses_tx = responses_tx.clone();
invokes.spawn(async move {
let result = invoker.invoke(&server_id, message, invoke_timeout).await;
let response = match result {
Ok(response) => with_id(response, id),
Err(err) => json_rpc_error(id, INTERNAL_ERROR_CODE, &err),
};
match serde_json::to_string(&response) {
Ok(mut encoded) => {
encoded.push('\n');
let _ = responses_tx.send(encoded).await;
}
Err(err) => tracing::warn!(%err, "acp mcp bridge: failed to serialize response"),
}
});
}
}
/// Serialize every server→client response onto the duplex through a single writer.
async fn write_responses(
mut server_to_client: DuplexStream,
mut responses_rx: tokio::sync::mpsc::Receiver<String>,
) {
while let Some(encoded) = responses_rx.recv().await {
if server_to_client
.write_all(encoded.as_bytes())
.await
.is_err()
|| server_to_client.flush().await.is_err()
{
break; // rmcp closed its end
}
}
}
/// Overwrite a JSON-RPC response object's `id` with the request id.
///
/// If the SDK response isn't a JSON object (so it has nowhere to carry an `id`),
/// rmcp can't correlate it and the waiting request would otherwise stall until its
/// timeout. In that case synthesize a properly-keyed JSON-RPC error instead, so the
/// waiting request fails fast and correctly.
fn with_id(mut response: Value, id: Value) -> Value {
match response.as_object_mut() {
Some(obj) => {
obj.insert("id".to_string(), id);
response
}
None => json_rpc_error(
id,
INTERNAL_ERROR_CODE,
"acp mcp bridge: server returned a non-object JSON-RPC response",
),
}
}
fn json_rpc_error(id: Value, code: i64, message: &str) -> Value {
serde_json::json!({
"jsonrpc": "2.0",
"id": id,
"error": { "code": code, "message": message },
})
}
#[cfg(test)]
mod tests {
use super::*;
/// Invoker that echoes the request's method back as the result, or fails for a
/// method named "boom".
struct EchoInvoker;
#[async_trait::async_trait]
impl AcpReverseInvoker for EchoInvoker {
async fn invoke(
&self,
_server_id: &str,
message: Value,
_timeout: Duration,
) -> Result<Value, String> {
let method = message.get("method").cloned().unwrap_or(Value::Null);
if method == "boom" {
return Err("server exploded".to_string());
}
let id = message.get("id").cloned().unwrap_or(Value::Null);
Ok(serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": { "method": method } }))
}
}
/// Drive the pump directly (no rmcp): write client→server lines, read back the
/// server→client lines.
fn spawn_pump() -> (DuplexStream, DuplexStream) {
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
tokio::spawn(pump(
"srv".to_string(),
Arc::new(EchoInvoker),
Duration::from_secs(60),
pump_read,
pump_write,
));
(test_write, test_read)
}
async fn read_line(reader: &mut BufReader<DuplexStream>) -> Value {
let mut line = String::new();
reader.read_line(&mut line).await.unwrap();
serde_json::from_str(line.trim()).unwrap()
}
#[tokio::test]
async fn request_gets_a_response_notification_does_not() {
let (mut to_server, from_server) = spawn_pump();
let mut reader = BufReader::new(from_server);
// A notification (no id) must NOT produce a response line...
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n")
.await
.unwrap();
// ...so the first line we read back is the request's response (id 1), proving
// the notification was silently consumed.
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\"}\n")
.await
.unwrap();
let response = read_line(&mut reader).await;
assert_eq!(response["id"], 1);
assert_eq!(response["result"]["method"], "tools/list");
}
/// A slow request must not block a later fast one: the fast response comes back
/// first even though its request was written second (head-of-line free).
#[tokio::test]
async fn a_slow_request_does_not_block_a_later_fast_one() {
struct DelayInvoker;
#[async_trait::async_trait]
impl AcpReverseInvoker for DelayInvoker {
async fn invoke(
&self,
_server_id: &str,
message: Value,
_timeout: Duration,
) -> Result<Value, String> {
let id = message.get("id").cloned().unwrap_or(Value::Null);
// id 1 is slow, id 2 is fast.
if id == 1 {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
Ok(serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": {} }))
}
}
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
tokio::spawn(pump(
"srv".to_string(),
Arc::new(DelayInvoker),
Duration::from_secs(60),
pump_read,
pump_write,
));
let mut to_server = test_write;
let mut reader = BufReader::new(test_read);
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"slow\"}\n")
.await
.unwrap();
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"fast\"}\n")
.await
.unwrap();
// The fast request (id 2) returns before the slow one (id 1).
assert_eq!(read_line(&mut reader).await["id"], 2);
assert_eq!(read_line(&mut reader).await["id"], 1);
}
/// Regression: a chunked request (JSON args exceed the read buffer) must still parse
/// when an in-flight invoke completes mid-read. The pre-fix `select!` reaped the invoke
/// and cleared the partially-read line, desyncing the stream; the cancellation-safe read
/// does not.
#[tokio::test]
async fn chunked_request_survives_an_invoke_completing_mid_read() {
/// id 1 completes after a short delay; everything else returns immediately.
struct DelayInvoker;
#[async_trait::async_trait]
impl AcpReverseInvoker for DelayInvoker {
async fn invoke(
&self,
_server_id: &str,
message: Value,
_timeout: Duration,
) -> Result<Value, String> {
let id = message.get("id").cloned().unwrap_or(Value::Null);
if id == 1 {
tokio::time::sleep(Duration::from_millis(50)).await;
}
Ok(serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": {} }))
}
}
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
tokio::spawn(pump(
"srv".to_string(),
Arc::new(DelayInvoker),
Duration::from_secs(60),
pump_read,
pump_write,
));
let mut to_server = test_write;
let mut reader = BufReader::new(test_read);
// In-flight invoke (id 1): its task will finish ~50ms from now.
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"slow\"}\n")
.await
.unwrap();
// Begin a second request (id 2) but withhold its closing brace + newline, so
// the reader blocks mid-message while id 1's invoke completes.
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":2,")
.await
.unwrap();
to_server.flush().await.unwrap();
// Let id 1's invoke complete *during* the pending chunked read.
tokio::time::sleep(Duration::from_millis(120)).await;
to_server
.write_all(b"\"method\":\"chunked\"}\n")
.await
.unwrap();
to_server.flush().await.unwrap();
// Both responses must arrive (order may vary). Critically, id 2 parsed — no
// desync from the mid-read completion of id 1.
let first = read_line(&mut reader).await;
let second = read_line(&mut reader).await;
let mut ids = [
first["id"].as_i64().unwrap(),
second["id"].as_i64().unwrap(),
];
ids.sort_unstable();
assert_eq!(ids, [1, 2]);
}
#[tokio::test]
async fn invoker_error_becomes_a_json_rpc_error_keyed_to_the_request_id() {
let (mut to_server, from_server) = spawn_pump();
let mut reader = BufReader::new(from_server);
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":7,\"method\":\"boom\"}\n")
.await
.unwrap();
let response = read_line(&mut reader).await;
assert_eq!(response["id"], 7);
assert_eq!(response["error"]["code"], -32603);
assert_eq!(response["error"]["message"], "server exploded");
}
/// A non-object SDK response can't carry an `id`, so rmcp couldn't correlate it.
/// The bridge synthesizes an id-keyed JSON-RPC error so the waiting request fails
/// fast instead of timing out.
#[tokio::test]
async fn non_object_response_becomes_a_json_rpc_error_keyed_to_the_request_id() {
/// Returns a JSON array (not an object) as its "response".
struct NonObjectInvoker;
#[async_trait::async_trait]
impl AcpReverseInvoker for NonObjectInvoker {
async fn invoke(
&self,
_server_id: &str,
_message: Value,
_timeout: Duration,
) -> Result<Value, String> {
Ok(serde_json::json!([1, 2, 3]))
}
}
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
tokio::spawn(pump(
"srv".to_string(),
Arc::new(NonObjectInvoker),
Duration::from_secs(60),
pump_read,
pump_write,
));
let mut to_server = test_write;
let mut reader = BufReader::new(test_read);
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":9,\"method\":\"tools/list\"}\n")
.await
.unwrap();
let response = read_line(&mut reader).await;
assert_eq!(response["id"], 9);
assert_eq!(response["error"]["code"], -32603);
}
/// The configured per-server timeout (not a hardcoded constant) must reach the
/// invoker for every reverse call, so the zero-IPC path can't silently shrink a
/// long tool's budget.
#[tokio::test]
async fn pump_forwards_the_configured_timeout_to_the_invoker() {
use std::sync::Mutex;
/// Records the timeout it was invoked with so the test can assert on it.
struct RecordingInvoker(Arc<Mutex<Option<Duration>>>);
#[async_trait::async_trait]
impl AcpReverseInvoker for RecordingInvoker {
async fn invoke(
&self,
_server_id: &str,
message: Value,
timeout: Duration,
) -> Result<Value, String> {
*self.0.lock().unwrap() = Some(timeout);
let id = message.get("id").cloned().unwrap_or(Value::Null);
Ok(serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": {} }))
}
}
let seen = Arc::new(Mutex::new(None));
let configured = Duration::from_secs(4242);
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
tokio::spawn(pump(
"srv".to_string(),
Arc::new(RecordingInvoker(seen.clone())),
configured,
pump_read,
pump_write,
));
let mut to_server = test_write;
let mut reader = BufReader::new(test_read);
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\"}\n")
.await
.unwrap();
// Wait for the response so the invoke has definitely run.
assert_eq!(read_line(&mut reader).await["id"], 1);
assert_eq!(*seen.lock().unwrap(), Some(configured));
}
/// Teardown (rmcp dropping the duplex) must ABORT an in-flight invoke promptly,
/// not wait out its timeout. We send a request whose invoke sleeps far longer than
/// the test budget, tear the transport down, and assert the pump self-terminates
/// quickly while the invoke neither completes nor lingers.
#[tokio::test]
async fn teardown_aborts_in_flight_invokes() {
use std::sync::atomic::{AtomicBool, Ordering};
/// Flips a flag when the invoke future is dropped (i.e. aborted).
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
struct SlowInvoker {
started: Arc<AtomicBool>,
completed: Arc<AtomicBool>,
dropped: Arc<AtomicBool>,
}
#[async_trait::async_trait]
impl AcpReverseInvoker for SlowInvoker {
async fn invoke(
&self,
_server_id: &str,
_message: Value,
_timeout: Duration,
) -> Result<Value, String> {
let _drop_flag = DropFlag(self.dropped.clone());
self.started.store(true, Ordering::SeqCst);
// Far longer than the per-call timeout AND the test's wait budget, so a
// "completed" or "timed out" outcome can only mean it wasn't aborted.
tokio::time::sleep(Duration::from_secs(3600)).await;
self.completed.store(true, Ordering::SeqCst);
Ok(Value::Null)
}
}
let started = Arc::new(AtomicBool::new(false));
let completed = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
let (test_write, pump_read) = tokio::io::duplex(BRIDGE_BUF);
let (pump_write, test_read) = tokio::io::duplex(BRIDGE_BUF);
let pump_handle = tokio::spawn(pump(
"srv".to_string(),
Arc::new(SlowInvoker {
started: started.clone(),
completed: completed.clone(),
dropped: dropped.clone(),
}),
Duration::from_secs(600),
pump_read,
pump_write,
));
let mut to_server = test_write;
to_server
.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"slow\"}\n")
.await
.unwrap();
// Wait until the invoke has actually started before tearing down.
for _ in 0..200 {
if started.load(Ordering::SeqCst) {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
assert!(started.load(Ordering::SeqCst), "invoke should have started");
// Tear down: dropping both client ends closes the duplex, exactly as rmcp does
// on shutdown.
drop(to_server);
drop(test_read);
// The pump must self-terminate well within the (600s) invoke timeout, proving
// the in-flight invoke was aborted rather than awaited.
tokio::time::timeout(Duration::from_secs(5), pump_handle)
.await
.expect("pump should self-terminate promptly after teardown")
.unwrap();
// The aborted invoke future must be dropped (abort is async, so poll briefly)
// and must never have run to completion.
for _ in 0..200 {
if dropped.load(Ordering::SeqCst) {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
assert!(
dropped.load(Ordering::SeqCst),
"aborted invoke future should be dropped"
);
assert!(
!completed.load(Ordering::SeqCst),
"aborted invoke must not run to completion"
);
}
/// A mock SDK MCP **server** behind the reverse channel. It speaks just enough
/// real MCP to satisfy an rmcp client: the `initialize` handshake, a `tools/list`
/// advertising one `echo` tool, and a `tools/call` that echoes its text argument.
/// Each `invoke` receives one JSON-RPC request and returns one JSON-RPC response
/// (the bridge overwrites the `id`), mirroring the real on-wire shapes.
struct MockSdkServer;
#[async_trait::async_trait]
impl AcpReverseInvoker for MockSdkServer {
async fn invoke(
&self,
_server_id: &str,
message: Value,
_timeout: Duration,
) -> Result<Value, String> {
let id = message.get("id").cloned().unwrap_or(Value::Null);
let method = message
.get("method")
.and_then(|m| m.as_str())
.unwrap_or_default();
let result = match method {
"initialize" => serde_json::json!({
// Echo the client's protocol version so the handshake is always compatible.
"protocolVersion": message["params"]["protocolVersion"],
"capabilities": { "tools": {} },
"serverInfo": { "name": "mock-sdk-server", "version": "0.0.0" },
}),
"tools/list" => serde_json::json!({
"tools": [{
"name": "echo",
"description": "Echoes its text argument back.",
"inputSchema": {
"type": "object",
"properties": { "text": { "type": "string" } },
"required": ["text"],
},
}],
}),
"tools/call" => {
let text = message["params"]["arguments"]["text"]
.as_str()
.unwrap_or_default();
serde_json::json!({
"content": [{ "type": "text", "text": text }],
"isError": false,
})
}
other => return Err(format!("mock SDK server: unexpected method {other}")),
};
Ok(serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": result }))
}
}
/// End-to-end: drive a REAL `rmcp` client (`RunningService<RoleClient, _>`) through
/// `acp_bridge_transport` against [`MockSdkServer`], proving the bridge speaks real
/// MCP — the full `initialize` handshake (including rmcp's id-less
/// `notifications/initialized`, which the bridge discards), `tools/list`, and
/// `tools/call` — then a clean cancel/teardown. This is the same client path
/// production uses in `servers.rs` (`client.serve(transport)`).
#[tokio::test]
async fn real_rmcp_client_handshakes_lists_and_calls_over_the_bridge() {
use rmcp::ServiceExt;
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams};
let transport = acp_bridge_transport(
"srv".to_string(),
Arc::new(MockSdkServer),
Duration::from_secs(60),
);
// `()` is rmcp's minimal `ClientHandler`; `serve` runs the real initialize
// handshake over our bridge transport and yields a live `RunningService`.
let client =
().serve(transport)
.await
.expect("rmcp handshake over the bridge should succeed");
let tools = client
.list_tools(Some(PaginatedRequestParams::default()))
.await
.expect("tools/list over the bridge");
assert_eq!(tools.tools.len(), 1);
assert_eq!(tools.tools[0].name.as_ref(), "echo");
let result = client
.call_tool(
CallToolRequestParams::new("echo").with_arguments(
serde_json::json!({ "text": "hello bridge" })
.as_object()
.cloned()
.expect("arguments object"),
),
)
.await
.expect("tools/call over the bridge");
let text = result.content[0]
.as_text()
.expect("text content")
.text
.clone();
assert_eq!(text, "hello bridge");
// Clean teardown: cancelling drops rmcp's duplex end; the pump observes EOF and
// self-terminates (the abort mechanics are covered by
// `teardown_aborts_in_flight_invokes`).
client.cancel().await.expect("clean teardown");
}
}

View file

@ -0,0 +1,435 @@
//! Persistent credential storage for MCP server OAuth tokens.
//!
//! Credentials are stored in `$GROK_HOME/mcp_credentials.json`, keyed by a
//! composite key derived from the server name and URL. This keeps MCP OAuth
//! tokens isolated from the user's xAI auth (`auth.json`).
//!
//! Stores rmcp's `StoredCredentials` type directly — the same type that
//! rmcp's `AuthorizationManager` uses internally.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use url::Url;
use crate::rmcp;
type Result<T> = std::result::Result<T, McpCredentialError>;
#[derive(Debug, thiserror::Error)]
pub enum McpCredentialError {
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("{0}")]
Other(String),
}
/// File name for the credential store inside `$GROK_HOME`.
const CREDENTIALS_FILENAME: &str = "mcp_credentials.json";
/// On-disk credential store: `$GROK_HOME/mcp_credentials.json`.
///
/// Stores rmcp `StoredCredentials` per MCP server, keyed by
/// `"{server_name}:{server_url}"`.
#[derive(Clone, Serialize, Deserialize, Default)]
pub struct McpCredentialStore {
#[serde(flatten)]
entries: BTreeMap<String, rmcp::transport::auth::StoredCredentials>,
}
impl std::fmt::Debug for McpCredentialStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("McpCredentialStore")
.field("entry_count", &self.entries.len())
.finish()
}
}
impl McpCredentialStore {
/// Build the composite key for a credential entry.
pub fn key(server_name: &str, server_url: &Url) -> String {
format!("{}:{}", server_name, server_url)
}
/// Load the credential store from the default path (`$GROK_HOME/mcp_credentials.json`).
///
/// Returns an empty store if the file does not exist.
pub fn load_default() -> Result<Self> {
match Self::default_path() {
Some(path) => Self::load_from(&path),
None => Ok(Self::default()),
}
}
/// Load from a specific path.
pub fn load_from(path: &Path) -> Result<Self> {
if !path.exists() {
return Ok(Self::default());
}
let content = std::fs::read_to_string(path)?;
let store: McpCredentialStore = serde_json::from_str(&content)?;
Ok(store)
}
/// Save the credential store to the default path.
pub fn save_default(&self) -> Result<()> {
let path = Self::default_path().ok_or_else(|| {
McpCredentialError::Other("no user grok home (set $GROK_HOME or $HOME)".into())
})?;
self.save_to(&path)
}
/// Atomically insert a credential and save — safe for concurrent use.
///
/// Instead of the caller doing `insert_rmcp` + `save_default` (which races
/// with other processes), this method:
/// 1. Acquires a file lock on `mcp_credentials.json.lock`
/// 2. Reloads the store from disk (picks up other processes' writes)
/// 3. Inserts the new entry
/// 4. Saves atomically (temp + rename)
/// 5. Updates `self` with the merged result
/// 6. Releases the lock
pub fn insert_and_save(
&mut self,
server_name: &str,
server_url: &url::Url,
creds: rmcp::transport::auth::StoredCredentials,
) -> Result<()> {
let path = Self::default_path().ok_or_else(|| {
McpCredentialError::Other("no user grok home (set $GROK_HOME or $HOME)".into())
})?;
let lock_path = path.with_extension("lock");
// Ensure parent dir exists.
if let Some(parent) = lock_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
#[cfg(unix)]
{
use std::os::unix::io::AsRawFd;
let lock_file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&lock_path)?;
let fd = lock_file.as_raw_fd();
loop {
if unsafe { libc::flock(fd, libc::LOCK_EX) } == 0 {
break;
}
let err = std::io::Error::last_os_error();
if err.kind() == std::io::ErrorKind::Interrupted {
continue; // Retry on EINTR.
}
// Lock failed for another reason — fall back to non-atomic insert.
self.insert_rmcp(server_name, server_url, creds);
return self.save_to(&path);
}
// Reload from disk under lock to merge with concurrent writes.
let mut fresh = Self::load_from(&path).unwrap_or_default();
fresh.insert_rmcp(server_name, server_url, creds);
fresh.save_to(&path)?;
*self = fresh;
// Lock released when lock_file is dropped.
}
#[cfg(not(unix))]
{
// No flock on non-unix — best-effort.
self.insert_rmcp(server_name, server_url, creds);
self.save_to(&path)?;
}
Ok(())
}
/// Save to a specific path.
///
/// Writes atomically via temp file + rename to prevent credential loss on
/// crash. On Unix, the temp file is created with 0600 permissions from the
/// start (no TOCTOU window where secrets are world-readable).
pub fn save_to(&self, path: &Path) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let content = serde_json::to_string_pretty(self)?;
let tmp_path = path.with_extension("tmp");
{
use std::io::Write;
#[cfg(unix)]
let file = {
use std::os::unix::fs::OpenOptionsExt;
std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(&tmp_path)?
};
#[cfg(not(unix))]
let file = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&tmp_path)?;
let mut writer = std::io::BufWriter::new(file);
writer.write_all(content.as_bytes())?;
writer.flush()?;
}
std::fs::rename(&tmp_path, path)?;
Ok(())
}
/// Look up credentials for a server.
pub fn get(
&self,
server_name: &str,
server_url: &Url,
) -> Option<&rmcp::transport::auth::StoredCredentials> {
self.entries.get(&Self::key(server_name, server_url))
}
/// Insert rmcp `StoredCredentials` for a server.
pub fn insert_rmcp(
&mut self,
server_name: &str,
server_url: &Url,
creds: rmcp::transport::auth::StoredCredentials,
) {
self.entries
.insert(Self::key(server_name, server_url), creds);
}
/// Check if credentials exist for a server (regardless of expiry).
pub fn has_credentials(&self, server_name: &str, server_url: &Url) -> bool {
self.entries
.contains_key(&Self::key(server_name, server_url))
}
/// Remove credentials for a server.
pub fn remove(&mut self, server_name: &str, server_url: &Url) {
self.entries.remove(&Self::key(server_name, server_url));
}
/// Remove all credentials for a server by name (any URL).
pub fn remove_by_server_name(&mut self, server_name: &str) -> usize {
let prefix = format!("{server_name}:");
let before = self.entries.len();
self.entries.retain(|k, _| !k.starts_with(&prefix));
before - self.entries.len()
}
/// Whether the store is empty.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Default path: `$GROK_HOME/mcp_credentials.json`.
fn default_path() -> Option<PathBuf> {
Some(xai_grok_config::user_grok_home()?.join(CREDENTIALS_FILENAME))
}
}
/// Adapter implementing rmcp's `CredentialStore` trait backed by the on-disk
/// `McpCredentialStore`. Each adapter instance is scoped to a single MCP server
/// (keyed by name + URL); rmcp's `AuthorizationManager` calls load/save/clear
/// transparently during token exchange and refresh.
pub struct McpCredentialStoreAdapter {
server_name: String,
server_url: url::Url,
}
impl McpCredentialStoreAdapter {
pub fn new(server_name: String, server_url: url::Url) -> Self {
Self {
server_name,
server_url,
}
}
}
#[async_trait::async_trait]
impl rmcp::transport::auth::CredentialStore for McpCredentialStoreAdapter {
async fn load(
&self,
) -> std::result::Result<
Option<rmcp::transport::auth::StoredCredentials>,
rmcp::transport::auth::AuthError,
> {
let name = self.server_name.clone();
let url = self.server_url.clone();
tokio::task::spawn_blocking(move || {
let store = McpCredentialStore::load_default()
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))?;
Ok(store.get(&name, &url).cloned())
})
.await
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))?
}
async fn save(
&self,
credentials: rmcp::transport::auth::StoredCredentials,
) -> std::result::Result<(), rmcp::transport::auth::AuthError> {
let name = self.server_name.clone();
let url = self.server_url.clone();
tokio::task::spawn_blocking(move || {
let mut store = McpCredentialStore::load_default().unwrap_or_default();
store
.insert_and_save(&name, &url, credentials)
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))
})
.await
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))?
}
async fn clear(&self) -> std::result::Result<(), rmcp::transport::auth::AuthError> {
let name = self.server_name.clone();
let url = self.server_url.clone();
tokio::task::spawn_blocking(move || {
let mut store = McpCredentialStore::load_default().unwrap_or_default();
store.remove(&name, &url);
store
.save_default()
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))
})
.await
.map_err(|e| rmcp::transport::auth::AuthError::InternalError(e.to_string()))?
}
}
#[cfg(test)]
mod tests {
use super::*;
fn test_stored_creds(client_id: &str) -> rmcp::transport::auth::StoredCredentials {
rmcp::transport::auth::StoredCredentials::new(client_id.to_string(), None, Vec::new(), None)
}
#[test]
fn insert_and_get() {
let mut store = McpCredentialStore::default();
let url = Url::parse("https://test.example.com/mcp").unwrap();
store.insert_rmcp("test", &url, test_stored_creds("test-client"));
assert!(store.get("test", &url).is_some());
assert_eq!(store.get("test", &url).unwrap().client_id, "test-client");
}
#[test]
fn remove_entry() {
let mut store = McpCredentialStore::default();
let url = Url::parse("https://test.example.com/mcp").unwrap();
store.insert_rmcp("test", &url, test_stored_creds("test-client"));
store.remove("test", &url);
assert!(store.get("test", &url).is_none());
}
#[test]
fn has_credentials() {
let mut store = McpCredentialStore::default();
let url = Url::parse("https://test.example.com/mcp").unwrap();
assert!(!store.has_credentials("test", &url));
store.insert_rmcp("test", &url, test_stored_creds("c"));
assert!(store.has_credentials("test", &url));
}
#[test]
fn roundtrip_serialization() {
let mut store = McpCredentialStore::default();
let url = Url::parse("https://test.example.com/mcp").unwrap();
store.insert_rmcp("test", &url, test_stored_creds("test-client"));
let json = serde_json::to_string(&store).unwrap();
let loaded: McpCredentialStore = serde_json::from_str(&json).unwrap();
assert!(loaded.get("test", &url).is_some());
}
/// Raw JSON fixture in the exact shape rmcp 0.17 persisted to
/// `$GROK_HOME/mcp_credentials.json`. Existing credential files must keep
/// loading across rmcp upgrades (2.1's `OAuthTokenResponse` gained vendor
/// extra token fields), so this must be a string literal — never JSON
/// serialized by the current code.
#[test]
fn legacy_on_disk_fixture_still_deserializes() {
use oauth2::TokenResponse as _;
let fixture = r#"{
"linear:https://mcp.example.com/mcp": {
"client_id": "legacy-client-id",
"token_response": {
"access_token": "at-123",
"token_type": "bearer",
"expires_in": 3600,
"refresh_token": "rt-456",
"scope": "read write"
},
"granted_scopes": ["read", "write"],
"token_received_at": 1730000000
},
"noauth:https://example.com/mcp": {
"client_id": "c2",
"token_response": null
}
}"#;
let store: McpCredentialStore = serde_json::from_str(fixture).unwrap();
let url = Url::parse("https://mcp.example.com/mcp").unwrap();
let creds = store.get("linear", &url).expect("legacy entry loads");
assert_eq!(creds.client_id, "legacy-client-id");
let token = creds.token_response.as_ref().expect("token loads");
assert_eq!(token.access_token().secret(), "at-123");
assert_eq!(token.refresh_token().unwrap().secret(), "rt-456");
assert_eq!(creds.granted_scopes, vec!["read", "write"]);
assert_eq!(creds.token_received_at, Some(1730000000));
// Entry without the `#[serde(default)]` fields on disk still loads.
let url2 = Url::parse("https://example.com/mcp").unwrap();
let creds2 = store.get("noauth", &url2).expect("minimal entry loads");
assert!(creds2.token_response.is_none());
assert!(creds2.granted_scopes.is_empty());
assert!(creds2.token_received_at.is_none());
// Round-trip through the current serializer and reload.
let json = serde_json::to_string(&store).unwrap();
let reloaded: McpCredentialStore = serde_json::from_str(&json).unwrap();
let re = reloaded
.get("linear", &url)
.expect("round-trip keeps entry");
assert_eq!(re.client_id, "legacy-client-id");
let re_token = re.token_response.as_ref().expect("round-trip keeps token");
assert_eq!(re_token.access_token().secret(), "at-123");
assert_eq!(re_token.refresh_token().unwrap().secret(), "rt-456");
assert_eq!(re.granted_scopes, vec!["read", "write"]);
assert_eq!(re.token_received_at, Some(1730000000));
}
#[test]
fn save_and_load_from_file() {
let dir = std::env::temp_dir().join("grok-mcp-credentials-test");
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("test_creds.json");
let mut store = McpCredentialStore::default();
let url = Url::parse("https://test.example.com/mcp").unwrap();
store.insert_rmcp("test", &url, test_stored_creds("test-client"));
store.save_to(&path).unwrap();
let loaded = McpCredentialStore::load_from(&path).unwrap();
assert!(loaded.get("test", &url).is_some());
let _ = std::fs::remove_file(&path);
let _ = std::fs::remove_dir(&dir);
}
}

View file

@ -0,0 +1,38 @@
//! MCP integration crate.
//!
//! Two responsibilities:
//!
//! 1. **Quarantines `rmcp` 2.1 and `reqwest` 0.13.** `rmcp` 2.1 requires
//! `reqwest >= 0.13.2`. The rest of the workspace consumes `reqwest` 0.12
//! and a transitive ecosystem (`opentelemetry-otlp`, `oauth2`,
//! `xai-mixpanel`, `xai-grok-tools`, ...) also pinned to 0.12. Bumping every
//! crate to 0.13 to satisfy `rmcp` triggers a cascade — an OpenTelemetry
//! `HttpClient` adapter and cross-version test breakage when a crate
//! carries both versions under a renamed `package = "reqwest"` alias.
//! reqwest 0.13 is now a fully private impl detail of [`servers`]; no
//! re-export. Consumers reach `rmcp` model types through this namespace
//! (`xai_grok_mcp::rmcp::*`).
//!
//! 2. **Owns MCP-specific integration code**:
//! - [`credentials`] -- on-disk `$GROK_HOME/mcp_credentials.json` store and
//! the rmcp `CredentialStore` adapter.
//! - [`oauth`] -- browser-based OAuth flow with cross-process + in-process
//! dedup.
//! - [`oauth_config`] -- BYO OAuth config types parsed out of `config.toml`.
//! - [`servers`] -- MCP transport layer (rmcp's `StreamableHttpClientTransport`
//! and `TokioChildProcess`) plus client lifecycle, tool invocation, error
//! classification, and managed-MCP refresh.
//! - [`mcp_http_client`] -- backoff wrapper around the HTTP client handed to
//! rmcp's streamable-HTTP transport (works around rmcp's zero-backoff SSE
//! reconnect loop).
pub use rmcp;
pub mod acp_transport;
pub mod credentials;
pub mod liveness;
pub mod mcp_http_client;
pub mod oauth;
pub mod oauth_config;
pub mod servers;
pub mod wire;

View file

@ -0,0 +1,354 @@
//! Per-`Ready`-client transport-closed poller.
//!
//! Each successful handshake spawns one [`TransportLivenessHandle`]
//! that polls the owning [`McpClient`]'s state machine on a small
//! interval (default 500 ms). On the **first observation of
//! `Ready` + `is_transport_closed() == true`** it emits a single
//! [`McpClientEvent::TransportClosed`] and exits.
//!
//! The poller is *one-shot*. The session-side dispatcher decides
//! what to do with the event — drop the dead client, surface
//! `unavailable` over ACP, and trigger a restart on a debounce.
//!
//! ## Watcher state machine
//!
//! Per-tick classification (single state-mutex acquisition via
//! [`McpClient::liveness_check`]):
//!
//! | State observed | Action | Emit? |
//! |-------------------------------|-------------------|------------------------|
//! | `Ready` + transport open | continue polling | no |
//! | `Ready` + transport closed | clear slot, exit | `TransportClosed` |
//! | `Initializing` (re-handshake) | clear slot, exit | no — silent withdrawal |
//! | `Pending` | clear slot, exit | no — silent withdrawal |
//! | `Empty` | clear slot, exit | no — silent withdrawal |
//!
//! This avoids the previous false-positive `TransportClosed` whenever
//! someone called `reset_transport()` or any other code path
//! moved the state away from `Ready`.
//!
//! ## Slot-clearing on exit
//!
//! Before exiting, the task clears
//! [`McpClient::liveness_handle`] so a subsequent
//! [`McpClient::arm_liveness_watcher`] call can install a fresh
//! handle. Without this, a dead-but-still-present
//! [`TransportLivenessHandle`] would silently block re-arming.
//!
//! ## Cancellation
//!
//! Dropping the [`TransportLivenessHandle`] cancels the spawned
//! task via [`tokio_util::sync::DropGuard`]. Both teardown paths
//! (slot-clear-from-inside, external drop) end with the same
//! handle-drop semantics.
//!
//! ## Why polling, not a `JoinHandle`-on-the-service-loop?
//!
//! rmcp 2.1's `RunningService` does not expose a future that
//! resolves on transport shutdown. The closest signal is
//! `Peer::is_transport_closed()` (a state inspection), which is the
//! same one [`McpClient::is_healthy`] reads. A `select!` on a
//! per-client `Notify` would require patching rmcp; polling avoids
//! that quarantine break and the overhead is negligible (one mutex
//! acquire + one atomic load per tick).
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc::UnboundedSender;
use tokio_util::sync::{CancellationToken, DropGuard};
use crate::servers::{LivenessCheck, McpClient, McpClientEvent, McpServerName};
/// Default poll interval. Picked to keep mean detection latency
/// under one second while polling is cheap (`Mutex::lock` +
/// `tokio::sync::mpsc::is_closed`). See module doc.
pub const DEFAULT_POLL_INTERVAL: Duration = Duration::from_millis(500);
/// Shared liveness-handle slot type — same Arc lives on the
/// [`McpClient`] and is passed into the polling task so the task
/// can clear the slot before exiting. Kept private to the crate to
/// discourage external mutation.
pub(crate) type SharedLivenessSlot = Arc<parking_lot::Mutex<Option<TransportLivenessHandle>>>;
/// Release the client's liveness slot, dropping any handle it held.
///
/// Both watcher-exit arms (transport closed, transient state drift)
/// clear the slot so a later [`McpClient::arm_liveness_watcher`] can
/// install a fresh handle. The taken handle is dropped outside the
/// critical section — the lock is held for nanoseconds.
fn clear_liveness_slot(slot: &SharedLivenessSlot) {
let stale_handle = slot.lock().take();
drop(stale_handle);
}
/// RAII handle for the per-client liveness task.
///
/// Drop semantics: drop → `DropGuard` cancels the `CancellationToken`
/// → the polling task wakes from `select!` on the next tick and
/// exits cleanly without emitting. There is no public `abort()` /
/// `stop()` — the contract is "tie the handle to the client".
pub struct TransportLivenessHandle {
/// Name of the server this handle is watching. Exposed for
/// diagnostics / log lines.
pub server_name: McpServerName,
/// On drop, cancels the spawned task. Field is held purely for
/// its `Drop`; never read.
_cancel: DropGuard,
}
impl std::fmt::Debug for TransportLivenessHandle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TransportLivenessHandle")
.field("server_name", &self.server_name)
.finish()
}
}
impl TransportLivenessHandle {
pub fn server_name(&self) -> &str {
&self.server_name
}
}
/// Spawn a one-shot transport-liveness poller for a `Ready` client.
///
/// # Parameters
///
/// - `server_name`: bound to emitted events.
/// - `client`: `Arc<McpClient>` whose `liveness_check` we poll.
/// - `poll_interval`: tick period.
/// - `on_event`: sink for `TransportClosed` if observed.
/// - `liveness_slot`: shared Arc to the owning `McpClient`'s
/// `liveness_handle` field. Cleared from inside the task before
/// exit.
///
/// # Contract
///
/// - Caller MUST have already observed the client transition to
/// [`crate::servers::ClientStateKind::Ready`]
/// ([`McpClient::arm_liveness_watcher`] enforces this).
/// - The poller exits silently on transient non-`Ready` states; only
/// `Ready` + closed transport produces an event.
/// - The send may fail if the dispatcher has dropped its receiver
/// (subagent teardown, session shutdown). That's logged at debug
/// and the task exits — there's no retry.
///
/// # Why `tokio::time::interval` and not `sleep_until`
///
/// `interval` ticks immediately on first poll, which gives us
/// instant detection of "the transport was already closed when the
/// handle was spawned" — a real failure mode if a handshake races a
/// shutdown event from the server (e.g. Ctrl+C against an stdio
/// server that died between `Ready` write and the spawn). The
/// `MissedTickBehavior::Skip` default is fine: the worst case under
/// a runtime stall is "we don't poll for a while", which only
/// delays detection.
pub fn spawn_transport_liveness(
server_name: McpServerName,
client: Arc<McpClient>,
poll_interval: Duration,
on_event: UnboundedSender<McpClientEvent>,
liveness_slot: SharedLivenessSlot,
) -> TransportLivenessHandle {
let token = CancellationToken::new();
let drop_guard = token.clone().drop_guard();
let server_name_for_task = server_name.clone();
tokio::spawn(async move {
let mut tick = tokio::time::interval(poll_interval);
// Skip missed ticks under runtime stall — see fn doc.
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = token.cancelled() => {
// Cancelled by the handle's `DropGuard`. The
// caller dropped the handle (e.g. McpClient
// teardown), so the slot has already been
// mutated externally — do not race the dropper
// by clearing the slot here.
tracing::trace!(
server = %server_name_for_task,
"transport liveness watcher cancelled by handle drop",
);
return;
}
_ = tick.tick() => {
match client.liveness_check().await {
LivenessCheck::Healthy => continue,
LivenessCheck::TransportClosed => {
tracing::info!(
server = %server_name_for_task,
"transport liveness watcher detected closed transport",
);
// Clear our own slot before exiting so a
// subsequent `arm_liveness_watcher` can
// install a fresh handle.
//
// Self-cancel-by-drop: clearing the slot
// drops the taken `TransportLivenessHandle`,
// whose `DropGuard` cancels the very
// `CancellationToken` this task is
// `select!`ing on. Benign because we
// `return` immediately — but DO NOT add any
// post-`return` work that re-enters the
// `select!`; it would race this self-cancel.
clear_liveness_slot(&liveness_slot);
if on_event
.send(McpClientEvent::TransportClosed {
server: server_name_for_task.clone(),
// Bind the event to THIS client
// instance so the dispatcher can
// skip evicting a replacement
// registered under the same name.
client_id: client.client_id(),
})
.is_err()
{
tracing::debug!(
server = %server_name_for_task,
"dispatcher receiver dropped; liveness watcher exiting silently",
);
}
return;
}
LivenessCheck::Transient => {
// State moved out of `Ready` (re-handshake
// started, or the transport was reset
// externally). The watcher detects
// *transport closure*, not state changes,
// so exit silently; the caller re-arms a
// fresh watcher when the new handshake
// completes.
tracing::debug!(
server = %server_name_for_task,
"transport liveness watcher: state drifted out of Ready, exiting silently",
);
clear_liveness_slot(&liveness_slot);
return;
}
}
}
}
}
});
TransportLivenessHandle {
server_name,
_cancel: drop_guard,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::servers::McpClient;
use tokio::sync::mpsc::unbounded_channel;
/// Stub client whose `liveness_check()` returns
/// `LivenessCheck::Transient`: `McpClient::stub` lands in
/// `ClientState::Empty`, which the liveness classifier treats as a
/// silent-withdrawal state (NOT `TransportClosed`).
fn make_stub_client() -> Arc<McpClient> {
Arc::new(McpClient::stub("test-server"))
}
/// Contract: a watcher whose owning client never reaches
/// `Ready+closed` (here the stub is `Empty`) exits **silently**
/// — no `TransportClosed` event, and the slot is cleared.
/// The watcher must not false-positive on non-`Ready` states.
#[tokio::test(start_paused = true)]
async fn poller_silent_exit_on_non_ready_state() {
let (tx, mut rx) = unbounded_channel::<McpClientEvent>();
let slot: SharedLivenessSlot = Arc::new(parking_lot::Mutex::new(None));
let client = make_stub_client();
let handle = spawn_transport_liveness(
"test-server".to_string(),
client,
Duration::from_millis(500),
tx,
Arc::clone(&slot),
);
// Pre-populate the slot so we can assert the watcher
// clears it on exit.
*slot.lock() = Some(handle);
// First `interval.tick()` fires immediately under paused
// time. The watcher classifies `Empty` as `Transient` and
// exits silently.
tokio::time::advance(Duration::from_millis(10)).await;
tokio::task::yield_now().await;
// No event emitted: the watcher exited silently.
assert!(
rx.try_recv().is_err(),
"non-Ready states must not produce TransportClosed",
);
// Slot is cleared so re-arming wouldn't be blocked.
assert!(
slot.lock().is_none(),
"watcher must clear its own slot on exit",
);
}
/// Contract: when the watcher emits `TransportClosed` it both
/// (a) sends the event and (b) clears the shared liveness
/// slot so the next `arm_liveness_watcher` succeeds.
///
/// We exercise this by constructing a client that *would*
/// classify as `Ready + closed` — but `McpClient::stub` is
/// `Empty`, which classifies as `Transient`, so this test
/// instead asserts the silent-exit path. The Ready+closed
/// path is covered by the integration test in `servers.rs`.
#[tokio::test(start_paused = true)]
async fn poller_clears_slot_on_exit() {
let (tx, mut rx) = unbounded_channel::<McpClientEvent>();
let slot: SharedLivenessSlot = Arc::new(parking_lot::Mutex::new(None));
let client = make_stub_client();
let handle = spawn_transport_liveness(
"test-server".to_string(),
client,
Duration::from_millis(500),
tx,
Arc::clone(&slot),
);
*slot.lock() = Some(handle);
tokio::time::advance(Duration::from_millis(10)).await;
tokio::task::yield_now().await;
// Advancing several intervals confirms the watcher exited
// (not just stuck in a loop without progress).
tokio::time::advance(Duration::from_secs(5)).await;
tokio::task::yield_now().await;
assert!(rx.try_recv().is_err());
assert!(
slot.lock().is_none(),
"slot must be cleared even on the silent-exit path",
);
}
/// Contract: dropping the handle stops the task without
/// emitting. Drops happen via the external `DropGuard` path,
/// distinct from the in-task slot-clear path tested above.
#[tokio::test(start_paused = true)]
async fn drop_cancels_task_before_first_tick() {
let (tx, mut rx) = unbounded_channel::<McpClientEvent>();
let slot: SharedLivenessSlot = Arc::new(parking_lot::Mutex::new(None));
let client = make_stub_client();
let handle = spawn_transport_liveness(
"test-server".to_string(),
client,
Duration::from_secs(60), // Long interval so the first tick is far away.
tx,
Arc::clone(&slot),
);
// Drop before the tick can fire — the `DropGuard` arm
// wins the `select!`.
drop(handle);
tokio::task::yield_now().await;
assert!(rx.try_recv().is_err());
}
}

View file

@ -0,0 +1,497 @@
//! MCP HTTP client wrapper that throttles SSE reconnects with exponential
//! backoff, working around rmcp's zero-backoff reconnect loop: when an
//! established SSE stream errors, rmcp re-issues the `GET` immediately with
//! its retry counter reset to 0, never consulting its `SseRetryPolicy`
//! (only connect failures and graceful EOF consult it). We ship rmcp 2.1;
//! still unfixed upstream as of rmcp 2.1.0:
//! <https://github.com/modelcontextprotocol/rust-sdk/blob/rmcp-v2.1.0/crates/rmcp/src/transport/common/client_side_sse.rs#L250-L261>
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use futures::stream::BoxStream;
use http::{HeaderName, HeaderValue};
use rmcp::model::ClientJsonRpcMessage;
use rmcp::transport::streamable_http_client::{
StreamableHttpClient, StreamableHttpError, StreamableHttpPostResponse,
};
use sse_stream::{Error as SseError, Sse};
/// A stream that survived this long is healthy and resets the backoff. Flood
/// lifetimes are sub-millisecond; healthy proxies/LBs recycle idle streams no
/// faster than ~25s.
const STABLE_STREAM_THRESHOLD: Duration = Duration::from_secs(2);
/// Delay for the n-th consecutive rapid death: `BASE_DELAY * 2^(n-2)`
/// (the first reconnects immediately), capped at [`MAX_DELAY`].
const BASE_DELAY: Duration = Duration::from_millis(500);
/// Caps a broken server's cost at ~2 attempts/min; a healed server gets its
/// stream back within 30s.
const MAX_DELAY: Duration = Duration::from_secs(30);
const WARN_COOLDOWN: Duration = Duration::from_secs(60 * 60);
/// How to log a throttled reconnect.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReconnectLog {
Warn,
SuppressedWarn,
Debug,
}
#[derive(Debug, Clone, Copy)]
struct BackoffPlan {
attempt: u32,
delay: Duration,
log: ReconnectLog,
}
/// Hold one per `McpClient`; clones share state, so transport rebuilds keep
/// the limit.
#[derive(Debug, Clone, Default)]
pub struct WarnBudget(Arc<parking_lot::Mutex<Option<Instant>>>);
impl WarnBudget {
/// Latches `now` and returns true when no warn fired within
/// `WARN_COOLDOWN`. Never acquire a `ThrottleState` lock while holding
/// this one.
fn try_consume(&self, now: Instant) -> bool {
let mut last_warn_at = self.0.lock();
let available = last_warn_at.is_none_or(|t| now.duration_since(t) >= WARN_COOLDOWN);
if available {
*last_warn_at = Some(now);
}
available
}
#[cfg(test)]
fn last_warn_at(&self) -> Option<Instant> {
*self.0.lock()
}
}
#[derive(Debug, Default)]
struct ThrottleState {
/// Age at the next `get_stream` approximates the previous stream's
/// lifetime, since reconnects follow deaths within a round trip.
last_established: Option<Instant>,
consecutive_rapid: u32,
warn_budget: WarnBudget,
/// Limits each episode to one warn; cleared when the episode resets.
/// The warn may fire late if the cooldown held it back at episode entry.
episode_warned: bool,
}
impl ThrottleState {
fn with_budget(warn_budget: WarnBudget) -> Self {
Self {
warn_budget,
..Self::default()
}
}
fn delay_for_attempt(attempt: u32) -> Duration {
// 2^6 * BASE_DELAY already saturates MAX_DELAY; clamp guards pow overflow.
let exp = attempt.saturating_sub(2).min(6);
(BASE_DELAY * 2u32.pow(exp)).min(MAX_DELAY)
}
fn plan_on_get_stream(&mut self, now: Instant) -> Option<BackoffPlan> {
let rapid = self
.last_established
.is_some_and(|t| now.duration_since(t) < STABLE_STREAM_THRESHOLD);
if rapid {
self.consecutive_rapid = self.consecutive_rapid.saturating_add(1);
} else {
self.consecutive_rapid = 0;
self.episode_warned = false;
}
let attempt = self.consecutive_rapid;
if attempt < 2 {
return None;
}
let log = if self.episode_warned {
ReconnectLog::Debug
} else if self.warn_budget.try_consume(now) {
self.episode_warned = true;
ReconnectLog::Warn
} else {
ReconnectLog::SuppressedWarn
};
Some(BackoffPlan {
attempt,
delay: Self::delay_for_attempt(attempt),
log,
})
}
fn mark_established(&mut self, at: Instant) {
self.last_established = Some(at);
}
}
/// Wraps any [`StreamableHttpClient`] and backs off `get_stream` reconnects;
/// `post_message` / `delete_session` delegate untouched. Clones share the
/// throttle state (rmcp clones the client per stream task / reconnect).
///
/// Backoff and episode state are per instance; the [`WarnBudget`] is the
/// caller's, so a rebuilt client does not warn again within the cooldown.
#[derive(Clone)]
pub struct McpHttpClient<C> {
inner: C,
server_name: Arc<str>,
state: Arc<parking_lot::Mutex<ThrottleState>>,
}
// No `Debug` derive: rmcp's `AuthClient` (an inner type) is not `Debug`.
impl<C> McpHttpClient<C> {
pub fn new(inner: C, server_name: impl Into<Arc<str>>, warn_budget: WarnBudget) -> Self {
Self {
inner,
server_name: server_name.into(),
state: Arc::new(parking_lot::Mutex::new(ThrottleState::with_budget(
warn_budget,
))),
}
}
}
/// The system clock in production; the paused clock under `start_paused`
/// tests. Use this for all throttle timing so timing tests stay
/// deterministic.
fn now() -> Instant {
tokio::time::Instant::now().into_std()
}
// `C: Sync` because the trait's `+ Send` futures borrow `&self`.
impl<C: StreamableHttpClient + Sync> StreamableHttpClient for McpHttpClient<C> {
type Error = C::Error;
async fn get_stream(
&self,
uri: Arc<str>,
session_id: Arc<str>,
last_event_id: Option<String>,
auth_token: Option<String>,
custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<BoxStream<'static, Result<Sse, SseError>>, StreamableHttpError<Self::Error>> {
let plan = {
let mut st = self.state.lock();
st.plan_on_get_stream(now())
};
if let Some(plan) = plan {
match plan.log {
ReconnectLog::Warn => {
tracing::warn!(
server = %self.server_name,
uri = %uri,
attempt = plan.attempt,
delay_ms = plan.delay.as_millis() as u64,
max_delay_ms = MAX_DELAY.as_millis() as u64,
cooldown_secs = WARN_COOLDOWN.as_secs(),
"MCP SSE stream keeps dying immediately after connect; \
backing off reconnects (capped, retries forever)"
);
}
ReconnectLog::SuppressedWarn | ReconnectLog::Debug => {
tracing::debug!(
server = %self.server_name,
uri = %uri,
attempt = plan.attempt,
delay_ms = plan.delay.as_millis() as u64,
suppressed_warn = plan.log == ReconnectLog::SuppressedWarn,
"MCP SSE reconnect backoff"
);
}
}
tokio::time::sleep(plan.delay).await;
}
let result = self
.inner
.get_stream(uri, session_id, last_event_id, auth_token, custom_headers)
.await;
if result.is_ok() {
self.state.lock().mark_established(now());
}
result
}
async fn post_message(
&self,
uri: Arc<str>,
message: ClientJsonRpcMessage,
session_id: Option<Arc<str>>,
auth_token: Option<String>,
custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<StreamableHttpPostResponse, StreamableHttpError<Self::Error>> {
self.inner
.post_message(uri, message, session_id, auth_token, custom_headers)
.await
}
async fn delete_session(
&self,
uri: Arc<str>,
session_id: Arc<str>,
auth_token: Option<String>,
custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<(), StreamableHttpError<Self::Error>> {
self.inner
.delete_session(uri, session_id, auth_token, custom_headers)
.await
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Simulates rapid stream deaths starting at `start` until the throttle
/// engages (attempt 2). Returns the throttle-entry time and its plan.
fn drive_to_first_throttle(st: &mut ThrottleState, start: Instant) -> (Instant, BackoffPlan) {
assert!(st.plan_on_get_stream(start).is_none());
st.mark_established(start);
let t1 = start + Duration::from_millis(10);
assert!(st.plan_on_get_stream(t1).is_none());
st.mark_established(t1);
let t2 = t1 + Duration::from_millis(10);
let plan = st.plan_on_get_stream(t2).expect("attempt 2 is throttled");
assert_eq!(plan.attempt, 2);
(t2, plan)
}
#[test]
fn warn_lifecycle_across_episodes_and_cooldowns() {
let mut st = ThrottleState::default();
// Episode 1: the entry attempt warns once; later attempts stay debug.
let (t2, p2) = drive_to_first_throttle(&mut st, Instant::now());
assert_eq!(p2.delay, BASE_DELAY);
assert_eq!(p2.log, ReconnectLog::Warn);
let first_warn_at = st.warn_budget.last_warn_at().expect("latched");
st.mark_established(t2);
let p3 = st
.plan_on_get_stream(t2 + Duration::from_millis(10))
.expect("throttled");
assert_eq!(p3.log, ReconnectLog::Debug);
st.mark_established(t2 + Duration::from_millis(10));
// Stable recovery, then a new outage inside the cooldown: the
// episode reset must not reset the cooldown, so entry is suppressed.
let (mut t, p_entry) = drive_to_first_throttle(&mut st, t2 + Duration::from_secs(30 * 60));
assert_eq!(p_entry.log, ReconnectLog::SuppressedWarn);
st.mark_established(t);
// The outage continues: every attempt is suppressed until the
// cooldown expires, then the episode's one warn fires late.
let step = STABLE_STREAM_THRESHOLD - Duration::from_millis(1);
let rearm_at = first_warn_at + WARN_COOLDOWN;
let late_warn_at = loop {
t += step;
let p = st.plan_on_get_stream(t).expect("throttled");
st.mark_established(t);
match p.log {
ReconnectLog::SuppressedWarn => assert!(t < rearm_at),
ReconnectLog::Warn => {
assert!(t >= rearm_at, "late warn must wait for the cooldown");
assert!(p.attempt > 2, "late warn fires mid-episode");
break t;
}
ReconnectLog::Debug => {
panic!("Debug event before the episode's warn fired")
}
}
};
// Same outage, another full cooldown: elapsed time alone must not
// produce more warns.
let past_next_cooldown = late_warn_at + WARN_COOLDOWN + Duration::from_secs(1);
let mut attempts = 0u32;
while t < past_next_cooldown {
t += step;
attempts += 1;
let p = st.plan_on_get_stream(t).expect("throttled");
assert_eq!(p.log, ReconnectLog::Debug);
st.mark_established(t);
}
let min_attempts = (WARN_COOLDOWN.as_millis() / step.as_millis()) as u32;
assert!(
attempts >= min_attempts,
"loop must cross the cooldown window"
);
}
#[test]
fn suppressed_episode_that_recovers_drops_its_warn() {
let mut st = ThrottleState::default();
let (t2, p2) = drive_to_first_throttle(&mut st, Instant::now());
assert_eq!(p2.log, ReconnectLog::Warn);
let first_warn_at = st.warn_budget.last_warn_at().expect("latched");
st.mark_established(t2);
let (t_ep2, p_ep2) = drive_to_first_throttle(&mut st, t2 + Duration::from_secs(30 * 60));
assert_eq!(p_ep2.log, ReconnectLog::SuppressedWarn);
st.mark_established(t_ep2);
// Land the third episode's throttle entry exactly on the cooldown
// boundary, which is inclusive.
let rearm_at = first_warn_at + WARN_COOLDOWN;
let (entry, p_ep3) = drive_to_first_throttle(&mut st, rearm_at - Duration::from_millis(20));
assert_eq!(entry, rearm_at);
assert_eq!(p_ep3.log, ReconnectLog::Warn);
}
/// A rebuilt client for the same server shares the warn budget, so it
/// does not warn again within the cooldown.
#[test]
fn rebuilt_client_shares_the_server_warn_budget() {
let budget = WarnBudget::default();
let mut st1 = ThrottleState::with_budget(budget.clone());
let (t2, p2) = drive_to_first_throttle(&mut st1, Instant::now());
assert_eq!(p2.log, ReconnectLog::Warn);
let mut st2 = ThrottleState::with_budget(budget);
let (_, p_rebuilt) = drive_to_first_throttle(&mut st2, t2 + Duration::from_secs(60));
assert_eq!(p_rebuilt.log, ReconnectLog::SuppressedWarn);
}
/// Inner client whose streams always succeed and end immediately,
/// simulating rapid stream deaths.
#[derive(Clone)]
struct MockInner;
impl StreamableHttpClient for MockInner {
type Error = std::io::Error;
async fn get_stream(
&self,
_uri: Arc<str>,
_session_id: Arc<str>,
_last_event_id: Option<String>,
_auth_token: Option<String>,
_custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<BoxStream<'static, Result<Sse, SseError>>, StreamableHttpError<Self::Error>>
{
Ok(Box::pin(futures::stream::empty()))
}
async fn post_message(
&self,
_uri: Arc<str>,
_message: ClientJsonRpcMessage,
_session_id: Option<Arc<str>>,
_auth_token: Option<String>,
_custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<StreamableHttpPostResponse, StreamableHttpError<Self::Error>> {
unimplemented!("not used by these tests")
}
async fn delete_session(
&self,
_uri: Arc<str>,
_session_id: Arc<str>,
_auth_token: Option<String>,
_custom_headers: HashMap<HeaderName, HeaderValue>,
) -> Result<(), StreamableHttpError<Self::Error>> {
unimplemented!("not used by these tests")
}
}
/// Counts this module's warn events and records each debug event's
/// `suppressed_warn` field.
#[derive(Clone, Default)]
struct LogCapture {
warns: Arc<std::sync::atomic::AtomicUsize>,
debug_suppressed_flags: Arc<parking_lot::Mutex<Vec<Option<bool>>>>,
}
struct SuppressedFlag(Option<bool>);
impl tracing::field::Visit for SuppressedFlag {
fn record_bool(&mut self, field: &tracing::field::Field, value: bool) {
if field.name() == "suppressed_warn" {
self.0 = Some(value);
}
}
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
}
impl tracing::Subscriber for LogCapture {
fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool {
metadata
.target()
.starts_with("xai_grok_mcp::mcp_http_client")
}
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
use std::sync::atomic::Ordering;
let level = *event.metadata().level();
if level == tracing::Level::WARN {
self.warns.fetch_add(1, Ordering::Relaxed);
} else if level == tracing::Level::DEBUG {
let mut flag = SuppressedFlag(None);
event.record(&mut flag);
self.debug_suppressed_flags.lock().push(flag.0);
}
}
fn enter(&self, _: &tracing::span::Id) {}
fn exit(&self, _: &tracing::span::Id) {}
}
async fn drive_once(client: &McpHttpClient<MockInner>) {
let stream = client
.get_stream(
"http://mock".into(),
"session".into(),
None,
None,
HashMap::new(),
)
.await
.expect("mock stream");
drop(stream);
}
// Paused tokio time drives both the backoff sleeps and the throttle
// clock.
#[tokio::test(start_paused = true)]
async fn get_stream_maps_warn_debug_and_suppressed_severities() {
use std::sync::atomic::Ordering;
let capture = LogCapture::default();
let _guard = tracing::subscriber::set_default(capture.clone());
let client = McpHttpClient::new(MockInner, "mock-server", WarnBudget::default());
// Attempts 0 and 1 are unthrottled; attempt 2 warns; attempt 3 is debug.
for _ in 0..4 {
drive_once(&client).await;
}
assert_eq!(capture.warns.load(Ordering::Relaxed), 1, "exactly one warn");
assert_eq!(
*capture.debug_suppressed_flags.lock(),
vec![Some(false)],
"in-episode debug is not a suppressed warning"
);
// A stable gap resets the episode; the next throttled attempt is a
// suppressed warning and must also log at debug, not warn.
tokio::time::advance(STABLE_STREAM_THRESHOLD + Duration::from_millis(1)).await;
for _ in 0..3 {
drive_once(&client).await;
}
assert_eq!(
capture.warns.load(Ordering::Relaxed),
1,
"suppressed warning must not log at warn"
);
assert_eq!(
*capture.debug_suppressed_flags.lock(),
vec![Some(false), Some(true)],
"suppressed entry logs at debug with suppressed_warn set"
);
}
}

View file

@ -0,0 +1,522 @@
//! OAuth flow orchestration for local MCP servers.
//!
//! Uses rmcp's `AuthorizationManager` for RFC-compliant discovery (RFC 8414 +
//! 9728), DCR, PKCE, and token exchange. Proactive discovery determines auth
//! requirements before connecting (not reactively after a 401), and `AuthClient`
//! wraps the transport for transparent token injection and refresh.
//!
//! This module handles the interactive browser-based consent flow and
//! cross-process dedup. Credential persistence is delegated to
//! [`crate::credentials::McpCredentialStoreAdapter`] (implements rmcp's
//! `CredentialStore` trait).
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use tokio::sync::{Mutex, oneshot, watch};
use crate::oauth_config::McpOAuthConfig;
use crate::rmcp::transport::auth::{AuthorizationManager, OAuthClientConfig};
/// Client name advertised to MCP servers during Dynamic Client Registration
/// (RFC 7591). Surfaces as the application name on third-party OAuth consent
/// screens (e.g. Linear, GitHub), so keep this human-recognizable.
const MCP_OAUTH_CLIENT_NAME: &str = "Grok";
/// How often the interactive OAuth flow polls the credential store to detect
/// a login completed in another window or process.
const CREDENTIAL_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);
// ---------------------------------------------------------------------------
// Two-layer dedup: prevents duplicate browser tabs both within one process
// (multiple async tasks / sessions) and across separate processes (leader
// mode disabled, multiple `grok` invocations).
//
// Layer 1 (cross-process): filesystem lock at $GROK_HOME/mcp_auth_{safe_name}.lock
// Layer 2 (in-process): watch channel so only one task runs the flow
// ---------------------------------------------------------------------------
/// In-process in-flight auth tracker. Keyed by server name.
/// Each entry has a generation counter so that when a forced override evicts
/// a stale leader, the old leader's cleanup doesn't clobber the new entry.
struct InFlightEntry {
rx: watch::Receiver<Option<Result<(), String>>>,
generation: u64,
}
#[allow(clippy::type_complexity)]
static IN_FLIGHT_AUTH: std::sync::LazyLock<Mutex<HashMap<String, InFlightEntry>>> =
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
static GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
/// Run the browser-based OAuth flow with dedup.
///
/// If another task or process is already running the flow for the same server,
/// waits for it instead of opening another browser tab. The provided
/// `AuthorizationManager` must already have metadata set (from
/// `discover_metadata`). On success, the manager's credentials are updated
/// and persisted via its `CredentialStore`.
///
/// When `force` is true (user-initiated auth), any existing in-flight entry
/// is evicted so a fresh browser flow starts immediately. The old leader's
/// browser tab becomes orphaned but its cleanup is generation-safe.
pub async fn authenticate_mcp_server_dedup(
server_name: &str,
server_url: &str,
auth_manager: &Arc<Mutex<AuthorizationManager>>,
byo_config: Option<&McpOAuthConfig>,
force: bool,
) -> Result<(), String> {
// --- Layer 2: in-process dedup via watch channel ---
let mut in_flight = IN_FLIGHT_AUTH.lock().await;
// Remove stale entries left by panicked leaders (sender dropped).
if in_flight
.get(server_name)
.is_some_and(|e| e.rx.has_changed().is_err())
{
in_flight.remove(server_name);
}
if let Some(entry) = in_flight.get(server_name) {
if force {
tracing::info!(
server = server_name,
"User-initiated auth override; evicting stale in-flight entry"
);
in_flight.remove(server_name);
} else {
let mut rx = entry.rx.clone();
drop(in_flight);
tracing::info!(
server = server_name,
"Another task in this process is already authenticating; waiting..."
);
loop {
let snapshot = rx.borrow_and_update().clone();
if let Some(result) = snapshot {
if result.is_ok() {
let mut mgr = auth_manager.lock().await;
let _ = mgr.initialize_from_store().await;
}
return result;
}
if rx.changed().await.is_err() {
return Err("Auth leader dropped".to_string());
}
}
}
}
// We are the in-process leader.
let generation = GENERATION.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let (tx, rx) = watch::channel::<Option<Result<(), String>>>(None);
in_flight.insert(server_name.to_string(), InFlightEntry { rx, generation });
drop(in_flight);
// --- Layer 1: cross-process dedup via filesystem lock (Unix only) ---
// When force is set, skip the fs lock — the old leader may still hold it
// and we don't want to block behind a stale browser flow.
#[cfg(unix)]
let result = if force {
run_browser_auth_flow(server_name, server_url, auth_manager, byo_config).await
} else {
authenticate_with_fs_lock(server_name, server_url, auth_manager, byo_config).await
};
#[cfg(not(unix))]
let result = run_browser_auth_flow(server_name, server_url, auth_manager, byo_config).await;
// Broadcast to in-process followers and clean up.
// Only remove if our generation is still current (a force override may
// have replaced us).
let _ = tx.send(Some(result.clone()));
let mut in_flight = IN_FLIGHT_AUTH.lock().await;
if in_flight
.get(server_name)
.is_some_and(|e| e.generation == generation)
{
in_flight.remove(server_name);
}
result
}
/// Acquire a filesystem lock, then either run the auth flow (if we're the
/// first process) or reload credentials from disk (if another process
/// already completed auth while we waited for the lock).
#[cfg(unix)]
async fn authenticate_with_fs_lock(
server_name: &str,
server_url: &str,
auth_manager: &Arc<Mutex<AuthorizationManager>>,
byo_config: Option<&McpOAuthConfig>,
) -> Result<(), String> {
use oauth2::TokenResponse as _;
let lock_path = auth_lock_path(server_name);
if let Some(parent) = lock_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
// Snapshot the current access token before waiting for the lock.
// After acquiring, we compare to detect if another process authed.
let token_before = {
let mgr = auth_manager.lock().await;
mgr.get_credentials()
.await
.ok()
.and_then(|(_, tok)| tok)
.map(|t| t.access_token().secret().to_string())
};
let lock_file = match std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&lock_path)
{
Ok(f) => f,
Err(e) => {
tracing::warn!(%e, "Failed to create auth lock file; proceeding without cross-process dedup");
return run_browser_auth_flow(server_name, server_url, auth_manager, byo_config).await;
}
};
let lock_file = tokio::task::spawn_blocking(move || {
use std::os::unix::io::AsRawFd;
let fd = lock_file.as_raw_fd();
loop {
if unsafe { libc::flock(fd, libc::LOCK_EX) } == 0 {
return Some(lock_file);
}
let err = std::io::Error::last_os_error();
if err.kind() == std::io::ErrorKind::Interrupted {
continue;
}
return None;
}
})
.await
.ok()
.flatten();
let Some(_lock_guard) = lock_file else {
tracing::warn!("Failed to acquire auth lock; proceeding without cross-process dedup");
return run_browser_auth_flow(server_name, server_url, auth_manager, byo_config).await;
};
// We hold the lock. Reload from disk and check if another process
// wrote a DIFFERENT token while we waited (not just any token).
{
let mut mgr = auth_manager.lock().await;
if let Ok(true) = mgr.initialize_from_store().await {
let token_after = mgr
.get_credentials()
.await
.ok()
.and_then(|(_, tok)| tok)
.map(|t| t.access_token().secret().to_string());
if token_after != token_before {
tracing::info!(
server = server_name,
"Another process already authenticated; reusing fresh token"
);
return Ok(());
}
}
}
run_browser_auth_flow(server_name, server_url, auth_manager, byo_config).await
}
#[cfg(unix)]
fn auth_lock_path(server_name: &str) -> std::path::PathBuf {
let safe: String = server_name
.chars()
.map(|c| {
if c.is_alphanumeric() || c == '-' || c == '_' {
c
} else {
'_'
}
})
.collect();
xai_grok_config::grok_home().join(format!("mcp_auth_{safe}.lock"))
}
/// Run the interactive browser-based OAuth flow.
///
/// Drives the `AuthorizationManager` (which must already have metadata set)
/// through client setup, authorization URL generation, browser consent, and
/// code exchange. Credentials are auto-persisted by the manager's
/// `CredentialStore` on successful token exchange.
async fn run_browser_auth_flow(
server_name: &str,
server_url: &str,
auth_manager: &Arc<Mutex<AuthorizationManager>>,
byo_config: Option<&McpOAuthConfig>,
) -> Result<(), String> {
// 1. Try token refresh first (no browser needed).
{
let mgr = auth_manager.lock().await;
match mgr.refresh_token().await {
Ok(_) => {
tracing::info!(
server = server_name,
"Token refreshed successfully (no browser)"
);
return Ok(());
}
Err(e) => {
tracing::info!(
server = server_name,
%e,
"Token refresh failed, falling through to browser auth"
);
}
}
}
// 2. Bind the loopback callback port (fixed BYO port if set, else ephemeral).
let requested_port = byo_config.and_then(|b| b.callback_port).unwrap_or(0);
let listener =
tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], requested_port)))
.await
.map_err(|e| format!("Failed to bind loopback port {requested_port}: {e}"))?;
let port = listener
.local_addr()
.map_err(|e| format!("Failed to get loopback port: {e}"))?
.port();
let redirect_uri = format!("http://127.0.0.1:{port}/callback");
// 3. Configure client and get authorization URL (lock held briefly).
let byo_scopes: Vec<String> = byo_config
.and_then(|b| b.scopes.as_ref())
.cloned()
.unwrap_or_default();
let auth_url = {
let mut mgr = auth_manager.lock().await;
let scopes: Vec<String>;
if let Some(byo) = byo_config
&& let Some(client_id) = byo.client_id.clone()
{
tracing::info!(
server = server_name,
"Using BYO client credentials (oauth_client_id from config)"
);
scopes = byo_scopes;
let mut config =
OAuthClientConfig::new(client_id, redirect_uri.clone()).with_scopes(scopes.clone());
config.client_secret = byo.client_secret.clone();
mgr.configure_client(config)
.map_err(|e| format!("Failed to configure BYO client: {e}"))?;
} else {
scopes = if byo_scopes.is_empty() {
mgr.select_scopes(None, &[])
} else {
byo_scopes
};
let scope_refs: Vec<&str> = scopes.iter().map(|s| s.as_str()).collect();
mgr.register_client(MCP_OAUTH_CLIENT_NAME, &redirect_uri, &scope_refs)
.await
.map_err(|e| format!("Dynamic client registration failed: {e}"))?;
}
let scopes: Vec<&str> = scopes.iter().map(|s| s.as_str()).collect();
mgr.get_authorization_url(&scopes)
.await
.map_err(|e| format!("Failed to get authorization URL: {e}"))?
};
// Lock released — browser flow can take minutes.
// Snapshot token before browser opens so we can detect if another flow
// (e.g. the old evicted leader, or another process) writes fresh tokens.
let token_before_browser = {
let mgr = auth_manager.lock().await;
mgr.get_credentials()
.await
.ok()
.and_then(|(_, tok)| tok)
.map(|t| {
use oauth2::TokenResponse as _;
t.access_token().secret().to_string()
})
};
// 4. Open browser for user consent.
tracing::info!(server = server_name, "Opening browser for OAuth consent");
if let Err(e) = webbrowser::open(&auth_url) {
// eprintln! corrupts the TUI alternate screen (in-process, fd 2).
// TODO: surface auth URL via ACP notification instead.
tracing::warn!(%e, url = %auth_url, "Failed to open browser for MCP OAuth; user must visit URL manually");
}
// 5. Wait for the OAuth callback OR for tokens to appear on disk.
// The credential store poll catches the case where a force-evicted
// old leader (or another process) completed auth via a different
// browser tab while we're waiting for our own callback.
//
// IMPORTANT: peek the on-disk file directly via `McpCredentialStore::load_default`
// rather than calling `mgr.initialize_from_store()`. The latter has the
// side effect of running `configure_client_id(stored.client_id)`, which
// replaces `oauth_client`'s freshly-DCR'd `client_id` and ephemeral
// `redirect_uri` with the *old* stored values (`redirect_uri = base_url`).
// If the user then completes the browser flow before another process
// writes new tokens, `exchange_code_for_token` would use the clobbered
// config and the server would reject with `invalid_grant: Invalid redirect_uri`.
let parsed_server_url = match url::Url::parse(server_url) {
Ok(u) => Some(u),
Err(e) => {
tracing::warn!(
server = server_name,
url = server_url,
error = %e,
"could not parse server URL for credential-store poll; falling back to callback-only auth-completion detection"
);
None
}
};
let server_name_for_poll = server_name.to_string();
let token_snapshot = token_before_browser.clone();
let poll_store = async move {
let Some(url) = parsed_server_url else {
// No credential-store key — disable the poll. Callback path still works.
std::future::pending::<()>().await;
return;
};
loop {
tokio::time::sleep(CREDENTIAL_POLL_INTERVAL).await;
let Ok(store) = crate::credentials::McpCredentialStore::load_default() else {
continue;
};
let token_now = store
.get(&server_name_for_poll, &url)
.and_then(|entry| entry.token_response.as_ref())
.map(|t| {
use oauth2::TokenResponse as _;
t.access_token().secret().to_string()
});
if token_now.is_some() && token_now != token_snapshot {
return;
}
}
};
let (callback_server, callback_rx) = start_oauth_callback_server(listener);
tokio::select! {
result = callback_rx => {
callback_server.abort();
let (code, csrf_state) = result
.map_err(|_| "Callback channel dropped".to_string())?
.map_err(|e| format!("OAuth callback failed: {e}"))?;
// 6. Exchange code for tokens (auto-persists via CredentialStore).
let mgr = auth_manager.lock().await;
mgr.exchange_code_for_token(&code, &csrf_state)
.await
.map_err(|e| format!("Token exchange failed: {e}"))?;
tracing::info!(server = server_name, "MCP OAuth authentication successful");
}
_ = poll_store => {
callback_server.abort();
tracing::info!(
server = server_name,
"Fresh tokens detected on disk from another auth flow; skipping callback wait"
);
}
}
Ok(())
}
fn html_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&#39;")
}
/// Start a loopback HTTP server for the OAuth callback.
///
/// Returns the server task handle (for cleanup) and a oneshot receiver
/// that resolves with `(code, state)` when the callback arrives.
///
/// The caller is responsible for aborting the server handle.
#[allow(clippy::type_complexity)]
fn start_oauth_callback_server(
listener: tokio::net::TcpListener,
) -> (
tokio::task::JoinHandle<()>,
oneshot::Receiver<Result<(String, String), String>>,
) {
use axum::{Router, extract::Query, response::Html, routing::get};
let (tx, rx) = oneshot::channel::<Result<(String, String), String>>();
let tx = Arc::new(tokio::sync::Mutex::new(Some(tx)));
let handler = {
let tx = tx.clone();
move |Query(params): Query<HashMap<String, String>>| {
let tx = tx.clone();
async move {
let result = if let Some(error) = params.get("error") {
let desc = params
.get("error_description")
.cloned()
.unwrap_or_else(|| "Unknown error".to_string());
Err(format!("OAuth error: {error} - {desc}"))
} else {
match (params.get("code"), params.get("state")) {
(Some(code), Some(state)) => Ok((code.clone(), state.clone())),
(None, _) => Err("Missing authorization code".to_string()),
(_, None) => Err("Missing state parameter".to_string()),
}
};
let html = match &result {
Ok(_) => {
r#"<!DOCTYPE html><html><head><title>Authorization Complete</title></head>
<body style="font-family: sans-serif; text-align: center; padding: 50px;">
<h1>Authorization Complete</h1>
<p>You can close this window and return to the terminal.</p>
<script>window.close();</script></body></html>"#
.to_string()
}
Err(e) => {
let msg = html_escape(e);
format!(
r#"<!DOCTYPE html><html><head><title>Authorization Failed</title></head>
<body style="font-family: sans-serif; text-align: center; padding: 50px;">
<h1>Authorization Failed</h1>
<p>{msg}</p>
<p>You can close this window and return to the terminal.</p>
</body></html>"#
)
}
};
if let Some(tx) = tx.lock().await.take() {
let _ = tx.send(result);
}
Html(html)
}
}
};
let app = Router::new().route("/callback", get(handler.clone()).post(handler));
let server = tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
(server, rx)
}

View file

@ -0,0 +1,27 @@
//! OAuth configuration types for MCP servers.
//!
//! Constructed by the host's TOML parsing (`McpServerConfig::oauth_config`)
//! and consumed by [`crate::oauth`].
use std::collections::HashMap;
/// OAuth configuration extracted from an MCP server's config.
///
/// Travels alongside `acp::McpServer` (which can't be extended since it's
/// an external crate type). Keyed by server name in [`McpOAuthConfigMap`].
#[derive(Debug, Clone, Default)]
pub struct McpOAuthConfig {
pub client_id: Option<String>,
pub client_secret: Option<String>,
pub scopes: Option<Vec<String>>,
pub callback_port: Option<u16>,
}
impl McpOAuthConfig {
pub fn is_configured(&self) -> bool {
self.client_id.is_some()
}
}
/// Per-server OAuth configuration map, keyed by MCP server name.
pub type McpOAuthConfigMap = HashMap<String, McpOAuthConfig>;

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,27 @@
//! Single source of truth for the `x.ai/mcp/*` ACP wire strings.
//!
//! These method/`_meta` keys are part of the cross-language MCP-over-ACP
//! protocol the SDK speaks (mirrors the SDK's `_mcp_wire.py` / `mcpWire.ts`).
//! Reference these constants instead of re-typing the literals so the agent and
//! SDK can't drift apart.
/// Forward tool-invocation method (client -> agent): `x.ai/mcp/call`.
///
/// The pager/client asks the agent to invoke an MCP tool on a server the agent is
/// connected to, outside the LLM loop. See `extensions::mcp::handle_call`.
pub const MCP_CALL: &str = "x.ai/mcp/call";
/// Reverse zero-IPC tool-invocation method (agent -> client): `x.ai/mcp/sdk_call`.
///
/// The agent invokes a tool that lives in the SDK's in-process MCP server by sending
/// the MCP JSON-RPC message back to the client over the ACP reverse channel. Distinct
/// from [`MCP_CALL`] so the two disjoint schemas don't share a method string for
/// metrics/tracing. See the agent-side ACP invoker that handles this method.
pub const MCP_SDK_CALL: &str = "x.ai/mcp/sdk_call";
/// `session/new` `_meta` key listing in-process SDK MCP servers: `x.ai/mcp/servers`.
pub const MCP_SERVERS: &str = "x.ai/mcp/servers";
/// `initialize` `_meta` capability flag advertising in-process SDK MCP support
/// (enables the SDK's `transport="acp"`): `x.ai/mcp/sdk`.
pub const MCP_SDK: &str = "x.ai/mcp/sdk";

View file

@ -0,0 +1,208 @@
//! Black-box repro of rmcp's zero-backoff SSE reconnect loop (and of
//! `McpHttpClient` bounding it), against a fake MCP streamable-HTTP server
//! whose standing-GET behavior is the variable under test. Each GET the fake
//! server counts corresponds to one rmcp `WARN sse stream error: ...` line.
//!
//! Run with: cargo test -p xai-grok-mcp --test repro_sse_flood -- --nocapture
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use axum::Json;
use axum::body::Body;
use axum::extract::State;
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use futures::StreamExt;
use serde_json::{Value, json};
use xai_grok_mcp::mcp_http_client::{McpHttpClient, WarnBudget};
use xai_grok_mcp::rmcp::ServiceExt;
use xai_grok_mcp::rmcp::transport::StreamableHttpClientTransport;
use xai_grok_mcp::rmcp::transport::streamable_http_client::StreamableHttpClientTransportConfig;
/// Gap letting the first body chunk flush before the abort, so the client
/// sees a *body* death (rmcp's zero-backoff flood path) rather than a failed
/// request (rmcp's policy-backed path).
const FLUSH_GAP: Duration = Duration::from_millis(5);
/// How the fake server ends the standing GET stream.
#[derive(Clone, Copy)]
enum GetBehavior {
/// 200, then the body is aborted mid-stream — the production failure
/// mode ("error decoding response body" -> the WARN flood path).
AbnormalBodyDeath,
/// 200, then the stream stays open (a working MCP server).
Healthy,
}
#[derive(Clone)]
struct ServerState {
behavior: GetBehavior,
gets: Arc<AtomicUsize>,
}
async fn handle_post(Json(req): Json<Value>) -> Response {
match req["method"].as_str() {
Some("initialize") => {
let result = json!({
"jsonrpc": "2.0",
"id": req["id"],
"result": {
"protocolVersion": req["params"]["protocolVersion"],
"capabilities": {},
"serverInfo": {"name": "fake", "version": "0.0.0"},
},
});
([("mcp-session-id", "fake-session-1")], Json(result)).into_response()
}
Some("tools/list") => {
let result = json!({
"jsonrpc": "2.0",
"id": req["id"],
"result": {"tools": [{"name": "echo", "inputSchema": {"type": "object"}}]},
});
Json(result).into_response()
}
// notifications/initialized and anything else.
_ => StatusCode::ACCEPTED.into_response(),
}
}
async fn handle_get(State(state): State<ServerState>) -> Response {
state.gets.fetch_add(1, Ordering::Relaxed);
let body = match state.behavior {
GetBehavior::AbnormalBodyDeath => Body::from_stream(
futures::stream::iter([Ok::<_, std::io::Error>(": partial\n".to_owned())]).chain(
futures::stream::once(async {
tokio::time::sleep(FLUSH_GAP).await;
Err(std::io::Error::other("body aborted mid-stream"))
}),
),
),
GetBehavior::Healthy => {
Body::from_stream(futures::stream::pending::<Result<String, std::io::Error>>())
}
};
([(header::CONTENT_TYPE, "text/event-stream")], body).into_response()
}
async fn spawn_fake_server(behavior: GetBehavior) -> (String, Arc<AtomicUsize>) {
let gets = Arc::new(AtomicUsize::new(0));
let app = axum::Router::new()
.route("/mcp", get(handle_get).post(handle_post))
.with_state(ServerState {
behavior,
gets: gets.clone(),
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
(format!("http://{addr}/mcp"), gets)
}
/// The flood: body death on the standing GET -> zero-backoff reconnect loop,
/// one WARN per GET (set RUST_LOG=rmcp=warn to see them).
#[tokio::test(flavor = "multi_thread")]
async fn repro_zero_backoff_reconnect_flood() {
let (url, gets) = spawn_fake_server(GetBehavior::AbnormalBodyDeath).await;
let transport = StreamableHttpClientTransport::from_uri(url.as_str());
let client = ().serve(transport).await.expect("handshake against fake server should succeed");
const OBSERVE: Duration = Duration::from_secs(3);
tokio::time::sleep(OBSERVE).await;
let n = gets.load(Ordering::Relaxed);
let _ = client.cancel().await;
eprintln!(
"[repro] {n} standing-GET reconnect attempts (= WARN log lines) in {OBSERVE:?} \
= {:.0}/sec",
n as f64 / OBSERVE.as_secs_f64()
);
// A backoff-respecting client would attempt ~3-5 in 3s; the bug produces
// hundreds-to-thousands.
assert!(
n > 20,
"expected a zero-backoff reconnect flood (>20 GETs in {OBSERVE:?}), got {n}; \
if this FAILS with a small n, the rmcp loop got fixed - delete mcp_http_client.rs"
);
}
/// The fix: same body-killing server, client wrapped in `McpHttpClient` —
/// the backoff schedule allows only a handful of reconnects.
#[tokio::test(flavor = "multi_thread")]
async fn throttled_client_bounds_the_flood() {
let (url, gets) = spawn_fake_server(GetBehavior::AbnormalBodyDeath).await;
let throttled = McpHttpClient::new(
reqwest::Client::default(),
"fake-server",
WarnBudget::default(),
);
let transport = StreamableHttpClientTransport::with_client(
throttled,
StreamableHttpClientTransportConfig::with_uri(url.as_str()),
);
let client = ().serve(transport).await.expect("handshake against fake server should succeed");
// Checkpoint: backoff must engage early (instant, instant, 0.5s => 3-4
// GETs by 1.2s; an unthrottled client would be in the hundreds).
tokio::time::sleep(Duration::from_millis(1200)).await;
let early = gets.load(Ordering::Relaxed);
assert!(
early <= 4,
"backoff must engage early; got {early} GETs by 1.2s"
);
const OBSERVE: Duration = Duration::from_secs(4);
tokio::time::sleep(OBSERVE - Duration::from_millis(1200)).await;
let n = gets.load(Ordering::Relaxed);
let _ = client.cancel().await;
eprintln!("[repro] {n} throttled reconnect attempts in {OBSERVE:?} (schedule allows ~5-6)");
assert!(
(3..=8).contains(&n),
"throttled reconnects should follow the backoff schedule (~5-6 in {OBSERVE:?}), got {n}"
);
}
/// A working server through the throttled client: handshake + tools/list
/// succeed and the standing GET opens exactly once — wrapper invisible.
#[tokio::test(flavor = "multi_thread")]
async fn throttled_client_does_not_affect_healthy_server() {
let (url, gets) = spawn_fake_server(GetBehavior::Healthy).await;
let throttled = McpHttpClient::new(
reqwest::Client::default(),
"fake-server",
WarnBudget::default(),
);
let client = ()
.serve(StreamableHttpClientTransport::with_client(
throttled,
StreamableHttpClientTransportConfig::with_uri(url.as_str()),
))
.await
.expect("handshake against healthy server should succeed");
let tools = client
.list_tools(Default::default())
.await
.expect("tools/list should succeed through the throttled client");
assert_eq!(tools.tools.len(), 1);
assert_eq!(tools.tools[0].name, "echo");
tokio::time::sleep(Duration::from_secs(3)).await;
let n = gets.load(Ordering::Relaxed);
let _ = client.cancel().await;
eprintln!("[repro] healthy server: {n} GET(s) in 3s, tools/list ok");
assert_eq!(
n, 1,
"healthy stream must be opened once and never reconnected"
);
}