Publish harness and TUI open-source
initial sync from the monorepo
This commit is contained in:
commit
c68e39f604
2734 changed files with 1437016 additions and 0 deletions
233
crates/codegen/xai-grok-telemetry/tests/external_otlp.rs
Normal file
233
crates/codegen/xai-grok-telemetry/tests/external_otlp.rs
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
//! Integration test for the external OTEL stream against an in-process OTLP
|
||||
//! collector: wire payloads, delta temporality, gates-off canary absence at
|
||||
//! the wire layer, flush-on-shutdown ≤ 2 s, and post-shutdown silence.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use otlp_collector as col;
|
||||
|
||||
const CANARY_MODEL: &str = "sk-CANARYabcdefghij1234567890";
|
||||
const CANARY_PROMPT: &str = "CANARY_PROMPT_TEXT do not export";
|
||||
const CANARY_MCP: &str = "canary-internal-mcp-server";
|
||||
|
||||
#[test]
|
||||
fn external_stream_end_to_end() {
|
||||
let collected = col::Collected::default();
|
||||
let endpoint = col::start_collector(collected.clone());
|
||||
|
||||
// Resolve through the real config path (double opt-in, gates off).
|
||||
let mut cfg = xai_grok_telemetry::external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
// Keep intervals short so the test is fast; flush() forces anyway.
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("200".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("double opt-in must resolve");
|
||||
cfg.client = xai_grok_telemetry::external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: "0.0.0-test".into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
|
||||
xai_grok_telemetry::external::init(Some(cfg));
|
||||
assert!(xai_grok_telemetry::external::is_active());
|
||||
|
||||
// Emit through the same funnel production uses — with the product events client
|
||||
// never initialized (TelemetryMode effectively Disabled) and no auth at
|
||||
// all, pinning the Disabled half of the G7 independence matrix at the
|
||||
// funnel level: the external sink fires anyway.
|
||||
assert!(!xai_grok_telemetry::is_enabled());
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionNew {
|
||||
session_id: "sess-int-1".into(),
|
||||
client_identifier: None,
|
||||
client_version: None,
|
||||
is_git_repo: true,
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionHarness {
|
||||
session_id: "sess-int-1".into(),
|
||||
client_identifier: Some("grok-pager".into()),
|
||||
model_id: "grok-4".into(),
|
||||
agent_name: "grok-build-plan".into(),
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
mcp_server_names: vec![CANARY_MCP.into()],
|
||||
plugin_names: vec![],
|
||||
skill_names: vec![],
|
||||
lsp_server_names: vec![],
|
||||
hook_names: vec![],
|
||||
agents_md_dir_names: vec![],
|
||||
memory_enabled: false,
|
||||
is_git_repo: true,
|
||||
auto_update: None,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: CANARY_PROMPT.len(),
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: Some(CANARY_PROMPT.into()),
|
||||
});
|
||||
// Model-id canary for the metrics body (increment-time scrub).
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::ModelResponseReceived {
|
||||
model_id: CANARY_MODEL.into(),
|
||||
duration_ms: 5,
|
||||
stop_reason: Some("stop".into()),
|
||||
prompt_tokens: Some(11),
|
||||
completion_tokens: Some(7),
|
||||
reasoning_tokens: None,
|
||||
cached_prompt_tokens: None,
|
||||
});
|
||||
|
||||
xai_grok_telemetry::external::flush();
|
||||
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
collected.logs_len() > 0 && collected.metrics_len() > 0
|
||||
}),
|
||||
"collector must receive both signals"
|
||||
);
|
||||
|
||||
// ── Logs payload ────────────────────────────────────────────────────
|
||||
let logs = col::decode_logs(&collected);
|
||||
let mut event_names: Vec<String> = vec![];
|
||||
let mut resource_service_name = None;
|
||||
for req in &logs {
|
||||
for rl in &req.resource_logs {
|
||||
if let Some(resource) = &rl.resource {
|
||||
for kv in &resource.attributes {
|
||||
if kv.key == "service.name"
|
||||
&& let Some(v) = &kv.value
|
||||
{
|
||||
resource_service_name = Some(format!("{v:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
for sl in &rl.scope_logs {
|
||||
assert_eq!(
|
||||
sl.scope.as_ref().map(|s| s.name.as_str()),
|
||||
Some("ai.xai.grok_code")
|
||||
);
|
||||
for record in &sl.log_records {
|
||||
event_names.push(record.event_name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
resource_service_name
|
||||
.as_deref()
|
||||
.is_some_and(|s| s.contains("grok-cli")),
|
||||
"service.name=grok-cli is a wire commitment: {resource_service_name:?}"
|
||||
);
|
||||
for expected in [
|
||||
"grok_code.session_start",
|
||||
"grok_code.user_prompt",
|
||||
"grok_code.api_request",
|
||||
] {
|
||||
assert!(
|
||||
event_names.iter().any(|n| n == expected),
|
||||
"missing {expected} in {event_names:?}"
|
||||
);
|
||||
}
|
||||
// session_start arrives exactly once per emission (no double-send from
|
||||
// the funnel).
|
||||
assert_eq!(
|
||||
event_names
|
||||
.iter()
|
||||
.filter(|n| *n == "grok_code.session_start")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
|
||||
// ── Metrics payload: names + Delta temporality + session.count == 1 ──
|
||||
let metrics = col::decode_metrics(&collected);
|
||||
let mut metric_names = vec![];
|
||||
let mut session_count_total = 0u64;
|
||||
for req in &metrics {
|
||||
for rm in &req.resource_metrics {
|
||||
for sm in &rm.scope_metrics {
|
||||
for metric in &sm.metrics {
|
||||
metric_names.push(metric.name.clone());
|
||||
use opentelemetry_proto::tonic::metrics::v1::metric::Data;
|
||||
if let Some(Data::Sum(sum)) = &metric.data {
|
||||
assert_eq!(
|
||||
sum.aggregation_temporality,
|
||||
opentelemetry_proto::tonic::metrics::v1::AggregationTemporality::Delta
|
||||
as i32,
|
||||
"default temporality must be Delta (CC parity)"
|
||||
);
|
||||
if metric.name == "grok_code.session.count" {
|
||||
for dp in &sum.data_points {
|
||||
if let Some(
|
||||
opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(v),
|
||||
) = dp.value
|
||||
{
|
||||
session_count_total += v as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
metric_names.iter().any(|n| n == "grok_code.session.count"),
|
||||
"missing session.count in {metric_names:?}"
|
||||
);
|
||||
assert!(metric_names.iter().any(|n| n == "grok_code.token.usage"));
|
||||
assert_eq!(
|
||||
session_count_total, 1,
|
||||
"session.count must increment exactly once per SessionNew"
|
||||
);
|
||||
|
||||
// ── Canary absence at the HTTP layer (raw bytes, both signals) ──────
|
||||
let raw_logs = collected.raw_logs();
|
||||
let raw_metrics = collected.raw_metrics();
|
||||
for (label, raw) in [("logs", &raw_logs), ("metrics", &raw_metrics)] {
|
||||
let haystack = String::from_utf8_lossy(raw);
|
||||
assert!(
|
||||
!haystack.contains("CANARY"),
|
||||
"canary reached the {label} wire: gates are off / scrub failed"
|
||||
);
|
||||
assert!(
|
||||
!haystack.contains(CANARY_MCP),
|
||||
"MCP server name reached the {label} wire"
|
||||
);
|
||||
}
|
||||
// Prompt length exported, text not (already covered by the canary scan).
|
||||
|
||||
// ── Shutdown: ≤ 2 s + post-shutdown silence ─────────────────────────
|
||||
let start = std::time::Instant::now();
|
||||
xai_grok_telemetry::external::shutdown();
|
||||
assert!(
|
||||
start.elapsed() <= std::time::Duration::from_millis(2500),
|
||||
"shutdown watchdog must bound exit at ~2s (took {:?})",
|
||||
start.elapsed()
|
||||
);
|
||||
assert!(!xai_grok_telemetry::external::is_active());
|
||||
|
||||
let logs_before = collected.logs_len();
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: 1,
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: None,
|
||||
});
|
||||
std::thread::sleep(std::time::Duration::from_millis(400));
|
||||
assert_eq!(
|
||||
collected.logs_len(),
|
||||
logs_before,
|
||||
"no exports after shutdown"
|
||||
);
|
||||
|
||||
// Idempotent shutdown: second call is a no-op, not an error/panic.
|
||||
xai_grok_telemetry::external::shutdown();
|
||||
}
|
||||
|
|
@ -0,0 +1,285 @@
|
|||
//! Wire test for the external OTEL stream with **both content gates ON** — the
|
||||
//! higher-risk privacy path, where prompt text and tool parameters actually
|
||||
//! leave the process. Asserts against an in-process OTLP collector that:
|
||||
//!
|
||||
//! - gated content (`prompt`, `tool_parameters`, `file_path`, verbatim
|
||||
//! `tool_name`/`mcp_server.name`) IS present when the gate is on,
|
||||
//! - planted secret shapes are STILL scrubbed inside that gated content
|
||||
//! (gates loosen *which fields* export, never the secret scrub),
|
||||
//! - identity attributes ride every record and metric once set,
|
||||
//! - `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=cumulative` and
|
||||
//! `OTEL_METRICS_INCLUDE_VERSION=1` take effect on the wire,
|
||||
//! - the remote fleet kill switch stops emission in-process.
|
||||
//!
|
||||
//! Single sequential `#[test]` because the `EXTERNAL` registry is a
|
||||
//! process-global `OnceLock`, so each init-config scenario is its own test
|
||||
//! binary.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use otlp_collector as col;
|
||||
use xai_grok_telemetry::external::{self, ExternalOtelRemotePolicy, IdentityAttrs};
|
||||
|
||||
// Secret shapes — MUST be scrubbed everywhere, even inside gated content.
|
||||
const SECRET_KEY: &str = "sk-LEAKaaaaaaaaaaaaaaaa1234567890";
|
||||
const SECRET_MODEL: &str = "grok-4-sk-LEAKmodel1234567890abcd";
|
||||
// Benign markers — with the gate ON these MUST appear on the wire (proving the
|
||||
// gated field is actually exported, not just that the scrub ran).
|
||||
const PROMPT_MARK: &str = "promptbodymarker";
|
||||
const PARAM_MARK: &str = "parammarker";
|
||||
const CLIENT_VERSION: &str = "9.9.9-cv";
|
||||
|
||||
#[test]
|
||||
fn external_stream_gates_on_end_to_end() {
|
||||
let collected = col::Collected::default();
|
||||
let endpoint = col::start_collector(collected.clone());
|
||||
|
||||
let mut cfg = external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
// Both content gates ON.
|
||||
"OTEL_LOG_USER_PROMPTS" | "OTEL_LOG_TOOL_DETAILS" => Some("1".into()),
|
||||
"OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE" => Some("cumulative".into()),
|
||||
"OTEL_METRICS_INCLUDE_VERSION" => Some("1".into()),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("200".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("double opt-in must resolve");
|
||||
assert!(cfg.gates.log_user_prompts && cfg.gates.log_tool_details);
|
||||
cfg.client = external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: CLIENT_VERSION.into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
|
||||
external::init(Some(cfg));
|
||||
assert!(external::is_active(), "gates-on config must activate");
|
||||
|
||||
// Identity attrs (plain ids — never tokens) ride every record + metric.
|
||||
external::set_identity(IdentityAttrs {
|
||||
user_id: Some("user-x".into()),
|
||||
organization_id: Some("org-acme".into()),
|
||||
team_id: Some("team-7".into()),
|
||||
deployment_id: Some("deploy-eu".into()),
|
||||
});
|
||||
|
||||
// Product events disabled — pins the "external active while product telemetry off"
|
||||
// half of the independence matrix through the real funnel.
|
||||
assert!(!xai_grok_telemetry::is_enabled());
|
||||
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionHarness {
|
||||
session_id: "sess-gates-on".into(),
|
||||
client_identifier: Some("grok-pager".into()),
|
||||
model_id: "grok-4".into(),
|
||||
agent_name: "grok-build-plan".into(),
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
mcp_server_names: vec!["internal-mcp".into()],
|
||||
plugin_names: vec![],
|
||||
skill_names: vec![],
|
||||
lsp_server_names: vec![],
|
||||
hook_names: vec![],
|
||||
agents_md_dir_names: vec![],
|
||||
memory_enabled: false,
|
||||
is_git_repo: true,
|
||||
auto_update: None,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: 100,
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: Some(format!("refactor {PROMPT_MARK} with key {SECRET_KEY} now")),
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::ModelResponseReceived {
|
||||
model_id: SECRET_MODEL.into(),
|
||||
duration_ms: 5,
|
||||
stop_reason: Some("stop".into()),
|
||||
prompt_tokens: Some(11),
|
||||
completion_tokens: Some(7),
|
||||
reasoning_tokens: Some(3),
|
||||
cached_prompt_tokens: Some(9),
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::ToolCallCompleted {
|
||||
tool_name: "github__create_issue".into(),
|
||||
outcome: xai_file_utils::events::types::ToolOutcome::Success,
|
||||
duration_ms: 12,
|
||||
file_path: Some("/tmp/projectdir/config.toml".into()),
|
||||
parameters: Some(serde_json::json!({
|
||||
"marker": PARAM_MARK,
|
||||
"token": SECRET_KEY,
|
||||
"deep": {"a": {"b": "c"}},
|
||||
})),
|
||||
});
|
||||
|
||||
external::flush();
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
!collected.logs.lock().unwrap().is_empty()
|
||||
&& !collected.metrics.lock().unwrap().is_empty()
|
||||
}),
|
||||
"collector must receive both signals"
|
||||
);
|
||||
|
||||
// ── Resource + scope ────────────────────────────────────────────────
|
||||
let records = col::log_records(&collected);
|
||||
let harness = col::find_event(&collected, "grok_code.session_start")
|
||||
.expect("session_start must be present");
|
||||
assert_eq!(harness.scope_name, "ai.xai.grok_code");
|
||||
assert_eq!(
|
||||
harness
|
||||
.resource
|
||||
.get("service.name")
|
||||
.and_then(|v| v.as_str()),
|
||||
Some("grok-cli"),
|
||||
"service.name=grok-cli is a wire commitment"
|
||||
);
|
||||
assert_eq!(
|
||||
harness
|
||||
.resource
|
||||
.get("grok_code.schema.version")
|
||||
.and_then(|v| v.as_str()),
|
||||
Some("v1")
|
||||
);
|
||||
// External records carry no free-text body.
|
||||
assert!(
|
||||
records.iter().all(|r| !r.has_body),
|
||||
"no record may carry a body"
|
||||
);
|
||||
|
||||
// ── Identity attrs on a record ──────────────────────────────────────
|
||||
assert_eq!(
|
||||
harness.attrs.get("user.id").and_then(|v| v.as_str()),
|
||||
Some("user-x")
|
||||
);
|
||||
assert_eq!(
|
||||
harness
|
||||
.attrs
|
||||
.get("organization.id")
|
||||
.and_then(|v| v.as_str()),
|
||||
Some("org-acme")
|
||||
);
|
||||
assert_eq!(
|
||||
harness.attrs.get("team.id").and_then(|v| v.as_str()),
|
||||
Some("team-7")
|
||||
);
|
||||
assert_eq!(
|
||||
harness.attrs.get("deployment.id").and_then(|v| v.as_str()),
|
||||
Some("deploy-eu")
|
||||
);
|
||||
|
||||
// ── Prompt gate ON: text present, secret still scrubbed ─────────────
|
||||
let prompt = col::find_event(&collected, "grok_code.user_prompt").expect("user_prompt present");
|
||||
let prompt_text = prompt
|
||||
.attrs
|
||||
.get("prompt")
|
||||
.and_then(|v| v.as_str())
|
||||
.expect("prompt attr present when OTEL_LOG_USER_PROMPTS=1");
|
||||
assert!(
|
||||
prompt_text.contains(PROMPT_MARK),
|
||||
"gated prompt body must export: {prompt_text:?}"
|
||||
);
|
||||
assert!(
|
||||
!prompt_text.contains(SECRET_KEY),
|
||||
"secret survived in prompt: {prompt_text:?}"
|
||||
);
|
||||
|
||||
// ── Tool details gate ON: verbatim name + gated path/params, scrubbed ─
|
||||
let tool = col::find_event(&collected, "grok_code.tool_result").expect("tool_result present");
|
||||
assert_eq!(
|
||||
tool.attrs.get("tool_name").and_then(|v| v.as_str()),
|
||||
Some("github__create_issue"),
|
||||
"details gate exposes the verbatim tool name"
|
||||
);
|
||||
assert_eq!(
|
||||
tool.attrs.get("file_extension").and_then(|v| v.as_str()),
|
||||
Some("toml"),
|
||||
"file_extension always exported"
|
||||
);
|
||||
assert!(
|
||||
tool.attrs.contains_key("file_path"),
|
||||
"full path exported under details gate"
|
||||
);
|
||||
let params = tool
|
||||
.attrs
|
||||
.get("tool_parameters")
|
||||
.and_then(|v| v.as_str())
|
||||
.expect("tool_parameters present under details gate");
|
||||
assert!(
|
||||
params.contains(PARAM_MARK),
|
||||
"gated params must export: {params:?}"
|
||||
);
|
||||
assert!(
|
||||
!params.contains(SECRET_KEY),
|
||||
"secret survived in params: {params:?}"
|
||||
);
|
||||
|
||||
// ── Metrics: cumulative temporality + app.version + scrubbed model ──
|
||||
let tokens = col::find_metric(&collected, "grok_code.token.usage");
|
||||
assert!(!tokens.is_empty(), "token.usage must export");
|
||||
for p in &tokens {
|
||||
assert_eq!(
|
||||
p.temporality,
|
||||
col::TEMPORALITY_CUMULATIVE,
|
||||
"cumulative requested"
|
||||
);
|
||||
assert_eq!(
|
||||
p.attrs.get("app.version").and_then(|v| v.as_str()),
|
||||
Some(CLIENT_VERSION),
|
||||
"OTEL_METRICS_INCLUDE_VERSION=1 attaches app.version"
|
||||
);
|
||||
assert_eq!(
|
||||
p.attrs.get("user.id").and_then(|v| v.as_str()),
|
||||
Some("user-x")
|
||||
);
|
||||
let model = p.attrs.get("model").and_then(|v| v.as_str()).unwrap_or("");
|
||||
assert!(
|
||||
!model.contains("sk-LEAKmodel"),
|
||||
"metric model must be scrubbed: {model:?}"
|
||||
);
|
||||
}
|
||||
let sessions = col::find_metric(&collected, "grok_code.session.count");
|
||||
// SessionHarness has no session.count metric; that comes from SessionNew —
|
||||
// not emitted here, so just confirm token.usage identity coverage above.
|
||||
let _ = sessions;
|
||||
|
||||
// ── Canary scan at the raw HTTP layer (both signals) ────────────────
|
||||
let raw = collected.raw_text();
|
||||
assert!(!raw.contains(SECRET_KEY), "secret key reached the wire");
|
||||
assert!(
|
||||
!raw.contains("sk-LEAKmodel"),
|
||||
"secret model shape reached the wire"
|
||||
);
|
||||
|
||||
// ── Remote fleet kill switch stops emission in-process ──────────────
|
||||
external::flush();
|
||||
col::wait_until(std::time::Duration::from_millis(500), || false);
|
||||
let logs_before = collected.logs_len();
|
||||
external::apply_remote_policy(ExternalOtelRemotePolicy {
|
||||
force_disable: true,
|
||||
lock_content_gates: false,
|
||||
});
|
||||
assert!(
|
||||
!external::is_active(),
|
||||
"kill switch must clear the emission gate"
|
||||
);
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: 1,
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: Some("post-kill".into()),
|
||||
});
|
||||
std::thread::sleep(std::time::Duration::from_millis(400));
|
||||
assert_eq!(
|
||||
collected.logs_len(),
|
||||
logs_before,
|
||||
"no exports after the remote kill switch"
|
||||
);
|
||||
|
||||
external::shutdown();
|
||||
}
|
||||
127
crates/codegen/xai-grok-telemetry/tests/external_otlp_grpc.rs
Normal file
127
crates/codegen/xai-grok-telemetry/tests/external_otlp_grpc.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//! gRPC transport coverage for the external OTEL stream. This mirrors the
|
||||
//! primary HTTP/protobuf wire test in `external_otlp.rs`, but must live in its
|
||||
//! own integration-test binary because the external telemetry registry is a
|
||||
//! process-global `OnceLock`.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use otlp_collector as col;
|
||||
|
||||
const CANARY_MODEL: &str = "sk-CANARYgrpcabcdefghij1234567890";
|
||||
const CANARY_PROMPT: &str = "CANARY_GRPC_PROMPT_TEXT do not export";
|
||||
const CANARY_MCP: &str = "canary-grpc-internal-mcp-server";
|
||||
|
||||
#[test]
|
||||
fn external_stream_grpc_end_to_end() {
|
||||
let collected = col::Collected::default();
|
||||
let endpoint =
|
||||
col::start_collector_with_protocol(collected.clone(), col::CollectorProtocol::Grpc);
|
||||
|
||||
let mut cfg = xai_grok_telemetry::external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL" => Some("grpc".into()),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("200".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("double opt-in must resolve");
|
||||
cfg.client = xai_grok_telemetry::external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: "0.0.0-test".into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
|
||||
xai_grok_telemetry::external::init(Some(cfg));
|
||||
assert!(xai_grok_telemetry::external::is_active());
|
||||
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionNew {
|
||||
session_id: "sess-grpc-1".into(),
|
||||
client_identifier: None,
|
||||
client_version: None,
|
||||
is_git_repo: true,
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionHarness {
|
||||
session_id: "sess-grpc-1".into(),
|
||||
client_identifier: Some("grok-pager".into()),
|
||||
model_id: "grok-4".into(),
|
||||
agent_name: "grok-build-plan".into(),
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
mcp_server_names: vec![CANARY_MCP.into()],
|
||||
plugin_names: vec![],
|
||||
skill_names: vec![],
|
||||
lsp_server_names: vec![],
|
||||
hook_names: vec![],
|
||||
agents_md_dir_names: vec![],
|
||||
memory_enabled: false,
|
||||
is_git_repo: true,
|
||||
auto_update: None,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: CANARY_PROMPT.len(),
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: Some(CANARY_PROMPT.into()),
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::ModelResponseReceived {
|
||||
model_id: CANARY_MODEL.into(),
|
||||
duration_ms: 5,
|
||||
stop_reason: Some("stop".into()),
|
||||
prompt_tokens: Some(11),
|
||||
completion_tokens: Some(7),
|
||||
reasoning_tokens: None,
|
||||
cached_prompt_tokens: None,
|
||||
});
|
||||
|
||||
xai_grok_telemetry::external::flush();
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
collected.logs_len() > 0 && collected.metrics_len() > 0
|
||||
}),
|
||||
"gRPC collector must receive both signals"
|
||||
);
|
||||
|
||||
let event_names = col::event_names(&collected);
|
||||
for expected in [
|
||||
"grok_code.session_start",
|
||||
"grok_code.user_prompt",
|
||||
"grok_code.api_request",
|
||||
] {
|
||||
assert!(
|
||||
event_names.iter().any(|n| n == expected),
|
||||
"missing {expected} in {event_names:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let metrics = col::metric_points(&collected);
|
||||
assert!(
|
||||
metrics.iter().any(|p| p.name == "grok_code.session.count"),
|
||||
"missing session.count in {metrics:?}"
|
||||
);
|
||||
assert!(
|
||||
metrics.iter().any(|p| p.name == "grok_code.token.usage"),
|
||||
"missing token.usage in {metrics:?}"
|
||||
);
|
||||
for point in metrics {
|
||||
assert_eq!(
|
||||
point.temporality,
|
||||
col::TEMPORALITY_DELTA,
|
||||
"default temporality must be Delta over gRPC"
|
||||
);
|
||||
}
|
||||
|
||||
let raw = collected.raw_text();
|
||||
assert!(!raw.contains("CANARY"), "canary reached the gRPC wire");
|
||||
assert!(
|
||||
!raw.contains(CANARY_MCP),
|
||||
"MCP server name reached the gRPC wire"
|
||||
);
|
||||
|
||||
xai_grok_telemetry::external::shutdown();
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
//! Wire test for the **no-double-send invariant** (the credential-leak guard).
|
||||
//!
|
||||
//! If the internal trace firehose resolved its endpoint/headers from the
|
||||
//! deprecated `OTEL_EXPORTER_OTLP_*` fallback, the shell sets
|
||||
//! `internal_pipeline_consumed_otel_vars = true`, and `external::init` MUST
|
||||
//! refuse to activate — otherwise the same standard vars could point both the
|
||||
//! internally-authed firehose and the customer collector at one endpoint,
|
||||
//! leaking xAI credentials. Here we prove the refusal end-to-end: even with a
|
||||
//! fully valid double opt-in pointed at a live collector, nothing is exported.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use otlp_collector as col;
|
||||
use xai_grok_telemetry::external;
|
||||
|
||||
#[test]
|
||||
fn refuses_to_activate_when_internal_consumed_standard_vars() {
|
||||
let collected = col::Collected::default();
|
||||
let endpoint = col::start_collector(collected.clone());
|
||||
|
||||
let mut cfg = external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("100".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("config resolves (the refusal happens at init, not resolution)");
|
||||
cfg.client = external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: "0.0.0-test".into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
// The flag the shell sets when the internal firehose consumed the standard
|
||||
// OTEL_* vars via the deprecated fallback.
|
||||
cfg.internal_pipeline_consumed_otel_vars = true;
|
||||
|
||||
external::init(Some(cfg));
|
||||
assert!(
|
||||
!external::is_active(),
|
||||
"external stream MUST refuse to activate to prevent credential leakage"
|
||||
);
|
||||
|
||||
// Emit through the real funnel; with the stream inert this must be a no-op.
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionNew {
|
||||
session_id: "sess-guard".into(),
|
||||
client_identifier: None,
|
||||
client_version: None,
|
||||
is_git_repo: true,
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
});
|
||||
external::flush();
|
||||
|
||||
// Give any (erroneously constructed) exporter ample time to phone home.
|
||||
std::thread::sleep(std::time::Duration::from_millis(600));
|
||||
assert_eq!(
|
||||
collected.logs_len(),
|
||||
0,
|
||||
"no logs may be exported when refused"
|
||||
);
|
||||
assert_eq!(
|
||||
collected.metrics_len(),
|
||||
0,
|
||||
"no metrics may be exported when refused"
|
||||
);
|
||||
|
||||
external::shutdown();
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
//! Wire test for ambient-context injection: when events are emitted inside a
|
||||
//! `with_session_ctx` scope, the external records must carry `session.id`,
|
||||
//! `turn_number`, `prompt.id`, and a monotonic `event.sequence` — and
|
||||
//! `prompt.id` must appear on events ONLY, never on metrics (unbounded
|
||||
//! cardinality). Complements the other wire tests, which emit outside any ctx.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use otlp_collector as col;
|
||||
use xai_grok_telemetry::external;
|
||||
|
||||
#[test]
|
||||
fn ambient_ctx_injects_session_turn_and_prompt_id() {
|
||||
let collected = col::Collected::default();
|
||||
let endpoint = col::start_collector(collected.clone());
|
||||
|
||||
let mut cfg = external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("150".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("double opt-in must resolve");
|
||||
cfg.client = external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: "0.0.0-test".into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
external::init(Some(cfg));
|
||||
assert!(external::is_active());
|
||||
|
||||
// Emit inside a session ctx (turn_number = 3) so the ambient snapshot is
|
||||
// populated. `log_event` is synchronous and runs within the task-local
|
||||
// scope of `with_session_ctx`.
|
||||
let ctx = xai_grok_telemetry::TelemetryCtx::new(
|
||||
"sess-ctx".to_owned(),
|
||||
Arc::new(tokio::sync::Mutex::new(3usize)),
|
||||
);
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.expect("current-thread runtime");
|
||||
rt.block_on(xai_grok_telemetry::with_session_ctx(ctx, async {
|
||||
xai_grok_telemetry::session_ctx::begin_prompt_id();
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::PromptSubmitted {
|
||||
prompt_length: 42,
|
||||
model_id: "grok-4".into(),
|
||||
client_identifier: None,
|
||||
screen_mode: None,
|
||||
prompt_text: None,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::ModelResponseReceived {
|
||||
model_id: "grok-4".into(),
|
||||
duration_ms: 5,
|
||||
stop_reason: Some("stop".into()),
|
||||
prompt_tokens: Some(11),
|
||||
completion_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_prompt_tokens: None,
|
||||
});
|
||||
}));
|
||||
|
||||
external::flush();
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
!collected.logs.lock().unwrap().is_empty()
|
||||
&& !collected.metrics.lock().unwrap().is_empty()
|
||||
}),
|
||||
"collector must receive both signals"
|
||||
);
|
||||
|
||||
// ── Event carries session.id, turn_number, prompt.id, event.sequence ──
|
||||
let prompt = col::find_event(&collected, "grok_code.user_prompt").expect("user_prompt present");
|
||||
assert_eq!(
|
||||
prompt.attrs.get("session.id").and_then(|v| v.as_str()),
|
||||
Some("sess-ctx"),
|
||||
"ambient session.id injected onto events"
|
||||
);
|
||||
assert_eq!(
|
||||
prompt.attrs.get("turn_number").and_then(|v| v.as_i64()),
|
||||
Some(3),
|
||||
"ambient turn_number injected onto events"
|
||||
);
|
||||
let prompt_id = prompt
|
||||
.attrs
|
||||
.get("prompt.id")
|
||||
.and_then(|v| v.as_str())
|
||||
.expect("prompt.id injected onto events");
|
||||
assert!(!prompt_id.is_empty(), "prompt.id must be a real uuid");
|
||||
assert!(
|
||||
prompt.attrs.contains_key("event.sequence"),
|
||||
"event.sequence injected onto every event"
|
||||
);
|
||||
|
||||
// ── prompt.id / turn_number NEVER on metrics ────────────────────────
|
||||
let tokens = col::find_metric(&collected, "grok_code.token.usage");
|
||||
assert!(!tokens.is_empty(), "token.usage must export");
|
||||
for p in &tokens {
|
||||
assert!(
|
||||
!p.attrs.contains_key("prompt.id"),
|
||||
"prompt.id must never reach metrics"
|
||||
);
|
||||
assert!(
|
||||
!p.attrs.contains_key("turn_number"),
|
||||
"turn_number must never reach metrics"
|
||||
);
|
||||
// session.id DOES flow to metrics from the ambient ctx (cardinality
|
||||
// opt-in, default on).
|
||||
assert_eq!(
|
||||
p.attrs.get("session.id").and_then(|v| v.as_str()),
|
||||
Some("sess-ctx")
|
||||
);
|
||||
}
|
||||
|
||||
external::shutdown();
|
||||
}
|
||||
89
crates/codegen/xai-grok-telemetry/tests/manual_auth_emit.rs
Normal file
89
crates/codegen/xai-grok-telemetry/tests/manual_auth_emit.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
//! Wire test: `log_event(ManualAuth)` must POST to the product events endpoint as
|
||||
//! `grok-shell-manual_auth` with the `reason`/`trigger`/`token_kind`/`principal`
|
||||
//! the `distinct(principal)` alert consumes. Mocks the observability backend
|
||||
//! (real HTTP collector) so the emit->wire path is checked, not just the struct.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use xai_grok_telemetry::client;
|
||||
use xai_grok_telemetry::config::{TelemetryConfig, TelemetryMode};
|
||||
use xai_grok_telemetry::events::{AuthTokenKind, ManualAuth, ManualAuthReason, ManualAuthSurface};
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn manual_auth_posts_to_events_endpoint_as_grok_shell_manual_auth() {
|
||||
let bodies: Arc<Mutex<Vec<serde_json::Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let captured = bodies.clone();
|
||||
let app = axum::Router::new().route(
|
||||
"/events",
|
||||
axum::routing::post(move |axum::Json(v): axum::Json<serde_json::Value>| {
|
||||
let captured = captured.clone();
|
||||
async move {
|
||||
captured.lock().unwrap().push(v);
|
||||
axum::http::StatusCode::OK
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/events", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
|
||||
client::init(
|
||||
TelemetryConfig {
|
||||
events_url: Some(url),
|
||||
events_api_key: Some("test-key".into()),
|
||||
mixpanel_enabled: false,
|
||||
..TelemetryConfig::default()
|
||||
},
|
||||
TelemetryMode::Enabled,
|
||||
Some("user-xyz".into()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"0.0.0-test".into(),
|
||||
None,
|
||||
reqwest::Client::new(),
|
||||
);
|
||||
|
||||
xai_grok_telemetry::log_event(ManualAuth {
|
||||
reason: ManualAuthReason::RefreshTokenRejected,
|
||||
trigger: ManualAuthSurface::Turn,
|
||||
token_kind: AuthTokenKind::OidcSession,
|
||||
principal: Some("user-xyz".into()),
|
||||
});
|
||||
|
||||
// The emit is fire-and-forget; poll the collector for the POST.
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let event = loop {
|
||||
let found = bodies.lock().unwrap().iter().find_map(|b| {
|
||||
let e = b.get("events")?.get(0)?;
|
||||
(e.get("event_name")?.as_str()? == "grok-shell-manual_auth").then(|| e.clone())
|
||||
});
|
||||
if let Some(e) = found {
|
||||
break e;
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"no grok-shell-manual_auth POST received"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
};
|
||||
|
||||
let meta = event.get("event_metadata").expect("event_metadata present");
|
||||
assert_eq!(
|
||||
meta.get("reason").and_then(|v| v.as_str()),
|
||||
Some("refresh_token_rejected"),
|
||||
);
|
||||
assert_eq!(meta.get("trigger").and_then(|v| v.as_str()), Some("turn"));
|
||||
assert_eq!(
|
||||
meta.get("token_kind").and_then(|v| v.as_str()),
|
||||
Some("oidc_session"),
|
||||
);
|
||||
assert_eq!(
|
||||
meta.get("principal").and_then(|v| v.as_str()),
|
||||
Some("user-xyz"),
|
||||
"principal must be a queryable top-level metadata field for distinct() counting",
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
383
crates/codegen/xai-grok-telemetry/tests/otlp_collector/mod.rs
Normal file
383
crates/codegen/xai-grok-telemetry/tests/otlp_collector/mod.rs
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
//! Shared in-process OTLP collector + decode helpers for the external-stream
|
||||
//! wire tests. Each integration-test binary that needs a collector does
|
||||
//! `mod otlp_collector;` and uses these.
|
||||
//!
|
||||
//! The collector runs on its own thread with its own current-thread runtime.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use prost::Message as _;
|
||||
|
||||
use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsService;
|
||||
use opentelemetry_proto::tonic::collector::logs::v1::{
|
||||
ExportLogsServiceRequest, ExportLogsServiceResponse,
|
||||
};
|
||||
use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsService;
|
||||
use opentelemetry_proto::tonic::collector::metrics::v1::{
|
||||
ExportMetricsServiceRequest, ExportMetricsServiceResponse,
|
||||
};
|
||||
|
||||
/// Delta / Cumulative aggregation-temporality enum values (OTLP metrics v1).
|
||||
pub const TEMPORALITY_DELTA: i32 = 1;
|
||||
pub const TEMPORALITY_CUMULATIVE: i32 = 2;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Collected {
|
||||
pub logs: Arc<Mutex<Vec<Vec<u8>>>>,
|
||||
pub metrics: Arc<Mutex<Vec<Vec<u8>>>>,
|
||||
}
|
||||
|
||||
impl Collected {
|
||||
pub fn logs_len(&self) -> usize {
|
||||
self.logs.lock().unwrap().len()
|
||||
}
|
||||
pub fn metrics_len(&self) -> usize {
|
||||
self.metrics.lock().unwrap().len()
|
||||
}
|
||||
pub fn raw_logs(&self) -> Vec<u8> {
|
||||
self.logs.lock().unwrap().concat()
|
||||
}
|
||||
pub fn raw_metrics(&self) -> Vec<u8> {
|
||||
self.metrics.lock().unwrap().concat()
|
||||
}
|
||||
/// Combined raw bytes of both signals, lossy-decoded to a string — for
|
||||
/// canary/leak scans at the HTTP layer.
|
||||
pub fn raw_text(&self) -> String {
|
||||
let mut bytes = self.raw_logs();
|
||||
bytes.extend(self.raw_metrics());
|
||||
String::from_utf8_lossy(&bytes).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum CollectorProtocol {
|
||||
HttpProtobuf,
|
||||
Grpc,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GrpcCollector {
|
||||
collected: Collected,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LogsService for GrpcCollector {
|
||||
async fn export(
|
||||
&self,
|
||||
request: tonic::Request<ExportLogsServiceRequest>,
|
||||
) -> Result<tonic::Response<ExportLogsServiceResponse>, tonic::Status> {
|
||||
let mut body = Vec::new();
|
||||
request
|
||||
.into_inner()
|
||||
.encode(&mut body)
|
||||
.expect("encode gRPC logs request");
|
||||
self.collected.logs.lock().unwrap().push(body);
|
||||
Ok(tonic::Response::new(ExportLogsServiceResponse::default()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MetricsService for GrpcCollector {
|
||||
async fn export(
|
||||
&self,
|
||||
request: tonic::Request<ExportMetricsServiceRequest>,
|
||||
) -> Result<tonic::Response<ExportMetricsServiceResponse>, tonic::Status> {
|
||||
let mut body = Vec::new();
|
||||
request
|
||||
.into_inner()
|
||||
.encode(&mut body)
|
||||
.expect("encode gRPC metrics request");
|
||||
self.collected.metrics.lock().unwrap().push(body);
|
||||
Ok(tonic::Response::new(ExportMetricsServiceResponse::default()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Start an HTTP/protobuf collector; returns its base URL
|
||||
/// (`http://127.0.0.1:PORT`).
|
||||
pub fn start_collector(collected: Collected) -> String {
|
||||
start_collector_with_protocol(collected, CollectorProtocol::HttpProtobuf)
|
||||
}
|
||||
|
||||
/// Start the collector for the requested OTLP transport; returns its base URL
|
||||
/// (`http://127.0.0.1:PORT`).
|
||||
pub fn start_collector_with_protocol(collected: Collected, protocol: CollectorProtocol) -> String {
|
||||
let (addr_tx, addr_rx) = std::sync::mpsc::channel::<SocketAddr>();
|
||||
std::thread::spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("collector runtime");
|
||||
rt.block_on(async move {
|
||||
match protocol {
|
||||
CollectorProtocol::HttpProtobuf => start_http_collector(collected, addr_tx).await,
|
||||
CollectorProtocol::Grpc => start_grpc_collector(collected, addr_tx).await,
|
||||
}
|
||||
});
|
||||
});
|
||||
let addr = addr_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("collector must start");
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
async fn start_http_collector(collected: Collected, addr_tx: std::sync::mpsc::Sender<SocketAddr>) {
|
||||
use axum::{Router, body::Bytes, extract::State, routing::post};
|
||||
async fn sink(
|
||||
State((store, which)): State<(Collected, &'static str)>,
|
||||
body: Bytes,
|
||||
) -> &'static str {
|
||||
let target = match which {
|
||||
"logs" => &store.logs,
|
||||
_ => &store.metrics,
|
||||
};
|
||||
target.lock().unwrap().push(body.to_vec());
|
||||
""
|
||||
}
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/v1/logs",
|
||||
post(sink).with_state((collected.clone(), "logs")),
|
||||
)
|
||||
.route(
|
||||
"/v1/metrics",
|
||||
post(sink).with_state((collected.clone(), "metrics")),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind collector");
|
||||
addr_tx
|
||||
.send(listener.local_addr().expect("collector addr"))
|
||||
.expect("send addr");
|
||||
axum::serve(listener, app).await.expect("collector serve");
|
||||
}
|
||||
|
||||
async fn start_grpc_collector(collected: Collected, addr_tx: std::sync::mpsc::Sender<SocketAddr>) {
|
||||
use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsServiceServer;
|
||||
use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsServiceServer;
|
||||
|
||||
let incoming = tonic::transport::server::TcpIncoming::bind(
|
||||
"127.0.0.1:0".parse().expect("collector bind addr"),
|
||||
)
|
||||
.expect("bind gRPC collector");
|
||||
addr_tx
|
||||
.send(incoming.local_addr().expect("collector addr"))
|
||||
.expect("send addr");
|
||||
let service = GrpcCollector { collected };
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(LogsServiceServer::new(service.clone()))
|
||||
.add_service(MetricsServiceServer::new(service))
|
||||
.serve_with_incoming(incoming)
|
||||
.await
|
||||
.expect("collector serve");
|
||||
}
|
||||
|
||||
pub fn decode_logs(
|
||||
collected: &Collected,
|
||||
) -> Vec<opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest> {
|
||||
collected
|
||||
.logs
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|body| {
|
||||
opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest::decode(
|
||||
body.as_slice(),
|
||||
)
|
||||
.expect("valid logs protobuf")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn decode_metrics(
|
||||
collected: &Collected,
|
||||
) -> Vec<opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest> {
|
||||
collected
|
||||
.metrics
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|body| {
|
||||
opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest::decode(
|
||||
body.as_slice(),
|
||||
)
|
||||
.expect("valid metrics protobuf")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Poll `check` until it is true or `deadline` elapses.
|
||||
pub fn wait_until(deadline: std::time::Duration, mut check: impl FnMut() -> bool) -> bool {
|
||||
let start = std::time::Instant::now();
|
||||
while start.elapsed() < deadline {
|
||||
if check() {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(25));
|
||||
}
|
||||
check()
|
||||
}
|
||||
|
||||
// ── Decoding ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Flattened attribute value (the external schema is flat: no arrays/maps).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum AttrVal {
|
||||
S(String),
|
||||
I(i64),
|
||||
B(bool),
|
||||
D(f64),
|
||||
Other,
|
||||
}
|
||||
|
||||
impl AttrVal {
|
||||
pub fn as_str(&self) -> Option<&str> {
|
||||
match self {
|
||||
AttrVal::S(s) => Some(s.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
pub fn as_i64(&self) -> Option<i64> {
|
||||
match self {
|
||||
AttrVal::I(i) => Some(*i),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn anyval(v: &opentelemetry_proto::tonic::common::v1::AnyValue) -> AttrVal {
|
||||
use opentelemetry_proto::tonic::common::v1::any_value::Value;
|
||||
match &v.value {
|
||||
Some(Value::StringValue(s)) => AttrVal::S(s.clone()),
|
||||
Some(Value::IntValue(i)) => AttrVal::I(*i),
|
||||
Some(Value::BoolValue(b)) => AttrVal::B(*b),
|
||||
Some(Value::DoubleValue(d)) => AttrVal::D(*d),
|
||||
_ => AttrVal::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn kvs_to_map(
|
||||
attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue],
|
||||
) -> HashMap<String, AttrVal> {
|
||||
attrs
|
||||
.iter()
|
||||
.filter_map(|kv| kv.value.as_ref().map(|v| (kv.key.clone(), anyval(v))))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// One decoded external log record.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RecordView {
|
||||
pub event_name: String,
|
||||
pub attrs: HashMap<String, AttrVal>,
|
||||
/// `service.name`, `grok_code.schema.version`, … from the owning resource.
|
||||
pub resource: HashMap<String, AttrVal>,
|
||||
pub scope_name: String,
|
||||
pub has_body: bool,
|
||||
}
|
||||
|
||||
pub fn log_records(c: &Collected) -> Vec<RecordView> {
|
||||
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
|
||||
let mut out = Vec::new();
|
||||
for body in c.logs.lock().unwrap().iter() {
|
||||
let req = ExportLogsServiceRequest::decode(body.as_slice()).expect("valid logs protobuf");
|
||||
for rl in &req.resource_logs {
|
||||
let resource = rl
|
||||
.resource
|
||||
.as_ref()
|
||||
.map(|r| kvs_to_map(&r.attributes))
|
||||
.unwrap_or_default();
|
||||
for sl in &rl.scope_logs {
|
||||
let scope_name = sl
|
||||
.scope
|
||||
.as_ref()
|
||||
.map(|s| s.name.clone())
|
||||
.unwrap_or_default();
|
||||
for r in &sl.log_records {
|
||||
out.push(RecordView {
|
||||
event_name: r.event_name.clone(),
|
||||
attrs: kvs_to_map(&r.attributes),
|
||||
resource: resource.clone(),
|
||||
scope_name: scope_name.clone(),
|
||||
has_body: r.body.is_some(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One decoded metric data point (sums only — the external schema is all
|
||||
/// monotonic counters).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetricPoint {
|
||||
pub name: String,
|
||||
pub temporality: i32,
|
||||
pub is_monotonic: bool,
|
||||
pub attrs: HashMap<String, AttrVal>,
|
||||
pub int_value: i64,
|
||||
pub scope_name: String,
|
||||
}
|
||||
|
||||
pub fn metric_points(c: &Collected) -> Vec<MetricPoint> {
|
||||
use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
|
||||
use opentelemetry_proto::tonic::metrics::v1::metric::Data;
|
||||
use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value;
|
||||
let mut out = Vec::new();
|
||||
for body in c.metrics.lock().unwrap().iter() {
|
||||
let req =
|
||||
ExportMetricsServiceRequest::decode(body.as_slice()).expect("valid metrics protobuf");
|
||||
for rm in &req.resource_metrics {
|
||||
for sm in &rm.scope_metrics {
|
||||
let scope_name = sm
|
||||
.scope
|
||||
.as_ref()
|
||||
.map(|s| s.name.clone())
|
||||
.unwrap_or_default();
|
||||
for metric in &sm.metrics {
|
||||
if let Some(Data::Sum(sum)) = &metric.data {
|
||||
for dp in &sum.data_points {
|
||||
let int_value = match dp.value {
|
||||
Some(Value::AsInt(i)) => i,
|
||||
Some(Value::AsDouble(d)) => d as i64,
|
||||
None => 0,
|
||||
};
|
||||
out.push(MetricPoint {
|
||||
name: metric.name.clone(),
|
||||
temporality: sum.aggregation_temporality,
|
||||
is_monotonic: sum.is_monotonic,
|
||||
attrs: kvs_to_map(&dp.attributes),
|
||||
int_value,
|
||||
scope_name: scope_name.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// All event names seen across the decoded log records.
|
||||
pub fn event_names(c: &Collected) -> Vec<String> {
|
||||
log_records(c).into_iter().map(|r| r.event_name).collect()
|
||||
}
|
||||
|
||||
/// First record matching `event_name`, if any.
|
||||
pub fn find_event(c: &Collected, event_name: &str) -> Option<RecordView> {
|
||||
log_records(c)
|
||||
.into_iter()
|
||||
.find(|r| r.event_name == event_name)
|
||||
}
|
||||
|
||||
/// All metric points for a given metric name.
|
||||
pub fn find_metric(c: &Collected, name: &str) -> Vec<MetricPoint> {
|
||||
metric_points(c)
|
||||
.into_iter()
|
||||
.filter(|m| m.name == name)
|
||||
.collect()
|
||||
}
|
||||
Loading…
Reference in a new issue