//! Opt-in, content-redacted **external OTEL** telemetry stream. //! //! Enterprise customers point the Grok CLI at *their own* OpenTelemetry //! collector (standard `OTEL_*` env vars + the `GROK_EXTERNAL_OTEL` master //! switch) and receive a curated, ZDR-safe schema: ~6 counters and ~17 //! log-record events fanned out from the same typed call sites that emit the //! product events ([`crate::session_ctx::log_event`]). //! //! Structural invariants (enforced by construction and tests): //! - The providers here are **never** registered with `opentelemetry::global` //! (the internal tracer provider owns the global slot); everything is //! handle-based through the [`EXTERNAL`] registry. //! - The exporters carry **only** customer headers/metadata from //! `OTEL_EXPORTER_OTLP_HEADERS` — this module has no dependency on //! `AuthCredentialProvider` and no code path that can attach internal auth //! headers. //! - Default **off**: with `GROK_EXTERNAL_OTEL` unset (or no exporter //! selected) nothing is constructed — zero allocation, zero threads, zero //! sockets. //! - Independent of `TelemetryMode`, GCS trace upload, and the data-collection / //! data-retention opt-outs (user-confirmed): those govern xAI-side //! retention; this stream ships only to the customer's own collector under //! the customer's own explicit double opt-in. //! //! This module is the second authoritative privacy boundary in this crate //! (alongside `otel_layer::redact`). pub mod config; mod emit; mod providers; mod redact; pub mod schema; pub mod truncate; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use opentelemetry::logs::LoggerProvider as _; use opentelemetry::metrics::MeterProvider as _; use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider}; use opentelemetry_sdk::metrics::SdkMeterProvider; pub use config::{ContentGates, ExternalOtelConfig, ExternalOtelFileConfig}; static EXTERNAL: OnceLock>> = OnceLock::new(); /// Identity *attributes* (plain id strings — never tokens). Derived from a /// `CredentialSnapshot` at the telemetry-client init sites; updated post-auth /// and on logout. #[derive(Debug, Clone, Default)] pub struct IdentityAttrs { pub user_id: Option, pub organization_id: Option, pub team_id: Option, pub deployment_id: Option, } impl IdentityAttrs { pub fn from_snapshot(snapshot: &xai_grok_auth::CredentialSnapshot) -> Self { Self { user_id: snapshot.user_id.clone(), organization_id: snapshot.organization_id.clone(), team_id: snapshot.team_id.clone(), deployment_id: snapshot.deployment_id.clone(), } } } /// Remote-settings policy for the external stream. **Restrictive-only by /// construction**: there is deliberately no enable direction (remote settings /// are fetched per-run and never persisted, so a remote "enable" could never /// reach init). #[derive(Debug, Clone, Copy, Default)] pub struct ExternalOtelRemotePolicy { /// Fleet kill switch: flush, then drop subsequent emissions in-process. pub force_disable: bool, /// Force the content gates off regardless of local env/config. pub lock_content_gates: bool, } /// The handle owning both providers. Never global; reached only through the /// [`EXTERNAL`] registry. pub struct ExternalTelemetry { logger_provider: Option, meter_provider: Option, logger: Option, instruments: Option, /// Emission gate; cleared by the remote force-disable. The single /// authority for "emitting right now". active: AtomicBool, /// Content gates; may only TIGHTEN post-init. gates: redact::SharedGates, identity: parking_lot::RwLock, /// `event.sequence` (monotonic, per-process). sequence: AtomicU64, shutdown_once: std::sync::Once, include_session_id_on_metrics: bool, include_version_on_metrics: bool, app_version: String, health: Arc, /// Init summary for the adoption meta-event (emitted once, post-auth). configured_meta: ConfiguredMeta, meta_event_once: std::sync::Once, } #[derive(Debug, Clone)] struct ConfiguredMeta { metrics_exporter: &'static str, logs_exporter: &'static str, logs_endpoint_origin: String, metrics_endpoint_origin: String, protocol: &'static str, prompts_gate: bool, details_gate: bool, source: &'static str, } fn exporter_label(sel: config::ExporterSelection) -> &'static str { match sel { config::ExporterSelection::None => "none", config::ExporterSelection::Otlp => "otlp", config::ExporterSelection::Console => "console", } } impl ExternalTelemetry { pub(crate) fn next_sequence(&self) -> u64 { self.sequence.fetch_add(1, Ordering::Relaxed) } } /// Initialize the external stream. Called once from binary startup after /// config resolution, **before auth** (no credentials needed). `None` records /// the dormant state — the default path allocates nothing. pub fn init(cfg: Option) { let value = cfg.and_then(build_handle); if EXTERNAL.set(value).is_err() { tracing::debug!("external otel: init called more than once; keeping first registration"); } } fn build_handle(cfg: ExternalOtelConfig) -> Option> { // No-double-send invariant, enforced in code (not release discipline): // if the internal firehose resolved its endpoint/headers from // `OTEL_EXPORTER_OTLP_*` (the deprecated fallback), refuse to activate. if cfg.internal_pipeline_consumed_otel_vars { tracing::warn!( "external otel: refusing to activate — the internal trace pipeline consumed \ OTEL_EXPORTER_OTLP_* (deprecated fallback). Migrate internal repointing to \ GROK_INTERNAL_OTLP_* to use the external stream." ); return None; } let gates: redact::SharedGates = Arc::new(parking_lot::RwLock::new(cfg.gates)); let health = Arc::new(redact::ExportHealth::default()); let built = match providers::build(&cfg, gates.clone(), health.clone()) { Ok(built) => built, Err(e) => { tracing::warn!(error = %e, "external otel: exporter construction failed; stream disabled"); return None; } }; if built.logger_provider.is_none() && built.meter_provider.is_none() { return None; } let logger = built .logger_provider .as_ref() .map(|p| p.logger(schema::SCOPE_NAME)); let instruments = built .meter_provider .as_ref() .map(|p| emit::Instruments::new(&p.meter(schema::SCOPE_NAME))); let configured_meta = ConfiguredMeta { metrics_exporter: exporter_label(cfg.metrics_exporter), logs_exporter: exporter_label(cfg.logs_exporter), logs_endpoint_origin: crate::redact_common::url_origin(&cfg.logs_endpoint).into_owned(), metrics_endpoint_origin: crate::redact_common::url_origin(&cfg.metrics_endpoint) .into_owned(), protocol: cfg.transport.as_protocol_str(), prompts_gate: cfg.gates.log_user_prompts, details_gate: cfg.gates.log_tool_details, source: cfg.enabled_source, }; tracing::debug!( metrics_exporter = configured_meta.metrics_exporter, logs_exporter = configured_meta.logs_exporter, "external otel: stream active" ); Some(Arc::new(ExternalTelemetry { logger_provider: built.logger_provider, meter_provider: built.meter_provider, logger, instruments, active: AtomicBool::new(true), gates, identity: parking_lot::RwLock::new(IdentityAttrs::default()), sequence: AtomicU64::new(0), shutdown_once: std::sync::Once::new(), include_session_id_on_metrics: cfg.include_session_id_on_metrics, include_version_on_metrics: cfg.include_version_on_metrics, app_version: cfg.client.client_version.clone(), health, configured_meta, meta_event_once: std::sync::Once::new(), })) } fn handle() -> Option> { EXTERNAL.get().and_then(|opt| opt.clone()) } fn active_handle() -> Option> { handle().filter(|ext| ext.active.load(Ordering::Relaxed)) } /// Cheap check used by the fan-out hook and the split-sink call sites: /// registry present AND the runtime emission gate set. A stale `true` read /// only costs a wasted mapping, never an export ([`emit`] re-checks). pub fn is_active() -> bool { matches!(EXTERNAL.get(), Some(Some(ext)) if ext.active.load(Ordering::Relaxed)) } /// Map and emit one typed telemetry event. No-op unless the stream is active /// and the event has an `external = …` mapping. Synchronous and cheap (the /// batch processor queues; nothing blocks on I/O). pub fn emit(data: &T) { let Some(ext) = active_handle() else { return; }; let Some(record) = data.external_record() else { return; }; emit::emit_record(&ext, record); } /// Update identity attrs when auth completes (called alongside the /// telemetry-client init sites). Also emits the one-shot internal adoption /// meta-event — post-auth, when the product events client is live. pub fn set_identity(attrs: IdentityAttrs) { let Some(ext) = handle() else { return; }; set_identity_on(&ext, attrs); } pub(crate) fn set_identity_on(ext: &ExternalTelemetry, attrs: IdentityAttrs) { *ext.identity.write() = attrs; ext.meta_event_once.call_once(|| { let meta = &ext.configured_meta; crate::session_ctx::log_session_event(crate::events::ExternalOtelConfigured { metrics_exporter: meta.metrics_exporter.to_owned(), logs_exporter: meta.logs_exporter.to_owned(), protocol: meta.protocol.to_owned(), logs_endpoint_origin: meta.logs_endpoint_origin.clone(), metrics_endpoint_origin: meta.metrics_endpoint_origin.clone(), prompts_gate: meta.prompts_gate, details_gate: meta.details_gate, source: meta.source.to_owned(), }); }); } /// Apply remote policy when `RemoteSettings` arrive (post-auth, alongside /// [`set_identity`]). **TIGHTEN-ONLY**: may clear `active` (fleet kill switch /// — flushes, then drops subsequent emissions) and may force content gates /// off; it can never enable a stream that env/config left off, and never /// loosens gates mid-run. pub fn apply_remote_policy(policy: ExternalOtelRemotePolicy) { let Some(ext) = handle() else { return; }; apply_remote_policy_on(&ext, policy); } pub(crate) fn apply_remote_policy_on(ext: &ExternalTelemetry, policy: ExternalOtelRemotePolicy) { if policy.lock_content_gates { let mut gates = ext.gates.write(); if gates.log_user_prompts || gates.log_tool_details { *gates = ContentGates::default(); drop(gates); crate::session_ctx::log_session_event(crate::events::ExternalOtelRemotePolicyApplied { action: "gates_locked".to_owned(), }); } } if policy.force_disable && ext.active.swap(false, Ordering::Relaxed) { flush_on(ext); crate::session_ctx::log_session_event(crate::events::ExternalOtelRemotePolicyApplied { action: "force_disable".to_owned(), }); tracing::debug!("external otel: force-disabled by remote policy"); } } /// Flush both providers (logout path: called *before* credentials are /// cleared, so post-logout records cannot carry the prior user's ids — /// follow with [`set_identity`] carrying the new/empty identity). pub fn flush() { let Some(ext) = handle() else { return; }; flush_on(&ext); } pub(crate) fn flush_on(ext: &ExternalTelemetry) { if let Some(p) = ext.logger_provider.as_ref() && let Err(e) = p.force_flush() { tracing::debug!(error = %e, "external otel: logger flush failed"); } if let Some(p) = ext.meter_provider.as_ref() && let Err(e) = p.force_flush() { tracing::debug!(error = %e, "external otel: meter flush failed"); } } /// Flush + shutdown both providers with a 2-second watchdog. Idempotent — /// reachable from every `shutdown_otel()` exit path (16 `OtelGuard` sites, /// the direct call, and the signal handler); subsequent calls are no-ops. pub fn shutdown() { let Some(ext) = handle() else { return; }; ext.shutdown_once.call_once(|| { ext.active.store(false, Ordering::Relaxed); emit_export_health(&ext); let logger_provider = ext.logger_provider.clone(); let meter_provider = ext.meter_provider.clone(); let (tx, rx) = std::sync::mpsc::channel::<()>(); // Detached thread + timed wait: a hung provider must not hang exit // (`std::thread::scope` is unusable here — it joins unconditionally). std::thread::spawn(move || { if let Some(p) = logger_provider && let Err(e) = p.shutdown() { tracing::debug!(error = %e, "external otel: logger shutdown failed"); } if let Some(p) = meter_provider && let Err(e) = p.shutdown() { tracing::debug!(error = %e, "external otel: meter shutdown failed"); } let _ = tx.send(()); }); if rx.recv_timeout(std::time::Duration::from_secs(2)).is_err() { tracing::debug!("external otel: shutdown watchdog expired; abandoning flush thread"); } }); } /// Best-effort product-events export-health meta-event (never exported /// externally — avoid feedback loops). Emitting needs a Tokio runtime /// (`emit_event` spawns); skip silently when exiting without one. fn emit_export_health(ext: &ExternalTelemetry) { let health = &ext.health; let snapshot = crate::events::ExternalOtelExportHealth { records_dropped: health.records_dropped.load(Ordering::Relaxed), metric_exports_dropped: health.metric_exports_dropped.load(Ordering::Relaxed), export_failures: health.export_failures.load(Ordering::Relaxed), export_successes: health.export_successes.load(Ordering::Relaxed), }; tracing::debug!( records_dropped = snapshot.records_dropped, metric_exports_dropped = snapshot.metric_exports_dropped, export_failures = snapshot.export_failures, export_successes = snapshot.export_successes, "external otel: export health" ); if tokio::runtime::Handle::try_current().is_ok() { crate::session_ctx::log_session_event(snapshot); } } #[cfg(test)] pub(crate) mod test_support { //! Build an [`ExternalTelemetry`] over in-memory exporters so unit tests //! can assert exactly what would reach the wire (post-validator). use super::*; use opentelemetry_sdk::logs::InMemoryLogExporter; use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader}; pub(crate) struct TestStream { pub ext: ExternalTelemetry, pub logs: InMemoryLogExporter, pub metrics: InMemoryMetricExporter, } pub(crate) fn build(gates: ContentGates) -> TestStream { let shared_gates: redact::SharedGates = Arc::new(parking_lot::RwLock::new(gates)); let health = Arc::new(redact::ExportHealth::default()); let logs = InMemoryLogExporter::default(); let metrics = InMemoryMetricExporter::default(); let logger_provider = SdkLoggerProvider::builder() .with_simple_exporter(redact::RedactingLogExporter::new( logs.clone(), shared_gates.clone(), health.clone(), )) .build(); let meter_provider = SdkMeterProvider::builder() .with_reader( PeriodicReader::builder(redact::ValidatingMetricExporter::new( metrics.clone(), health.clone(), )) .build(), ) .build(); let logger = logger_provider.logger(schema::SCOPE_NAME); let instruments = emit::Instruments::new(&meter_provider.meter(schema::SCOPE_NAME)); let ext = ExternalTelemetry { logger_provider: Some(logger_provider), meter_provider: Some(meter_provider), logger: Some(logger), instruments: Some(instruments), active: AtomicBool::new(true), gates: shared_gates, identity: parking_lot::RwLock::new(IdentityAttrs::default()), sequence: AtomicU64::new(0), shutdown_once: std::sync::Once::new(), include_session_id_on_metrics: true, include_version_on_metrics: false, app_version: String::new(), health, configured_meta: ConfiguredMeta { metrics_exporter: "test", logs_exporter: "test", logs_endpoint_origin: String::new(), metrics_endpoint_origin: String::new(), protocol: "test", prompts_gate: gates.log_user_prompts, details_gate: gates.log_tool_details, source: "env", }, meta_event_once: std::sync::Once::new(), }; TestStream { ext, logs, metrics } } pub(crate) fn emit_into(stream: &TestStream, record: schema::ExternalRecord) { emit::emit_record(&stream.ext, record); stream .ext .logger_provider .as_ref() .expect("test logger provider") .force_flush() .expect("flush logs"); stream .ext .meter_provider .as_ref() .expect("test meter provider") .force_flush() .expect("flush metrics"); } pub(crate) fn emit_event_into( stream: &TestStream, event: &T, ) { if let Some(record) = event.external_record() { emit_into(stream, record); } } } #[cfg(test)] mod tests;