Synced from monorepo
Synced from monorepo Changes: - grok-shell: send an expired external-provider credential to the sign-in flow, not a 401 loop - pager: clickable ▲ jumps to the top of the response being read - grok-shell: keep a large task log from making the completion message too long - Plan viewer scrollbar: widen grab zone to the border column; fix striped thumb in Terminal.app - pager: poll the tmux probe teardown grace instead of sleeping it - security: vendor-compat MCP kill switch is now actually enforced when reported as on - grok-shell: restore session eviction when a leader client disconnects - Bump rust-toolchain to 1.93.0 - workspace: lexical-normalize permission path patterns before glob matching - pager: reject garbage Enter in the /resume picker - pager: show Mermaid affordances in plan mode preview - pager: drop manage-account link from /session-info - workspace: auto-approve read-only git queries; defer write floor to auto classifier - Add free-form pattern editor to the "Always allow" command prompt - grok-shell: fix /btw caching - pager: Tab walks answers in the ask_user_question card - External-provider auth refresh: single 7s attempt instead of 3×5s - pager: don't resurrect finished background tasks as Running when completion arrives first - pager: report tmux truecolor clamping in Doctor - Fix plan viewer scrollbar click+drag hijacked by comment gutter - pager/shell: stop double Recap after the same last turn - sampler: preserve x-should-retry through stream collection - pager: clear plan-mode indicator immediately when the user approves a plan - pager: tmux does not re-read its config on reattach Source-Revision: 64c4de99cc822b25ce9c54ab5a4f372093d0885d
This commit is contained in:
grokkybara[bot]
parent
a422116582
commit
780d1388ff
323 changed files with 12258 additions and 7226 deletions
|
|
@ -70,8 +70,20 @@ opentelemetry_sdk = { workspace = true, features = [
|
|||
"experimental_metrics_periodicreader_with_async_runtime",
|
||||
"rt-tokio-current-thread",
|
||||
] }
|
||||
opentelemetry-otlp = { workspace = true }
|
||||
# `tls-aws-lc` compiles opentelemetry-otlp's TLS support for the gRPC
|
||||
# transport. The workspace default is only `tls-roots`, which since otlp 0.32
|
||||
# no longer implies a TLS provider feature — without one, every `https://`
|
||||
# gRPC collector endpoint is rejected at exporter build time (GB-4580).
|
||||
# aws-lc matches the TLS backend the workspace `tonic` already uses.
|
||||
opentelemetry-otlp = { workspace = true, features = ["tls-aws-lc"] }
|
||||
opentelemetry-http = { workspace = true }
|
||||
# Embedded Mozilla trust anchors for the gRPC TLS fallback: keeps `https://`
|
||||
# collectors working on hosts with no readable system CA store (parity with
|
||||
# the HTTP transport's embedded-roots reqwest client in `otlp_http.rs`).
|
||||
webpki-roots = { workspace = true }
|
||||
# Re-encodes the validated GROK_EXTRA_CA_BUNDLE DERs as PEM for tonic's
|
||||
# `Certificate::from_pem` (the gRPC transport's extra-CA parity with HTTP).
|
||||
base64 = { workspace = true }
|
||||
http = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
|
|
@ -83,6 +95,12 @@ tracing-opentelemetry = { workspace = true }
|
|||
# Pre-main unified-log redirect for this crate's own test binary.
|
||||
ctor = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport"] }
|
||||
# Self-signed CA + server cert for the TLS collector fixture
|
||||
# (tests/external_otlp_grpc_tls.rs). `rustls` pins the process-default crypto
|
||||
# provider there: the test binary links both ring and aws-lc-rs, so rustls
|
||||
# can't auto-select one for the server-side acceptor.
|
||||
rcgen = { workspace = true }
|
||||
rustls = { workspace = true }
|
||||
# In-memory log/metric exporters for the external-stream wire-shape tests.
|
||||
opentelemetry_sdk = { workspace = true, features = ["testing"] }
|
||||
# In-process OTLP collector fixture (tests/external_otlp.rs).
|
||||
|
|
|
|||
|
|
@ -988,6 +988,66 @@ pub struct NonGitDecisionEvent {
|
|||
// Prompt Latency (every turn)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Why a [`ProcessResourceUsage`] was sampled, so a mid-life reading is not
|
||||
/// read as a post-teardown one.
|
||||
#[derive(Serialize, Clone, Copy)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ResourceReportTrigger {
|
||||
SessionClose,
|
||||
Periodic,
|
||||
}
|
||||
|
||||
/// The ceilings this process runs under. The denominator for
|
||||
/// `ProcessResourceUsage`: usage against limits is headroom.
|
||||
#[derive(Serialize)]
|
||||
pub struct ProcessResourceLimits {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nofile_soft: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nofile_hard: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nproc_soft: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nproc_hard: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub available_parallelism: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cgroup_pids_max: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cgroup_memory_max: Option<String>,
|
||||
}
|
||||
|
||||
/// Emitted when the jemalloc heap monitor crosses a configured threshold.
|
||||
/// The acute signal that a build is growing without bound.
|
||||
#[derive(Serialize)]
|
||||
pub struct HeapThresholdCrossed {
|
||||
pub threshold_bytes: u64,
|
||||
pub resident_bytes: u64,
|
||||
pub allocated_bytes: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rss_peak_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// What this process still holds just after a session was removed. Aggregated
|
||||
/// per release, a rising tail is a leak; `resident_sessions` separates leader
|
||||
/// mode, where one process serves many sessions and a leak compounds.
|
||||
#[derive(Serialize)]
|
||||
pub struct ProcessResourceUsage {
|
||||
pub trigger: ResourceReportTrigger,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rss_bytes: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub peak_rss_bytes: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub footprint_bytes: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub threads: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub open_files: Option<u64>,
|
||||
pub resident_sessions: usize,
|
||||
pub session_threads: usize,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PromptLatency {
|
||||
pub turn_index: u32,
|
||||
|
|
@ -1579,6 +1639,9 @@ pub enum ManualAuthReason {
|
|||
RefreshTokenRejected,
|
||||
/// Token type has no refresh authority (API key / legacy / OIDC sans refresh token).
|
||||
NoRefreshAuthority,
|
||||
/// The operator's auth-provider command could not mint a credential
|
||||
/// unattended, so only an interactive run of it can restore the session.
|
||||
ProviderInteractiveRequired,
|
||||
RecoveryExhausted,
|
||||
TokenExpiredNoRefresh,
|
||||
/// Recovered session violated the `force_login_team_uuid` pin.
|
||||
|
|
@ -1762,6 +1825,9 @@ telemetry_event!(MultiAgentDiscard, "multi_agent_discard");
|
|||
telemetry_event!(RepoChanges, "repo_changes");
|
||||
telemetry_event!(NonGitDecisionEvent, "non_git_decision");
|
||||
telemetry_event!(PromptLatency, "prompt_latency");
|
||||
telemetry_event!(HeapThresholdCrossed, "heap_threshold_crossed");
|
||||
telemetry_event!(ProcessResourceUsage, "process_resource_usage");
|
||||
telemetry_event!(ProcessResourceLimits, "process_resource_limits");
|
||||
telemetry_event!(
|
||||
TurnCompleted,
|
||||
"turn_completed",
|
||||
|
|
|
|||
|
|
@ -158,6 +158,13 @@ pub struct ExternalOtelConfig {
|
|||
/// `OTEL_EXPORTER_OTLP_HEADERS` plus `OTEL_EXPORTER_OTLP_METRICS_HEADERS`.
|
||||
/// The **only** headers the external metric exporter ever sends.
|
||||
pub metrics_headers: Vec<(String, String)>,
|
||||
/// PEM file with additional trusted CA certificate(s) for verifying the
|
||||
/// logs collector (`OTEL_EXPORTER_OTLP_CERTIFICATE`, overridden by
|
||||
/// `OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE`). Additive to the default roots.
|
||||
pub logs_ca_certificate: Option<String>,
|
||||
/// Same for the metrics collector (`OTEL_EXPORTER_OTLP_CERTIFICATE`,
|
||||
/// overridden by `OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE`).
|
||||
pub metrics_ca_certificate: Option<String>,
|
||||
/// `OTEL_EXPORTER_OTLP_TIMEOUT` (ms). Default 10 s.
|
||||
pub timeout: Duration,
|
||||
/// `OTEL_METRIC_EXPORT_INTERVAL` (ms). Default 60 s.
|
||||
|
|
@ -350,6 +357,21 @@ impl ExternalOtelConfig {
|
|||
let logs_headers = resolve_signal_headers("OTEL_EXPORTER_OTLP_LOGS_HEADERS");
|
||||
let metrics_headers = resolve_signal_headers("OTEL_EXPORTER_OTLP_METRICS_HEADERS");
|
||||
|
||||
// Collector CA certificate (OTLP spec): base var with per-signal
|
||||
// overrides. A path, not a secret — but env-only like headers, so
|
||||
// resolution stays a pure function of the standard OTEL_* interface.
|
||||
let base_certificate =
|
||||
getenv("OTEL_EXPORTER_OTLP_CERTIFICATE").filter(|s| !s.trim().is_empty());
|
||||
let resolve_signal_certificate = |signal_var: &str| {
|
||||
getenv(signal_var)
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.or_else(|| base_certificate.clone())
|
||||
.map(|s| s.trim().to_string())
|
||||
};
|
||||
let logs_ca_certificate = resolve_signal_certificate("OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE");
|
||||
let metrics_ca_certificate =
|
||||
resolve_signal_certificate("OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE");
|
||||
|
||||
let gates = ContentGates {
|
||||
log_user_prompts: getenv("OTEL_LOG_USER_PROMPTS")
|
||||
.as_deref()
|
||||
|
|
@ -380,6 +402,8 @@ impl ExternalOtelConfig {
|
|||
metrics_endpoint,
|
||||
logs_headers,
|
||||
metrics_headers,
|
||||
logs_ca_certificate,
|
||||
metrics_ca_certificate,
|
||||
timeout: parse_ms(
|
||||
getenv("OTEL_EXPORTER_OTLP_TIMEOUT"),
|
||||
Duration::from_millis(10_000),
|
||||
|
|
@ -634,6 +658,43 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ca_certificate_resolved_with_signal_overrides() {
|
||||
let cfg = ExternalOtelConfig::resolve_with(
|
||||
env(&[
|
||||
("GROK_EXTERNAL_OTEL", "1"),
|
||||
("OTEL_LOGS_EXPORTER", "otlp"),
|
||||
("OTEL_METRICS_EXPORTER", "otlp"),
|
||||
("OTEL_EXPORTER_OTLP_CERTIFICATE", "/etc/ssl/corp-ca.pem"),
|
||||
(
|
||||
"OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE",
|
||||
"/etc/ssl/metrics-ca.pem",
|
||||
),
|
||||
]),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
cfg.logs_ca_certificate.as_deref(),
|
||||
Some("/etc/ssl/corp-ca.pem")
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.metrics_ca_certificate.as_deref(),
|
||||
Some("/etc/ssl/metrics-ca.pem")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ca_certificate_defaults_to_none() {
|
||||
let cfg = ExternalOtelConfig::resolve_with(
|
||||
env(&[("GROK_EXTERNAL_OTEL", "1"), ("OTEL_LOGS_EXPORTER", "otlp")]),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(cfg.logs_ca_certificate, None);
|
||||
assert_eq!(cfg.metrics_ca_certificate, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_gates_default_off_env_enables() {
|
||||
let cfg = ExternalOtelConfig::resolve_with(
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ pub mod truncate;
|
|||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use opentelemetry::logs::LoggerProvider as _;
|
||||
use opentelemetry::metrics::MeterProvider as _;
|
||||
|
|
@ -220,21 +221,54 @@ fn active_handle() -> Option<Arc<ExternalTelemetry>> {
|
|||
}
|
||||
|
||||
/// Fail-closed OTEL gate. Defaults open; the leader closes it before init and
|
||||
/// re-opens it when settings arrive (or immediately for a pure env-API-key
|
||||
/// leader, which has no remote policy to fetch).
|
||||
/// re-opens it when settings resolve.
|
||||
///
|
||||
/// On the leader, opening is the synchronizing event: `OtelGate::apply_and_open`
|
||||
/// applies the remote force-disable (`active = false`) and then opens here, so
|
||||
/// an emitter whose `Acquire` read observes the `Release` open also observes
|
||||
/// Opening is the synchronizing event: `OtelGate::apply_and_open` applies the
|
||||
/// remote force-disable (`active = false`) and then opens here, so an emitter
|
||||
/// whose `Acquire` read observes the `Release` open also observes
|
||||
/// `active = false`; the emit-path `active` load can therefore stay `Relaxed`.
|
||||
/// Closing is fail-safe and stays `Relaxed`. The follower path force-disables
|
||||
/// without re-opening and relies on eventual visibility, acceptable because the
|
||||
/// policy is tighten-only.
|
||||
/// The window-expiry open has no such pairing and relies on eventual
|
||||
/// visibility, acceptable because the policy is tighten-only.
|
||||
static SETTINGS_RESOLVED: AtomicBool = AtomicBool::new(true);
|
||||
|
||||
const DEFAULT_SETTINGS_GATE_MAX_WAIT: Duration = Duration::from_secs(30);
|
||||
|
||||
static SETTINGS_GATE_MAX_WAIT_MS: AtomicU64 =
|
||||
AtomicU64::new(DEFAULT_SETTINGS_GATE_MAX_WAIT.as_millis() as u64);
|
||||
|
||||
static GATE_CLOSED_AT_MS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn process_uptime_ms() -> u64 {
|
||||
static START: OnceLock<std::time::Instant> = OnceLock::new();
|
||||
u64::try_from(
|
||||
START
|
||||
.get_or_init(std::time::Instant::now)
|
||||
.elapsed()
|
||||
.as_millis(),
|
||||
)
|
||||
.unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
/// Set the bound on the fail-closed window.
|
||||
pub fn set_settings_gate_max_wait(max_wait: Duration) {
|
||||
SETTINGS_GATE_MAX_WAIT_MS.store(
|
||||
u64::try_from(max_wait.as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
/// The current bound on the fail-closed window.
|
||||
pub fn settings_gate_max_wait() -> Duration {
|
||||
Duration::from_millis(SETTINGS_GATE_MAX_WAIT_MS.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Close the gate (leader preinit + account switch).
|
||||
pub fn suppress_external_otel_until_settings() {
|
||||
SETTINGS_RESOLVED.store(false, Ordering::Relaxed);
|
||||
GATE_CLOSED_AT_MS.store(process_uptime_ms(), Ordering::Relaxed);
|
||||
// `Release`: a reader that observes the close must also observe the
|
||||
// timestamp published just above, or it would measure this window from an
|
||||
// earlier close and open immediately.
|
||||
SETTINGS_RESOLVED.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Open the gate. `Release` publishes the force-disable applied just before it.
|
||||
|
|
@ -244,10 +278,28 @@ pub fn mark_external_otel_settings_resolved() {
|
|||
}
|
||||
}
|
||||
|
||||
/// Read the gate. `Acquire` pairs with the `Release` open.
|
||||
/// Read the gate. `Acquire` pairs with the `Release` open (and with the
|
||||
/// `Release` close that publishes the window start).
|
||||
#[inline]
|
||||
pub fn is_settings_gate_open() -> bool {
|
||||
SETTINGS_RESOLVED.load(Ordering::Acquire)
|
||||
SETTINGS_RESOLVED.load(Ordering::Acquire) || settings_gate_window_expired()
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn settings_gate_window_expired() -> bool {
|
||||
let waited = process_uptime_ms().saturating_sub(GATE_CLOSED_AT_MS.load(Ordering::Relaxed));
|
||||
if waited < SETTINGS_GATE_MAX_WAIT_MS.load(Ordering::Relaxed) {
|
||||
return false;
|
||||
}
|
||||
static LOGGED: std::sync::Once = std::sync::Once::new();
|
||||
LOGGED.call_once(|| {
|
||||
tracing::warn!(
|
||||
waited_ms = waited,
|
||||
"external otel: no fleet policy arrived within the bounded window; \
|
||||
emitting under local configuration (a policy that arrives later still applies)"
|
||||
);
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Cheap check used by the fan-out hook and the split-sink call sites:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ use std::time::Duration;
|
|||
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use opentelemetry_otlp::{
|
||||
Protocol, WithExportConfig, WithHttpConfig, WithTonicConfig, tonic_types::metadata::MetadataMap,
|
||||
Protocol, WithExportConfig, WithHttpConfig, WithTonicConfig,
|
||||
tonic_types::metadata::MetadataMap,
|
||||
tonic_types::transport::{Certificate, ClientTlsConfig},
|
||||
};
|
||||
use opentelemetry_sdk::logs::{
|
||||
BatchConfig, BatchConfigBuilder, BatchLogProcessor as ThreadBatchLogProcessor,
|
||||
|
|
@ -254,6 +256,117 @@ pub(crate) struct BuiltProviders {
|
|||
pub meter_provider: Option<SdkMeterProvider>,
|
||||
}
|
||||
|
||||
/// TLS configurations to try, in order, when building a gRPC exporter.
|
||||
///
|
||||
/// `opentelemetry-otlp` 0.32 must be handed an explicit `ClientTlsConfig` for
|
||||
/// `https://` endpoints: its own fallback is `ClientTlsConfig::new()`, whose
|
||||
/// root store is **empty** in tonic 0.14 (`Endpoint::from_shared` never
|
||||
/// auto-enables roots), so every handshake would fail with `UnknownIssuer`.
|
||||
///
|
||||
/// For https endpoints this returns two candidates:
|
||||
/// 1. system CA store + embedded webpki roots (+ the customer CA, if any);
|
||||
/// 2. embedded webpki roots only (+ the customer CA, if any) — the fallback
|
||||
/// for hosts whose native store is missing or unreadable, where tonic
|
||||
/// fails candidate 1 at build time (`NativeCertsNotFound`). This keeps
|
||||
/// parity with the HTTP transport's embedded-roots reqwest client.
|
||||
///
|
||||
/// For plain `http://` endpoints it returns a single `None` (no TLS).
|
||||
/// `true` when the bytes contain at least one PEM certificate block.
|
||||
///
|
||||
/// The emptiness gate for fail-closed CA handling: a readable but cert-less
|
||||
/// bundle must fail exporter construction, not silently fall back to the
|
||||
/// default roots. Malformed blocks are caught later by the TLS stack's own
|
||||
/// parser (also fail-closed, at exporter build).
|
||||
pub(crate) fn pem_contains_certificate(pem: &[u8]) -> bool {
|
||||
const MARKER: &[u8] = b"-----BEGIN CERTIFICATE-----";
|
||||
pem.windows(MARKER.len()).any(|window| window == MARKER)
|
||||
}
|
||||
|
||||
/// Re-encode validated DER roots as one multi-block PEM string (tonic's
|
||||
/// `Certificate::from_pem` parses every block in a single certificate).
|
||||
/// `None` when `ders` is empty.
|
||||
fn ders_to_pem_bundle(ders: &[Vec<u8>]) -> Option<String> {
|
||||
use base64::Engine as _;
|
||||
if ders.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut pem = String::new();
|
||||
for der in ders {
|
||||
pem.push_str("-----BEGIN CERTIFICATE-----\n");
|
||||
pem.push_str(&base64::engine::general_purpose::STANDARD.encode(der));
|
||||
pem.push_str("\n-----END CERTIFICATE-----\n");
|
||||
}
|
||||
Some(pem)
|
||||
}
|
||||
|
||||
fn grpc_tls_candidates(
|
||||
endpoint: &str,
|
||||
ca_certificate_path: Option<&str>,
|
||||
) -> BuildResult<Vec<Option<ClientTlsConfig>>> {
|
||||
// Mirror opentelemetry-otlp's own `is_https` detection exactly (parsed
|
||||
// URI scheme == https, tonic/mod.rs) so its empty-root-store
|
||||
// `ClientTlsConfig::new()` fallback for https endpoints is unreachable:
|
||||
// every endpoint it treats as https gets an explicit config from us.
|
||||
// Schemeless or non-http(s) endpoints get no TLS config here and fail
|
||||
// fail-closed inside the exporter ("invalid URL, scheme is missing").
|
||||
let is_https = endpoint
|
||||
.parse::<http::Uri>()
|
||||
.ok()
|
||||
.and_then(|uri| uri.scheme().cloned())
|
||||
.is_some_and(|scheme| scheme == http::uri::Scheme::HTTPS);
|
||||
if !is_https {
|
||||
return Ok(vec![None]);
|
||||
}
|
||||
let mut base =
|
||||
ClientTlsConfig::new().trust_anchors(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
|
||||
// Process-wide `GROK_EXTRA_CA_BUNDLE` roots (fail-open by that crate's
|
||||
// contract), matching what the HTTP transport applies via
|
||||
// `with_extra_root_certificates_blocking` — the same corporate/MITM CA
|
||||
// must work on both transports.
|
||||
if let Some(extra_pem) = ders_to_pem_bundle(xai_grok_extra_ca::extra_root_ders()) {
|
||||
base = base.ca_certificate(Certificate::from_pem(extra_pem));
|
||||
}
|
||||
let base = match ca_certificate_path {
|
||||
// Fail closed on an unreadable or certificate-less customer CA (the
|
||||
// caller warns and disables the stream): silently exporting without
|
||||
// the configured trust anchor would be worse than not exporting.
|
||||
Some(path) => {
|
||||
let pem = std::fs::read(path).map_err(|e| {
|
||||
opentelemetry_otlp::ExporterBuildError::InternalFailure(format!(
|
||||
"reading OTEL_EXPORTER_OTLP_CERTIFICATE {path:?}: {e}"
|
||||
))
|
||||
})?;
|
||||
if !pem_contains_certificate(&pem) {
|
||||
return Err(opentelemetry_otlp::ExporterBuildError::InternalFailure(
|
||||
format!("OTEL_EXPORTER_OTLP_CERTIFICATE {path:?} contains no certificates"),
|
||||
));
|
||||
}
|
||||
base.ca_certificate(Certificate::from_pem(pem))
|
||||
}
|
||||
None => base,
|
||||
};
|
||||
Ok(vec![Some(base.clone().with_native_roots()), Some(base)])
|
||||
}
|
||||
|
||||
/// Run `build` with each TLS candidate in order, returning the first success.
|
||||
fn build_with_tls_fallback<T>(
|
||||
candidates: Vec<Option<ClientTlsConfig>>,
|
||||
mut build: impl FnMut(Option<ClientTlsConfig>) -> BuildResult<T>,
|
||||
) -> BuildResult<T> {
|
||||
debug_assert!(!candidates.is_empty());
|
||||
let mut last_err = None;
|
||||
for candidate in candidates {
|
||||
match build(candidate) {
|
||||
Ok(exporter) => return Ok(exporter),
|
||||
Err(e) => {
|
||||
tracing::debug!(error = %e, "external otel: gRPC exporter TLS candidate failed");
|
||||
last_err = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_err.expect("at least one TLS candidate is always supplied"))
|
||||
}
|
||||
|
||||
enum OtlpExportTransport<'a> {
|
||||
HttpProtobuf(&'a crate::otlp_http::BlockingOtlpClient),
|
||||
Grpc(&'a DedicatedRuntime),
|
||||
|
|
@ -293,13 +406,20 @@ impl OtlpExportFactory for OtlpLogExporterBuilder<'_> {
|
|||
let endpoint = self.cfg.logs_endpoint.clone();
|
||||
let timeout = self.cfg.timeout;
|
||||
let metadata = customer_metadata(&self.cfg.logs_headers);
|
||||
let tls_candidates =
|
||||
grpc_tls_candidates(&endpoint, self.cfg.logs_ca_certificate.as_deref())?;
|
||||
runtime.run(move || {
|
||||
opentelemetry_otlp::LogExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint)
|
||||
.with_timeout(timeout)
|
||||
.with_metadata(metadata)
|
||||
.build()
|
||||
build_with_tls_fallback(tls_candidates, |tls| {
|
||||
let mut builder = opentelemetry_otlp::LogExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint.clone())
|
||||
.with_timeout(timeout)
|
||||
.with_metadata(metadata.clone());
|
||||
if let Some(tls) = tls {
|
||||
builder = builder.with_tls_config(tls);
|
||||
}
|
||||
builder.build()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -333,14 +453,21 @@ impl OtlpExportFactory for OtlpMetricExporterBuilder<'_> {
|
|||
let timeout = self.cfg.timeout;
|
||||
let metadata = customer_metadata(&self.cfg.metrics_headers);
|
||||
let temporality = self.temporality;
|
||||
let tls_candidates =
|
||||
grpc_tls_candidates(&endpoint, self.cfg.metrics_ca_certificate.as_deref())?;
|
||||
runtime.run(move || {
|
||||
opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint)
|
||||
.with_timeout(timeout)
|
||||
.with_metadata(metadata)
|
||||
.with_temporality(temporality)
|
||||
.build()
|
||||
build_with_tls_fallback(tls_candidates, |tls| {
|
||||
let mut builder = opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint.clone())
|
||||
.with_timeout(timeout)
|
||||
.with_metadata(metadata.clone())
|
||||
.with_temporality(temporality);
|
||||
if let Some(tls) = tls {
|
||||
builder = builder.with_tls_config(tls);
|
||||
}
|
||||
builder.build()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -473,7 +600,25 @@ pub(crate) fn build(
|
|||
&& (cfg.logs_exporter == ExporterSelection::Otlp
|
||||
|| cfg.metrics_exporter == ExporterSelection::Otlp);
|
||||
let http_client = needs_http_client
|
||||
.then(|| crate::otlp_http::build_blocking_client(cfg.timeout))
|
||||
.then(|| {
|
||||
// One client serves both signals, so trust the union of the
|
||||
// per-signal customer CA bundles (additive roots) — but only for
|
||||
// signals actually exporting over OTLP: a bad CA override on an
|
||||
// inactive signal must not fail-closed the active one.
|
||||
let mut ca_files: Vec<&str> = [
|
||||
(cfg.logs_exporter == ExporterSelection::Otlp)
|
||||
.then_some(cfg.logs_ca_certificate.as_deref())
|
||||
.flatten(),
|
||||
(cfg.metrics_exporter == ExporterSelection::Otlp)
|
||||
.then_some(cfg.metrics_ca_certificate.as_deref())
|
||||
.flatten(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
ca_files.dedup();
|
||||
crate::otlp_http::build_blocking_client(cfg.timeout, &ca_files)
|
||||
})
|
||||
.transpose()
|
||||
.map_err(opentelemetry_otlp::ExporterBuildError::InternalFailure)?;
|
||||
|
||||
|
|
@ -592,6 +737,160 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// Regression test for GB-4580: with the opentelemetry-otlp 0.32 bump,
|
||||
/// the `tls-roots` feature stopped implying a TLS provider feature and
|
||||
/// every `https://` gRPC endpoint was rejected at exporter build time
|
||||
/// ("uses HTTPS but no TLS feature is enabled"), silently disabling the
|
||||
/// external stream. The exporters must build for https endpoints.
|
||||
#[test]
|
||||
fn grpc_exporters_build_for_https_endpoints() {
|
||||
let cfg = ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL" => Some("grpc".into()),
|
||||
// Nothing listens here: gRPC channels connect lazily, so
|
||||
// exporter construction must still succeed.
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some("https://localhost:1".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("config must resolve");
|
||||
let gates: SharedGates = Arc::new(parking_lot::RwLock::new(Default::default()));
|
||||
let health = Arc::new(ExportHealth::default());
|
||||
let built = super::build(&cfg, gates, health)
|
||||
.expect("https gRPC exporters must build (GB-4580 regression)");
|
||||
assert!(built.logger_provider.is_some());
|
||||
assert!(built.meter_provider.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_tls_candidates_plain_http_has_no_tls() {
|
||||
let candidates =
|
||||
grpc_tls_candidates("http://localhost:4317", None).expect("http must resolve");
|
||||
assert_eq!(candidates.len(), 1);
|
||||
assert!(candidates[0].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_tls_candidates_https_tries_native_then_embedded_roots() {
|
||||
let candidates =
|
||||
grpc_tls_candidates("https://collector.corp.example:4317", None).expect("https");
|
||||
assert_eq!(
|
||||
candidates.len(),
|
||||
2,
|
||||
"native-roots candidate + embedded fallback"
|
||||
);
|
||||
assert!(candidates.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
/// Endpoints without a scheme must not get a TLS config: they agree with
|
||||
/// opentelemetry-otlp's own https detection (parsed scheme), and the
|
||||
/// exporter rejects them at connect time ("scheme is missing") rather
|
||||
/// than handshaking with an empty root store.
|
||||
#[test]
|
||||
fn grpc_tls_candidates_schemeless_endpoint_gets_no_tls() {
|
||||
let candidates =
|
||||
grpc_tls_candidates("collector.corp.example:4317", None).expect("schemeless");
|
||||
assert_eq!(candidates.len(), 1);
|
||||
assert!(candidates[0].is_none());
|
||||
}
|
||||
|
||||
/// Scheme detection is on the parsed URI, so case differences cannot
|
||||
/// diverge from the exporter's own https check.
|
||||
#[test]
|
||||
fn grpc_tls_candidates_uppercase_https_scheme_detected() {
|
||||
let candidates =
|
||||
grpc_tls_candidates("HTTPS://collector.corp.example:4317", None).expect("https");
|
||||
assert_eq!(candidates.len(), 2);
|
||||
assert!(candidates.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
/// A CA override on an *inactive* signal must not take down the shared
|
||||
/// HTTP client (and with it the whole stream) for the active signal.
|
||||
#[test]
|
||||
fn inactive_signal_ca_does_not_disable_http_stream() {
|
||||
let cfg = ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
// Only metrics export; logs are off but carry a broken CA.
|
||||
"OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE" => {
|
||||
Some("/nonexistent/inactive-signal-ca.pem".into())
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("config must resolve");
|
||||
assert_eq!(cfg.logs_exporter, ExporterSelection::None);
|
||||
let gates: SharedGates = Arc::new(parking_lot::RwLock::new(Default::default()));
|
||||
let health = Arc::new(ExportHealth::default());
|
||||
let built = super::build(&cfg, gates, health)
|
||||
.expect("inactive signal's CA must not fail the active stream");
|
||||
assert!(built.logger_provider.is_none());
|
||||
assert!(built.meter_provider.is_some());
|
||||
}
|
||||
|
||||
/// A readable but certificate-less CA bundle must fail closed, not build
|
||||
/// exporters that verify without the configured trust anchor.
|
||||
#[test]
|
||||
fn grpc_tls_candidates_fail_closed_on_empty_ca_file() {
|
||||
let file = tempfile::NamedTempFile::new().expect("temp CA file");
|
||||
std::fs::write(file.path(), "# readable, but no PEM certificate blocks\n")
|
||||
.expect("write empty bundle");
|
||||
let err = grpc_tls_candidates(
|
||||
"https://collector.corp.example:4317",
|
||||
Some(file.path().to_str().expect("utf-8 path")),
|
||||
)
|
||||
.expect_err("certificate-less bundle must fail exporter construction");
|
||||
assert!(err.to_string().contains("no certificates"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pem_certificate_detection() {
|
||||
assert!(pem_contains_certificate(
|
||||
b"-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n"
|
||||
));
|
||||
assert!(!pem_contains_certificate(b""));
|
||||
// A non-certificate PEM block (CSR) must not count — note it does not
|
||||
// contain the exact `-----BEGIN CERTIFICATE-----` marker.
|
||||
assert!(!pem_contains_certificate(
|
||||
b"-----BEGIN CERTIFICATE REQUEST-----\nAAAA\n-----END CERTIFICATE REQUEST-----\n"
|
||||
));
|
||||
}
|
||||
|
||||
/// The DER→PEM re-encode used for `GROK_EXTRA_CA_BUNDLE` must produce a
|
||||
/// bundle other PEM parsers can read back, one block per DER.
|
||||
#[test]
|
||||
fn ders_to_pem_bundle_roundtrips() {
|
||||
assert!(ders_to_pem_bundle(&[]).is_none());
|
||||
let key = rcgen::KeyPair::generate().expect("key");
|
||||
let cert = rcgen::CertificateParams::new(vec!["localhost".into()])
|
||||
.expect("params")
|
||||
.self_signed(&key)
|
||||
.expect("cert");
|
||||
let der = cert.der().to_vec();
|
||||
let pem = ders_to_pem_bundle(&[der.clone(), der]).expect("bundle");
|
||||
let parsed = reqwest::Certificate::from_pem_bundle(pem.as_bytes()).expect("parse back");
|
||||
assert_eq!(parsed.len(), 2);
|
||||
assert!(pem_contains_certificate(pem.as_bytes()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_tls_candidates_fail_closed_on_missing_ca_file() {
|
||||
let err = grpc_tls_candidates(
|
||||
"https://collector.corp.example:4317",
|
||||
Some("/nonexistent/corp-ca.pem"),
|
||||
)
|
||||
.expect_err("missing CA bundle must fail exporter construction");
|
||||
assert!(
|
||||
err.to_string().contains("OTEL_EXPORTER_OTLP_CERTIFICATE"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exporter_metadata_is_customer_headers_only() {
|
||||
let cfg = cfg_with_headers(vec![
|
||||
|
|
|
|||
|
|
@ -998,6 +998,40 @@ fn settings_gate_suppresses_until_resolved() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_gate_opens_when_the_bounded_window_expires() {
|
||||
let _serial = GATE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
struct RestoreGate(std::time::Duration);
|
||||
impl Drop for RestoreGate {
|
||||
fn drop(&mut self) {
|
||||
super::set_settings_gate_max_wait(self.0);
|
||||
super::mark_external_otel_settings_resolved();
|
||||
}
|
||||
}
|
||||
let _restore = RestoreGate(super::settings_gate_max_wait());
|
||||
|
||||
super::set_settings_gate_max_wait(std::time::Duration::from_secs(600));
|
||||
super::suppress_external_otel_until_settings();
|
||||
assert!(
|
||||
!super::is_settings_gate_open(),
|
||||
"inside the window the gate stays fail-closed"
|
||||
);
|
||||
|
||||
super::set_settings_gate_max_wait(std::time::Duration::ZERO);
|
||||
assert!(
|
||||
super::is_settings_gate_open(),
|
||||
"an expired window must resolve the gate open onto local policy"
|
||||
);
|
||||
|
||||
super::set_settings_gate_max_wait(std::time::Duration::from_secs(600));
|
||||
super::suppress_external_otel_until_settings();
|
||||
assert!(
|
||||
!super::is_settings_gate_open(),
|
||||
"re-closing must restart the window, not stay open"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Metric increment derivation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -421,7 +421,7 @@ fn build_server_provider(client: OtelClientInfo, config: OtelLayerConfig) -> Sdk
|
|||
.exporter
|
||||
.timeout
|
||||
.unwrap_or(std::time::Duration::from_secs(10));
|
||||
let http_client = match crate::otlp_http::build_blocking_client(timeout) {
|
||||
let http_client = match crate::otlp_http::build_blocking_client(timeout, &[]) {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "otel: OTLP HTTP client build failed; span export disabled");
|
||||
|
|
@ -554,9 +554,10 @@ mod tests {
|
|||
static_headers: Arc::new(std::collections::HashMap::new()),
|
||||
credentials: provider,
|
||||
last_token: parking_lot::Mutex::new(last_token.to_string()),
|
||||
http_client: crate::otlp_http::build_blocking_client(std::time::Duration::from_secs(
|
||||
30,
|
||||
))
|
||||
http_client: crate::otlp_http::build_blocking_client(
|
||||
std::time::Duration::from_secs(30),
|
||||
&[],
|
||||
)
|
||||
.expect("test OTLP HTTP client must build"),
|
||||
resource: parking_lot::Mutex::new(opentelemetry_sdk::Resource::builder().build()),
|
||||
token_header_value: Arc::from("xai-grok-cli"),
|
||||
|
|
|
|||
|
|
@ -45,18 +45,50 @@ impl HttpClient for BlockingOtlpClient {
|
|||
/// The blocking client can't be built inside a Tokio runtime, and the batch
|
||||
/// processors drive exports from non-Tokio threads — building on a fresh
|
||||
/// thread avoids the "no reactor" panic for every caller.
|
||||
///
|
||||
/// `extra_ca_pem_files` are PEM bundle paths whose certificates are added to
|
||||
/// the trusted roots (the external stream's `OTEL_EXPORTER_OTLP_CERTIFICATE`,
|
||||
/// for customer collectors behind a private CA). Errors reading or parsing a
|
||||
/// listed bundle fail construction — exporting without a CA the user
|
||||
/// explicitly configured would silently verify against the wrong trust set.
|
||||
pub(crate) fn build_blocking_client(
|
||||
timeout: std::time::Duration,
|
||||
extra_ca_pem_files: &[&str],
|
||||
) -> Result<BlockingOtlpClient, String> {
|
||||
let mut extra_roots = Vec::new();
|
||||
for path in extra_ca_pem_files {
|
||||
let pem = std::fs::read(path)
|
||||
.map_err(|e| format!("reading OTEL_EXPORTER_OTLP_CERTIFICATE {path:?}: {e}"))?;
|
||||
let certs = reqwest::Certificate::from_pem_bundle(&pem)
|
||||
.map_err(|e| format!("parsing OTEL_EXPORTER_OTLP_CERTIFICATE {path:?}: {e}"))?;
|
||||
// A readable but certificate-less bundle must fail closed too:
|
||||
// building a client that verifies without the configured CA would
|
||||
// silently use the wrong trust set.
|
||||
if certs.is_empty() {
|
||||
return Err(format!(
|
||||
"OTEL_EXPORTER_OTLP_CERTIFICATE {path:?} contains no certificates"
|
||||
));
|
||||
}
|
||||
extra_roots.extend(certs);
|
||||
}
|
||||
std::thread::Builder::new()
|
||||
.name("otlp-client-build".into())
|
||||
.spawn(move || {
|
||||
xai_grok_extra_ca::with_extra_root_certificates_blocking(
|
||||
// Two additive trust sources on top of the embedded webpki
|
||||
// roots: the process-wide `GROK_EXTRA_CA_BUNDLE` (fail-open,
|
||||
// handled inside xai-grok-extra-ca) and the external stream's
|
||||
// per-call `OTEL_EXPORTER_OTLP_CERTIFICATE` files (fail-closed,
|
||||
// validated above).
|
||||
let mut builder = xai_grok_extra_ca::with_extra_root_certificates_blocking(
|
||||
reqwest::blocking::Client::builder().timeout(timeout),
|
||||
)
|
||||
.build()
|
||||
.map(BlockingOtlpClient)
|
||||
.map_err(|e| format!("building blocking OTLP HTTP client: {e}"))
|
||||
);
|
||||
for cert in extra_roots {
|
||||
builder = builder.add_root_certificate(cert);
|
||||
}
|
||||
builder
|
||||
.build()
|
||||
.map(BlockingOtlpClient)
|
||||
.map_err(|e| format!("building blocking OTLP HTTP client: {e}"))
|
||||
})
|
||||
.map_err(|e| format!("spawning OTLP client builder thread: {e}"))?
|
||||
.join()
|
||||
|
|
@ -72,7 +104,35 @@ mod tests {
|
|||
/// with no system CA store.
|
||||
#[test]
|
||||
fn blocking_otlp_client_builds_with_embedded_roots() {
|
||||
build_blocking_client(std::time::Duration::from_secs(5))
|
||||
build_blocking_client(std::time::Duration::from_secs(5), &[])
|
||||
.expect("client with embedded webpki roots must build on any host");
|
||||
}
|
||||
|
||||
/// A configured-but-unreadable customer CA must fail construction (the
|
||||
/// caller degrades by disabling the stream) instead of silently building
|
||||
/// a client that verifies against the wrong trust set.
|
||||
#[test]
|
||||
fn blocking_otlp_client_fails_closed_on_missing_ca_file() {
|
||||
let err = build_blocking_client(
|
||||
std::time::Duration::from_secs(5),
|
||||
&["/nonexistent/corp-ca.pem"],
|
||||
)
|
||||
.expect_err("missing CA bundle must fail construction");
|
||||
assert!(err.contains("OTEL_EXPORTER_OTLP_CERTIFICATE"), "{err}");
|
||||
}
|
||||
|
||||
/// A readable but certificate-less bundle must also fail closed instead
|
||||
/// of building a client that verifies against the default roots only.
|
||||
#[test]
|
||||
fn blocking_otlp_client_fails_closed_on_empty_ca_bundle() {
|
||||
let file = tempfile::NamedTempFile::new().expect("temp CA file");
|
||||
std::fs::write(file.path(), "# readable, but no PEM certificate blocks\n")
|
||||
.expect("write empty bundle");
|
||||
let err = build_blocking_client(
|
||||
std::time::Duration::from_secs(5),
|
||||
&[file.path().to_str().expect("utf-8 path")],
|
||||
)
|
||||
.expect_err("certificate-less bundle must fail construction");
|
||||
assert!(err.contains("no certificates"), "{err}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
//! HTTPS (TLS) gRPC transport coverage for the external OTEL stream —
|
||||
//! regression test for GB-4580, where `https://` collector endpoints were
|
||||
//! rejected at exporter build time and the stream silently disabled itself.
|
||||
//!
|
||||
//! The collector presents a certificate signed by a freshly generated CA and
|
||||
//! the client trusts it via the standard `OTEL_EXPORTER_OTLP_CERTIFICATE`
|
||||
//! variable, so the full TLS handshake + OTLP export path is exercised.
|
||||
//! Lives in its own integration-test binary because the external telemetry
|
||||
//! registry is a process-global `OnceLock`.
|
||||
|
||||
mod otlp_collector;
|
||||
|
||||
use otlp_collector as col;
|
||||
|
||||
#[test]
|
||||
fn external_stream_grpc_over_tls_end_to_end() {
|
||||
let tls = col::generate_tls_material();
|
||||
let ca_file = tempfile::NamedTempFile::new().expect("CA temp file");
|
||||
std::fs::write(ca_file.path(), &tls.ca_cert_pem).expect("write CA pem");
|
||||
let ca_path = ca_file.path().to_str().expect("utf-8 CA path").to_string();
|
||||
|
||||
let collected = col::Collected::default();
|
||||
let endpoint = col::start_grpc_tls_collector(
|
||||
collected.clone(),
|
||||
tls.server_cert_pem.clone(),
|
||||
tls.server_key_pem.clone(),
|
||||
);
|
||||
assert!(endpoint.starts_with("https://"), "{endpoint}");
|
||||
|
||||
let mut cfg = xai_grok_telemetry::external::ExternalOtelConfig::resolve_with(
|
||||
|name| match name {
|
||||
"GROK_EXTERNAL_OTEL" => Some("1".into()),
|
||||
"OTEL_LOGS_EXPORTER" | "OTEL_METRICS_EXPORTER" => Some("otlp".into()),
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT" => Some(endpoint.clone()),
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL" => Some("grpc".into()),
|
||||
"OTEL_EXPORTER_OTLP_CERTIFICATE" => Some(ca_path.clone()),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL" => Some("200".into()),
|
||||
"OTEL_BLRP_SCHEDULE_DELAY" => Some("100".into()),
|
||||
_ => None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("double opt-in must resolve");
|
||||
assert_eq!(cfg.logs_ca_certificate.as_deref(), Some(ca_path.as_str()));
|
||||
cfg.client = xai_grok_telemetry::external::config::ExternalClientInfo {
|
||||
service_version: "0.0.0-test".into(),
|
||||
client_version: "0.0.0-test".into(),
|
||||
app_entrypoint: "cli".into(),
|
||||
};
|
||||
|
||||
xai_grok_telemetry::external::init(Some(cfg));
|
||||
assert!(
|
||||
xai_grok_telemetry::external::is_active(),
|
||||
"https gRPC exporters must build and activate the stream (GB-4580)"
|
||||
);
|
||||
|
||||
// `SessionNew` maps to the `session.count` metric; `SessionHarness` maps
|
||||
// to the `session_start` log record — emit both so each signal's TLS
|
||||
// export path is exercised.
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionNew {
|
||||
session_id: "sess-grpc-tls-1".into(),
|
||||
client_identifier: None,
|
||||
client_version: None,
|
||||
is_git_repo: true,
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
});
|
||||
xai_grok_telemetry::log_event(xai_grok_telemetry::events::SessionHarness {
|
||||
session_id: "sess-grpc-tls-1".into(),
|
||||
client_identifier: Some("grok-pager".into()),
|
||||
model_id: "grok-4".into(),
|
||||
agent_name: "grok-build-plan".into(),
|
||||
permission_mode: xai_grok_telemetry::enums::PermissionMode::Ask,
|
||||
mcp_server_names: vec![],
|
||||
plugin_names: vec![],
|
||||
skill_names: vec![],
|
||||
lsp_server_names: vec![],
|
||||
hook_names: vec![],
|
||||
agents_md_dir_names: vec![],
|
||||
memory_enabled: false,
|
||||
is_git_repo: true,
|
||||
auto_update: None,
|
||||
});
|
||||
|
||||
xai_grok_telemetry::external::flush();
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
collected.logs_len() > 0
|
||||
}),
|
||||
"log records must arrive over TLS"
|
||||
);
|
||||
let names = col::event_names(&collected);
|
||||
assert!(
|
||||
names.iter().any(|n| n == "grok_code.session_start"),
|
||||
"expected grok_code.session_start in {names:?}"
|
||||
);
|
||||
|
||||
// Metrics ride the same TLS channel config; make sure at least one
|
||||
// periodic export lands too.
|
||||
assert!(
|
||||
col::wait_until(std::time::Duration::from_secs(10), || {
|
||||
collected.metrics_len() > 0
|
||||
}),
|
||||
"metric exports must arrive over TLS"
|
||||
);
|
||||
|
||||
xai_grok_telemetry::external::shutdown();
|
||||
}
|
||||
|
|
@ -174,6 +174,87 @@ async fn start_grpc_collector(collected: Collected, addr_tx: std::sync::mpsc::Se
|
|||
.expect("collector serve");
|
||||
}
|
||||
|
||||
/// A freshly generated CA plus a `localhost` server certificate signed by it,
|
||||
/// all PEM-encoded — for the TLS collector variants.
|
||||
pub struct TestTlsMaterial {
|
||||
pub ca_cert_pem: String,
|
||||
pub server_cert_pem: String,
|
||||
pub server_key_pem: String,
|
||||
}
|
||||
|
||||
/// Generate a self-signed CA and a `localhost`/`127.0.0.1` server certificate
|
||||
/// signed by it.
|
||||
pub fn generate_tls_material() -> TestTlsMaterial {
|
||||
use rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair};
|
||||
|
||||
let ca_key = KeyPair::generate().expect("generate CA key");
|
||||
let mut ca_params = CertificateParams::new(Vec::new()).expect("CA params");
|
||||
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
let ca_cert = ca_params.self_signed(&ca_key).expect("self-sign CA");
|
||||
|
||||
let server_key = KeyPair::generate().expect("generate server key");
|
||||
let server_params =
|
||||
CertificateParams::new(vec!["localhost".to_string(), "127.0.0.1".to_string()])
|
||||
.expect("server params");
|
||||
let server_cert = server_params
|
||||
.signed_by(&server_key, &ca_cert, &ca_key)
|
||||
.expect("sign server cert");
|
||||
|
||||
TestTlsMaterial {
|
||||
ca_cert_pem: ca_cert.pem(),
|
||||
server_cert_pem: server_cert.pem(),
|
||||
server_key_pem: server_key.serialize_pem(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a **TLS** gRPC collector presenting `server_cert_pem`; returns its
|
||||
/// base URL (`https://localhost:PORT`).
|
||||
pub fn start_grpc_tls_collector(
|
||||
collected: Collected,
|
||||
server_cert_pem: String,
|
||||
server_key_pem: String,
|
||||
) -> String {
|
||||
use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsServiceServer;
|
||||
use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsServiceServer;
|
||||
|
||||
// The test binary links both ring and aws-lc-rs, so rustls cannot pick a
|
||||
// process default on its own; the server-side acceptor needs one pinned.
|
||||
// (The production client is unaffected: tonic passes a provider
|
||||
// explicitly.)
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let (addr_tx, addr_rx) = std::sync::mpsc::channel::<SocketAddr>();
|
||||
std::thread::spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("collector runtime");
|
||||
rt.block_on(async move {
|
||||
let incoming = tonic::transport::server::TcpIncoming::bind(
|
||||
"127.0.0.1:0".parse().expect("collector bind addr"),
|
||||
)
|
||||
.expect("bind gRPC TLS collector");
|
||||
addr_tx
|
||||
.send(incoming.local_addr().expect("collector addr"))
|
||||
.expect("send addr");
|
||||
let identity = tonic::transport::Identity::from_pem(server_cert_pem, server_key_pem);
|
||||
let service = GrpcCollector { collected };
|
||||
tonic::transport::Server::builder()
|
||||
.tls_config(tonic::transport::ServerTlsConfig::new().identity(identity))
|
||||
.expect("collector TLS config")
|
||||
.add_service(LogsServiceServer::new(service.clone()))
|
||||
.add_service(MetricsServiceServer::new(service))
|
||||
.serve_with_incoming(incoming)
|
||||
.await
|
||||
.expect("collector serve");
|
||||
});
|
||||
});
|
||||
let addr = addr_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("collector must start");
|
||||
format!("https://localhost:{}", addr.port())
|
||||
}
|
||||
|
||||
pub fn decode_logs(
|
||||
collected: &Collected,
|
||||
) -> Vec<opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest> {
|
||||
|
|
|
|||
Loading…
Reference in a new issue