Make telemetry configurable

Also allows opting-out of the OTLP exporter to remove the dependency to
protoc when building.

crates/cli/Cargo.toml

@@ -21,12 +21,18 @@ serde_yaml = "0.8.21"
 warp = "0.3.1"
 argon2 = { version = "0.3.1", features = ["password-hash"] }
 opentelemetry = { version = "0.16.0", features = ["trace", "metrics", "rt-tokio"] }
-opentelemetry-otlp = { version = "0.9.0", features = ["trace", "metrics"] }
+opentelemetry-otlp = { version = "0.9.0", features = ["trace", "metrics"], optional = true }
 opentelemetry-semantic-conventions = "0.8.0"
 tracing-opentelemetry = "0.15.0"
+url = "2.2.2"
 
 mas-config = { path = "../config" }
 mas-core = { path = "../core" }
 
 [dev-dependencies]
 indoc = "1.0.3"
+
+[features]
+default = ["otlp"]
+# Enable Opentelemetry OTLP exporter. Requires "protoc"
+otlp = ["opentelemetry-otlp"]

crates/cli/src/main.rs

@@ -22,8 +22,11 @@ use std::path::PathBuf;
 
 use anyhow::Context;
 use clap::Clap;
-use mas_config::ConfigurationSection;
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
+use mas_config::{ConfigurationSection, TelemetryConfig};
+use tracing_subscriber::{
+    filter::LevelFilter, layer::SubscriberExt, reload, util::SubscriberInitExt, EnvFilter, Layer,
+    Registry,
+};
 
 use self::{
     config::ConfigCommand, database::DatabaseCommand, manage::ManageCommand, server::ServerCommand,
@@ -107,14 +110,17 @@ async fn try_main() -> anyhow::Result<()> {
         // Display the error if it is something other than the .env file not existing
         .or_else(|e| if e.not_found() { Ok(None) } else { Err(e) })?;
 
-    // Setup logging & tracing
-    let (tracer, _meter) = telemetry::setup()?;
-    let telemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
-
+    // Setup logging
     // This writes logs to stderr
     let fmt_layer = tracing_subscriber::fmt::layer().with_writer(std::io::stderr);
     let filter_layer = EnvFilter::try_from_default_env().or_else(|_| EnvFilter::try_new("info"))?;
 
+    // Don't fill the telemetry layer for now, we want to configure it based on the
+    // app config, so we need to delay that a bit
+    let (telemetry_layer, handle) = reload::Layer::new(None);
+    // We only want "INFO" level spans to go through OpenTelemetry
+    let telemetry_layer = telemetry_layer.with_filter(LevelFilter::INFO);
+
     let subscriber = Registry::default()
         .with(telemetry_layer)
         .with(filter_layer)
@@ -132,6 +138,22 @@ async fn try_main() -> anyhow::Result<()> {
     // Parse the CLI arguments
     let opts = RootCommand::parse();
 
+    // Telemetry config could fail to load, but that's probably OK, since the whole
+    // config will be loaded afterwards, and crash if there is a problem.
+    // Falling back to default.
+    let telemetry_config: TelemetryConfig = opts.load_config().unwrap_or_default();
+
+    // Setup OpenTelemtry tracing and metrics
+    let tracer = telemetry::setup(&telemetry_config)?;
+    if let Some(tracer) = tracer {
+        // Now we can swap out the actual opentelemetry tracing layer
+        handle.reload(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_tracked_inactivity(false),
+        )?;
+    }
+
     // And run the command
     tracing::trace!(?opts, "Running command");
     opts.run().await?;

crates/cli/src/telemetry.rs

@@ -15,29 +15,33 @@
 use std::time::Duration;
 
 use futures::stream::{Stream, StreamExt};
+use mas_config::{MetricsConfig, TelemetryConfig, TracingConfig};
 use opentelemetry::{
     global,
-    sdk::{
-        self,
-        metrics::{self, PushController},
-        trace::{self, Tracer},
-        Resource,
-    },
+    sdk::{self, trace::Tracer, Resource},
 };
 use opentelemetry_semantic_conventions as semcov;
 
-pub fn setup() -> anyhow::Result<(Tracer, PushController)> {
+pub fn setup(config: &TelemetryConfig) -> anyhow::Result<Option<Tracer>> {
     global::set_error_handler(|e| tracing::error!("{}", e))?;
 
-    Ok((tracer()?, meter()?))
+    let tracer = tracer(&config.tracing)?;
+    meter(&config.metrics)?;
+    Ok(tracer)
 }
 
 pub fn shutdown() {
     global::shutdown_tracer_provider();
 }
 
-fn tracer() -> anyhow::Result<Tracer> {
-    let exporter = opentelemetry_otlp::new_exporter().tonic();
+#[cfg(feature = "otlp")]
+fn otlp_tracer(endpoint: &Option<url::Url>) -> anyhow::Result<Tracer> {
+    use opentelemetry_otlp::WithExportConfig;
+
+    let mut exporter = opentelemetry_otlp::new_exporter().tonic();
+    if let Some(endpoint) = endpoint {
+        exporter = exporter.with_endpoint(endpoint.to_string());
+    }
 
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()
@@ -48,25 +52,74 @@ fn tracer() -> anyhow::Result<Tracer> {
     Ok(tracer)
 }
 
+#[cfg(not(feature = "otlp"))]
+fn otlp_tracer(_endpoint: &Option<url::Url>) -> anyhow::Result<Tracer> {
+    anyhow::bail!("The service was compiled without OTLP exporter support, but config exports traces via OTLP.")
+}
+
+fn stdout_tracer() -> Tracer {
+    sdk::export::trace::stdout::new_pipeline()
+        .with_pretty_print(true)
+        .with_trace_config(trace_config())
+        .install_simple()
+}
+
+fn tracer(config: &TracingConfig) -> anyhow::Result<Option<Tracer>> {
+    let tracer = match config {
+        TracingConfig::None => return Ok(None),
+        TracingConfig::Stdout => stdout_tracer(),
+        TracingConfig::Otlp { endpoint } => otlp_tracer(endpoint)?,
+    };
+
+    Ok(Some(tracer))
+}
+
 fn interval(duration: Duration) -> impl Stream<Item = tokio::time::Instant> {
     // Skip first immediate tick from tokio
     opentelemetry::util::tokio_interval_stream(duration).skip(1)
 }
 
-fn meter() -> anyhow::Result<PushController> {
-    let exporter = opentelemetry_otlp::new_exporter().tonic();
+#[cfg(feature = "otlp")]
+fn otlp_meter(endpoint: &Option<url::Url>) -> anyhow::Result<()> {
+    use opentelemetry_otlp::WithExportConfig;
 
-    let meter = opentelemetry_otlp::new_pipeline()
+    let mut exporter = opentelemetry_otlp::new_exporter().tonic();
+    if let Some(endpoint) = endpoint {
+        exporter = exporter.with_endpoint(endpoint.to_string());
+    }
+
+    opentelemetry_otlp::new_pipeline()
         .metrics(tokio::spawn, interval)
         .with_exporter(exporter)
-        .with_aggregator_selector(metrics::selectors::simple::Selector::Exact)
+        .with_aggregator_selector(sdk::metrics::selectors::simple::Selector::Exact)
         .build()?;
 
-    Ok(meter)
+    Ok(())
 }
 
-fn trace_config() -> trace::Config {
-    trace::config().with_resource(resource())
+#[cfg(not(feature = "otlp"))]
+fn otlp_meter(_endpoint: &Option<url::Url>) -> anyhow::Result<()> {
+    anyhow::bail!("The service was compiled without OTLP exporter support, but config exports metrics via OTLP.")
+}
+
+fn stdout_meter() {
+    sdk::export::metrics::stdout(tokio::spawn, interval)
+        .with_pretty_print(true)
+        .init();
+}
+
+fn meter(config: &MetricsConfig) -> anyhow::Result<()> {
+    match config {
+        MetricsConfig::None => {}
+        MetricsConfig::Stdout => stdout_meter(),
+        MetricsConfig::Otlp { endpoint } => otlp_meter(endpoint)?,
+    };
+
+    Ok(())
+}
+
+fn trace_config() -> sdk::trace::Config {
+    sdk::trace::config().with_resource(resource())
 }
 
 fn resource() -> Resource {