Replace the OTEL-based tracing layer with tracing based layers

crates/cli/Cargo.toml

@@ -57,6 +57,7 @@ mas-storage = { path = "../storage" }
 mas-storage-pg = { path = "../storage-pg" }
 mas-tasks = { path = "../tasks" }
 mas-templates = { path = "../templates" }
+mas-tower = { path = "../tower" }
 oauth2-types = { path = "../oauth2-types" }
 
 [dev-dependencies]

crates/cli/src/commands/debug.rs

@@ -75,7 +75,7 @@ impl Options {
                 url,
             } => {
                 let _span = info_span!("cli.debug.http").entered();
-                let mut client = http_client_factory.client("cli-debug-http").await?;
+                let mut client = http_client_factory.client().await?;
                 let request = hyper::Request::builder()
                     .uri(url)
                     .body(hyper::Body::empty())?;
@@ -101,7 +101,7 @@ impl Options {
             } => {
                 let _span = info_span!("cli.debug.http").entered();
                 let mut client = http_client_factory
-                    .client("cli-debug-http")
+                    .client()
                     .await?
                     .response_body_to_bytes()
                     .json_response();

crates/cli/src/server.rs

@@ -13,27 +13,153 @@
 // limitations under the License.
 
 use std::{
+    borrow::Cow,
     future::ready,
     net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
     os::unix::net::UnixListener,
 };
 
 use anyhow::Context;
-use axum::{body::HttpBody, error_handling::HandleErrorLayer, extract::FromRef, Extension, Router};
-use hyper::StatusCode;
+use axum::{
+    body::HttpBody,
+    error_handling::HandleErrorLayer,
+    extract::{FromRef, MatchedPath},
+    Extension, Router,
+};
+use hyper::{Method, Request, Response, StatusCode, Version};
 use listenfd::ListenFd;
 use mas_config::{HttpBindConfig, HttpResource, HttpTlsConfig, UnixOrTcp};
 use mas_handlers::AppState;
-use mas_http::otel::TraceLayer;
 use mas_listener::{unix_or_tcp::UnixOrTcpListener, ConnectionInfo};
 use mas_router::Route;
 use mas_spa::ViteManifestService;
 use mas_templates::Templates;
-use opentelemetry::KeyValue;
+use mas_tower::{
+    make_span_fn, metrics_attributes_fn, DurationRecorderLayer, InFlightCounterLayer, TraceLayer,
+    KV,
+};
+use opentelemetry::{Key, KeyValue};
+use opentelemetry_http::HeaderExtractor;
+use opentelemetry_semantic_conventions::trace::{
+    HTTP_METHOD, HTTP_ROUTE, HTTP_SCHEME, HTTP_STATUS_CODE,
+};
 use rustls::ServerConfig;
 use sentry_tower::{NewSentryLayer, SentryHttpLayer};
 use tower::Layer;
 use tower_http::{compression::CompressionLayer, services::ServeDir};
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+const NET_PROTOCOL_NAME: Key = Key::from_static_str("net.protocol.name");
+const NET_PROTOCOL_VERSION: Key = Key::from_static_str("net.protocol.version");
+const MAS_LISTENER_NAME: Key = Key::from_static_str("mas.listener.name");
+
+#[inline]
+fn otel_http_method<B>(request: &Request<B>) -> Cow<'static, str> {
+    match request.method() {
+        &Method::OPTIONS => "OPTIONS".into(),
+        &Method::GET => "GET".into(),
+        &Method::POST => "POST".into(),
+        &Method::PUT => "PUT".into(),
+        &Method::DELETE => "DELETE".into(),
+        &Method::HEAD => "HEAD".into(),
+        &Method::TRACE => "TRACE".into(),
+        &Method::CONNECT => "CONNECT".into(),
+        &Method::PATCH => "PATCH".into(),
+        other => other.to_string().into(),
+    }
+}
+
+#[inline]
+fn otel_net_protocol_version<B>(request: &Request<B>) -> Cow<'static, str> {
+    match request.version() {
+        Version::HTTP_09 => "0.9".into(),
+        Version::HTTP_10 => "1.0".into(),
+        Version::HTTP_11 => "1.1".into(),
+        Version::HTTP_2 => "2.0".into(),
+        Version::HTTP_3 => "3.0".into(),
+        other => format!("{other:?}").into(),
+    }
+}
+
+fn otel_http_route<B>(request: &Request<B>) -> Option<&str> {
+    request
+        .extensions()
+        .get::<MatchedPath>()
+        .map(MatchedPath::as_str)
+}
+
+fn otel_http_target<B>(request: &Request<B>) -> &str {
+    request.uri().path_and_query().map_or("", |p| p.as_str())
+}
+
+fn otel_http_scheme<B>(request: &Request<B>) -> &'static str {
+    // XXX: maybe we should panic if the connection info was not injected in the
+    // request extensions
+    request
+        .extensions()
+        .get::<ConnectionInfo>()
+        .map_or("http", |conn_info| {
+            if conn_info.get_tls_ref().is_some() {
+                "https"
+            } else {
+                "http"
+            }
+        })
+}
+
+fn make_http_span<B>(req: &Request<B>) -> Span {
+    let method = otel_http_method(req);
+    let route = otel_http_route(req);
+
+    let span_name = if let Some(route) = route.as_ref() {
+        format!("{method} {route}")
+    } else {
+        format!("{method}")
+    };
+
+    let span = tracing::info_span!(
+        "http.server.request",
+        "otel.kind" = "server",
+        "otel.name" = span_name,
+        "otel.status_code" = tracing::field::Empty,
+        "net.protocol.name" = "http",
+        "net.protocol.version" = otel_net_protocol_version(req).as_ref(),
+        "http.scheme" = otel_http_scheme(req),
+        "http.method" = method.as_ref(),
+        "http.route" = tracing::field::Empty,
+        "http.target" = otel_http_target(req),
+        "http.status_code" = tracing::field::Empty,
+    );
+
+    if let Some(route) = route.as_ref() {
+        span.record("http.route", route);
+    }
+
+    // Extract the parent span context from the request headers
+    let parent_context = opentelemetry::global::get_text_map_propagator(|propagator| {
+        let extractor = HeaderExtractor(req.headers());
+        let context = opentelemetry::Context::new();
+        propagator.extract_with_context(&context, &extractor)
+    });
+    span.set_parent(parent_context);
+
+    span
+}
+
+fn on_http_request_labels<B>(request: &Request<B>) -> Vec<KeyValue> {
+    vec![
+        NET_PROTOCOL_NAME.string("http"),
+        NET_PROTOCOL_VERSION.string(otel_net_protocol_version(request)),
+        HTTP_METHOD.string(otel_http_method(request)),
+        HTTP_SCHEME.string(otel_http_scheme(request).as_ref()),
+        HTTP_ROUTE.string(otel_http_route(request).unwrap_or("FALLBACK").to_owned()),
+    ]
+}
+
+fn on_http_response_labels<B>(res: &Response<B>) -> Vec<KeyValue> {
+    vec![HTTP_STATUS_CODE.i64(res.status().as_u16().into())]
+}
 
 pub fn build_router<B>(
     state: AppState,
@@ -112,16 +238,34 @@ where
         }
     }
 
-    let mut trace_layer = TraceLayer::axum();
-
-    if let Some(name) = name {
-        trace_layer = trace_layer.with_static_attribute(KeyValue::new("listener", name.to_owned()));
-    }
-
     router
+        .layer(
+            InFlightCounterLayer::new("http.server.active_requests").on_request((
+                name.map(|name| MAS_LISTENER_NAME.string(name.to_owned())),
+                metrics_attributes_fn(on_http_request_labels),
+            )),
+        )
+        .layer(
+            DurationRecorderLayer::new("http.server.duration")
+                .on_request((
+                    name.map(|name| MAS_LISTENER_NAME.string(name.to_owned())),
+                    metrics_attributes_fn(on_http_request_labels),
+                ))
+                .on_response_fn(on_http_response_labels),
+        )
+        .layer(
+            TraceLayer::new((
+                make_span_fn(make_http_span),
+                name.map(|name| KV("mas.listener.name", name.to_owned())),
+            ))
+            .on_response_fn(|span: &Span, response: &Response<_>| {
+                let status_code = response.status().as_u16();
+                span.record("http.status_code", status_code);
+                span.record("otel.status_code", "OK");
+            }),
+        )
         .layer(SentryHttpLayer::new())
         .layer(NewSentryLayer::new_from_top())
-        .layer(trace_layer)
         .layer(CompressionLayer::new())
         .with_state(state)
 }

crates/cli/src/telemetry.rs

@@ -375,8 +375,9 @@ fn prometheus_meter() -> anyhow::Result<BasicController> {
 fn prometheus_meter() -> anyhow::Result<BasicController> {
     let controller = sdk::metrics::controllers::basic(
         sdk::metrics::processors::factory(
+            // All histogram metrics are in milliseconds. Each bucket is ~2x the previous one.
             sdk::metrics::selectors::simple::histogram([
-                0.01, 0.02, 0.05, 0.10, 0.20, 0.50, 1.0, 2.0, 5.0,
+                1.0, 3.0, 5.0, 10.0, 30.0, 50.0, 100.0, 300.0, 1000.0,
             ]),
             sdk::export::metrics::aggregation::cumulative_temporality_selector(),
         )