matrix/authentication-service
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-authentication-service.git synced 2025-07-29 22:01:14 +03:00
Code Activity

Export Prometheus metrics on regular listeners

Browse Source
This commit is contained in:
Quentin Gliech
2022-10-04 10:52:52 +02:00
parent 84ac87f551
commit 014a8366ed
6 changed files with 101 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/cli/Cargo.toml
View File

@ -38,7 +38,7 @@ prometheus = { version = "0.13.2", optional = true }
mas-config = { path = "../config" }
mas-email = { path = "../email" }
mas-handlers = { path = "../handlers", default-features = false }
mas-http = { path = "../http", features = ["axum", "client", "native-roots"] }
mas-http = { path = "../http", default-features = false, features = ["axum", "client"] }
mas-policy = { path = "../policy" }
mas-router = { path = "../router" }
mas-static-files = { path = "../static-files" }
@ -62,9 +62,9 @@ webpki-roots = ["mas-http/webpki-roots", "mas-handlers/webpki-roots"]
dev = ["mas-templates/dev", "mas-static-files/dev"]
# Enable OpenTelemetry OTLP exporter.
otlp = ["dep:opentelemetry-otlp", "dep:opentelemetry-http"]
otlp = ["dep:opentelemetry-otlp"]
# Enable OpenTelemetry Jaeger exporter and propagator.
jaeger = ["dep:opentelemetry-jaeger"]
jaeger = ["dep:opentelemetry-jaeger", "dep:opentelemetry-http"]
# Enable OpenTelemetry Zipkin exporter and B3 propagator.
zipkin = ["dep:opentelemetry-zipkin", "dep:opentelemetry-http"]
# Enable OpenTelemetry Prometheus exporter. Requires "protoc"

31
crates/cli/src/commands/server.rs
View File

@ -32,7 +32,7 @@ use mas_storage::MIGRATOR;
use mas_tasks::TaskQueue;
use mas_templates::Templates;
use tokio::io::AsyncRead;
use tracing::{error, info};
use tracing::{error, info, log::warn};
#[derive(Parser, Debug, Default)]
pub(super) struct Options {
@ -271,11 +271,28 @@ impl Options {
let mut router = mas_handlers::empty_router(state.clone());
for resource in config.resources {
let is_tls = config.tls.is_some();
let adresses: Vec<String> = listeners.iter().map(|listener| {
let addr = listener.local_addr();
let proto = if is_tls { "https" } else { "http" };
if let Ok(addr) = addr {
format!("{proto}://{addr:?}")
} else {
warn!("Could not get local address for listener, something might be wrong!");
format!("{proto}://???")
}
}).collect();
info!("Listening on {adresses:?} with resources {resources:?}", resources = &config.resources);
for resource in &config.resources {
router = match resource {
mas_config::HttpResource::Health => {
router.merge(mas_handlers::healthcheck_router(state.clone()))
}
mas_config::HttpResource::Prometheus => {
router.route_service("/metrics", crate::telemetry::prometheus_service())
}
mas_config::HttpResource::Discovery => {
router.merge(mas_handlers::discovery_router(state.clone()))
}
@ -319,16 +336,6 @@ impl Options {
.try_for_each_concurrent(None, move |listener| {
let listener = MaybeTlsAcceptor::new(tls_config.clone(), listener);
// Unless there is something really bad happening, we should be able to
// grab the local_addr here. Panicking here if it is not the case is
// probably fine.
let addr = listener.local_addr().unwrap();
if listener.is_secure() {
info!("Listening on https://{addr:?}");
} else {
info!("Listening on http://{addr:?}");
}
Server::builder(listener)
.serve(router.clone().into_make_service())
.with_graceful_shutdown(signal.clone())

134
crates/cli/src/telemetry.rs
View File

@ -15,6 +15,7 @@
use std::time::Duration;
use anyhow::{bail, Context as _};
use hyper::{header::CONTENT_TYPE, Body, Response};
use mas_config::{
JaegerExporterProtocolConfig, MetricsExporterConfig, Propagator, TelemetryConfig,
TracingExporterConfig,
@ -33,13 +34,18 @@ use opentelemetry::{
};
#[cfg(feature = "jaeger")]
use opentelemetry_jaeger::Propagator as JaegerPropagator;
#[cfg(feature = "prometheus")]
use opentelemetry_prometheus::PrometheusExporter;
use opentelemetry_semantic_conventions as semcov;
#[cfg(feature = "zipkin")]
use opentelemetry_zipkin::{B3Encoding, Propagator as ZipkinPropagator};
use tokio::sync::OnceCell;
use url::Url;
static METRICS_BASIC_CONTROLLER: OnceCell<Option<BasicController>> = OnceCell::const_new();
static METRICS_BASIC_CONTROLLER: OnceCell<BasicController> = OnceCell::const_new();
#[cfg(feature = "prometheus")]
static PROMETHEUS_EXPORTER: OnceCell<PrometheusExporter> = OnceCell::const_new();
pub async fn setup(
config: &TelemetryConfig,
@ -55,8 +61,11 @@ pub async fn setup(
let tracer = tracer(&config.tracing.exporter)
.await
.context("Failed to configure traces exporter")?;
let meter = meter(&config.metrics.exporter).context("Failed to configure metrics exporter")?;
METRICS_BASIC_CONTROLLER.set(meter.clone())?;
if let Some(meter) = meter.as_ref() {
METRICS_BASIC_CONTROLLER.set(meter.clone())?;
}
Ok((tracer, meter))
}
@ -64,7 +73,7 @@ pub async fn setup(
pub fn shutdown() {
global::shutdown_tracer_provider();
if let Some(Some(controller)) = METRICS_BASIC_CONTROLLER.get() {
if let Some(controller) = METRICS_BASIC_CONTROLLER.get() {
let cx = Context::new();
controller.stop(&cx).unwrap();
}
@ -103,7 +112,7 @@ fn propagator(propagators: &[Propagator]) -> anyhow::Result<impl TextMapPropagat
Ok(TextMapCompositePropagator::new(propagators?))
}
#[cfg(any(feature = "otlp", feature = "jaeger"))]
#[cfg(any(feature = "zipkin", feature = "jaeger"))]
async fn http_client() -> anyhow::Result<impl opentelemetry_http::HttpClient + 'static> {
let client = mas_http::make_untraced_client()
.await
@ -303,21 +312,67 @@ fn stdout_meter() -> anyhow::Result<BasicController> {
}
#[cfg(not(feature = "prometheus"))]
fn prometheus_meter(address: &str) -> anyhow::Result<BasicController> {
let _ = address;
pub fn prometheus_service<T>() -> tower::util::ServiceFn<
impl FnMut(T) -> std::future::Ready<Result<Response<Body>, std::convert::Infallible>> + Clone,
> {
tracing::warn!("Prometheus exporter was not enabled at compilation time, but the Prometheus resource was mounted on a listener");
tower::service_fn(move |_req| {
let response = Response::builder()
.status(500)
.header(CONTENT_TYPE, "text/plain")
.body(Body::from(
"Prometheus exporter was not enabled at compilation time",
))
.unwrap();
std::future::ready(Ok(response))
})
}
#[cfg(feature = "prometheus")]
pub fn prometheus_service<T>() -> tower::util::ServiceFn<
impl FnMut(T) -> std::future::Ready<Result<Response<Body>, std::convert::Infallible>> + Clone,
> {
use prometheus::{Encoder, TextEncoder};
if !PROMETHEUS_EXPORTER.initialized() {
tracing::warn!("A Prometheus resource was mounted on a listener, but the Prometheus exporter was not setup in the config");
}
tower::service_fn(move |_req| {
let response = if let Some(exporter) = PROMETHEUS_EXPORTER.get() {
let mut buffer = vec![];
let encoder = TextEncoder::new();
let metric_families = exporter.registry().gather();
// That shouldn't panic, unless we're constructing invalid labels
encoder.encode(&metric_families, &mut buffer).unwrap();
Response::builder()
.status(200)
.header(CONTENT_TYPE, encoder.format_type())
.body(Body::from(buffer))
.unwrap()
} else {
Response::builder()
.status(500)
.header(CONTENT_TYPE, "text/plain")
.body(Body::from("Prometheus exporter was not enabled in config"))
.unwrap()
};
std::future::ready(Ok(response))
})
}
#[cfg(not(feature = "prometheus"))]
fn prometheus_meter() -> anyhow::Result<BasicController> {
anyhow::bail!("The service was compiled without Prometheus exporter support, but config exports metrics via Prometheus.")
}
#[cfg(feature = "prometheus")]
fn prometheus_meter(address: &str) -> anyhow::Result<BasicController> {
use std::{
convert::Infallible,
net::{SocketAddr, TcpListener},
};
use hyper::{header::CONTENT_TYPE, service::make_service_fn, Body, Method, Request, Response};
use prometheus::{Encoder, TextEncoder};
fn prometheus_meter() -> anyhow::Result<BasicController> {
let controller = sdk::metrics::controllers::basic(
sdk::metrics::processors::factory(
sdk::metrics::selectors::simple::histogram([
@ -331,52 +386,7 @@ fn prometheus_meter(address: &str) -> anyhow::Result<BasicController> {
.build();
let exporter = opentelemetry_prometheus::exporter(controller.clone()).try_init()?;
let make_svc = make_service_fn(move |_conn| {
let exporter = exporter.clone();
async move {
Ok::<_, Infallible>(tower::service_fn(move |req: Request<Body>| {
let exporter = exporter.clone();
async move {
let response = match (req.method(), req.uri().path()) {
(&Method::GET, "/metrics") => {
let mut buffer = vec![];
let encoder = TextEncoder::new();
let metric_families = exporter.registry().gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
Response::builder()
.status(200)
.header(CONTENT_TYPE, encoder.format_type())
.body(Body::from(buffer))
.unwrap()
}
_ => Response::builder()
.status(404)
.body(Body::from("Metrics are exposed on /metrics"))
.unwrap(),
};
Ok::<_, Infallible>(response)
}
}))
}
});
let address: SocketAddr = address
.parse()
.context("could not parse listener address")?;
let listener = TcpListener::bind(address).context("could not bind address")?;
tracing::info!(
"Prometheus exporter listening on on http://{}/metrics",
listener.local_addr().unwrap()
);
let server = hyper::server::Server::from_tcp(listener)
.context("Failed to start HTTP server for the Prometheus metrics exporter")?
.serve(make_svc);
tokio::spawn(server);
PROMETHEUS_EXPORTER.set(exporter)?;
Ok(controller)
}
@ -386,7 +396,7 @@ fn meter(config: &MetricsExporterConfig) -> anyhow::Result<Option<BasicControlle
MetricsExporterConfig::None => None,
MetricsExporterConfig::Stdout => Some(stdout_meter()?),
MetricsExporterConfig::Otlp { endpoint } => Some(otlp_meter(endpoint)?),
MetricsExporterConfig::Prometheus { address } => Some(prometheus_meter(address)?),
MetricsExporterConfig::Prometheus => Some(prometheus_meter()?),
};
Ok(controller)