Load the configuration from a common Figment instance

This should avoid loading the same files multiple times.
It should also make it easier to do post-processing on the
configuration, like validation.

This does deprecate one undocumented feature: the ability to override
some fields during the configuration generation using environment
variables.

Cargo.lock

@@ -3023,6 +3023,7 @@ dependencies = [
  "camino",
  "clap",
  "dotenvy",
+ "figment",
  "httpdate",
  "hyper",
  "ipnetwork",

Cargo.toml

@@ -78,6 +78,11 @@ features = ["serde", "clock"]
 version = "4.5.3"
 features = ["derive"]
 
+# Configuration loading
+[workspace.dependencies.figment]
+version = "0.10.15"
+features = ["env", "yaml", "test"]
+
 # HTTP headers
 [workspace.dependencies.headers]
 version = "0.3.9"

crates/cli/Cargo.toml

@@ -17,6 +17,7 @@ axum = "0.6.20"
 camino.workspace = true
 clap.workspace = true
 dotenvy = "0.15.7"
+figment.workspace = true
 httpdate = "1.0.3"
 hyper.workspace = true
 ipnetwork = "0.20.0"

crates/cli/src/commands/config.rs

@@ -15,6 +15,7 @@
 use anyhow::Context;
 use camino::Utf8PathBuf;
 use clap::Parser;
+use figment::Figment;
 use mas_config::{ConfigurationSection, RootConfig, SyncConfig};
 use mas_storage::SystemClock;
 use mas_storage_pg::MIGRATOR;
@@ -67,13 +68,13 @@ enum Subcommand {
 }
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         match self.subcommand {
             SC::Dump { output } => {
                 let _span = info_span!("cli.config.dump").entered();
 
-                let config: RootConfig = root.load_config()?;
+                let config = RootConfig::extract(figment)?;
                 let config = serde_yaml::to_string(&config)?;
 
                 if let Some(output) = output {
@@ -89,8 +90,8 @@ impl Options {
             SC::Check => {
                 let _span = info_span!("cli.config.check").entered();
 
-                let _config: RootConfig = root.load_config()?;
+                let _config = RootConfig::extract(figment)?;
-                info!(path = ?root.config, "Configuration file looks good");
+                info!("Configuration file looks good");
             }
 
             SC::Generate { output } => {
@@ -98,7 +99,7 @@ impl Options {
 
                 // XXX: we should disallow SeedableRng::from_entropy
                 let rng = rand_chacha::ChaChaRng::from_entropy();
-                let config = RootConfig::load_and_generate(rng).await?;
+                let config = RootConfig::generate(rng).await?;
                 let config = serde_yaml::to_string(&config)?;
 
                 if let Some(output) = output {
@@ -112,7 +113,7 @@ impl Options {
             }
 
             SC::Sync { prune, dry_run } => {
-                let config: SyncConfig = root.load_config()?;
+                let config = SyncConfig::extract(figment)?;
                 let clock = SystemClock::default();
                 let encrypter = config.secrets.encrypter();
 

crates/cli/src/commands/database.rs

@@ -14,7 +14,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::DatabaseConfig;
+use figment::Figment;
+use mas_config::{ConfigurationSection, DatabaseConfig};
 use mas_storage_pg::MIGRATOR;
 use tracing::{info_span, Instrument};
 
@@ -33,9 +34,9 @@ enum Subcommand {
 }
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let _span = info_span!("cli.database.migrate").entered();
-        let config: DatabaseConfig = root.load_config()?;
+        let config = DatabaseConfig::extract(figment)?;
         let mut conn = database_connection_from_config(&config).await?;
 
         // Run pending migrations

crates/cli/src/commands/debug.rs

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use clap::Parser;
+use figment::Figment;
 use hyper::{Response, Uri};
-use mas_config::PolicyConfig;
+use mas_config::{ConfigurationSection, PolicyConfig};
 use mas_handlers::HttpClientFactory;
 use mas_http::HttpServiceExt;
 use tokio::io::AsyncWriteExt;
@@ -65,7 +66,7 @@ fn print_headers(parts: &hyper::http::response::Parts) {
 
 impl Options {
     #[tracing::instrument(skip_all)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         let http_client_factory = HttpClientFactory::new();
         match self.subcommand {
@@ -120,7 +121,7 @@ impl Options {
 
             SC::Policy => {
                 let _span = info_span!("cli.debug.policy").entered();
-                let config: PolicyConfig = root.load_config()?;
+                let config = PolicyConfig::extract(figment)?;
                 info!("Loading and compiling the policy module");
                 let policy_factory = policy_factory_from_config(&config).await?;
 

crates/cli/src/commands/doctor.rs

@@ -19,7 +19,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::RootConfig;
+use figment::Figment;
+use mas_config::{ConfigurationSection, RootConfig};
 use mas_handlers::HttpClientFactory;
 use mas_http::HttpServiceExt;
 use tower::{Service, ServiceExt};
@@ -34,11 +35,11 @@ pub(super) struct Options {}
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let _span = info_span!("cli.doctor").entered();
         info!("💡 Running diagnostics, make sure that both MAS and Synapse are running, and that MAS is using the same configuration files as this tool.");
 
-        let config: RootConfig = root.load_config()?;
+        let config = RootConfig::extract(figment)?;
 
         // We'll need an HTTP client
         let http_client_factory = HttpClientFactory::new();

crates/cli/src/commands/manage.rs

@@ -14,7 +14,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::{DatabaseConfig, PasswordsConfig};
+use figment::Figment;
+use mas_config::{ConfigurationSection, DatabaseConfig, PasswordsConfig};
 use mas_data_model::{Device, TokenType};
 use mas_storage::{
     compat::{CompatAccessTokenRepository, CompatSessionRepository},
@@ -89,7 +90,7 @@ enum Subcommand {
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         let clock = SystemClock::default();
         // XXX: we should disallow SeedableRng::from_entropy
@@ -100,8 +101,8 @@ impl Options {
                 let _span =
                     info_span!("cli.manage.set_password", user.username = %username).entered();
 
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
-                let passwords_config: PasswordsConfig = root.load_config()?;
+                let passwords_config = PasswordsConfig::extract(figment)?;
 
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let password_manager = password_manager_from_config(&passwords_config).await?;
@@ -136,7 +137,7 @@ impl Options {
                 )
                 .entered();
 
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -170,7 +171,7 @@ impl Options {
                 admin,
                 device_id,
             } => {
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -215,7 +216,7 @@ impl Options {
 
             SC::ProvisionAllUsers => {
                 let _span = info_span!("cli.manage.provision_all_users").entered();
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let mut txn = conn.begin().await?;
 
@@ -241,7 +242,7 @@ impl Options {
             SC::KillSessions { username, dry_run } => {
                 let _span =
                     info_span!("cli.manage.kill_sessions", user.username = username).entered();
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -361,7 +362,7 @@ impl Options {
                 deactivate,
             } => {
                 let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
-                let config: DatabaseConfig = root.load_config()?;
+                let config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -393,7 +394,7 @@ impl Options {
 
             SC::UnlockUser { username } => {
                 let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
-                let config: DatabaseConfig = root.load_config()?;
+                let config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);

crates/cli/src/commands/mod.rs

@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context;
 use camino::Utf8PathBuf;
 use clap::Parser;
-use mas_config::ConfigurationSection;
+use figment::{
+    providers::{Env, Format, Yaml},
+    Figment,
+};
 
 mod config;
 mod database;
@@ -65,22 +67,23 @@ pub struct Options {
 }
 
 impl Options {
-    pub async fn run(mut self) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as S;
-        match self.subcommand.take() {
+        match self.subcommand {
-            Some(S::Config(c)) => c.run(&self).await,
+            Some(S::Config(c)) => c.run(figment).await,
-            Some(S::Database(c)) => c.run(&self).await,
+            Some(S::Database(c)) => c.run(figment).await,
-            Some(S::Server(c)) => c.run(&self).await,
+            Some(S::Server(c)) => c.run(figment).await,
-            Some(S::Worker(c)) => c.run(&self).await,
+            Some(S::Worker(c)) => c.run(figment).await,
-            Some(S::Manage(c)) => c.run(&self).await,
+            Some(S::Manage(c)) => c.run(figment).await,
-            Some(S::Templates(c)) => c.run(&self).await,
+            Some(S::Templates(c)) => c.run(figment).await,
-            Some(S::Debug(c)) => c.run(&self).await,
+            Some(S::Debug(c)) => c.run(figment).await,
-            Some(S::Doctor(c)) => c.run(&self).await,
+            Some(S::Doctor(c)) => c.run(figment).await,
-            None => self::server::Options::default().run(&self).await,
+            None => self::server::Options::default().run(figment).await,
         }
     }
 
-    pub fn load_config<T: ConfigurationSection>(&self) -> anyhow::Result<T> {
+    /// Get a [`Figment`] instance with the configuration loaded
+    pub fn figment(&self) -> Figment {
         let configs = if self.config.is_empty() {
             // Read the MAS_CONFIG environment variable
             std::env::var("MAS_CONFIG")
@@ -93,7 +96,10 @@ impl Options {
         } else {
             self.config.clone()
         };
+        let base = Figment::new().merge(Env::prefixed("MAS_").split("_"));
 
-        T::load_from_files(&configs).context("could not load configuration")
+        configs
+            .into_iter()
+            .fold(base, |f, path| f.merge(Yaml::file(path)))
     }
 }

crates/cli/src/commands/server.rs

@@ -16,8 +16,9 @@ use std::{collections::BTreeSet, sync::Arc, time::Duration};
 
 use anyhow::Context;
 use clap::Parser;
+use figment::Figment;
 use itertools::Itertools;
-use mas_config::AppConfig;
+use mas_config::{AppConfig, ClientsConfig, ConfigurationSection, UpstreamOAuth2Config};
 use mas_handlers::{ActivityTracker, CookieManager, HttpClientFactory, MetadataCache, SiteConfig};
 use mas_listener::{server::Server, shutdown::ShutdownStream};
 use mas_matrix_synapse::SynapseConnection;
@@ -63,9 +64,9 @@ pub(super) struct Options {
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let span = info_span!("cli.run.init").entered();
-        let config: AppConfig = root.load_config()?;
+        let config = AppConfig::extract(figment)?;
 
         if self.migrate {
             warn!("The `--migrate` flag is deprecated and will be removed in a future release. Please use `--no-migrate` to disable automatic migrations on startup.");
@@ -101,8 +102,8 @@ impl Options {
         } else {
             // Sync the configuration with the database
             let mut conn = pool.acquire().await?;
-            let clients_config = root.load_config()?;
+            let clients_config = ClientsConfig::extract(figment)?;
-            let upstream_oauth2_config = root.load_config()?;
+            let upstream_oauth2_config = UpstreamOAuth2Config::extract(figment)?;
 
             crate::sync::config_sync(
                 upstream_oauth2_config,

crates/cli/src/commands/templates.rs

@@ -13,7 +13,8 @@
 // limitations under the License.
 
 use clap::Parser;
-use mas_config::{BrandingConfig, MatrixConfig, TemplatesConfig};
+use figment::Figment;
+use mas_config::{BrandingConfig, ConfigurationSection, MatrixConfig, TemplatesConfig};
 use mas_storage::{Clock, SystemClock};
 use rand::SeedableRng;
 use tracing::info_span;
@@ -33,15 +34,15 @@ enum Subcommand {
 }
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         match self.subcommand {
             SC::Check => {
                 let _span = info_span!("cli.templates.check").entered();
 
-                let template_config: TemplatesConfig = root.load_config()?;
+                let template_config = TemplatesConfig::extract(figment)?;
-                let branding_config: BrandingConfig = root.load_config()?;
+                let branding_config = BrandingConfig::extract(figment)?;
-                let matrix_config: MatrixConfig = root.load_config()?;
+                let matrix_config = MatrixConfig::extract(figment)?;
 
                 let clock = SystemClock::default();
                 // XXX: we should disallow SeedableRng::from_entropy

crates/cli/src/commands/worker.rs

@@ -13,7 +13,8 @@
 // limitations under the License.
 
 use clap::Parser;
-use mas_config::AppConfig;
+use figment::Figment;
+use mas_config::{AppConfig, ConfigurationSection};
 use mas_handlers::HttpClientFactory;
 use mas_matrix_synapse::SynapseConnection;
 use mas_router::UrlBuilder;
@@ -29,9 +30,9 @@ use crate::util::{database_pool_from_config, mailer_from_config, templates_from_
 pub(super) struct Options {}
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let span = info_span!("cli.worker.init").entered();
-        let config: AppConfig = root.load_config()?;
+        let config = AppConfig::extract(figment)?;
 
         // Connect to the database
         info!("Connecting to the database");

crates/cli/src/main.rs

@@ -18,7 +18,7 @@ use std::{io::IsTerminal, sync::Arc};
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::TelemetryConfig;
+use mas_config::{ConfigurationSection, TelemetryConfig};
 use sentry_tracing::EventFilter;
 use tracing_subscriber::{
     filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer, Registry,
@@ -67,10 +67,13 @@ async fn try_main() -> anyhow::Result<()> {
     // Parse the CLI arguments
     let opts = self::commands::Options::parse();
 
+    // Load the base configuration files
+    let figment = opts.figment();
+
     // Telemetry config could fail to load, but that's probably OK, since the whole
     // config will be loaded afterwards, and crash if there is a problem.
     // Falling back to default.
-    let telemetry_config: TelemetryConfig = opts.load_config().unwrap_or_default();
+    let telemetry_config = TelemetryConfig::extract(&figment).unwrap_or_default();
 
     // Setup Sentry
     let sentry = sentry::init((
@@ -126,7 +129,7 @@ async fn try_main() -> anyhow::Result<()> {
 
     // And run the command
     tracing::trace!(?opts, "Running command");
-    opts.run().await?;
+    opts.run(&figment).await?;
 
     Ok(())
 }

crates/config/Cargo.toml

@@ -21,7 +21,7 @@ anyhow.workspace = true
 
 camino = { workspace = true, features = ["serde1"] }
 chrono.workspace = true
-figment = { version = "0.10.15", features = ["env", "yaml", "test"] }
+figment.workspace = true
 ipnetwork = { version = "0.20.0", features = ["serde", "schemars"] }
 schemars.workspace = true
 ulid.workspace = true

crates/config/src/sections/clients.rs

@@ -201,7 +201,10 @@ impl ConfigurationSection for ClientsConfig {
 mod tests {
     use std::str::FromStr;
 
-    use figment::Jail;
+    use figment::{
+        providers::{Format, Yaml},
+        Figment, Jail,
+    };
 
     use super::*;
 
@@ -249,7 +252,9 @@ mod tests {
                 "#,
             )?;
 
-            let config = ClientsConfig::load_from_file("config.yaml")?;
+            let config = Figment::new()
+                .merge(Yaml::file("config.yaml"))
+                .extract_inner::<ClientsConfig>("clients")?;
 
             assert_eq!(config.0.len(), 5);
 

crates/config/src/sections/database.rs

@@ -164,7 +164,10 @@ impl ConfigurationSection for DatabaseConfig {
 
 #[cfg(test)]
 mod tests {
-    use figment::Jail;
+    use figment::{
+        providers::{Format, Yaml},
+        Figment, Jail,
+    };
 
     use super::*;
 
@@ -179,7 +182,9 @@ mod tests {
                 ",
             )?;
 
-            let config = DatabaseConfig::load_from_file("config.yaml")?;
+            let config = Figment::new()
+                .merge(Yaml::file("config.yaml"))
+                .extract_inner::<DatabaseConfig>("database")?;
 
             assert_eq!(
                 config.options,

crates/config/src/sections/matrix.rs

@@ -76,7 +76,10 @@ impl ConfigurationSection for MatrixConfig {
 
 #[cfg(test)]
 mod tests {
-    use figment::Jail;
+    use figment::{
+        providers::{Format, Yaml},
+        Figment, Jail,
+    };
 
     use super::*;
 
@@ -92,10 +95,12 @@ mod tests {
                 ",
             )?;
 
-            let config = MatrixConfig::load_from_file("config.yaml")?;
+            let config = Figment::new()
+                .merge(Yaml::file("config.yaml"))
+                .extract_inner::<MatrixConfig>("matrix")?;
 
-            assert_eq!(config.homeserver, "matrix.org".to_owned());
+            assert_eq!(&config.homeserver, "matrix.org");
-            assert_eq!(config.secret, "test".to_owned());
+            assert_eq!(&config.secret, "test");
 
             Ok(())
         });

crates/config/src/util.rs

@@ -12,14 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context;
 use async_trait::async_trait;
-use camino::Utf8Path;
+use figment::{error::Error as FigmentError, Figment};
-use figment::{
-    error::Error as FigmentError,
-    providers::{Env, Format, Serialized, Yaml},
-    Figment, Profile,
-};
 use rand::Rng;
 use serde::{de::DeserializeOwned, Serialize};
 
@@ -35,64 +29,13 @@ pub trait ConfigurationSection: Sized + DeserializeOwned + Serialize {
     where
         R: Rng + Send;
 
-    /// Generate a sample configuration and override it with environment
+    /// Extract configuration from a Figment instance.
-    /// variables.
-    ///
-    /// This is what backs the `config generate` subcommand, allowing to
-    /// programmatically generate a configuration file, e.g.
-    ///
-    /// ```sh
-    /// export MAS_OAUTH2_ISSUER=https://example.com/
-    /// export MAS_HTTP_ADDRESS=127.0.0.1:1234
-    /// matrix-authentication-service config generate
-    /// ```
-    ///
-    /// # Errors
-    ///
-    /// Returns an error if the configuration could not be generated or if the
-    /// existing configuration could not be loaded
-    async fn load_and_generate<R>(rng: R) -> anyhow::Result<Self>
-    where
-        R: Rng + Send,
-    {
-        let base = Self::generate(rng)
-            .await
-            .context("could not generate configuration")?;
-
-        Figment::new()
-            .merge(Serialized::from(&base, Profile::Default))
-            .merge(Env::prefixed("MAS_").split("_"))
-            .extract_inner(Self::path())
-            .context("could not load configuration")
-    }
-
-    /// Load configuration from a list of files and environment variables.
     ///
     /// # Errors
     ///
     /// Returns an error if the configuration could not be loaded
-    fn load_from_files<P>(paths: &[P]) -> Result<Self, FigmentError>
+    fn extract(figment: &Figment) -> Result<Self, FigmentError> {
-    where
+        figment.extract_inner(Self::path())
-        P: AsRef<Utf8Path>,
-    {
-        let base = Figment::new().merge(Env::prefixed("MAS_").split("_"));
-
-        paths
-            .iter()
-            .fold(base, |f, path| f.merge(Yaml::file(path.as_ref())))
-            .extract_inner(Self::path())
-    }
-
-    /// Load configuration from a file and environment variables.
-    ///
-    /// # Errors
-    ///
-    /// Returns an error if the configuration could not be loaded
-    fn load_from_file<P>(path: P) -> Result<Self, FigmentError>
-    where
-        P: AsRef<Utf8Path>,
-    {
-        Self::load_from_files(&[path])
     }
 
     /// Generate config used in unit tests