diff --git a/Cargo.lock b/Cargo.lock index 0380941f..40c97f34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3023,6 +3023,7 @@ dependencies = [ "camino", "clap", "dotenvy", + "figment", "httpdate", "hyper", "ipnetwork", diff --git a/Cargo.toml b/Cargo.toml index f3e1ee4d..101bd83d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,11 @@ features = ["serde", "clock"] version = "4.5.3" features = ["derive"] +# Configuration loading +[workspace.dependencies.figment] +version = "0.10.15" +features = ["env", "yaml", "test"] + # HTTP headers [workspace.dependencies.headers] version = "0.3.9" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5d387461..b603b989 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -17,6 +17,7 @@ axum = "0.6.20" camino.workspace = true clap.workspace = true dotenvy = "0.15.7" +figment.workspace = true httpdate = "1.0.3" hyper.workspace = true ipnetwork = "0.20.0" diff --git a/crates/cli/src/commands/config.rs b/crates/cli/src/commands/config.rs index fc1d47fa..51508d24 100644 --- a/crates/cli/src/commands/config.rs +++ b/crates/cli/src/commands/config.rs @@ -15,6 +15,7 @@ use anyhow::Context; use camino::Utf8PathBuf; use clap::Parser; +use figment::Figment; use mas_config::{ConfigurationSection, RootConfig, SyncConfig}; use mas_storage::SystemClock; use mas_storage_pg::MIGRATOR; @@ -67,13 +68,13 @@ enum Subcommand { } impl Options { - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { use Subcommand as SC; match self.subcommand { SC::Dump { output } => { let _span = info_span!("cli.config.dump").entered(); - let config: RootConfig = root.load_config()?; + let config = RootConfig::extract(figment)?; let config = serde_yaml::to_string(&config)?; if let Some(output) = output { @@ -89,8 +90,8 @@ impl Options { SC::Check => { let _span = info_span!("cli.config.check").entered(); - let _config: RootConfig = root.load_config()?; - info!(path = ?root.config, "Configuration file looks good"); + let _config = RootConfig::extract(figment)?; + info!("Configuration file looks good"); } SC::Generate { output } => { @@ -98,7 +99,7 @@ impl Options { // XXX: we should disallow SeedableRng::from_entropy let rng = rand_chacha::ChaChaRng::from_entropy(); - let config = RootConfig::load_and_generate(rng).await?; + let config = RootConfig::generate(rng).await?; let config = serde_yaml::to_string(&config)?; if let Some(output) = output { @@ -112,7 +113,7 @@ impl Options { } SC::Sync { prune, dry_run } => { - let config: SyncConfig = root.load_config()?; + let config = SyncConfig::extract(figment)?; let clock = SystemClock::default(); let encrypter = config.secrets.encrypter(); diff --git a/crates/cli/src/commands/database.rs b/crates/cli/src/commands/database.rs index 9c84535f..fc683040 100644 --- a/crates/cli/src/commands/database.rs +++ b/crates/cli/src/commands/database.rs @@ -14,7 +14,8 @@ use anyhow::Context; use clap::Parser; -use mas_config::DatabaseConfig; +use figment::Figment; +use mas_config::{ConfigurationSection, DatabaseConfig}; use mas_storage_pg::MIGRATOR; use tracing::{info_span, Instrument}; @@ -33,9 +34,9 @@ enum Subcommand { } impl Options { - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { let _span = info_span!("cli.database.migrate").entered(); - let config: DatabaseConfig = root.load_config()?; + let config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&config).await?; // Run pending migrations diff --git a/crates/cli/src/commands/debug.rs b/crates/cli/src/commands/debug.rs index b5c9a2c7..740d0cc4 100644 --- a/crates/cli/src/commands/debug.rs +++ b/crates/cli/src/commands/debug.rs @@ -13,8 +13,9 @@ // limitations under the License. use clap::Parser; +use figment::Figment; use hyper::{Response, Uri}; -use mas_config::PolicyConfig; +use mas_config::{ConfigurationSection, PolicyConfig}; use mas_handlers::HttpClientFactory; use mas_http::HttpServiceExt; use tokio::io::AsyncWriteExt; @@ -65,7 +66,7 @@ fn print_headers(parts: &hyper::http::response::Parts) { impl Options { #[tracing::instrument(skip_all)] - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { use Subcommand as SC; let http_client_factory = HttpClientFactory::new(); match self.subcommand { @@ -120,7 +121,7 @@ impl Options { SC::Policy => { let _span = info_span!("cli.debug.policy").entered(); - let config: PolicyConfig = root.load_config()?; + let config = PolicyConfig::extract(figment)?; info!("Loading and compiling the policy module"); let policy_factory = policy_factory_from_config(&config).await?; diff --git a/crates/cli/src/commands/doctor.rs b/crates/cli/src/commands/doctor.rs index 5df0e1a2..6458c44b 100644 --- a/crates/cli/src/commands/doctor.rs +++ b/crates/cli/src/commands/doctor.rs @@ -19,7 +19,8 @@ use anyhow::Context; use clap::Parser; -use mas_config::RootConfig; +use figment::Figment; +use mas_config::{ConfigurationSection, RootConfig}; use mas_handlers::HttpClientFactory; use mas_http::HttpServiceExt; use tower::{Service, ServiceExt}; @@ -34,11 +35,11 @@ pub(super) struct Options {} impl Options { #[allow(clippy::too_many_lines)] - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { let _span = info_span!("cli.doctor").entered(); info!("💡 Running diagnostics, make sure that both MAS and Synapse are running, and that MAS is using the same configuration files as this tool."); - let config: RootConfig = root.load_config()?; + let config = RootConfig::extract(figment)?; // We'll need an HTTP client let http_client_factory = HttpClientFactory::new(); diff --git a/crates/cli/src/commands/manage.rs b/crates/cli/src/commands/manage.rs index eca27926..0f63f506 100644 --- a/crates/cli/src/commands/manage.rs +++ b/crates/cli/src/commands/manage.rs @@ -14,7 +14,8 @@ use anyhow::Context; use clap::Parser; -use mas_config::{DatabaseConfig, PasswordsConfig}; +use figment::Figment; +use mas_config::{ConfigurationSection, DatabaseConfig, PasswordsConfig}; use mas_data_model::{Device, TokenType}; use mas_storage::{ compat::{CompatAccessTokenRepository, CompatSessionRepository}, @@ -89,7 +90,7 @@ enum Subcommand { impl Options { #[allow(clippy::too_many_lines)] - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { use Subcommand as SC; let clock = SystemClock::default(); // XXX: we should disallow SeedableRng::from_entropy @@ -100,8 +101,8 @@ impl Options { let _span = info_span!("cli.manage.set_password", user.username = %username).entered(); - let database_config: DatabaseConfig = root.load_config()?; - let passwords_config: PasswordsConfig = root.load_config()?; + let database_config = DatabaseConfig::extract(figment)?; + let passwords_config = PasswordsConfig::extract(figment)?; let mut conn = database_connection_from_config(&database_config).await?; let password_manager = password_manager_from_config(&passwords_config).await?; @@ -136,7 +137,7 @@ impl Options { ) .entered(); - let database_config: DatabaseConfig = root.load_config()?; + let database_config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&database_config).await?; let txn = conn.begin().await?; let mut repo = PgRepository::from_conn(txn); @@ -170,7 +171,7 @@ impl Options { admin, device_id, } => { - let database_config: DatabaseConfig = root.load_config()?; + let database_config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&database_config).await?; let txn = conn.begin().await?; let mut repo = PgRepository::from_conn(txn); @@ -215,7 +216,7 @@ impl Options { SC::ProvisionAllUsers => { let _span = info_span!("cli.manage.provision_all_users").entered(); - let database_config: DatabaseConfig = root.load_config()?; + let database_config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&database_config).await?; let mut txn = conn.begin().await?; @@ -241,7 +242,7 @@ impl Options { SC::KillSessions { username, dry_run } => { let _span = info_span!("cli.manage.kill_sessions", user.username = username).entered(); - let database_config: DatabaseConfig = root.load_config()?; + let database_config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&database_config).await?; let txn = conn.begin().await?; let mut repo = PgRepository::from_conn(txn); @@ -361,7 +362,7 @@ impl Options { deactivate, } => { let _span = info_span!("cli.manage.lock_user", user.username = username).entered(); - let config: DatabaseConfig = root.load_config()?; + let config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&config).await?; let txn = conn.begin().await?; let mut repo = PgRepository::from_conn(txn); @@ -393,7 +394,7 @@ impl Options { SC::UnlockUser { username } => { let _span = info_span!("cli.manage.lock_user", user.username = username).entered(); - let config: DatabaseConfig = root.load_config()?; + let config = DatabaseConfig::extract(figment)?; let mut conn = database_connection_from_config(&config).await?; let txn = conn.begin().await?; let mut repo = PgRepository::from_conn(txn); diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 8c487804..da78bdd6 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Context; use camino::Utf8PathBuf; use clap::Parser; -use mas_config::ConfigurationSection; +use figment::{ + providers::{Env, Format, Yaml}, + Figment, +}; mod config; mod database; @@ -65,22 +67,23 @@ pub struct Options { } impl Options { - pub async fn run(mut self) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { use Subcommand as S; - match self.subcommand.take() { - Some(S::Config(c)) => c.run(&self).await, - Some(S::Database(c)) => c.run(&self).await, - Some(S::Server(c)) => c.run(&self).await, - Some(S::Worker(c)) => c.run(&self).await, - Some(S::Manage(c)) => c.run(&self).await, - Some(S::Templates(c)) => c.run(&self).await, - Some(S::Debug(c)) => c.run(&self).await, - Some(S::Doctor(c)) => c.run(&self).await, - None => self::server::Options::default().run(&self).await, + match self.subcommand { + Some(S::Config(c)) => c.run(figment).await, + Some(S::Database(c)) => c.run(figment).await, + Some(S::Server(c)) => c.run(figment).await, + Some(S::Worker(c)) => c.run(figment).await, + Some(S::Manage(c)) => c.run(figment).await, + Some(S::Templates(c)) => c.run(figment).await, + Some(S::Debug(c)) => c.run(figment).await, + Some(S::Doctor(c)) => c.run(figment).await, + None => self::server::Options::default().run(figment).await, } } - pub fn load_config(&self) -> anyhow::Result { + /// Get a [`Figment`] instance with the configuration loaded + pub fn figment(&self) -> Figment { let configs = if self.config.is_empty() { // Read the MAS_CONFIG environment variable std::env::var("MAS_CONFIG") @@ -93,7 +96,10 @@ impl Options { } else { self.config.clone() }; + let base = Figment::new().merge(Env::prefixed("MAS_").split("_")); - T::load_from_files(&configs).context("could not load configuration") + configs + .into_iter() + .fold(base, |f, path| f.merge(Yaml::file(path))) } } diff --git a/crates/cli/src/commands/server.rs b/crates/cli/src/commands/server.rs index 4ecf5466..7c5f9219 100644 --- a/crates/cli/src/commands/server.rs +++ b/crates/cli/src/commands/server.rs @@ -16,8 +16,9 @@ use std::{collections::BTreeSet, sync::Arc, time::Duration}; use anyhow::Context; use clap::Parser; +use figment::Figment; use itertools::Itertools; -use mas_config::AppConfig; +use mas_config::{AppConfig, ClientsConfig, ConfigurationSection, UpstreamOAuth2Config}; use mas_handlers::{ActivityTracker, CookieManager, HttpClientFactory, MetadataCache, SiteConfig}; use mas_listener::{server::Server, shutdown::ShutdownStream}; use mas_matrix_synapse::SynapseConnection; @@ -63,9 +64,9 @@ pub(super) struct Options { impl Options { #[allow(clippy::too_many_lines)] - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { let span = info_span!("cli.run.init").entered(); - let config: AppConfig = root.load_config()?; + let config = AppConfig::extract(figment)?; if self.migrate { warn!("The `--migrate` flag is deprecated and will be removed in a future release. Please use `--no-migrate` to disable automatic migrations on startup."); @@ -101,8 +102,8 @@ impl Options { } else { // Sync the configuration with the database let mut conn = pool.acquire().await?; - let clients_config = root.load_config()?; - let upstream_oauth2_config = root.load_config()?; + let clients_config = ClientsConfig::extract(figment)?; + let upstream_oauth2_config = UpstreamOAuth2Config::extract(figment)?; crate::sync::config_sync( upstream_oauth2_config, diff --git a/crates/cli/src/commands/templates.rs b/crates/cli/src/commands/templates.rs index 35f1d978..9a0ce28d 100644 --- a/crates/cli/src/commands/templates.rs +++ b/crates/cli/src/commands/templates.rs @@ -13,7 +13,8 @@ // limitations under the License. use clap::Parser; -use mas_config::{BrandingConfig, MatrixConfig, TemplatesConfig}; +use figment::Figment; +use mas_config::{BrandingConfig, ConfigurationSection, MatrixConfig, TemplatesConfig}; use mas_storage::{Clock, SystemClock}; use rand::SeedableRng; use tracing::info_span; @@ -33,15 +34,15 @@ enum Subcommand { } impl Options { - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { use Subcommand as SC; match self.subcommand { SC::Check => { let _span = info_span!("cli.templates.check").entered(); - let template_config: TemplatesConfig = root.load_config()?; - let branding_config: BrandingConfig = root.load_config()?; - let matrix_config: MatrixConfig = root.load_config()?; + let template_config = TemplatesConfig::extract(figment)?; + let branding_config = BrandingConfig::extract(figment)?; + let matrix_config = MatrixConfig::extract(figment)?; let clock = SystemClock::default(); // XXX: we should disallow SeedableRng::from_entropy diff --git a/crates/cli/src/commands/worker.rs b/crates/cli/src/commands/worker.rs index e1564a41..ecb236ac 100644 --- a/crates/cli/src/commands/worker.rs +++ b/crates/cli/src/commands/worker.rs @@ -13,7 +13,8 @@ // limitations under the License. use clap::Parser; -use mas_config::AppConfig; +use figment::Figment; +use mas_config::{AppConfig, ConfigurationSection}; use mas_handlers::HttpClientFactory; use mas_matrix_synapse::SynapseConnection; use mas_router::UrlBuilder; @@ -29,9 +30,9 @@ use crate::util::{database_pool_from_config, mailer_from_config, templates_from_ pub(super) struct Options {} impl Options { - pub async fn run(self, root: &super::Options) -> anyhow::Result<()> { + pub async fn run(self, figment: &Figment) -> anyhow::Result<()> { let span = info_span!("cli.worker.init").entered(); - let config: AppConfig = root.load_config()?; + let config = AppConfig::extract(figment)?; // Connect to the database info!("Connecting to the database"); diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index fad847b4..775d4da3 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -18,7 +18,7 @@ use std::{io::IsTerminal, sync::Arc}; use anyhow::Context; use clap::Parser; -use mas_config::TelemetryConfig; +use mas_config::{ConfigurationSection, TelemetryConfig}; use sentry_tracing::EventFilter; use tracing_subscriber::{ filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer, Registry, @@ -67,10 +67,13 @@ async fn try_main() -> anyhow::Result<()> { // Parse the CLI arguments let opts = self::commands::Options::parse(); + // Load the base configuration files + let figment = opts.figment(); + // Telemetry config could fail to load, but that's probably OK, since the whole // config will be loaded afterwards, and crash if there is a problem. // Falling back to default. - let telemetry_config: TelemetryConfig = opts.load_config().unwrap_or_default(); + let telemetry_config = TelemetryConfig::extract(&figment).unwrap_or_default(); // Setup Sentry let sentry = sentry::init(( @@ -126,7 +129,7 @@ async fn try_main() -> anyhow::Result<()> { // And run the command tracing::trace!(?opts, "Running command"); - opts.run().await?; + opts.run(&figment).await?; Ok(()) } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 14326d40..6131971e 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -21,7 +21,7 @@ anyhow.workspace = true camino = { workspace = true, features = ["serde1"] } chrono.workspace = true -figment = { version = "0.10.15", features = ["env", "yaml", "test"] } +figment.workspace = true ipnetwork = { version = "0.20.0", features = ["serde", "schemars"] } schemars.workspace = true ulid.workspace = true diff --git a/crates/config/src/sections/clients.rs b/crates/config/src/sections/clients.rs index 1cd36fde..0d69d871 100644 --- a/crates/config/src/sections/clients.rs +++ b/crates/config/src/sections/clients.rs @@ -201,7 +201,10 @@ impl ConfigurationSection for ClientsConfig { mod tests { use std::str::FromStr; - use figment::Jail; + use figment::{ + providers::{Format, Yaml}, + Figment, Jail, + }; use super::*; @@ -249,7 +252,9 @@ mod tests { "#, )?; - let config = ClientsConfig::load_from_file("config.yaml")?; + let config = Figment::new() + .merge(Yaml::file("config.yaml")) + .extract_inner::("clients")?; assert_eq!(config.0.len(), 5); diff --git a/crates/config/src/sections/database.rs b/crates/config/src/sections/database.rs index 5ad79fe4..a7bbf1ab 100644 --- a/crates/config/src/sections/database.rs +++ b/crates/config/src/sections/database.rs @@ -164,7 +164,10 @@ impl ConfigurationSection for DatabaseConfig { #[cfg(test)] mod tests { - use figment::Jail; + use figment::{ + providers::{Format, Yaml}, + Figment, Jail, + }; use super::*; @@ -179,7 +182,9 @@ mod tests { ", )?; - let config = DatabaseConfig::load_from_file("config.yaml")?; + let config = Figment::new() + .merge(Yaml::file("config.yaml")) + .extract_inner::("database")?; assert_eq!( config.options, diff --git a/crates/config/src/sections/matrix.rs b/crates/config/src/sections/matrix.rs index 13a6418c..9d072978 100644 --- a/crates/config/src/sections/matrix.rs +++ b/crates/config/src/sections/matrix.rs @@ -76,7 +76,10 @@ impl ConfigurationSection for MatrixConfig { #[cfg(test)] mod tests { - use figment::Jail; + use figment::{ + providers::{Format, Yaml}, + Figment, Jail, + }; use super::*; @@ -92,10 +95,12 @@ mod tests { ", )?; - let config = MatrixConfig::load_from_file("config.yaml")?; + let config = Figment::new() + .merge(Yaml::file("config.yaml")) + .extract_inner::("matrix")?; - assert_eq!(config.homeserver, "matrix.org".to_owned()); - assert_eq!(config.secret, "test".to_owned()); + assert_eq!(&config.homeserver, "matrix.org"); + assert_eq!(&config.secret, "test"); Ok(()) }); diff --git a/crates/config/src/util.rs b/crates/config/src/util.rs index b6b6849c..f3a2fb7e 100644 --- a/crates/config/src/util.rs +++ b/crates/config/src/util.rs @@ -12,14 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Context; use async_trait::async_trait; -use camino::Utf8Path; -use figment::{ - error::Error as FigmentError, - providers::{Env, Format, Serialized, Yaml}, - Figment, Profile, -}; +use figment::{error::Error as FigmentError, Figment}; use rand::Rng; use serde::{de::DeserializeOwned, Serialize}; @@ -35,64 +29,13 @@ pub trait ConfigurationSection: Sized + DeserializeOwned + Serialize { where R: Rng + Send; - /// Generate a sample configuration and override it with environment - /// variables. - /// - /// This is what backs the `config generate` subcommand, allowing to - /// programmatically generate a configuration file, e.g. - /// - /// ```sh - /// export MAS_OAUTH2_ISSUER=https://example.com/ - /// export MAS_HTTP_ADDRESS=127.0.0.1:1234 - /// matrix-authentication-service config generate - /// ``` - /// - /// # Errors - /// - /// Returns an error if the configuration could not be generated or if the - /// existing configuration could not be loaded - async fn load_and_generate(rng: R) -> anyhow::Result - where - R: Rng + Send, - { - let base = Self::generate(rng) - .await - .context("could not generate configuration")?; - - Figment::new() - .merge(Serialized::from(&base, Profile::Default)) - .merge(Env::prefixed("MAS_").split("_")) - .extract_inner(Self::path()) - .context("could not load configuration") - } - - /// Load configuration from a list of files and environment variables. + /// Extract configuration from a Figment instance. /// /// # Errors /// /// Returns an error if the configuration could not be loaded - fn load_from_files

(paths: &[P]) -> Result - where - P: AsRef, - { - let base = Figment::new().merge(Env::prefixed("MAS_").split("_")); - - paths - .iter() - .fold(base, |f, path| f.merge(Yaml::file(path.as_ref()))) - .extract_inner(Self::path()) - } - - /// Load configuration from a file and environment variables. - /// - /// # Errors - /// - /// Returns an error if the configuration could not be loaded - fn load_from_file

(path: P) -> Result - where - P: AsRef, - { - Self::load_from_files(&[path]) + fn extract(figment: &Figment) -> Result { + figment.extract_inner(Self::path()) } /// Generate config used in unit tests