matrix/authentication-service
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-authentication-service.git synced 2025-08-06 06:02:40 +03:00
Code Activity

Load the configuration from a common Figment instance

Browse Source 
This should avoid loading the same files multiple times.
It should also make it easier to do post-processing on the
configuration, like validation.

This does deprecate one undocumented feature: the ability to override
some fields during the configuration generation using environment
variables.
This commit is contained in:
Quentin Gliech
2024-03-20 15:16:35 +01:00
parent d20e579290
commit 1cf283337b
18 changed files with 108 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -3023,6 +3023,7 @@ dependencies = [
"camino",
"clap",
"dotenvy",
"figment",
"httpdate",
"hyper",
"ipnetwork",

5
Cargo.toml
View File

@@ -78,6 +78,11 @@ features = ["serde", "clock"]
version = "4.5.3"
features = ["derive"]
# Configuration loading
[workspace.dependencies.figment]
version = "0.10.15"
features = ["env", "yaml", "test"]
# HTTP headers
[workspace.dependencies.headers]
version = "0.3.9"

1
crates/cli/Cargo.toml
View File

@@ -17,6 +17,7 @@ axum = "0.6.20"
camino.workspace = true
clap.workspace = true
dotenvy = "0.15.7"
figment.workspace = true
httpdate = "1.0.3"
hyper.workspace = true
ipnetwork = "0.20.0"

13
crates/cli/src/commands/config.rs
View File

@@ -15,6 +15,7 @@
use anyhow::Context;
use camino::Utf8PathBuf;
use clap::Parser;
use figment::Figment;
use mas_config::{ConfigurationSection, RootConfig, SyncConfig};
use mas_storage::SystemClock;
use mas_storage_pg::MIGRATOR;
@@ -67,13 +68,13 @@ enum Subcommand {
}
impl Options {
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
use Subcommand as SC;
match self.subcommand {
SC::Dump { output } => {
let _span = info_span!("cli.config.dump").entered();
let config: RootConfig = root.load_config()?;
let config = RootConfig::extract(figment)?;
let config = serde_yaml::to_string(&config)?;
if let Some(output) = output {
@@ -89,8 +90,8 @@ impl Options {
SC::Check => {
let _span = info_span!("cli.config.check").entered();
let _config: RootConfig = root.load_config()?;
info!(path = ?root.config, "Configuration file looks good");
let _config = RootConfig::extract(figment)?;
info!("Configuration file looks good");
}
SC::Generate { output } => {
@@ -98,7 +99,7 @@ impl Options {
// XXX: we should disallow SeedableRng::from_entropy
let rng = rand_chacha::ChaChaRng::from_entropy();
let config = RootConfig::load_and_generate(rng).await?;
let config = RootConfig::generate(rng).await?;
let config = serde_yaml::to_string(&config)?;
if let Some(output) = output {
@@ -112,7 +113,7 @@ impl Options {
}
SC::Sync { prune, dry_run } => {
let config: SyncConfig = root.load_config()?;
let config = SyncConfig::extract(figment)?;
let clock = SystemClock::default();
let encrypter = config.secrets.encrypter();

7
crates/cli/src/commands/database.rs
View File

@@ -14,7 +14,8 @@
use anyhow::Context;
use clap::Parser;
use mas_config::DatabaseConfig;
use figment::Figment;
use mas_config::{ConfigurationSection, DatabaseConfig};
use mas_storage_pg::MIGRATOR;
use tracing::{info_span, Instrument};
@@ -33,9 +34,9 @@ enum Subcommand {
}
impl Options {
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
let _span = info_span!("cli.database.migrate").entered();
let config: DatabaseConfig = root.load_config()?;
let config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&config).await?;
// Run pending migrations

7
crates/cli/src/commands/debug.rs
View File

@@ -13,8 +13,9 @@
// limitations under the License.
use clap::Parser;
use figment::Figment;
use hyper::{Response, Uri};
use mas_config::PolicyConfig;
use mas_config::{ConfigurationSection, PolicyConfig};
use mas_handlers::HttpClientFactory;
use mas_http::HttpServiceExt;
use tokio::io::AsyncWriteExt;
@@ -65,7 +66,7 @@ fn print_headers(parts: &hyper::http::response::Parts) {
impl Options {
#[tracing::instrument(skip_all)]
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
use Subcommand as SC;
let http_client_factory = HttpClientFactory::new();
match self.subcommand {
@@ -120,7 +121,7 @@ impl Options {
SC::Policy => {
let _span = info_span!("cli.debug.policy").entered();
let config: PolicyConfig = root.load_config()?;
let config = PolicyConfig::extract(figment)?;
info!("Loading and compiling the policy module");
let policy_factory = policy_factory_from_config(&config).await?;

7
crates/cli/src/commands/doctor.rs
View File

@@ -19,7 +19,8 @@
use anyhow::Context;
use clap::Parser;
use mas_config::RootConfig;
use figment::Figment;
use mas_config::{ConfigurationSection, RootConfig};
use mas_handlers::HttpClientFactory;
use mas_http::HttpServiceExt;
use tower::{Service, ServiceExt};
@@ -34,11 +35,11 @@ pub(super) struct Options {}
impl Options {
#[allow(clippy::too_many_lines)]
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
let _span = info_span!("cli.doctor").entered();
info!("💡 Running diagnostics, make sure that both MAS and Synapse are running, and that MAS is using the same configuration files as this tool.");
let config: RootConfig = root.load_config()?;
let config = RootConfig::extract(figment)?;
// We'll need an HTTP client
let http_client_factory = HttpClientFactory::new();

21
crates/cli/src/commands/manage.rs
View File

@@ -14,7 +14,8 @@
use anyhow::Context;
use clap::Parser;
use mas_config::{DatabaseConfig, PasswordsConfig};
use figment::Figment;
use mas_config::{ConfigurationSection, DatabaseConfig, PasswordsConfig};
use mas_data_model::{Device, TokenType};
use mas_storage::{
compat::{CompatAccessTokenRepository, CompatSessionRepository},
@@ -89,7 +90,7 @@ enum Subcommand {
impl Options {
#[allow(clippy::too_many_lines)]
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
use Subcommand as SC;
let clock = SystemClock::default();
// XXX: we should disallow SeedableRng::from_entropy
@@ -100,8 +101,8 @@ impl Options {
let _span =
info_span!("cli.manage.set_password", user.username = %username).entered();
let database_config: DatabaseConfig = root.load_config()?;
let passwords_config: PasswordsConfig = root.load_config()?;
let database_config = DatabaseConfig::extract(figment)?;
let passwords_config = PasswordsConfig::extract(figment)?;
let mut conn = database_connection_from_config(&database_config).await?;
let password_manager = password_manager_from_config(&passwords_config).await?;
@@ -136,7 +137,7 @@ impl Options {
)
.entered();
let database_config: DatabaseConfig = root.load_config()?;
let database_config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&database_config).await?;
let txn = conn.begin().await?;
let mut repo = PgRepository::from_conn(txn);
@@ -170,7 +171,7 @@ impl Options {
admin,
device_id,
} => {
let database_config: DatabaseConfig = root.load_config()?;
let database_config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&database_config).await?;
let txn = conn.begin().await?;
let mut repo = PgRepository::from_conn(txn);
@@ -215,7 +216,7 @@ impl Options {
SC::ProvisionAllUsers => {
let _span = info_span!("cli.manage.provision_all_users").entered();
let database_config: DatabaseConfig = root.load_config()?;
let database_config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&database_config).await?;
let mut txn = conn.begin().await?;
@@ -241,7 +242,7 @@ impl Options {
SC::KillSessions { username, dry_run } => {
let _span =
info_span!("cli.manage.kill_sessions", user.username = username).entered();
let database_config: DatabaseConfig = root.load_config()?;
let database_config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&database_config).await?;
let txn = conn.begin().await?;
let mut repo = PgRepository::from_conn(txn);
@@ -361,7 +362,7 @@ impl Options {
deactivate,
} => {
let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
let config: DatabaseConfig = root.load_config()?;
let config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&config).await?;
let txn = conn.begin().await?;
let mut repo = PgRepository::from_conn(txn);
@@ -393,7 +394,7 @@ impl Options {
SC::UnlockUser { username } => {
let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
let config: DatabaseConfig = root.load_config()?;
let config = DatabaseConfig::extract(figment)?;
let mut conn = database_connection_from_config(&config).await?;
let txn = conn.begin().await?;
let mut repo = PgRepository::from_conn(txn);

36
crates/cli/src/commands/mod.rs
View File

@@ -12,10 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use anyhow::Context;
use camino::Utf8PathBuf;
use clap::Parser;
use mas_config::ConfigurationSection;
use figment::{
providers::{Env, Format, Yaml},
Figment,
};
mod config;
mod database;
@@ -65,22 +67,23 @@ pub struct Options {
}
impl Options {
pub async fn run(mut self) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
use Subcommand as S;
match self.subcommand.take() {
Some(S::Config(c)) => c.run(&self).await,
Some(S::Database(c)) => c.run(&self).await,
Some(S::Server(c)) => c.run(&self).await,
Some(S::Worker(c)) => c.run(&self).await,
Some(S::Manage(c)) => c.run(&self).await,
Some(S::Templates(c)) => c.run(&self).await,
Some(S::Debug(c)) => c.run(&self).await,
Some(S::Doctor(c)) => c.run(&self).await,
None => self::server::Options::default().run(&self).await,
match self.subcommand {
Some(S::Config(c)) => c.run(figment).await,
Some(S::Database(c)) => c.run(figment).await,
Some(S::Server(c)) => c.run(figment).await,
Some(S::Worker(c)) => c.run(figment).await,
Some(S::Manage(c)) => c.run(figment).await,
Some(S::Templates(c)) => c.run(figment).await,
Some(S::Debug(c)) => c.run(figment).await,
Some(S::Doctor(c)) => c.run(figment).await,
None => self::server::Options::default().run(figment).await,
}
}
pub fn load_config<T: ConfigurationSection>(&self) -> anyhow::Result<T> {
/// Get a [`Figment`] instance with the configuration loaded
pub fn figment(&self) -> Figment {
let configs = if self.config.is_empty() {
// Read the MAS_CONFIG environment variable
std::env::var("MAS_CONFIG")
@@ -93,7 +96,10 @@ impl Options {
} else {
self.config.clone()
};
let base = Figment::new().merge(Env::prefixed("MAS_").split("_"));
T::load_from_files(&configs).context("could not load configuration")
configs
.into_iter()
.fold(base, |f, path| f.merge(Yaml::file(path)))
}
}

11
crates/cli/src/commands/server.rs
View File

@@ -16,8 +16,9 @@ use std::{collections::BTreeSet, sync::Arc, time::Duration};
use anyhow::Context;
use clap::Parser;
use figment::Figment;
use itertools::Itertools;
use mas_config::AppConfig;
use mas_config::{AppConfig, ClientsConfig, ConfigurationSection, UpstreamOAuth2Config};
use mas_handlers::{ActivityTracker, CookieManager, HttpClientFactory, MetadataCache, SiteConfig};
use mas_listener::{server::Server, shutdown::ShutdownStream};
use mas_matrix_synapse::SynapseConnection;
@@ -63,9 +64,9 @@ pub(super) struct Options {
impl Options {
#[allow(clippy::too_many_lines)]
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
let span = info_span!("cli.run.init").entered();
let config: AppConfig = root.load_config()?;
let config = AppConfig::extract(figment)?;
if self.migrate {
warn!("The `--migrate` flag is deprecated and will be removed in a future release. Please use `--no-migrate` to disable automatic migrations on startup.");
@@ -101,8 +102,8 @@ impl Options {
} else {
// Sync the configuration with the database
let mut conn = pool.acquire().await?;
let clients_config = root.load_config()?;
let upstream_oauth2_config = root.load_config()?;
let clients_config = ClientsConfig::extract(figment)?;
let upstream_oauth2_config = UpstreamOAuth2Config::extract(figment)?;
crate::sync::config_sync(
upstream_oauth2_config,

11
crates/cli/src/commands/templates.rs
View File

@@ -13,7 +13,8 @@
// limitations under the License.
use clap::Parser;
use mas_config::{BrandingConfig, MatrixConfig, TemplatesConfig};
use figment::Figment;
use mas_config::{BrandingConfig, ConfigurationSection, MatrixConfig, TemplatesConfig};
use mas_storage::{Clock, SystemClock};
use rand::SeedableRng;
use tracing::info_span;
@@ -33,15 +34,15 @@ enum Subcommand {
}
impl Options {
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
use Subcommand as SC;
match self.subcommand {
SC::Check => {
let _span = info_span!("cli.templates.check").entered();
let template_config: TemplatesConfig = root.load_config()?;
let branding_config: BrandingConfig = root.load_config()?;
let matrix_config: MatrixConfig = root.load_config()?;
let template_config = TemplatesConfig::extract(figment)?;
let branding_config = BrandingConfig::extract(figment)?;
let matrix_config = MatrixConfig::extract(figment)?;
let clock = SystemClock::default();
// XXX: we should disallow SeedableRng::from_entropy

7
crates/cli/src/commands/worker.rs
View File

@@ -13,7 +13,8 @@
// limitations under the License.
use clap::Parser;
use mas_config::AppConfig;
use figment::Figment;
use mas_config::{AppConfig, ConfigurationSection};
use mas_handlers::HttpClientFactory;
use mas_matrix_synapse::SynapseConnection;
use mas_router::UrlBuilder;
@@ -29,9 +30,9 @@ use crate::util::{database_pool_from_config, mailer_from_config, templates_from_
pub(super) struct Options {}
impl Options {
pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
let span = info_span!("cli.worker.init").entered();
let config: AppConfig = root.load_config()?;
let config = AppConfig::extract(figment)?;
// Connect to the database
info!("Connecting to the database");

9
crates/cli/src/main.rs
View File

@@ -18,7 +18,7 @@ use std::{io::IsTerminal, sync::Arc};
use anyhow::Context;
use clap::Parser;
use mas_config::TelemetryConfig;
use mas_config::{ConfigurationSection, TelemetryConfig};
use sentry_tracing::EventFilter;
use tracing_subscriber::{
filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer, Registry,
@@ -67,10 +67,13 @@ async fn try_main() -> anyhow::Result<()> {
// Parse the CLI arguments
let opts = self::commands::Options::parse();
// Load the base configuration files
let figment = opts.figment();
// Telemetry config could fail to load, but that's probably OK, since the whole
// config will be loaded afterwards, and crash if there is a problem.
// Falling back to default.
let telemetry_config: TelemetryConfig = opts.load_config().unwrap_or_default();
let telemetry_config = TelemetryConfig::extract(&figment).unwrap_or_default();
// Setup Sentry
let sentry = sentry::init((
@@ -126,7 +129,7 @@ async fn try_main() -> anyhow::Result<()> {
// And run the command
tracing::trace!(?opts, "Running command");
opts.run().await?;
opts.run(&figment).await?;
Ok(())
}

2
crates/config/Cargo.toml
View File

@@ -21,7 +21,7 @@ anyhow.workspace = true
camino = { workspace = true, features = ["serde1"] }
chrono.workspace = true
figment = { version = "0.10.15", features = ["env", "yaml", "test"] }
figment.workspace = true
ipnetwork = { version = "0.20.0", features = ["serde", "schemars"] }
schemars.workspace = true
ulid.workspace = true

9
crates/config/src/sections/clients.rs
View File

@@ -201,7 +201,10 @@ impl ConfigurationSection for ClientsConfig {
mod tests {
use std::str::FromStr;
use figment::Jail;
use figment::{
providers::{Format, Yaml},
Figment, Jail,
};
use super::*;
@@ -249,7 +252,9 @@ mod tests {
"#,
)?;
let config = ClientsConfig::load_from_file("config.yaml")?;
let config = Figment::new()
.merge(Yaml::file("config.yaml"))
.extract_inner::<ClientsConfig>("clients")?;
assert_eq!(config.0.len(), 5);

9
crates/config/src/sections/database.rs
View File

@@ -164,7 +164,10 @@ impl ConfigurationSection for DatabaseConfig {
#[cfg(test)]
mod tests {
use figment::Jail;
use figment::{
providers::{Format, Yaml},
Figment, Jail,
};
use super::*;
@@ -179,7 +182,9 @@ mod tests {
",
)?;
let config = DatabaseConfig::load_from_file("config.yaml")?;
let config = Figment::new()
.merge(Yaml::file("config.yaml"))
.extract_inner::<DatabaseConfig>("database")?;
assert_eq!(
config.options,

13
crates/config/src/sections/matrix.rs
View File

@@ -76,7 +76,10 @@ impl ConfigurationSection for MatrixConfig {
#[cfg(test)]
mod tests {
use figment::Jail;
use figment::{
providers::{Format, Yaml},
Figment, Jail,
};
use super::*;
@@ -92,10 +95,12 @@ mod tests {
",
)?;
let config = MatrixConfig::load_from_file("config.yaml")?;
let config = Figment::new()
.merge(Yaml::file("config.yaml"))
.extract_inner::<MatrixConfig>("matrix")?;
assert_eq!(config.homeserver, "matrix.org".to_owned());
assert_eq!(config.secret, "test".to_owned());
assert_eq!(&config.homeserver, "matrix.org");
assert_eq!(&config.secret, "test");
Ok(())
});

65
crates/config/src/util.rs
View File

@@ -12,14 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use anyhow::Context;
use async_trait::async_trait;
use camino::Utf8Path;
use figment::{
error::Error as FigmentError,
providers::{Env, Format, Serialized, Yaml},
Figment, Profile,
};
use figment::{error::Error as FigmentError, Figment};
use rand::Rng;
use serde::{de::DeserializeOwned, Serialize};
@@ -35,64 +29,13 @@ pub trait ConfigurationSection: Sized + DeserializeOwned + Serialize {
where
R: Rng + Send;
/// Generate a sample configuration and override it with environment
/// variables.
///
/// This is what backs the `config generate` subcommand, allowing to
/// programmatically generate a configuration file, e.g.
///
/// ```sh
/// export MAS_OAUTH2_ISSUER=https://example.com/
/// export MAS_HTTP_ADDRESS=127.0.0.1:1234
/// matrix-authentication-service config generate
/// ```
///
/// # Errors
///
/// Returns an error if the configuration could not be generated or if the
/// existing configuration could not be loaded
async fn load_and_generate<R>(rng: R) -> anyhow::Result<Self>
where
R: Rng + Send,
{
let base = Self::generate(rng)
.await
.context("could not generate configuration")?;
Figment::new()
.merge(Serialized::from(&base, Profile::Default))
.merge(Env::prefixed("MAS_").split("_"))
.extract_inner(Self::path())
.context("could not load configuration")
}
/// Load configuration from a list of files and environment variables.
/// Extract configuration from a Figment instance.
///
/// # Errors
///
/// Returns an error if the configuration could not be loaded
fn load_from_files<P>(paths: &[P]) -> Result<Self, FigmentError>
where
P: AsRef<Utf8Path>,
{
let base = Figment::new().merge(Env::prefixed("MAS_").split("_"));
paths
.iter()
.fold(base, |f, path| f.merge(Yaml::file(path.as_ref())))
.extract_inner(Self::path())
}
/// Load configuration from a file and environment variables.
///
/// # Errors
///
/// Returns an error if the configuration could not be loaded
fn load_from_file<P>(path: P) -> Result<Self, FigmentError>
where
P: AsRef<Utf8Path>,
{
Self::load_from_files(&[path])
fn extract(figment: &Figment) -> Result<Self, FigmentError> {
figment.extract_inner(Self::path())
}
/// Generate config used in unit tests