Reorganise CLI crate

crates/cli/src/commands/config.rs

@@ -0,0 +1,78 @@
+// Copyright 2021, 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+use mas_config::{ConfigurationSection, RootConfig};
+use schemars::gen::SchemaSettings;
+use tracing::info;
+
+#[derive(Parser, Debug)]
+pub(super) struct Options {
+    #[clap(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    /// Dump the current config as YAML
+    Dump,
+
+    /// Print the JSON Schema that validates configuration files
+    Schema,
+
+    /// Check a config file
+    Check,
+
+    /// Generate a new config file
+    Generate,
+}
+
+impl Options {
+    pub async fn run(&self, root: &super::Options) -> anyhow::Result<()> {
+        use Subcommand as SC;
+        match &self.subcommand {
+            SC::Dump => {
+                let config: RootConfig = root.load_config()?;
+
+                serde_yaml::to_writer(std::io::stdout(), &config)?;
+
+                Ok(())
+            }
+            SC::Schema => {
+                let settings = SchemaSettings::draft07().with(|s| {
+                    s.option_nullable = false;
+                    s.option_add_null_type = false;
+                });
+                let gen = settings.into_generator();
+                let schema = gen.into_root_schema_for::<RootConfig>();
+
+                serde_yaml::to_writer(std::io::stdout(), &schema)?;
+
+                Ok(())
+            }
+            SC::Check => {
+                let _config: RootConfig = root.load_config()?;
+                info!(path = ?root.config, "Configuration file looks good");
+                Ok(())
+            }
+            SC::Generate => {
+                let config = RootConfig::load_and_generate().await?;
+
+                serde_yaml::to_writer(std::io::stdout(), &config)?;
+
+                Ok(())
+            }
+        }
+    }
+}

crates/cli/src/commands/database.rs

@@ -0,0 +1,45 @@
+// Copyright 2021, 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Context;
+use clap::Parser;
+use mas_config::DatabaseConfig;
+use mas_storage::MIGRATOR;
+
+#[derive(Parser, Debug)]
+pub(super) struct Options {
+    #[clap(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    /// Run database migrations
+    Migrate,
+}
+
+impl Options {
+    pub async fn run(&self, root: &super::Options) -> anyhow::Result<()> {
+        let config: DatabaseConfig = root.load_config()?;
+        let pool = config.connect().await?;
+
+        // Run pending migrations
+        MIGRATOR
+            .run(&pool)
+            .await
+            .context("could not run migrations")?;
+
+        Ok(())
+    }
+}

crates/cli/src/commands/manage.rs

@@ -0,0 +1,78 @@
+// Copyright 2021, 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use argon2::Argon2;
+use clap::Parser;
+use mas_config::DatabaseConfig;
+use mas_storage::user::{
+    lookup_user_by_username, lookup_user_email, mark_user_email_as_verified, register_user,
+};
+use tracing::{info, warn};
+
+#[derive(Parser, Debug)]
+pub(super) struct Options {
+    #[clap(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    /// Register a new user
+    Register { username: String, password: String },
+
+    /// List active users
+    Users,
+
+    /// Mark email address as verified
+    VerifyEmail { username: String, email: String },
+}
+
+impl Options {
+    pub async fn run(&self, root: &super::Options) -> anyhow::Result<()> {
+        use Subcommand as SC;
+        match &self.subcommand {
+            SC::Register { username, password } => {
+                let config: DatabaseConfig = root.load_config()?;
+                let pool = config.connect().await?;
+                let mut txn = pool.begin().await?;
+                let hasher = Argon2::default();
+
+                let user = register_user(&mut txn, hasher, username, password).await?;
+                txn.commit().await?;
+                info!(?user, "User registered");
+
+                Ok(())
+            }
+            SC::Users => {
+                warn!("Not implemented yet");
+
+                Ok(())
+            }
+            SC::VerifyEmail { username, email } => {
+                let config: DatabaseConfig = root.load_config()?;
+                let pool = config.connect().await?;
+                let mut txn = pool.begin().await?;
+
+                let user = lookup_user_by_username(&mut txn, username).await?;
+                let email = lookup_user_email(&mut txn, &user, email).await?;
+                let email = mark_user_email_as_verified(&mut txn, email).await?;
+
+                txn.commit().await?;
+                info!(?email, "Email marked as verified");
+
+                Ok(())
+            }
+        }
+    }
+}

crates/cli/src/commands/mod.rs

@@ -0,0 +1,77 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+
+use anyhow::Context;
+use clap::Parser;
+use mas_config::ConfigurationSection;
+
+mod config;
+mod database;
+mod manage;
+mod server;
+mod templates;
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    /// Configuration-related commands
+    Config(self::config::Options),
+
+    /// Manage the database
+    Database(self::database::Options),
+
+    /// Runs the web server
+    Server(self::server::Options),
+
+    /// Manage the instance
+    Manage(self::manage::Options),
+
+    /// Templates-related commands
+    Templates(self::templates::Options),
+}
+
+#[derive(Parser, Debug)]
+pub struct Options {
+    /// Path to the configuration file
+    #[clap(
+        short,
+        long,
+        global = true,
+        default_value = "config.yaml",
+        multiple_occurrences(true)
+    )]
+    config: Vec<PathBuf>,
+
+    #[clap(subcommand)]
+    subcommand: Option<Subcommand>,
+}
+
+impl Options {
+    pub async fn run(&self) -> anyhow::Result<()> {
+        use Subcommand as S;
+        match &self.subcommand {
+            Some(S::Config(c)) => c.run(self).await,
+            Some(S::Database(c)) => c.run(self).await,
+            Some(S::Server(c)) => c.run(self).await,
+            Some(S::Manage(c)) => c.run(self).await,
+            Some(S::Templates(c)) => c.run(self).await,
+            None => self::server::Options::default().run(self).await,
+        }
+    }
+
+    pub fn load_config<'de, T: ConfigurationSection<'de>>(&self) -> anyhow::Result<T> {
+        T::load_from_files(&self.config).context("could not load configuration")
+    }
+}

crates/cli/src/commands/server.rs

@@ -0,0 +1,246 @@
+// Copyright 2021, 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    net::{SocketAddr, TcpListener},
+    sync::Arc,
+    time::Duration,
+};
+
+use anyhow::Context;
+use clap::Parser;
+use futures::{future::TryFutureExt, stream::TryStreamExt};
+use hyper::{header, Server};
+use mas_config::RootConfig;
+use mas_email::{MailTransport, Mailer};
+use mas_storage::MIGRATOR;
+use mas_tasks::TaskQueue;
+use mas_templates::Templates;
+use tower::{make::Shared, ServiceBuilder};
+use tower_http::{
+    compression::CompressionLayer, sensitive_headers::SetSensitiveHeadersLayer, trace::TraceLayer,
+};
+use tracing::{error, info};
+
+use crate::telemetry::{OtelMakeSpan, OtelOnResponse};
+
+#[derive(Parser, Debug, Default)]
+pub(super) struct Options {
+    /// Automatically apply pending migrations
+    #[clap(long)]
+    migrate: bool,
+
+    /// Watch for changes for templates on the filesystem
+    #[clap(short, long)]
+    watch: bool,
+}
+
+#[cfg(not(unix))]
+async fn shutdown_signal() {
+    // Wait for the CTRL+C signal
+    tokio::signal::ctrl_c()
+        .await
+        .expect("failed to install Ctrl+C signal handler");
+
+    tracing::info!("Got Ctrl+C, shutting down");
+}
+
+#[cfg(unix)]
+async fn shutdown_signal() {
+    use tokio::signal::unix::{signal, SignalKind};
+
+    // Wait for SIGTERM and SIGINT signals
+    // This might panic but should be fine
+    let mut term =
+        signal(SignalKind::terminate()).expect("failed to install SIGTERM signal handler");
+    let mut int = signal(SignalKind::interrupt()).expect("failed to install SIGINT signal handler");
+
+    tokio::select! {
+        _ = term.recv() => tracing::info!("Got SIGTERM, shutting down"),
+        _ = int.recv() => tracing::info!("Got SIGINT, shutting down"),
+    };
+}
+
+/// Watch for changes in the templates folders
+async fn watch_templates(
+    client: &watchman_client::Client,
+    templates: &Templates,
+) -> anyhow::Result<()> {
+    use watchman_client::{
+        fields::NameOnly,
+        pdu::{QueryResult, SubscribeRequest},
+        CanonicalPath, SubscriptionData,
+    };
+
+    let templates = templates.clone();
+
+    // Find which roots we're supposed to watch
+    let roots = templates.watch_roots().await;
+    let mut streams = Vec::new();
+
+    for root in roots {
+        // For each root, create a subscription
+        let resolved = client
+            .resolve_root(CanonicalPath::canonicalize(root)?)
+            .await?;
+
+        // TODO: we could subscribe to less, properly filter here
+        let (subscription, _) = client
+            .subscribe::<NameOnly>(&resolved, SubscribeRequest::default())
+            .await?;
+
+        // Create a stream out of that subscription
+        let stream = futures::stream::try_unfold(subscription, |mut sub| async move {
+            let next = sub.next().await?;
+            anyhow::Ok(Some((next, sub)))
+        });
+
+        streams.push(Box::pin(stream));
+    }
+
+    let files_changed_stream =
+        futures::stream::select_all(streams).try_filter_map(|event| async move {
+            match event {
+                SubscriptionData::FilesChanged(QueryResult {
+                    files: Some(files), ..
+                }) => {
+                    let files: Vec<_> = files.into_iter().map(|f| f.name.into_inner()).collect();
+                    Ok(Some(files))
+                }
+                _ => Ok(None),
+            }
+        });
+
+    let fut = files_changed_stream
+        .try_for_each(move |files| {
+            let templates = templates.clone();
+            async move {
+                info!(?files, "Files changed, reloading templates");
+
+                templates
+                    .clone()
+                    .reload()
+                    .await
+                    .context("Could not reload templates")
+            }
+        })
+        .inspect_err(|err| error!(%err, "Error while watching templates, stop watching"));
+
+    tokio::spawn(fut);
+
+    Ok(())
+}
+
+impl Options {
+    pub async fn run(&self, root: &super::Options) -> anyhow::Result<()> {
+        let config: RootConfig = root.load_config()?;
+
+        let addr: SocketAddr = config
+            .http
+            .address
+            .parse()
+            .context("could not parse listener address")?;
+        let listener = TcpListener::bind(addr).context("could not bind address")?;
+
+        // Connect to the mail server
+        let mail_transport = MailTransport::from_config(&config.email.transport).await?;
+        mail_transport.test_connection().await?;
+
+        // Connect to the database
+        let pool = config.database.connect().await?;
+
+        if self.migrate {
+            info!("Running pending migrations");
+            MIGRATOR
+                .run(&pool)
+                .await
+                .context("could not run migrations")?;
+        }
+
+        info!("Starting task scheduler");
+        let queue = TaskQueue::default();
+        queue.recuring(Duration::from_secs(15), mas_tasks::cleanup_expired(&pool));
+        queue.start();
+
+        // Initialize the key store
+        let key_store = config
+            .secrets
+            .key_store()
+            .await
+            .context("could not import keys from config")?;
+        // Wrap the key store in an Arc
+        let key_store = Arc::new(key_store);
+
+        let encrypter = config.secrets.encrypter();
+
+        // Load and compile the templates
+        let templates = Templates::load_from_config(&config.templates)
+            .await
+            .context("could not load templates")?;
+
+        let mailer = Mailer::new(
+            &templates,
+            &mail_transport,
+            &config.email.from,
+            &config.email.reply_to,
+        );
+
+        // Watch for changes in templates if the --watch flag is present
+        if self.watch {
+            let client = watchman_client::Connector::new()
+                .connect()
+                .await
+                .context("could not connect to watchman")?;
+
+            watch_templates(&client, &templates)
+                .await
+                .context("could not watch for templates changes")?;
+        }
+
+        // Start the server
+        let root = mas_handlers::root(&pool, &templates, &key_store, &encrypter, &mailer, &config);
+
+        // Explicitely the config to properly zeroize secret keys
+        drop(config);
+
+        let warp_service = warp::service(root);
+
+        let service = ServiceBuilder::new()
+            // Add high level tracing/logging to all requests
+            .layer(
+                TraceLayer::new_for_http()
+                    .make_span_with(OtelMakeSpan)
+                    .on_response(OtelOnResponse),
+            )
+            // Set a timeout
+            .timeout(Duration::from_secs(10))
+            // Compress responses
+            .layer(CompressionLayer::new())
+            // Mark the `Authorization` and `Cookie` headers as sensitive so it doesn't show in logs
+            .layer(SetSensitiveHeadersLayer::new(vec![
+                header::AUTHORIZATION,
+                header::COOKIE,
+            ]))
+            .service(warp_service);
+
+        info!("Listening on http://{}", listener.local_addr().unwrap());
+
+        Server::from_tcp(listener)?
+            .serve(Shared::new(service))
+            .with_graceful_shutdown(shutdown_signal())
+            .await?;
+
+        Ok(())
+    }
+}

crates/cli/src/commands/templates.rs

@@ -0,0 +1,72 @@
+// Copyright 2021, 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+
+use clap::Parser;
+use mas_config::TemplatesConfig;
+use mas_templates::Templates;
+
+#[derive(Parser, Debug)]
+pub(super) struct Options {
+    #[clap(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    /// Save the builtin templates to a folder
+    Save {
+        /// Where the templates should be saved
+        path: PathBuf,
+
+        /// Overwrite existing template files
+        #[clap(long)]
+        overwrite: bool,
+    },
+
+    /// Check for template validity at given path.
+    Check {
+        /// Path where the templates are
+        path: String,
+
+        /// Skip loading builtin templates
+        #[clap(long)]
+        skip_builtin: bool,
+    },
+}
+
+impl Options {
+    pub async fn run(&self, _root: &super::Options) -> anyhow::Result<()> {
+        use Subcommand as SC;
+        match &self.subcommand {
+            SC::Save { path, overwrite } => {
+                Templates::save(path, *overwrite).await?;
+
+                Ok(())
+            }
+
+            SC::Check { path, skip_builtin } => {
+                let config = TemplatesConfig {
+                    path: Some(path.to_string()),
+                    builtin: !skip_builtin,
+                };
+                let templates = Templates::load_from_config(&config).await?;
+                templates.check_render().await?;
+
+                Ok(())
+            }
+        }
+    }
+}