From c53318eca07eb05797c9f637dae5e4c7bcaec980 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Thu, 9 Dec 2021 14:07:09 +0100 Subject: [PATCH] Refactor templates loading & implement templates hot-reload --- Cargo.lock | 103 +++++++-- crates/cli/Cargo.toml | 2 + crates/cli/src/server.rs | 94 +++++++- crates/cli/src/templates.rs | 9 +- crates/config/src/templates.rs | 2 +- .../core/src/handlers/oauth2/authorization.rs | 14 +- crates/core/src/handlers/views/index.rs | 2 +- crates/core/src/handlers/views/login.rs | 4 +- crates/core/src/handlers/views/reauth.rs | 2 +- crates/core/src/handlers/views/register.rs | 2 +- crates/templates/Cargo.toml | 3 + crates/templates/src/lib.rs | 211 +++++++++++++----- crates/templates/src/macros.rs | 29 ++- 13 files changed, 375 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 558bda5c..516cb43d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,11 +319,24 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +dependencies = [ + "serde", +] [[package]] name = "cc" @@ -843,6 +856,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.18" @@ -931,6 +950,7 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e" dependencies = [ + "futures 0.1.31", "futures-channel", "futures-core", "futures-io", @@ -1014,7 +1034,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd819562fcebdac5afc5c113c3ec36f902840b70fd4fc458799c8ce4607ae55" dependencies = [ - "bytes", + "bytes 1.1.0", "fnv", "futures-core", "futures-sink", @@ -1073,7 +1093,7 @@ checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" dependencies = [ "base64 0.13.0", "bitflags", - "bytes", + "bytes 1.1.0", "headers-core", "http", "httpdate", @@ -1130,7 +1150,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ - "bytes", + "bytes 1.1.0", "fnv", "itoa", ] @@ -1141,7 +1161,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ - "bytes", + "bytes 1.1.0", "http", "pin-project-lite", ] @@ -1176,7 +1196,7 @@ version = "0.14.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436ec0091e4f20e655156a30a0df3770fe2900aa301e548e08446ec794b6953c" dependencies = [ - "bytes", + "bytes 1.1.0", "futures-channel", "futures-core", "futures-util", @@ -1294,6 +1314,15 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.3.1" @@ -1435,7 +1464,7 @@ dependencies = [ "argon2", "clap", "dotenv", - "futures", + "futures 0.3.18", "hyper", "indoc", "mas-config", @@ -1458,6 +1487,7 @@ dependencies = [ "tracing-subscriber", "url", "warp", + "watchman_client", ] [[package]] @@ -1826,7 +1856,7 @@ dependencies = [ "crossbeam-channel 0.5.1", "dashmap", "fnv", - "futures", + "futures 0.3.18", "js-sys", "lazy_static", "percent-encoding", @@ -1844,7 +1874,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d50ceb0b0e8b75cb3e388a2571a807c8228dabc5d6670f317b6eb21301095373" dependencies = [ "async-trait", - "bytes", + "bytes 1.1.0", "futures-util", "http", "opentelemetry", @@ -1877,7 +1907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f19d4b43842433c420c548c985d158f5628bba5b518e0be64627926d19889992" dependencies = [ "async-trait", - "futures", + "futures 0.3.18", "http", "opentelemetry", "prost", @@ -2265,7 +2295,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ - "bytes", + "bytes 1.1.0", "prost-derive", ] @@ -2275,7 +2305,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" dependencies = [ - "bytes", + "bytes 1.1.0", "heck", "itertools", "log", @@ -2306,7 +2336,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ - "bytes", + "bytes 1.1.0", "prost", ] @@ -2432,7 +2462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bea77bc708afa10e59905c3d4af7c8fd43c9214251673095ff8b14345fcbc5" dependencies = [ "base64 0.13.0", - "bytes", + "bytes 1.1.0", "encoding_rs", "futures-core", "futures-util", @@ -2695,6 +2725,19 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b929ea725591083cbca8b8ea178ed6efc918eccd40b784e199ce88967104199" +dependencies = [ + "anyhow", + "byteorder", + "bytes 0.4.12", + "serde", + "thiserror", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -2944,7 +2987,7 @@ dependencies = [ "base64 0.13.0", "bitflags", "byteorder", - "bytes", + "bytes 1.1.0", "chrono", "crc", "crossbeam-channel 0.5.1", @@ -3327,7 +3370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" dependencies = [ "autocfg 1.0.1", - "bytes", + "bytes 1.1.0", "libc", "memchr", "mio", @@ -3413,11 +3456,13 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ - "bytes", + "bytes 1.1.0", "futures-core", + "futures-io", "futures-sink", "log", "pin-project-lite", + "slab", "tokio", ] @@ -3430,7 +3475,7 @@ dependencies = [ "async-stream", "async-trait", "base64 0.13.0", - "bytes", + "bytes 1.1.0", "futures-core", "futures-util", "h2", @@ -3495,7 +3540,7 @@ dependencies = [ "async-compression", "base64 0.13.0", "bitflags", - "bytes", + "bytes 1.1.0", "futures-core", "futures-util", "http", @@ -3625,7 +3670,7 @@ checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" dependencies = [ "base64 0.13.0", "byteorder", - "bytes", + "bytes 1.1.0", "http", "httparse", "log", @@ -3855,7 +3900,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cef4e1e9114a4b7f1ac799f16ce71c14de5778500c5450ec6b7b920c55b587e" dependencies = [ - "bytes", + "bytes 1.1.0", "futures-channel", "futures-util", "headers", @@ -3951,6 +3996,24 @@ version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" +[[package]] +name = "watchman_client" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1afbab1186833c9b34f64132b80ed4b373ed4eab6f9efa1f55430835200f0a28" +dependencies = [ + "anyhow", + "bytes 1.1.0", + "futures 0.3.18", + "maplit", + "serde", + "serde_bser", + "thiserror", + "tokio", + "tokio-util", + "winapi", +] + [[package]] name = "web-sys" version = "0.3.55" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index cf79017f..e3645c8b 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -34,12 +34,14 @@ opentelemetry-zipkin = { version = "0.14.0", features = ["reqwest-client", "reqw mas-config = { path = "../config" } mas-core = { path = "../core" } mas-templates = { path = "../templates" } +watchman_client = "0.7.1" [dev-dependencies] indoc = "1.0.3" [features] default = ["otlp", "jaeger", "zipkin"] +dev = ["mas-templates/dev"] # Enable OpenTelemetry OTLP exporter. Requires "protoc" otlp = ["opentelemetry-otlp"] # Enable OpenTelemetry Jaeger exporter and propagator. diff --git a/crates/cli/src/server.rs b/crates/cli/src/server.rs index 28843d98..4a7cb1f9 100644 --- a/crates/cli/src/server.rs +++ b/crates/cli/src/server.rs @@ -19,6 +19,7 @@ use std::{ use anyhow::Context; use clap::Parser; +use futures::{future::TryFutureExt, stream::TryStreamExt}; use hyper::{header, Server, Version}; use mas_config::RootConfig; use mas_core::{ @@ -33,7 +34,7 @@ use tower_http::{ sensitive_headers::SetSensitiveHeadersLayer, trace::{MakeSpan, OnResponse, TraceLayer}, }; -use tracing::{field, info}; +use tracing::{error, field, info}; use super::RootCommand; @@ -42,6 +43,10 @@ pub(super) struct ServerCommand { /// Automatically apply pending migrations #[clap(long)] migrate: bool, + + /// Watch for changes for templates on the filesystem + #[clap(short, long)] + watch: bool, } #[derive(Debug, Clone, Default)] @@ -136,6 +141,76 @@ async fn shutdown_signal() { }; } +/// Watch for changes in the templates folders +async fn watch_templates( + client: &watchman_client::Client, + templates: &Templates, +) -> anyhow::Result<()> { + use watchman_client::{ + fields::NameOnly, + pdu::{QueryResult, SubscribeRequest}, + CanonicalPath, SubscriptionData, + }; + + let templates = templates.clone(); + + // Find which roots we're supposed to watch + let roots = templates.watch_roots().await; + let mut streams = Vec::new(); + + for root in roots { + // For each root, create a subscription + let resolved = client + .resolve_root(CanonicalPath::canonicalize(root)?) + .await?; + + // TODO: we could subscribe to less, properly filter here + let (subscription, _) = client + .subscribe::(&resolved, SubscribeRequest::default()) + .await?; + + // Create a stream out of that subscription + let stream = futures::stream::try_unfold(subscription, |mut sub| async move { + let next = sub.next().await?; + anyhow::Ok(Some((next, sub))) + }); + + streams.push(Box::pin(stream)); + } + + let files_changed_stream = + futures::stream::select_all(streams).try_filter_map(|event| async move { + match event { + SubscriptionData::FilesChanged(QueryResult { + files: Some(files), .. + }) => { + let files: Vec<_> = files.into_iter().map(|f| f.name.into_inner()).collect(); + Ok(Some(files)) + } + _ => Ok(None), + } + }); + + let fut = files_changed_stream + .try_for_each(move |files| { + let templates = templates.clone(); + async move { + info!(?files, "Files changed, reloading templates"); + + templates + .clone() + .reload() + .await + .context("Could not reload templates") + } + }) + .inspect_err(|err| error!(%err, "Error while watching templates, stop watching")); + + tokio::spawn(fut); + + Ok(()) +} + impl ServerCommand { pub async fn run(&self, root: &RootCommand) -> anyhow::Result<()> { let config: RootConfig = root.load_config()?; @@ -160,8 +235,21 @@ impl ServerCommand { queue.start(); // Load and compile the templates - let templates = - Templates::load_from_config(&config.templates).context("could not load templates")?; + let templates = Templates::load_from_config(&config.templates) + .await + .context("could not load templates")?; + + // Watch for changes in templates if the --watch flag is present + if self.watch { + let client = watchman_client::Connector::new() + .connect() + .await + .context("could not connect to watchman")?; + + watch_templates(&client, &templates) + .await + .context("could not watch for templates changes")?; + } // Start the server let root = mas_core::handlers::root(&pool, &templates, &config); diff --git a/crates/cli/src/templates.rs b/crates/cli/src/templates.rs index 661b74bc..fb0fd0e0 100644 --- a/crates/cli/src/templates.rs +++ b/crates/cli/src/templates.rs @@ -15,6 +15,7 @@ use std::path::PathBuf; use clap::Parser; +use mas_config::TemplatesConfig; use mas_templates::Templates; use super::RootCommand; @@ -59,8 +60,12 @@ impl TemplatesCommand { } SC::Check { path, skip_builtin } => { - let templates = Templates::load(Some(path), !skip_builtin)?; - templates.check_render()?; + let config = TemplatesConfig { + path: Some(path.to_string()), + builtin: !skip_builtin, + }; + let templates = Templates::load_from_config(&config).await?; + templates.check_render().await?; Ok(()) } diff --git a/crates/config/src/templates.rs b/crates/config/src/templates.rs index 0d40f1bf..42bfac40 100644 --- a/crates/config/src/templates.rs +++ b/crates/config/src/templates.rs @@ -22,7 +22,7 @@ fn default_builtin() -> bool { true } -#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)] pub struct TemplatesConfig { /// Path to the folder that holds the custom templates #[serde(default)] diff --git a/crates/core/src/handlers/oauth2/authorization.rs b/crates/core/src/handlers/oauth2/authorization.rs index e314463e..ecadbf97 100644 --- a/crates/core/src/handlers/oauth2/authorization.rs +++ b/crates/core/src/handlers/oauth2/authorization.rs @@ -96,7 +96,7 @@ enum ReplyOrBackToClient { Error(Box), } -fn back_to_client( +async fn back_to_client( mut redirect_uri: Url, response_mode: ResponseMode, state: Option, @@ -175,7 +175,7 @@ where ResponseMode::FormPost => { let merged = ParamsWithState { state, params }; let ctx = FormPostContext::new(redirect_uri, merged); - let rendered = templates.render_form_post(&ctx)?; + let rendered = templates.render_form_post(&ctx).await?; Ok(Box::new(html(rendered))) } } @@ -288,19 +288,19 @@ async fn actually_reply( let client = match client { Some(client) => client, - None => return Ok(Box::new(html(templates.render_error(&error.into())?))), + None => return Ok(Box::new(html(templates.render_error(&error.into()).await?))), }; let redirect_uri: Result, _> = redirect_uri.map(|r| r.parse()).transpose(); let redirect_uri = match redirect_uri { Ok(r) => r, - Err(_) => return Ok(Box::new(html(templates.render_error(&error.into())?))), + Err(_) => return Ok(Box::new(html(templates.render_error(&error.into()).await?))), }; let redirect_uri = client.resolve_redirect_uri(&redirect_uri); let redirect_uri = match redirect_uri { Ok(r) => r, - Err(_) => return Ok(Box::new(html(templates.render_error(&error.into())?))), + Err(_) => return Ok(Box::new(html(templates.render_error(&error.into()).await?))), }; let reply: ErrorResponse = error.into(); @@ -310,7 +310,9 @@ async fn actually_reply( } }; - back_to_client(redirect_uri, response_mode, state, params, &templates).wrap_error() + back_to_client(redirect_uri, response_mode, state, params, &templates) + .await + .wrap_error() } async fn get( diff --git a/crates/core/src/handlers/views/index.rs b/crates/core/src/handlers/views/index.rs index 0b14d9aa..1506f06e 100644 --- a/crates/core/src/handlers/views/index.rs +++ b/crates/core/src/handlers/views/index.rs @@ -58,7 +58,7 @@ async fn get( .maybe_with_session(maybe_session) .with_csrf(csrf_token.form_value()); - let content = templates.render_index(&ctx)?; + let content = templates.render_index(&ctx).await?; let reply = html(content); let reply = cookie_saver.save_encrypted(&csrf_token, reply)?; Ok(Box::new(reply)) diff --git a/crates/core/src/handlers/views/login.rs b/crates/core/src/handlers/views/login.rs index 681e8916..5fe60612 100644 --- a/crates/core/src/handlers/views/login.rs +++ b/crates/core/src/handlers/views/login.rs @@ -138,7 +138,7 @@ async fn get( None => ctx, }; let ctx = ctx.with_csrf(csrf_token.form_value()); - let content = templates.render_login(&ctx)?; + let content = templates.render_login(&ctx).await?; let reply = html(content); let reply = cookie_saver.save_encrypted(&csrf_token, reply)?; Ok(Box::new(reply)) @@ -171,7 +171,7 @@ async fn post( let ctx = LoginContext::default() .with_form_error(errored_form) .with_csrf(csrf_token.form_value()); - let content = templates.render_login(&ctx)?; + let content = templates.render_login(&ctx).await?; let reply = html(content); let reply = cookie_saver.save_encrypted(&csrf_token, reply)?; Ok(Box::new(reply)) diff --git a/crates/core/src/handlers/views/reauth.rs b/crates/core/src/handlers/views/reauth.rs index 377ba526..ceadc113 100644 --- a/crates/core/src/handlers/views/reauth.rs +++ b/crates/core/src/handlers/views/reauth.rs @@ -132,7 +132,7 @@ async fn get( }; let ctx = ctx.with_session(session).with_csrf(csrf_token.form_value()); - let content = templates.render_reauth(&ctx)?; + let content = templates.render_reauth(&ctx).await?; let reply = html(content); let reply = cookie_saver.save_encrypted(&csrf_token, reply)?; Ok(reply) diff --git a/crates/core/src/handlers/views/register.rs b/crates/core/src/handlers/views/register.rs index 3af36ae9..5cb72445 100644 --- a/crates/core/src/handlers/views/register.rs +++ b/crates/core/src/handlers/views/register.rs @@ -117,7 +117,7 @@ async fn get( Ok(Box::new(query.redirect()?)) } else { let ctx = EmptyContext.with_csrf(csrf_token.form_value()); - let content = templates.render_register(&ctx)?; + let content = templates.render_register(&ctx).await?; let reply = html(content); let reply = cookie_saver.save_encrypted(&csrf_token, reply)?; Ok(Box::new(reply)) diff --git a/crates/templates/Cargo.toml b/crates/templates/Cargo.toml index 63a1c19e..0936c3e5 100644 --- a/crates/templates/Cargo.toml +++ b/crates/templates/Cargo.toml @@ -5,6 +5,9 @@ authors = ["Quentin Gliech "] edition = "2018" license = "Apache-2.0" +[features] +dev = [] + [dependencies] tracing = "0.1.29" tokio = "1.14.0" diff --git a/crates/templates/src/lib.rs b/crates/templates/src/lib.rs index dbf2cf6f..81acd55e 100644 --- a/crates/templates/src/lib.rs +++ b/crates/templates/src/lib.rs @@ -23,14 +23,20 @@ //! Templates rendering -use std::{collections::HashSet, io::Cursor, path::Path, string::ToString, sync::Arc}; +use std::{ + collections::HashSet, + io::Cursor, + path::{Path, PathBuf}, + string::ToString, + sync::Arc, +}; -use anyhow::Context as _; +use anyhow::{bail, Context as _}; use mas_config::TemplatesConfig; use serde::Serialize; use tera::{Context, Error as TeraError, Tera}; use thiserror::Error; -use tokio::{fs::OpenOptions, io::AsyncWriteExt}; +use tokio::{fs::OpenOptions, io::AsyncWriteExt, sync::RwLock, task::JoinError}; use tracing::{debug, info, warn}; #[allow(missing_docs)] // TODO @@ -47,7 +53,10 @@ pub use self::context::{ /// Wrapper around [`tera::Tera`] helping rendering the various templates #[derive(Debug, Clone)] -pub struct Templates(Arc); +pub struct Templates { + tera: Arc>, + config: TemplatesConfig, +} /// There was an issue while loading the templates #[derive(Error, Debug)] @@ -56,6 +65,10 @@ pub enum TemplateLoadingError { #[error("could not load and compile some templates")] Compile(#[from] TeraError), + /// Could not join blocking task + #[error("error from async runtime")] + Runtime(#[from] JoinError), + /// There are essential templates missing #[error("missing templates {missing:?}")] MissingTemplates { @@ -67,44 +80,104 @@ pub enum TemplateLoadingError { } impl Templates { - /// Load the templates from [the config][`TemplatesConfig`] - pub fn load_from_config(config: &TemplatesConfig) -> Result { - Self::load(config.path.as_deref(), config.builtin) + /// List directories to watch + pub async fn watch_roots(&self) -> Vec { + Self::roots(self.config.path.as_deref(), self.config.builtin) + .await + .into_iter() + .filter_map(Result::ok) + .collect() } - /// Load the templates and check all needed templates are properly loaded - /// - /// # Arguments - /// - /// * `path` - An optional path to where templates should be loaded - /// * `builtin` - Set to `true` to load the builtin templates as well - pub fn load(path: Option<&str>, builtin: bool) -> Result { - let tera = { - let mut tera = Tera::default(); + async fn roots(path: Option<&str>, builtin: bool) -> Vec> { + let mut paths = Vec::new(); + if builtin && cfg!(feature = "dev") { + paths.push(PathBuf::from(format!( + "{}/src/res", + env!("CARGO_MANIFEST_DIR") + ))); + } - if builtin { - info!("Loading builtin templates"); + if let Some(path) = path { + paths.push(PathBuf::from(path)); + } - for (name, source) in EXTRA_TEMPLATES { - tera.add_raw_template(name, source)?; - } + let mut ret = Vec::new(); + for path in paths { + ret.push(tokio::fs::read_dir(&path).await.map(|_| path)); + } - for (name, source) in TEMPLATES { - tera.add_raw_template(name, source)?; - } + ret + } + + fn load_builtin() -> Result { + let mut tera = Tera::default(); + info!("Loading builtin templates"); + + for (name, source) in EXTRA_TEMPLATES { + if let Some(source) = source { + tera.add_raw_template(name, source)?; } + } - if let Some(path) = path { - let path = format!("{}/**/*.{{html,txt}}", path); + for (name, source) in TEMPLATES { + if let Some(source) = source { + tera.add_raw_template(name, source)?; + } + } + + Ok(tera) + } + + /// Load the templates from [the config][`TemplatesConfig`] + pub async fn load_from_config(config: &TemplatesConfig) -> Result { + let tera = Self::load(config.path.as_deref(), config.builtin).await?; + + Ok(Self { + tera: Arc::new(RwLock::new(tera)), + config: config.clone(), + }) + } + + async fn load(path: Option<&str>, builtin: bool) -> Result { + let mut teras = Vec::new(); + + let roots = Self::roots(path, builtin).await; + for maybe_root in roots { + let root = match maybe_root { + Ok(root) => root, + Err(err) => { + warn!(%err, "Could not open a template folder, skipping it"); + continue; + } + }; + + // This uses blocking I/Os, do that in a blocking task + let tera = tokio::task::spawn_blocking(move || { + // Using `to_string_lossy` here is probably fine + let path = format!("{}/**/*.{{html,txt}}", root.to_string_lossy()); info!(%path, "Loading templates from filesystem"); - tera.extend(&Tera::parse(&path)?)?; - } + Tera::parse(&path) + }) + .await??; - tera.build_inheritance_chains()?; - tera.check_macro_files()?; + teras.push(tera); + } - tera - }; + if builtin { + teras.push(Self::load_builtin()?); + } + + // Merging all Tera instances into a single one + let mut tera = teras + .into_iter() + .try_fold(Tera::default(), |mut acc, tera| { + acc.extend(&tera)?; + Ok::<_, TemplateLoadingError>(acc) + })?; + + tera.build_inheritance_chains()?; + tera.check_macro_files()?; let loaded: HashSet<_> = tera.get_template_names().collect(); let needed: HashSet<_> = std::array::IntoIter::new(TEMPLATES) @@ -114,7 +187,7 @@ impl Templates { let missing: HashSet<_> = needed.difference(&loaded).collect(); if missing.is_empty() { - Ok(Self(Arc::new(tera))) + Ok(tera) } else { let missing = missing.into_iter().map(ToString::to_string).collect(); let loaded = loaded.into_iter().map(ToString::to_string).collect(); @@ -122,8 +195,23 @@ impl Templates { } } + /// Reload the templates on disk + pub async fn reload(&self) -> anyhow::Result<()> { + // Prepare the new Tera instance + let new_tera = Self::load(self.config.path.as_deref(), self.config.builtin).await?; + + // Swap it + *self.tera.write().await = new_tera; + + Ok(()) + } + /// Save the builtin templates to a folder pub async fn save(path: &Path, overwrite: bool) -> anyhow::Result<()> { + if cfg!(feature = "dev") { + bail!("Builtin templates are not included in dev binaries") + } + tokio::fs::create_dir_all(&path) .await .context("could not create destination folder")?; @@ -140,22 +228,24 @@ impl Templates { }; for (name, source) in templates { - let path = path.join(name); + if let Some(source) = source { + let path = path.join(name); - let mut file = match options.open(&path).await { - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Not overwriting a template is a soft error - warn!(?path, "Not overwriting template"); - continue; - } - x => x.context(format!("could not open file {:?}", path))?, - }; + let mut file = match options.open(&path).await { + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // Not overwriting a template is a soft error + warn!(?path, "Not overwriting template"); + continue; + } + x => x.context(format!("could not open file {:?}", path))?, + }; - let mut buffer = Cursor::new(source); - file.write_all_buf(&mut buffer) - .await - .context(format!("could not write file {:?}", path))?; - info!(?path, "Wrote template"); + let mut buffer = Cursor::new(source); + file.write_all_buf(&mut buffer) + .await + .context(format!("could not write file {:?}", path))?; + info!(?path, "Wrote template"); + } } Ok(()) @@ -215,13 +305,13 @@ register_templates! { impl Templates { /// Render all templates with the generated samples to check if they render /// properly - pub fn check_render(&self) -> anyhow::Result<()> { - check::render_login(self)?; - check::render_register(self)?; - check::render_index(self)?; - check::render_reauth(self)?; - check::render_form_post::(self)?; - check::render_error(self)?; + pub async fn check_render(&self) -> anyhow::Result<()> { + check::render_login(self).await?; + check::render_register(self).await?; + check::render_index(self).await?; + check::render_reauth(self).await?; + check::render_form_post::(self).await?; + check::render_error(self).await?; Ok(()) } } @@ -230,9 +320,14 @@ impl Templates { mod tests { use super::*; - #[test] - fn check_builtin_templates() { - let templates = Templates::load(None, true).unwrap(); - templates.check_render().unwrap(); + #[tokio::test] + async fn check_builtin_templates() { + let config = TemplatesConfig { + path: None, + builtin: true, + }; + + let templates = Templates::load_from_config(&config).await.unwrap(); + templates.check_render().await.unwrap(); } } diff --git a/crates/templates/src/macros.rs b/crates/templates/src/macros.rs index ea6ac0a2..33c9e7b0 100644 --- a/crates/templates/src/macros.rs +++ b/crates/templates/src/macros.rs @@ -49,26 +49,40 @@ macro_rules! register_templates { )* } => { /// List of registered templates - static TEMPLATES: [(&'static str, &'static str); count!( $( $template )* )] = [ - $( ($template, include_str!(concat!("res/", $template))) ),* + static TEMPLATES: [(&'static str, Option<&'static str>); count!( $( $template )* )] = [ + $( ( + $template, + if cfg!(feature = "dev") { + None + } else { + Some(include_str!(concat!("res/", $template))) + } + ) ),* ]; /// List of extra templates used by other templates - static EXTRA_TEMPLATES: [(&'static str, &'static str); count!( $( $( $extra_template )* )? )] = [ - $( $( ($extra_template, include_str!(concat!("res/", $extra_template))) ),* )? + static EXTRA_TEMPLATES: [(&'static str, Option<&'static str>); count!( $( $( $extra_template )* )? )] = [ + $( $( ( + $extra_template, + if cfg!(feature = "dev") { + None + } else { + Some(include_str!(concat!("res/", $extra_template))) + } + ) ),* )? ]; impl Templates { $( $(#[$attr])? - pub fn $name + pub async fn $name $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? (&self, context: &$param) -> Result { let ctx = Context::from_serialize(context) .map_err(|source| TemplateError::Context { template: $template, source })?; - self.0.render($template, &ctx) + self.tera.read().await.render($template, &ctx) .map_err(|source| TemplateError::Render { template: $template, source }) } )* @@ -80,7 +94,7 @@ macro_rules! register_templates { $( #[doc = concat!("Render the `", $template, "` template with sample contexts")] - pub fn $name + pub async fn $name $(< $( $lt $( : $clt $(+ $dlt )* + TemplateContext )? ),+ >)? (templates: &Templates) -> anyhow::Result<()> { @@ -91,6 +105,7 @@ macro_rules! register_templates { let context = serde_json::to_value(&sample)?; ::tracing::info!(name, %context, "Rendering template"); templates. $name (&sample) + .await .with_context(|| format!("Failed to render template {:?} with context {}", name, context))?; }