templates: add translations function

.github/workflows/build.yaml

@@ -121,6 +121,7 @@ jobs:
             frontend/dist/
             policies/policy.wasm
             templates/
+            translations/
             LICENSE
 
 
@@ -251,6 +252,7 @@ jobs:
           mv artifacts/frontend/dist/manifest.json dist/share/manifest.json
           mv artifacts/frontend/dist/ dist/share/assets
           mv artifacts/templates/ dist/share/templates
+          mv artifacts/translations/ dist/share/translations
           mv artifacts/LICENSE dist/LICENSE
           chmod -R u=rwX,go=rX dist/
 

Cargo.lock

@@ -3293,6 +3293,7 @@ dependencies = [
  "chrono",
  "http",
  "mas-data-model",
+ "mas-i18n",
  "mas-router",
  "mas-spa",
  "minijinja",

Dockerfile

@@ -157,6 +157,7 @@ FROM --platform=${BUILDPLATFORM} scratch AS share
 COPY --from=frontend /share /share
 COPY --from=policy /app/policies/policy.wasm /share/policy.wasm
 COPY ./templates/ /share/templates
+COPY ./translations/ /share/translations
 
 ##################################
 ## Runtime stage, debug variant ##

crates/cli/src/util.rs

@@ -122,6 +122,7 @@ pub async fn templates_from_config(
         config.path.clone(),
         url_builder.clone(),
         config.assets_manifest.clone(),
+        config.translations_path.clone(),
     )
     .await
 }

crates/config/src/sections/templates.rs

@@ -50,6 +50,21 @@ fn default_assets_path() -> Utf8PathBuf {
     "./share/manifest.json".into()
 }
 
+#[cfg(not(any(feature = "docker", feature = "dist")))]
+fn default_translations_path() -> Utf8PathBuf {
+    "./translations/".into()
+}
+
+#[cfg(feature = "docker")]
+fn default_translations_path() -> Utf8PathBuf {
+    "/usr/local/share/mas-cli/translations/".into()
+}
+
+#[cfg(feature = "dist")]
+fn default_translations_path() -> Utf8PathBuf {
+    "./share/translations/".into()
+}
+
 /// Configuration related to templates
 #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
 pub struct TemplatesConfig {
@@ -62,6 +77,11 @@ pub struct TemplatesConfig {
     #[serde(default = "default_assets_path")]
     #[schemars(with = "Option<String>")]
     pub assets_manifest: Utf8PathBuf,
+
+    /// Path to the translations
+    #[serde(default = "default_translations_path")]
+    #[schemars(with = "Option<String>")]
+    pub translations_path: Utf8PathBuf,
 }
 
 impl Default for TemplatesConfig {
@@ -69,6 +89,7 @@ impl Default for TemplatesConfig {
         Self {
             path: default_path(),
             assets_manifest: default_assets_path(),
+            translations_path: default_translations_path(),
         }
     }
 }

crates/handlers/src/test_utils.rs

@@ -119,6 +119,7 @@ impl TestState {
             workspace_root.join("templates"),
             url_builder.clone(),
             workspace_root.join("frontend/dist/manifest.json"),
+            workspace_root.join("translations"),
         )
         .await?;
 

crates/i18n-scan/src/key.rs

@@ -35,7 +35,8 @@ impl Context {
         &self.func
     }
 
-    pub fn add_missing(&self, translation_tree: &mut TranslationTree) {
+    pub fn add_missing(&self, translation_tree: &mut TranslationTree) -> usize {
+        let mut count = 0;
         for translatable in &self.keys {
             let message = Message::from_literal(translatable.default_value());
             let key = translatable
@@ -47,8 +48,11 @@ impl Context {
                     None
                 });
 
-            translation_tree.set_if_not_defined(key, message);
+            if translation_tree.set_if_not_defined(key, message) {
+                count += 1;
+            }
         }
+        count
     }
 }
 

crates/i18n-scan/src/main.rs

@@ -42,6 +42,11 @@ struct Options {
     /// The name of the translation function
     #[clap(long, default_value = "_")]
     function: String,
+
+    /// Whether the existing translation file should be updated with missing
+    /// keys in-place
+    #[clap(long)]
+    update: bool,
 }
 
 fn main() {
@@ -50,7 +55,7 @@ fn main() {
     let options = Options::parse();
 
     // Open the existing translation file if one was provided
-    let mut tree = if let Some(path) = options.existing {
+    let mut tree = if let Some(path) = &options.existing {
         let mut file = File::open(path).expect("Failed to open existing translation file");
         serde_json::from_reader(&mut file).expect("Failed to parse existing translation file")
     } else {
@@ -86,10 +91,30 @@ fn main() {
         }
     }
 
-    context.add_missing(&mut tree);
+    let count = context.add_missing(&mut tree);
 
-    serde_json::to_writer_pretty(std::io::stdout(), &tree)
+    match count {
-        .expect("Failed to write translation tree");
+        0 => tracing::debug!("No missing keys"),
+        1 => tracing::info!("Added 1 missing key"),
+        count => tracing::info!("Added {} missing keys", count),
+    }
+
+    if options.update {
+        let mut file = File::options()
+            .write(true)
+            .read(false)
+            .open(
+                options
+                    .existing
+                    .expect("--update requires an existing translation file"),
+            )
+            .expect("Failed to open existing translation file");
+
+        serde_json::to_writer_pretty(&mut file, &tree).expect("Failed to write translation tree");
+    } else {
+        serde_json::to_writer_pretty(std::io::stdout(), &tree)
+            .expect("Failed to write translation tree");
+    }
 
     // Just to make sure we don't end up with a trailing newline
     println!();

crates/i18n/Cargo.toml

@@ -10,10 +10,10 @@ repository.workspace = true
 [dependencies]
 camino.workspace = true
 icu_list = { version = "1.3.0", features = ["compiled_data", "std"] }
-icu_locid = { version = "1.3.0", features = ["std"] }
+icu_locid = { version = "1.3.0", features = ["std",] }
 icu_locid_transform = { version = "1.3.0", features = ["compiled_data", "std"] }
 icu_plurals = { version = "1.3.0", features = ["compiled_data", "std"] }
-icu_provider = { version = "1.3.0", features = ["std"] }
+icu_provider = { version = "1.3.0", features = ["std", "sync"] }
 icu_provider_adapters = { version = "1.3.0", features = ["std"] }
 pad = "0.1.6"
 pest = "2.7.3"

crates/i18n/src/lib.rs

@@ -19,7 +19,9 @@ pub mod sprintf;
 pub mod translations;
 mod translator;
 
+pub use icu_provider::DataLocale;
+
 pub use self::{
     sprintf::{Argument, ArgumentList, Message},
-    translator::Translator,
+    translator::{LoadError, Translator},
 };

crates/i18n/src/sprintf/formatter.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Formatter;
+
 use pad::{Alignment, PadStr};
 use serde::Serialize;
 use serde_json::{ser::PrettyFormatter, Value};
@@ -482,6 +484,33 @@ fn format_value(value: &Value, placeholder: &Placeholder) -> Result<String, Form
     }
 }
 
+enum FormattedMessagePart<'a> {
+    Text(&'a str),
+    Placeholder(String),
+}
+
+impl std::fmt::Display for FormattedMessagePart<'_> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FormattedMessagePart::Text(text) => write!(f, "{text}"),
+            FormattedMessagePart::Placeholder(placeholder) => write!(f, "{placeholder}"),
+        }
+    }
+}
+
+pub struct FormattedMessage<'a> {
+    parts: Vec<FormattedMessagePart<'a>>,
+}
+
+impl std::fmt::Display for FormattedMessage<'_> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        for part in &self.parts {
+            write!(f, "{part}")?;
+        }
+        Ok(())
+    }
+}
+
 impl Message {
     /// Format the message with the given arguments.
     ///
@@ -490,19 +519,20 @@ impl Message {
     /// Returns an error if the message can't be formatted with the given
     /// arguments.
     pub fn format(&self, arguments: &ArgumentList) -> Result<String, FormatError> {
-        let mut buffer = String::new();
+        self.format_(arguments).map(|fm| fm.to_string())
+    }
+
+    #[doc(hidden)]
+    pub fn format_(&self, arguments: &ArgumentList) -> Result<FormattedMessage<'_>, FormatError> {
+        let mut parts = Vec::with_capacity(self.parts().len());
 
         // Holds the current index of the placeholder we are formatting, which is used
         // by non-named, non-indexed placeholders
         let mut current_placeholder = 0usize;
         for part in self.parts() {
-            match part {
+            let formatted = match part {
-                Part::Percent => {
+                Part::Percent => FormattedMessagePart::Text("%"),
-                    buffer.push('%');
+                Part::Text(text) => FormattedMessagePart::Text(text),
-                }
-                Part::Text(text) => {
-                    buffer.push_str(text);
-                }
                 Part::Placeholder(placeholder) => {
                     let value = find_value(
                         arguments,
@@ -529,13 +559,13 @@ impl Message {
                         formatted
                     };
 
-                    buffer.push_str(&formatted);
-
                     current_placeholder += 1;
+                    FormattedMessagePart::Placeholder(formatted)
                 }
-            }
+            };
+            parts.push(formatted);
         }
 
-        Ok(buffer)
+        Ok(FormattedMessage { parts })
     }
 }

crates/i18n/src/translations.rs

@@ -85,25 +85,23 @@ impl TranslationTree {
         &mut self,
         path: I,
         value: Message,
-    ) {
+    ) -> bool {
         let mut path = path.into_iter();
         let Some(next) = path.next() else {
             if let TranslationTree::Message(_) = self {
-                return;
+                return false;
             }
 
             *self = TranslationTree::Message(value);
-            return;
+            return true;
         };
 
         match self {
             TranslationTree::Message(_) => panic!("cannot set a value on a message node"),
-            TranslationTree::Children(children) => {
+            TranslationTree::Children(children) => children
-                children
+                .entry(next.deref().to_owned())
-                    .entry(next.deref().to_owned())
+                .or_default()
-                    .or_default()
+                .set_if_not_defined(path, value),
-                    .set_if_not_defined(path, value);
-            }
         }
     }
 

crates/i18n/src/translator.rs

@@ -297,6 +297,45 @@ impl Translator {
         let list = formatter.format_to_string(items);
         Ok(list)
     }
+
+    /// Get a list of available locales.
+    #[must_use]
+    pub fn available_locales(&self) -> Vec<&DataLocale> {
+        self.translations.keys().collect()
+    }
+
+    /// Check if a locale is available.
+    #[must_use]
+    pub fn has_locale(&self, locale: &DataLocale) -> bool {
+        self.translations.contains_key(locale)
+    }
+
+    /// Choose the best available locale from a list of candidates.
+    #[must_use]
+    pub fn choose_locale<'a>(
+        &self,
+        iter: impl Iterator<Item = &'a DataLocale>,
+    ) -> Option<DataLocale> {
+        for locale in iter {
+            let mut fallbacker = self
+                .fallbacker
+                .for_config(LocaleFallbackConfig::default())
+                .fallback_for(locale.clone());
+
+            loop {
+                if fallbacker.get().is_und() {
+                    break;
+                }
+
+                if self.has_locale(fallbacker.get()) {
+                    return Some(fallbacker.take());
+                }
+                fallbacker.step();
+            }
+        }
+
+        None
+    }
 }
 
 #[cfg(test)]

crates/templates/Cargo.toml

@@ -31,5 +31,6 @@ rand.workspace = true
 
 oauth2-types = { path = "../oauth2-types" }
 mas-data-model = { path = "../data-model" }
+mas-i18n = { path = "../i18n" }
 mas-router = { path = "../router" }
 mas-spa = { path = "../spa" }

crates/templates/src/functions.rs

@@ -19,10 +19,13 @@
 
 use std::{
     collections::{BTreeSet, HashMap},
+    fmt::Formatter,
     str::FromStr,
+    sync::Arc,
 };
 
 use camino::Utf8Path;
+use mas_i18n::{Argument, ArgumentList, DataLocale, Translator};
 use mas_router::UrlBuilder;
 use mas_spa::ViteManifest;
 use minijinja::{
@@ -35,6 +38,7 @@ pub fn register(
     env: &mut minijinja::Environment,
     url_builder: UrlBuilder,
     vite_manifest: ViteManifest,
+    translator: Translator,
 ) {
     env.add_test("empty", self::tester_empty);
     env.add_test("starting_with", tester_starting_with);
@@ -50,6 +54,12 @@ pub fn register(
             vite_manifest,
         }),
     );
+    env.add_global(
+        "_",
+        Value::from_object(Translate {
+            translations: Arc::new(translator),
+        }),
+    );
 }
 
 fn tester_empty(seq: &dyn SeqObject) -> bool {
@@ -182,12 +192,95 @@ fn function_add_params_to_url(
     Ok(uri.to_string())
 }
 
-#[derive(Debug)]
+struct Translate {
+    translations: Arc<Translator>,
+}
+
+impl std::fmt::Debug for Translate {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Translate")
+            .field("translations", &"..")
+            .finish()
+    }
+}
+
+impl std::fmt::Display for Translate {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("translate")
+    }
+}
+
+impl Object for Translate {
+    fn call(&self, state: &State, args: &[Value]) -> Result<Value, Error> {
+        let lang = state.lookup("lang").ok_or(minijinja::Error::new(
+            ErrorKind::UndefinedError,
+            "`lang` is not set",
+        ))?;
+
+        let lang = lang.as_str().ok_or(minijinja::Error::new(
+            ErrorKind::InvalidOperation,
+            "`lang` is not a string",
+        ))?;
+
+        let lang: DataLocale = lang.parse().map_err(|e| {
+            Error::new(ErrorKind::InvalidOperation, "Could not parse `lang`").with_source(e)
+        })?;
+
+        let (key, kwargs): (&str, Kwargs) = from_args(args)?;
+
+        let (message, _locale) = if let Some(count) = kwargs.get("count")? {
+            self.translations
+                .plural_with_fallback(lang, key, count)
+                .ok_or(Error::new(
+                    ErrorKind::InvalidOperation,
+                    "Missing translation",
+                ))?
+        } else {
+            self.translations
+                .message_with_fallback(lang, key)
+                .ok_or(Error::new(
+                    ErrorKind::InvalidOperation,
+                    "Missing translation",
+                ))?
+        };
+
+        let res: Result<ArgumentList, Error> = kwargs
+            .args()
+            .map(|name| {
+                let value: Value = kwargs.get(name)?;
+                let value = serde_json::to_value(value).map_err(|e| {
+                    Error::new(ErrorKind::InvalidOperation, "Could not serialize argument")
+                        .with_source(e)
+                })?;
+
+                Ok::<_, Error>(Argument::named(name.to_owned(), value))
+            })
+            .collect();
+        let list = res?;
+
+        let formatted = message.format(&list).map_err(|e| {
+            Error::new(ErrorKind::InvalidOperation, "Could not format message").with_source(e)
+        })?;
+
+        // TODO: escape
+        Ok(Value::from_safe_string(formatted))
+    }
+}
+
 struct IncludeAsset {
     url_builder: UrlBuilder,
     vite_manifest: ViteManifest,
 }
 
+impl std::fmt::Debug for IncludeAsset {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IncludeAsset")
+            .field("url_builder", &self.url_builder.assets_base())
+            .field("vite_manifest", &"..")
+            .finish()
+    }
+}
+
 impl std::fmt::Display for IncludeAsset {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str("include_asset")

crates/templates/src/lib.rs

@@ -29,6 +29,7 @@ use std::{collections::HashSet, sync::Arc};
 use anyhow::Context as _;
 use arc_swap::ArcSwap;
 use camino::{Utf8Path, Utf8PathBuf};
+use mas_i18n::Translator;
 use mas_router::UrlBuilder;
 use mas_spa::ViteManifest;
 use rand::Rng;
@@ -72,6 +73,7 @@ pub struct Templates {
     environment: Arc<ArcSwap<minijinja::Environment<'static>>>,
     url_builder: UrlBuilder,
     vite_manifest_path: Utf8PathBuf,
+    translations_path: Utf8PathBuf,
     path: Utf8PathBuf,
 }
 
@@ -90,6 +92,10 @@ pub enum TemplateLoadingError {
     #[error("invalid assets manifest")]
     ViteManifest(#[from] serde_json::Error),
 
+    /// Failed to load the translations
+    #[error("failed to load the translations")]
+    Translations(#[from] mas_i18n::LoadError),
+
     /// Failed to traverse the filesystem
     #[error("failed to traverse the filesystem")]
     WalkDir(#[from] walkdir::Error),
@@ -143,13 +149,21 @@ impl Templates {
         path: Utf8PathBuf,
         url_builder: UrlBuilder,
         vite_manifest_path: Utf8PathBuf,
+        translations_path: Utf8PathBuf,
     ) -> Result<Self, TemplateLoadingError> {
-        let environment = Self::load_(&path, url_builder.clone(), &vite_manifest_path).await?;
+        let environment = Self::load_(
+            &path,
+            url_builder.clone(),
+            &vite_manifest_path,
+            &translations_path,
+        )
+        .await?;
         Ok(Self {
             environment: Arc::new(ArcSwap::from_pointee(environment)),
             path,
             url_builder,
             vite_manifest_path,
+            translations_path,
         })
     }
 
@@ -157,6 +171,7 @@ impl Templates {
         path: &Utf8Path,
         url_builder: UrlBuilder,
         vite_manifest_path: &Utf8Path,
+        translations_path: &Utf8Path,
     ) -> Result<minijinja::Environment<'static>, TemplateLoadingError> {
         let path = path.to_owned();
         let span = tracing::Span::current();
@@ -170,6 +185,11 @@ impl Templates {
         let vite_manifest: ViteManifest =
             serde_json::from_slice(&vite_manifest).map_err(TemplateLoadingError::ViteManifest)?;
 
+        let translations_path = translations_path.to_owned();
+        let translator =
+            tokio::task::spawn_blocking(move || Translator::load_from_path(&translations_path))
+                .await??;
+
         let (loaded, mut env) = tokio::task::spawn_blocking(move || {
             span.in_scope(move || {
                 let mut loaded: HashSet<_> = HashSet::new();
@@ -203,7 +223,7 @@ impl Templates {
         })
         .await??;
 
-        self::functions::register(&mut env, url_builder, vite_manifest);
+        self::functions::register(&mut env, url_builder, vite_manifest, translator);
 
         let needed: HashSet<_> = TEMPLATES.into_iter().map(ToOwned::to_owned).collect();
         debug!(?loaded, ?needed, "Templates loaded");
@@ -228,6 +248,7 @@ impl Templates {
             &self.path,
             self.url_builder.clone(),
             &self.vite_manifest_path,
+            &self.translations_path,
         )
         .await?;
 
@@ -373,7 +394,9 @@ mod tests {
         let url_builder = UrlBuilder::new("https://example.com/".parse().unwrap(), None, None);
         let vite_manifest_path =
             Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist/manifest.json");
-        let templates = Templates::load(path, url_builder, vite_manifest_path)
+        let translations_path =
+            Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../translations");
+        let templates = Templates::load(path, url_builder, vite_manifest_path, translations_path)
             .await
             .unwrap();
         templates.check_render(now, &mut rng).unwrap();

docs/config.schema.json

@@ -199,7 +199,8 @@
       "description": "Configuration related to templates",
       "default": {
         "assets_manifest": "./frontend/dist/manifest.json",
-        "path": "./templates/"
+        "path": "./templates/",
+        "translations_path": "./translations/"
       },
       "allOf": [
         {
@@ -1918,6 +1919,11 @@
           "description": "Path to the folder which holds the templates",
           "default": "./templates/",
           "type": "string"
+        },
+        "translations_path": {
+          "description": "Path to the translations",
+          "default": "./translations/",
+          "type": "string"
         }
       }
     },

misc/update.sh

@@ -11,6 +11,7 @@ POLICIES_SCHEMA="${BASE_DIR}/policies/schema/"
 set -x
 cargo run -p mas-config > "${CONFIG_SCHEMA}"
 cargo run -p mas-graphql > "${GRAPHQL_SCHEMA}"
+cargo run -p mas-i18n-scan -- --update "${BASE_DIR}/templates/" "${BASE_DIR}/translations/en.json"
 OUT_DIR="${POLICIES_SCHEMA}" cargo run -p mas-policy --features jsonschema
 
 cd "${BASE_DIR}/frontend"

templates/app.html

@@ -15,13 +15,15 @@ limitations under the License.
 #}
 
 {# Must be kept in sync with frontend/index.html #}
+{# TODO: detect the locale from the browser #}
+{% set lang = "en-US" %}
 
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>matrix-authentication-service</title>
+    <title>{{ _("app.name") }}</title>
     <script>
       window.APP_CONFIG = JSON.parse("{{ app_config | tojson | add_slashes | safe }}");
       (function () {

templates/base.html

@@ -22,12 +22,14 @@ limitations under the License.
 {% import "components/errors.html" as errors %}
 {% import "components/icon.html" as icon %}
 {% import "components/scope.html" as scope %}
+{# TODO: detect the locale from the browser #}
+{% set lang = "en-US" %}
 
 <!DOCTYPE html>
 <html>
   <head>
     <meta charset="utf-8">
-    <title>{% block title %}matrix-authentication-service{% endblock title %}</title>
+    <title>{% block title %}{{ _("app.name") }}{% endblock title %}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     {{ include_asset('src/templates.css', preload=true) | indent(4) | safe }}
   </head>

templates/pages/index.html

@@ -20,7 +20,7 @@ limitations under the License.
   {{ navbar.top() }}
   <section class="flex-1 flex flex-col items-center justify-center">
     <div class="my-2 mx-8">
-      <h1 class="my-2 text-5xl font-semibold leading-tight">Matrix Authentication Service</h1>
+      <h1 class="my-2 text-5xl font-semibold leading-tight">{{ _("app.human_name") }}</h1>
       <p class="text-lg">
         OpenID Connect discovery document:
         <a class="cpd-link" data-kind="primary" href="{{ discovery_url }}">{{ discovery_url }}</a>

translations/en.json

@@ -0,0 +1,6 @@
+{
+  "app": {
+    "human_name": "Matrix Authentication Service",
+    "name": "matrix-authentication-service"
+  }
+}