diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ea3e893e..3414e324 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -121,6 +121,7 @@ jobs: frontend/dist/ policies/policy.wasm templates/ + translations/ LICENSE @@ -251,6 +252,7 @@ jobs: mv artifacts/frontend/dist/manifest.json dist/share/manifest.json mv artifacts/frontend/dist/ dist/share/assets mv artifacts/templates/ dist/share/templates + mv artifacts/translations/ dist/share/translations mv artifacts/LICENSE dist/LICENSE chmod -R u=rwX,go=rX dist/ diff --git a/Cargo.lock b/Cargo.lock index 9f0205af..31efe91f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3293,6 +3293,7 @@ dependencies = [ "chrono", "http", "mas-data-model", + "mas-i18n", "mas-router", "mas-spa", "minijinja", diff --git a/Dockerfile b/Dockerfile index 82387cbc..11966a71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -157,6 +157,7 @@ FROM --platform=${BUILDPLATFORM} scratch AS share COPY --from=frontend /share /share COPY --from=policy /app/policies/policy.wasm /share/policy.wasm COPY ./templates/ /share/templates +COPY ./translations/ /share/translations ################################## ## Runtime stage, debug variant ## diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 4ca4a126..e0ad6965 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -122,6 +122,7 @@ pub async fn templates_from_config( config.path.clone(), url_builder.clone(), config.assets_manifest.clone(), + config.translations_path.clone(), ) .await } diff --git a/crates/config/src/sections/templates.rs b/crates/config/src/sections/templates.rs index b2b44ef0..40e99b35 100644 --- a/crates/config/src/sections/templates.rs +++ b/crates/config/src/sections/templates.rs @@ -50,6 +50,21 @@ fn default_assets_path() -> Utf8PathBuf { "./share/manifest.json".into() } +#[cfg(not(any(feature = "docker", feature = "dist")))] +fn default_translations_path() -> Utf8PathBuf { + "./translations/".into() +} + +#[cfg(feature = "docker")] +fn default_translations_path() -> Utf8PathBuf { + "/usr/local/share/mas-cli/translations/".into() +} + +#[cfg(feature = "dist")] +fn default_translations_path() -> Utf8PathBuf { + "./share/translations/".into() +} + /// Configuration related to templates #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)] pub struct TemplatesConfig { @@ -62,6 +77,11 @@ pub struct TemplatesConfig { #[serde(default = "default_assets_path")] #[schemars(with = "Option")] pub assets_manifest: Utf8PathBuf, + + /// Path to the translations + #[serde(default = "default_translations_path")] + #[schemars(with = "Option")] + pub translations_path: Utf8PathBuf, } impl Default for TemplatesConfig { @@ -69,6 +89,7 @@ impl Default for TemplatesConfig { Self { path: default_path(), assets_manifest: default_assets_path(), + translations_path: default_translations_path(), } } } diff --git a/crates/handlers/src/test_utils.rs b/crates/handlers/src/test_utils.rs index af088c4c..fe9b146a 100644 --- a/crates/handlers/src/test_utils.rs +++ b/crates/handlers/src/test_utils.rs @@ -119,6 +119,7 @@ impl TestState { workspace_root.join("templates"), url_builder.clone(), workspace_root.join("frontend/dist/manifest.json"), + workspace_root.join("translations"), ) .await?; diff --git a/crates/i18n-scan/src/key.rs b/crates/i18n-scan/src/key.rs index e75e9254..723177a3 100644 --- a/crates/i18n-scan/src/key.rs +++ b/crates/i18n-scan/src/key.rs @@ -35,7 +35,8 @@ impl Context { &self.func } - pub fn add_missing(&self, translation_tree: &mut TranslationTree) { + pub fn add_missing(&self, translation_tree: &mut TranslationTree) -> usize { + let mut count = 0; for translatable in &self.keys { let message = Message::from_literal(translatable.default_value()); let key = translatable @@ -47,8 +48,11 @@ impl Context { None }); - translation_tree.set_if_not_defined(key, message); + if translation_tree.set_if_not_defined(key, message) { + count += 1; + } } + count } } diff --git a/crates/i18n-scan/src/main.rs b/crates/i18n-scan/src/main.rs index d032873f..ff6dc347 100644 --- a/crates/i18n-scan/src/main.rs +++ b/crates/i18n-scan/src/main.rs @@ -42,6 +42,11 @@ struct Options { /// The name of the translation function #[clap(long, default_value = "_")] function: String, + + /// Whether the existing translation file should be updated with missing + /// keys in-place + #[clap(long)] + update: bool, } fn main() { @@ -50,7 +55,7 @@ fn main() { let options = Options::parse(); // Open the existing translation file if one was provided - let mut tree = if let Some(path) = options.existing { + let mut tree = if let Some(path) = &options.existing { let mut file = File::open(path).expect("Failed to open existing translation file"); serde_json::from_reader(&mut file).expect("Failed to parse existing translation file") } else { @@ -86,10 +91,30 @@ fn main() { } } - context.add_missing(&mut tree); + let count = context.add_missing(&mut tree); - serde_json::to_writer_pretty(std::io::stdout(), &tree) - .expect("Failed to write translation tree"); + match count { + 0 => tracing::debug!("No missing keys"), + 1 => tracing::info!("Added 1 missing key"), + count => tracing::info!("Added {} missing keys", count), + } + + if options.update { + let mut file = File::options() + .write(true) + .read(false) + .open( + options + .existing + .expect("--update requires an existing translation file"), + ) + .expect("Failed to open existing translation file"); + + serde_json::to_writer_pretty(&mut file, &tree).expect("Failed to write translation tree"); + } else { + serde_json::to_writer_pretty(std::io::stdout(), &tree) + .expect("Failed to write translation tree"); + } // Just to make sure we don't end up with a trailing newline println!(); diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index e2c292c8..ab312e8a 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -10,10 +10,10 @@ repository.workspace = true [dependencies] camino.workspace = true icu_list = { version = "1.3.0", features = ["compiled_data", "std"] } -icu_locid = { version = "1.3.0", features = ["std"] } +icu_locid = { version = "1.3.0", features = ["std",] } icu_locid_transform = { version = "1.3.0", features = ["compiled_data", "std"] } icu_plurals = { version = "1.3.0", features = ["compiled_data", "std"] } -icu_provider = { version = "1.3.0", features = ["std"] } +icu_provider = { version = "1.3.0", features = ["std", "sync"] } icu_provider_adapters = { version = "1.3.0", features = ["std"] } pad = "0.1.6" pest = "2.7.3" diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 728b07d2..2aba9271 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -19,7 +19,9 @@ pub mod sprintf; pub mod translations; mod translator; +pub use icu_provider::DataLocale; + pub use self::{ sprintf::{Argument, ArgumentList, Message}, - translator::Translator, + translator::{LoadError, Translator}, }; diff --git a/crates/i18n/src/sprintf/formatter.rs b/crates/i18n/src/sprintf/formatter.rs index 8b6a2d65..e369b7e5 100644 --- a/crates/i18n/src/sprintf/formatter.rs +++ b/crates/i18n/src/sprintf/formatter.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::Formatter; + use pad::{Alignment, PadStr}; use serde::Serialize; use serde_json::{ser::PrettyFormatter, Value}; @@ -482,6 +484,33 @@ fn format_value(value: &Value, placeholder: &Placeholder) -> Result { + Text(&'a str), + Placeholder(String), +} + +impl std::fmt::Display for FormattedMessagePart<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FormattedMessagePart::Text(text) => write!(f, "{text}"), + FormattedMessagePart::Placeholder(placeholder) => write!(f, "{placeholder}"), + } + } +} + +pub struct FormattedMessage<'a> { + parts: Vec>, +} + +impl std::fmt::Display for FormattedMessage<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for part in &self.parts { + write!(f, "{part}")?; + } + Ok(()) + } +} + impl Message { /// Format the message with the given arguments. /// @@ -490,19 +519,20 @@ impl Message { /// Returns an error if the message can't be formatted with the given /// arguments. pub fn format(&self, arguments: &ArgumentList) -> Result { - let mut buffer = String::new(); + self.format_(arguments).map(|fm| fm.to_string()) + } + + #[doc(hidden)] + pub fn format_(&self, arguments: &ArgumentList) -> Result, FormatError> { + let mut parts = Vec::with_capacity(self.parts().len()); // Holds the current index of the placeholder we are formatting, which is used // by non-named, non-indexed placeholders let mut current_placeholder = 0usize; for part in self.parts() { - match part { - Part::Percent => { - buffer.push('%'); - } - Part::Text(text) => { - buffer.push_str(text); - } + let formatted = match part { + Part::Percent => FormattedMessagePart::Text("%"), + Part::Text(text) => FormattedMessagePart::Text(text), Part::Placeholder(placeholder) => { let value = find_value( arguments, @@ -529,13 +559,13 @@ impl Message { formatted }; - buffer.push_str(&formatted); - current_placeholder += 1; + FormattedMessagePart::Placeholder(formatted) } - } + }; + parts.push(formatted); } - Ok(buffer) + Ok(FormattedMessage { parts }) } } diff --git a/crates/i18n/src/translations.rs b/crates/i18n/src/translations.rs index 111c7d22..16f9719c 100644 --- a/crates/i18n/src/translations.rs +++ b/crates/i18n/src/translations.rs @@ -85,25 +85,23 @@ impl TranslationTree { &mut self, path: I, value: Message, - ) { + ) -> bool { let mut path = path.into_iter(); let Some(next) = path.next() else { if let TranslationTree::Message(_) = self { - return; + return false; } *self = TranslationTree::Message(value); - return; + return true; }; match self { TranslationTree::Message(_) => panic!("cannot set a value on a message node"), - TranslationTree::Children(children) => { - children - .entry(next.deref().to_owned()) - .or_default() - .set_if_not_defined(path, value); - } + TranslationTree::Children(children) => children + .entry(next.deref().to_owned()) + .or_default() + .set_if_not_defined(path, value), } } diff --git a/crates/i18n/src/translator.rs b/crates/i18n/src/translator.rs index 8650d738..60fa1d5a 100644 --- a/crates/i18n/src/translator.rs +++ b/crates/i18n/src/translator.rs @@ -297,6 +297,45 @@ impl Translator { let list = formatter.format_to_string(items); Ok(list) } + + /// Get a list of available locales. + #[must_use] + pub fn available_locales(&self) -> Vec<&DataLocale> { + self.translations.keys().collect() + } + + /// Check if a locale is available. + #[must_use] + pub fn has_locale(&self, locale: &DataLocale) -> bool { + self.translations.contains_key(locale) + } + + /// Choose the best available locale from a list of candidates. + #[must_use] + pub fn choose_locale<'a>( + &self, + iter: impl Iterator, + ) -> Option { + for locale in iter { + let mut fallbacker = self + .fallbacker + .for_config(LocaleFallbackConfig::default()) + .fallback_for(locale.clone()); + + loop { + if fallbacker.get().is_und() { + break; + } + + if self.has_locale(fallbacker.get()) { + return Some(fallbacker.take()); + } + fallbacker.step(); + } + } + + None + } } #[cfg(test)] diff --git a/crates/templates/Cargo.toml b/crates/templates/Cargo.toml index 1813ed2f..a16e5221 100644 --- a/crates/templates/Cargo.toml +++ b/crates/templates/Cargo.toml @@ -31,5 +31,6 @@ rand.workspace = true oauth2-types = { path = "../oauth2-types" } mas-data-model = { path = "../data-model" } +mas-i18n = { path = "../i18n" } mas-router = { path = "../router" } mas-spa = { path = "../spa" } diff --git a/crates/templates/src/functions.rs b/crates/templates/src/functions.rs index 55cc8395..219c5483 100644 --- a/crates/templates/src/functions.rs +++ b/crates/templates/src/functions.rs @@ -19,10 +19,13 @@ use std::{ collections::{BTreeSet, HashMap}, + fmt::Formatter, str::FromStr, + sync::Arc, }; use camino::Utf8Path; +use mas_i18n::{Argument, ArgumentList, DataLocale, Translator}; use mas_router::UrlBuilder; use mas_spa::ViteManifest; use minijinja::{ @@ -35,6 +38,7 @@ pub fn register( env: &mut minijinja::Environment, url_builder: UrlBuilder, vite_manifest: ViteManifest, + translator: Translator, ) { env.add_test("empty", self::tester_empty); env.add_test("starting_with", tester_starting_with); @@ -50,6 +54,12 @@ pub fn register( vite_manifest, }), ); + env.add_global( + "_", + Value::from_object(Translate { + translations: Arc::new(translator), + }), + ); } fn tester_empty(seq: &dyn SeqObject) -> bool { @@ -182,12 +192,95 @@ fn function_add_params_to_url( Ok(uri.to_string()) } -#[derive(Debug)] +struct Translate { + translations: Arc, +} + +impl std::fmt::Debug for Translate { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Translate") + .field("translations", &"..") + .finish() + } +} + +impl std::fmt::Display for Translate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("translate") + } +} + +impl Object for Translate { + fn call(&self, state: &State, args: &[Value]) -> Result { + let lang = state.lookup("lang").ok_or(minijinja::Error::new( + ErrorKind::UndefinedError, + "`lang` is not set", + ))?; + + let lang = lang.as_str().ok_or(minijinja::Error::new( + ErrorKind::InvalidOperation, + "`lang` is not a string", + ))?; + + let lang: DataLocale = lang.parse().map_err(|e| { + Error::new(ErrorKind::InvalidOperation, "Could not parse `lang`").with_source(e) + })?; + + let (key, kwargs): (&str, Kwargs) = from_args(args)?; + + let (message, _locale) = if let Some(count) = kwargs.get("count")? { + self.translations + .plural_with_fallback(lang, key, count) + .ok_or(Error::new( + ErrorKind::InvalidOperation, + "Missing translation", + ))? + } else { + self.translations + .message_with_fallback(lang, key) + .ok_or(Error::new( + ErrorKind::InvalidOperation, + "Missing translation", + ))? + }; + + let res: Result = kwargs + .args() + .map(|name| { + let value: Value = kwargs.get(name)?; + let value = serde_json::to_value(value).map_err(|e| { + Error::new(ErrorKind::InvalidOperation, "Could not serialize argument") + .with_source(e) + })?; + + Ok::<_, Error>(Argument::named(name.to_owned(), value)) + }) + .collect(); + let list = res?; + + let formatted = message.format(&list).map_err(|e| { + Error::new(ErrorKind::InvalidOperation, "Could not format message").with_source(e) + })?; + + // TODO: escape + Ok(Value::from_safe_string(formatted)) + } +} + struct IncludeAsset { url_builder: UrlBuilder, vite_manifest: ViteManifest, } +impl std::fmt::Debug for IncludeAsset { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IncludeAsset") + .field("url_builder", &self.url_builder.assets_base()) + .field("vite_manifest", &"..") + .finish() + } +} + impl std::fmt::Display for IncludeAsset { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("include_asset") diff --git a/crates/templates/src/lib.rs b/crates/templates/src/lib.rs index b3b9ca0c..42fd242b 100644 --- a/crates/templates/src/lib.rs +++ b/crates/templates/src/lib.rs @@ -29,6 +29,7 @@ use std::{collections::HashSet, sync::Arc}; use anyhow::Context as _; use arc_swap::ArcSwap; use camino::{Utf8Path, Utf8PathBuf}; +use mas_i18n::Translator; use mas_router::UrlBuilder; use mas_spa::ViteManifest; use rand::Rng; @@ -72,6 +73,7 @@ pub struct Templates { environment: Arc>>, url_builder: UrlBuilder, vite_manifest_path: Utf8PathBuf, + translations_path: Utf8PathBuf, path: Utf8PathBuf, } @@ -90,6 +92,10 @@ pub enum TemplateLoadingError { #[error("invalid assets manifest")] ViteManifest(#[from] serde_json::Error), + /// Failed to load the translations + #[error("failed to load the translations")] + Translations(#[from] mas_i18n::LoadError), + /// Failed to traverse the filesystem #[error("failed to traverse the filesystem")] WalkDir(#[from] walkdir::Error), @@ -143,13 +149,21 @@ impl Templates { path: Utf8PathBuf, url_builder: UrlBuilder, vite_manifest_path: Utf8PathBuf, + translations_path: Utf8PathBuf, ) -> Result { - let environment = Self::load_(&path, url_builder.clone(), &vite_manifest_path).await?; + let environment = Self::load_( + &path, + url_builder.clone(), + &vite_manifest_path, + &translations_path, + ) + .await?; Ok(Self { environment: Arc::new(ArcSwap::from_pointee(environment)), path, url_builder, vite_manifest_path, + translations_path, }) } @@ -157,6 +171,7 @@ impl Templates { path: &Utf8Path, url_builder: UrlBuilder, vite_manifest_path: &Utf8Path, + translations_path: &Utf8Path, ) -> Result, TemplateLoadingError> { let path = path.to_owned(); let span = tracing::Span::current(); @@ -170,6 +185,11 @@ impl Templates { let vite_manifest: ViteManifest = serde_json::from_slice(&vite_manifest).map_err(TemplateLoadingError::ViteManifest)?; + let translations_path = translations_path.to_owned(); + let translator = + tokio::task::spawn_blocking(move || Translator::load_from_path(&translations_path)) + .await??; + let (loaded, mut env) = tokio::task::spawn_blocking(move || { span.in_scope(move || { let mut loaded: HashSet<_> = HashSet::new(); @@ -203,7 +223,7 @@ impl Templates { }) .await??; - self::functions::register(&mut env, url_builder, vite_manifest); + self::functions::register(&mut env, url_builder, vite_manifest, translator); let needed: HashSet<_> = TEMPLATES.into_iter().map(ToOwned::to_owned).collect(); debug!(?loaded, ?needed, "Templates loaded"); @@ -228,6 +248,7 @@ impl Templates { &self.path, self.url_builder.clone(), &self.vite_manifest_path, + &self.translations_path, ) .await?; @@ -373,7 +394,9 @@ mod tests { let url_builder = UrlBuilder::new("https://example.com/".parse().unwrap(), None, None); let vite_manifest_path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist/manifest.json"); - let templates = Templates::load(path, url_builder, vite_manifest_path) + let translations_path = + Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../translations"); + let templates = Templates::load(path, url_builder, vite_manifest_path, translations_path) .await .unwrap(); templates.check_render(now, &mut rng).unwrap(); diff --git a/docs/config.schema.json b/docs/config.schema.json index 30245b37..4ca08ee5 100644 --- a/docs/config.schema.json +++ b/docs/config.schema.json @@ -199,7 +199,8 @@ "description": "Configuration related to templates", "default": { "assets_manifest": "./frontend/dist/manifest.json", - "path": "./templates/" + "path": "./templates/", + "translations_path": "./translations/" }, "allOf": [ { @@ -1918,6 +1919,11 @@ "description": "Path to the folder which holds the templates", "default": "./templates/", "type": "string" + }, + "translations_path": { + "description": "Path to the translations", + "default": "./translations/", + "type": "string" } } }, diff --git a/misc/update.sh b/misc/update.sh index 91e275df..39a8c602 100644 --- a/misc/update.sh +++ b/misc/update.sh @@ -11,6 +11,7 @@ POLICIES_SCHEMA="${BASE_DIR}/policies/schema/" set -x cargo run -p mas-config > "${CONFIG_SCHEMA}" cargo run -p mas-graphql > "${GRAPHQL_SCHEMA}" +cargo run -p mas-i18n-scan -- --update "${BASE_DIR}/templates/" "${BASE_DIR}/translations/en.json" OUT_DIR="${POLICIES_SCHEMA}" cargo run -p mas-policy --features jsonschema cd "${BASE_DIR}/frontend" diff --git a/templates/app.html b/templates/app.html index c6f899c0..702773f6 100644 --- a/templates/app.html +++ b/templates/app.html @@ -15,13 +15,15 @@ limitations under the License. #} {# Must be kept in sync with frontend/index.html #} +{# TODO: detect the locale from the browser #} +{% set lang = "en-US" %} - matrix-authentication-service + {{ _("app.name") }}