matrix/authentication-service
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-authentication-service.git synced 2025-08-09 04:22:45 +03:00
Code Activity

templates: add translations function

Browse Source
This commit is contained in:
Quentin Gliech
2023-10-02 18:06:13 +02:00
parent 50a7f81ffc
commit 15ad89aa82
22 changed files with 297 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build.yaml vendored
View File

@@ -121,6 +121,7 @@ jobs:
frontend/dist/
policies/policy.wasm
templates/
translations/
LICENSE
@@ -251,6 +252,7 @@ jobs:
mv artifacts/frontend/dist/manifest.json dist/share/manifest.json
mv artifacts/frontend/dist/ dist/share/assets
mv artifacts/templates/ dist/share/templates
mv artifacts/translations/ dist/share/translations
mv artifacts/LICENSE dist/LICENSE
chmod -R u=rwX,go=rX dist/

1
Cargo.lock generated
View File

@@ -3293,6 +3293,7 @@ dependencies = [
"chrono",
"http",
"mas-data-model",
"mas-i18n",
"mas-router",
"mas-spa",
"minijinja",

1
Dockerfile
View File

@@ -157,6 +157,7 @@ FROM --platform=${BUILDPLATFORM} scratch AS share
COPY --from=frontend /share /share
COPY --from=policy /app/policies/policy.wasm /share/policy.wasm
COPY ./templates/ /share/templates
COPY ./translations/ /share/translations
##################################
## Runtime stage, debug variant ##

1
crates/cli/src/util.rs
View File

@@ -122,6 +122,7 @@ pub async fn templates_from_config(
config.path.clone(),
url_builder.clone(),
config.assets_manifest.clone(),
config.translations_path.clone(),
)
.await
}

21
crates/config/src/sections/templates.rs
View File

@@ -50,6 +50,21 @@ fn default_assets_path() -> Utf8PathBuf {
"./share/manifest.json".into()
}
#[cfg(not(any(feature = "docker", feature = "dist")))]
fn default_translations_path() -> Utf8PathBuf {
"./translations/".into()
}
#[cfg(feature = "docker")]
fn default_translations_path() -> Utf8PathBuf {
"/usr/local/share/mas-cli/translations/".into()
}
#[cfg(feature = "dist")]
fn default_translations_path() -> Utf8PathBuf {
"./share/translations/".into()
}
/// Configuration related to templates
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
pub struct TemplatesConfig {
@@ -62,6 +77,11 @@ pub struct TemplatesConfig {
#[serde(default = "default_assets_path")]
#[schemars(with = "Option<String>")]
pub assets_manifest: Utf8PathBuf,
/// Path to the translations
#[serde(default = "default_translations_path")]
#[schemars(with = "Option<String>")]
pub translations_path: Utf8PathBuf,
}
impl Default for TemplatesConfig {
@@ -69,6 +89,7 @@ impl Default for TemplatesConfig {
Self {
path: default_path(),
assets_manifest: default_assets_path(),
translations_path: default_translations_path(),
}
}
}

1
crates/handlers/src/test_utils.rs
View File

@@ -119,6 +119,7 @@ impl TestState {
workspace_root.join("templates"),
url_builder.clone(),
workspace_root.join("frontend/dist/manifest.json"),
workspace_root.join("translations"),
)
.await?;

8
crates/i18n-scan/src/key.rs
View File

@@ -35,7 +35,8 @@ impl Context {
&self.func
}
pub fn add_missing(&self, translation_tree: &mut TranslationTree) {
pub fn add_missing(&self, translation_tree: &mut TranslationTree) -> usize {
let mut count = 0;
for translatable in &self.keys {
let message = Message::from_literal(translatable.default_value());
let key = translatable
@@ -47,8 +48,11 @@ impl Context {
None
});
translation_tree.set_if_not_defined(key, message);
if translation_tree.set_if_not_defined(key, message) {
count += 1;
}
}
count
}
}

33
crates/i18n-scan/src/main.rs
View File

@@ -42,6 +42,11 @@ struct Options {
/// The name of the translation function
#[clap(long, default_value = "_")]
function: String,
/// Whether the existing translation file should be updated with missing
/// keys in-place
#[clap(long)]
update: bool,
}
fn main() {
@@ -50,7 +55,7 @@ fn main() {
let options = Options::parse();
// Open the existing translation file if one was provided
let mut tree = if let Some(path) = options.existing {
let mut tree = if let Some(path) = &options.existing {
let mut file = File::open(path).expect("Failed to open existing translation file");
serde_json::from_reader(&mut file).expect("Failed to parse existing translation file")
} else {
@@ -86,10 +91,30 @@ fn main() {
}
}
context.add_missing(&mut tree);
let count = context.add_missing(&mut tree);
serde_json::to_writer_pretty(std::io::stdout(), &tree)
.expect("Failed to write translation tree");
match count {
0 => tracing::debug!("No missing keys"),
1 => tracing::info!("Added 1 missing key"),
count => tracing::info!("Added {} missing keys", count),
}
if options.update {
let mut file = File::options()
.write(true)
.read(false)
.open(
options
.existing
.expect("--update requires an existing translation file"),
)
.expect("Failed to open existing translation file");
serde_json::to_writer_pretty(&mut file, &tree).expect("Failed to write translation tree");
} else {
serde_json::to_writer_pretty(std::io::stdout(), &tree)
.expect("Failed to write translation tree");
}
// Just to make sure we don't end up with a trailing newline
println!();

4
crates/i18n/Cargo.toml
View File

@@ -10,10 +10,10 @@ repository.workspace = true
[dependencies]
camino.workspace = true
icu_list = { version = "1.3.0", features = ["compiled_data", "std"] }
icu_locid = { version = "1.3.0", features = ["std"] }
icu_locid = { version = "1.3.0", features = ["std",] }
icu_locid_transform = { version = "1.3.0", features = ["compiled_data", "std"] }
icu_plurals = { version = "1.3.0", features = ["compiled_data", "std"] }
icu_provider = { version = "1.3.0", features = ["std"] }
icu_provider = { version = "1.3.0", features = ["std", "sync"] }
icu_provider_adapters = { version = "1.3.0", features = ["std"] }
pad = "0.1.6"
pest = "2.7.3"

4
crates/i18n/src/lib.rs
View File

@@ -19,7 +19,9 @@ pub mod sprintf;
pub mod translations;
mod translator;
pub use icu_provider::DataLocale;
pub use self::{
sprintf::{Argument, ArgumentList, Message},
translator::Translator,
translator::{LoadError, Translator},
};

54
crates/i18n/src/sprintf/formatter.rs
View File

@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Formatter;
use pad::{Alignment, PadStr};
use serde::Serialize;
use serde_json::{ser::PrettyFormatter, Value};
@@ -482,6 +484,33 @@ fn format_value(value: &Value, placeholder: &Placeholder) -> Result<String, Form
}
}
enum FormattedMessagePart<'a> {
Text(&'a str),
Placeholder(String),
}
impl std::fmt::Display for FormattedMessagePart<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
FormattedMessagePart::Text(text) => write!(f, "{text}"),
FormattedMessagePart::Placeholder(placeholder) => write!(f, "{placeholder}"),
}
}
}
pub struct FormattedMessage<'a> {
parts: Vec<FormattedMessagePart<'a>>,
}
impl std::fmt::Display for FormattedMessage<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
for part in &self.parts {
write!(f, "{part}")?;
}
Ok(())
}
}
impl Message {
/// Format the message with the given arguments.
///
@@ -490,19 +519,20 @@ impl Message {
/// Returns an error if the message can't be formatted with the given
/// arguments.
pub fn format(&self, arguments: &ArgumentList) -> Result<String, FormatError> {
let mut buffer = String::new();
self.format_(arguments).map(|fm| fm.to_string())
}
#[doc(hidden)]
pub fn format_(&self, arguments: &ArgumentList) -> Result<FormattedMessage<'_>, FormatError> {
let mut parts = Vec::with_capacity(self.parts().len());
// Holds the current index of the placeholder we are formatting, which is used
// by non-named, non-indexed placeholders
let mut current_placeholder = 0usize;
for part in self.parts() {
match part {
Part::Percent => {
buffer.push('%');
}
Part::Text(text) => {
buffer.push_str(text);
}
let formatted = match part {
Part::Percent => FormattedMessagePart::Text("%"),
Part::Text(text) => FormattedMessagePart::Text(text),
Part::Placeholder(placeholder) => {
let value = find_value(
arguments,
@@ -529,13 +559,13 @@ impl Message {
formatted
};
buffer.push_str(&formatted);
current_placeholder += 1;
FormattedMessagePart::Placeholder(formatted)
}
}
};
parts.push(formatted);
}
Ok(buffer)
Ok(FormattedMessage { parts })
}
}

16
crates/i18n/src/translations.rs
View File

@@ -85,25 +85,23 @@ impl TranslationTree {
&mut self,
path: I,
value: Message,
) {
) -> bool {
let mut path = path.into_iter();
let Some(next) = path.next() else {
if let TranslationTree::Message(_) = self {
return;
return false;
}
*self = TranslationTree::Message(value);
return;
return true;
};
match self {
TranslationTree::Message(_) => panic!("cannot set a value on a message node"),
TranslationTree::Children(children) => {
children
.entry(next.deref().to_owned())
.or_default()
.set_if_not_defined(path, value);
}
TranslationTree::Children(children) => children
.entry(next.deref().to_owned())
.or_default()
.set_if_not_defined(path, value),
}
}

39
crates/i18n/src/translator.rs
View File

@@ -297,6 +297,45 @@ impl Translator {
let list = formatter.format_to_string(items);
Ok(list)
}
/// Get a list of available locales.
#[must_use]
pub fn available_locales(&self) -> Vec<&DataLocale> {
self.translations.keys().collect()
}
/// Check if a locale is available.
#[must_use]
pub fn has_locale(&self, locale: &DataLocale) -> bool {
self.translations.contains_key(locale)
}
/// Choose the best available locale from a list of candidates.
#[must_use]
pub fn choose_locale<'a>(
&self,
iter: impl Iterator<Item = &'a DataLocale>,
) -> Option<DataLocale> {
for locale in iter {
let mut fallbacker = self
.fallbacker
.for_config(LocaleFallbackConfig::default())
.fallback_for(locale.clone());
loop {
if fallbacker.get().is_und() {
break;
}
if self.has_locale(fallbacker.get()) {
return Some(fallbacker.take());
}
fallbacker.step();
}
}
None
}
}
#[cfg(test)]

1
crates/templates/Cargo.toml
View File

@@ -31,5 +31,6 @@ rand.workspace = true
oauth2-types = { path = "../oauth2-types" }
mas-data-model = { path = "../data-model" }
mas-i18n = { path = "../i18n" }
mas-router = { path = "../router" }
mas-spa = { path = "../spa" }

95
crates/templates/src/functions.rs
View File

@@ -19,10 +19,13 @@
use std::{
collections::{BTreeSet, HashMap},
fmt::Formatter,
str::FromStr,
sync::Arc,
};
use camino::Utf8Path;
use mas_i18n::{Argument, ArgumentList, DataLocale, Translator};
use mas_router::UrlBuilder;
use mas_spa::ViteManifest;
use minijinja::{
@@ -35,6 +38,7 @@ pub fn register(
env: &mut minijinja::Environment,
url_builder: UrlBuilder,
vite_manifest: ViteManifest,
translator: Translator,
) {
env.add_test("empty", self::tester_empty);
env.add_test("starting_with", tester_starting_with);
@@ -50,6 +54,12 @@ pub fn register(
vite_manifest,
}),
);
env.add_global(
"_",
Value::from_object(Translate {
translations: Arc::new(translator),
}),
);
}
fn tester_empty(seq: &dyn SeqObject) -> bool {
@@ -182,12 +192,95 @@ fn function_add_params_to_url(
Ok(uri.to_string())
}
#[derive(Debug)]
struct Translate {
translations: Arc<Translator>,
}
impl std::fmt::Debug for Translate {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Translate")
.field("translations", &"..")
.finish()
}
}
impl std::fmt::Display for Translate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("translate")
}
}
impl Object for Translate {
fn call(&self, state: &State, args: &[Value]) -> Result<Value, Error> {
let lang = state.lookup("lang").ok_or(minijinja::Error::new(
ErrorKind::UndefinedError,
"`lang` is not set",
))?;
let lang = lang.as_str().ok_or(minijinja::Error::new(
ErrorKind::InvalidOperation,
"`lang` is not a string",
))?;
let lang: DataLocale = lang.parse().map_err(|e| {
Error::new(ErrorKind::InvalidOperation, "Could not parse `lang`").with_source(e)
})?;
let (key, kwargs): (&str, Kwargs) = from_args(args)?;
let (message, _locale) = if let Some(count) = kwargs.get("count")? {
self.translations
.plural_with_fallback(lang, key, count)
.ok_or(Error::new(
ErrorKind::InvalidOperation,
"Missing translation",
))?
} else {
self.translations
.message_with_fallback(lang, key)
.ok_or(Error::new(
ErrorKind::InvalidOperation,
"Missing translation",
))?
};
let res: Result<ArgumentList, Error> = kwargs
.args()
.map(|name| {
let value: Value = kwargs.get(name)?;
let value = serde_json::to_value(value).map_err(|e| {
Error::new(ErrorKind::InvalidOperation, "Could not serialize argument")
.with_source(e)
})?;
Ok::<_, Error>(Argument::named(name.to_owned(), value))
})
.collect();
let list = res?;
let formatted = message.format(&list).map_err(|e| {
Error::new(ErrorKind::InvalidOperation, "Could not format message").with_source(e)
})?;
// TODO: escape
Ok(Value::from_safe_string(formatted))
}
}
struct IncludeAsset {
url_builder: UrlBuilder,
vite_manifest: ViteManifest,
}
impl std::fmt::Debug for IncludeAsset {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IncludeAsset")
.field("url_builder", &self.url_builder.assets_base())
.field("vite_manifest", &"..")
.finish()
}
}
impl std::fmt::Display for IncludeAsset {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("include_asset")

29
crates/templates/src/lib.rs
View File

@@ -29,6 +29,7 @@ use std::{collections::HashSet, sync::Arc};
use anyhow::Context as _;
use arc_swap::ArcSwap;
use camino::{Utf8Path, Utf8PathBuf};
use mas_i18n::Translator;
use mas_router::UrlBuilder;
use mas_spa::ViteManifest;
use rand::Rng;
@@ -72,6 +73,7 @@ pub struct Templates {
environment: Arc<ArcSwap<minijinja::Environment<'static>>>,
url_builder: UrlBuilder,
vite_manifest_path: Utf8PathBuf,
translations_path: Utf8PathBuf,
path: Utf8PathBuf,
}
@@ -90,6 +92,10 @@ pub enum TemplateLoadingError {
#[error("invalid assets manifest")]
ViteManifest(#[from] serde_json::Error),
/// Failed to load the translations
#[error("failed to load the translations")]
Translations(#[from] mas_i18n::LoadError),
/// Failed to traverse the filesystem
#[error("failed to traverse the filesystem")]
WalkDir(#[from] walkdir::Error),
@@ -143,13 +149,21 @@ impl Templates {
path: Utf8PathBuf,
url_builder: UrlBuilder,
vite_manifest_path: Utf8PathBuf,
translations_path: Utf8PathBuf,
) -> Result<Self, TemplateLoadingError> {
let environment = Self::load_(&path, url_builder.clone(), &vite_manifest_path).await?;
let environment = Self::load_(
&path,
url_builder.clone(),
&vite_manifest_path,
&translations_path,
)
.await?;
Ok(Self {
environment: Arc::new(ArcSwap::from_pointee(environment)),
path,
url_builder,
vite_manifest_path,
translations_path,
})
}
@@ -157,6 +171,7 @@ impl Templates {
path: &Utf8Path,
url_builder: UrlBuilder,
vite_manifest_path: &Utf8Path,
translations_path: &Utf8Path,
) -> Result<minijinja::Environment<'static>, TemplateLoadingError> {
let path = path.to_owned();
let span = tracing::Span::current();
@@ -170,6 +185,11 @@ impl Templates {
let vite_manifest: ViteManifest =
serde_json::from_slice(&vite_manifest).map_err(TemplateLoadingError::ViteManifest)?;
let translations_path = translations_path.to_owned();
let translator =
tokio::task::spawn_blocking(move || Translator::load_from_path(&translations_path))
.await??;
let (loaded, mut env) = tokio::task::spawn_blocking(move || {
span.in_scope(move || {
let mut loaded: HashSet<_> = HashSet::new();
@@ -203,7 +223,7 @@ impl Templates {
})
.await??;
self::functions::register(&mut env, url_builder, vite_manifest);
self::functions::register(&mut env, url_builder, vite_manifest, translator);
let needed: HashSet<_> = TEMPLATES.into_iter().map(ToOwned::to_owned).collect();
debug!(?loaded, ?needed, "Templates loaded");
@@ -228,6 +248,7 @@ impl Templates {
&self.path,
self.url_builder.clone(),
&self.vite_manifest_path,
&self.translations_path,
)
.await?;
@@ -373,7 +394,9 @@ mod tests {
let url_builder = UrlBuilder::new("https://example.com/".parse().unwrap(), None, None);
let vite_manifest_path =
Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist/manifest.json");
let templates = Templates::load(path, url_builder, vite_manifest_path)
let translations_path =
Utf8Path::new(env!("CARGO_MANIFEST_DIR")).join("../../translations");
let templates = Templates::load(path, url_builder, vite_manifest_path, translations_path)
.await
.unwrap();
templates.check_render(now, &mut rng).unwrap();

8
docs/config.schema.json
View File

@@ -199,7 +199,8 @@
"description": "Configuration related to templates",
"default": {
"assets_manifest": "./frontend/dist/manifest.json",
"path": "./templates/"
"path": "./templates/",
"translations_path": "./translations/"
},
"allOf": [
{
@@ -1918,6 +1919,11 @@
"description": "Path to the folder which holds the templates",
"default": "./templates/",
"type": "string"
},
"translations_path": {
"description": "Path to the translations",
"default": "./translations/",
"type": "string"
}
}
},

1
misc/update.sh
View File

@@ -11,6 +11,7 @@ POLICIES_SCHEMA="${BASE_DIR}/policies/schema/"
set -x
cargo run -p mas-config > "${CONFIG_SCHEMA}"
cargo run -p mas-graphql > "${GRAPHQL_SCHEMA}"
cargo run -p mas-i18n-scan -- --update "${BASE_DIR}/templates/" "${BASE_DIR}/translations/en.json"
OUT_DIR="${POLICIES_SCHEMA}" cargo run -p mas-policy --features jsonschema
cd "${BASE_DIR}/frontend"

4
templates/app.html
View File

@@ -15,13 +15,15 @@ limitations under the License.
#}
{# Must be kept in sync with frontend/index.html #}
{# TODO: detect the locale from the browser #}
{% set lang = "en-US" %}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>matrix-authentication-service</title>
<title>{{ _("app.name") }}</title>
<script>
window.APP_CONFIG = JSON.parse("{{ app_config | tojson | add_slashes | safe }}");
(function () {

4
templates/base.html
View File

@@ -22,12 +22,14 @@ limitations under the License.
{% import "components/errors.html" as errors %}
{% import "components/icon.html" as icon %}
{% import "components/scope.html" as scope %}
{# TODO: detect the locale from the browser #}
{% set lang = "en-US" %}
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>{% block title %}matrix-authentication-service{% endblock title %}</title>
<title>{% block title %}{{ _("app.name") }}{% endblock title %}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
{{ include_asset('src/templates.css', preload=true) | indent(4) | safe }}
</head>

2
templates/pages/index.html
View File

@@ -20,7 +20,7 @@ limitations under the License.
{{ navbar.top() }}
<section class="flex-1 flex flex-col items-center justify-center">
<div class="my-2 mx-8">
<h1 class="my-2 text-5xl font-semibold leading-tight">Matrix Authentication Service</h1>
<h1 class="my-2 text-5xl font-semibold leading-tight">{{ _("app.human_name") }}</h1>
<p class="text-lg">
OpenID Connect discovery document:
<a class="cpd-link" data-kind="primary" href="{{ discovery_url }}">{{ discovery_url }}</a>

6
translations/en.json Normal file
View File

@@ -0,0 +1,6 @@
{
"app": {
"human_name": "Matrix Authentication Service",
"name": "matrix-authentication-service"
}
}