From b44c3fc68f82f6ae81e69f7595c2059d8bbd3098 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Mon, 2 Oct 2023 15:28:02 +0200 Subject: [PATCH] i18n-scan: remove tera support & cleanup minijinja support --- Cargo.lock | 118 --------- Cargo.toml | 5 - crates/i18n-scan/Cargo.toml | 1 - crates/i18n-scan/src/key.rs | 64 +++-- crates/i18n-scan/src/main.rs | 67 +++-- crates/i18n-scan/src/minijinja.rs | 299 +++++++++++++++------- crates/i18n-scan/src/tera.rs | 400 ------------------------------ crates/templates/src/functions.rs | 15 -- 8 files changed, 279 insertions(+), 690 deletions(-) delete mode 100644 crates/i18n-scan/src/tera.rs diff --git a/Cargo.lock b/Cargo.lock index c1c82407..08dc7c04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -718,16 +718,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "bstr" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "bumpalo" version = "3.14.0" @@ -1881,30 +1871,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "globset" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" -dependencies = [ - "aho-corasick", - "bstr", - "fnv", - "log", - "regex", -] - -[[package]] -name = "globwalk" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" -dependencies = [ - "bitflags 1.3.2", - "ignore", - "walkdir", -] - [[package]] name = "gloo-timers" version = "0.2.6" @@ -2381,23 +2347,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "ignore" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" -dependencies = [ - "globset", - "lazy_static", - "log", - "memchr", - "regex", - "same-file", - "thread_local", - "walkdir", - "winapi-util", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -3033,7 +2982,6 @@ dependencies = [ "mas-i18n", "minijinja", "serde_json", - "tera", "tracing", "tracing-subscriber", "walkdir", @@ -5666,22 +5614,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tera" -version = "1.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "970dff17c11e884a4a09bc76e3a17ef71e01bb13447a11e85226e254fe6d10b8" -dependencies = [ - "globwalk", - "lazy_static", - "pest", - "pest_derive", - "regex", - "serde", - "serde_json", - "unic-segment", -] - [[package]] name = "thiserror" version = "1.0.48" @@ -6149,56 +6081,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "unic-char-property" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" -dependencies = [ - "unic-char-range", -] - -[[package]] -name = "unic-char-range" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" - -[[package]] -name = "unic-common" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" - -[[package]] -name = "unic-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" -dependencies = [ - "unic-ucd-segment", -] - -[[package]] -name = "unic-ucd-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-version" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" -dependencies = [ - "unic-common", -] - [[package]] name = "unicase" version = "2.7.0" diff --git a/Cargo.toml b/Cargo.toml index af30208d..4b18de08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,11 +53,6 @@ features = ["derive"] # Most of the time, if we need serde, we need derive [workspace.dependencies.serde_json] version = "1.0.107" -# Templates -[workspace.dependencies.tera] -version = "1.19.1" -default-features = false - # Custom error types [workspace.dependencies.thiserror] version = "1.0.48" diff --git a/crates/i18n-scan/Cargo.toml b/crates/i18n-scan/Cargo.toml index 8f22b773..cf8e74d4 100644 --- a/crates/i18n-scan/Cargo.toml +++ b/crates/i18n-scan/Cargo.toml @@ -12,7 +12,6 @@ camino.workspace = true clap.workspace = true minijinja = { workspace = true, features = ["unstable_machinery"] } serde_json.workspace = true -tera.workspace = true tracing-subscriber.workspace = true tracing.workspace = true walkdir = "2.4.0" diff --git a/crates/i18n-scan/src/key.rs b/crates/i18n-scan/src/key.rs index 5e7b8531..e75e9254 100644 --- a/crates/i18n-scan/src/key.rs +++ b/crates/i18n-scan/src/key.rs @@ -14,43 +14,65 @@ use mas_i18n::{translations::TranslationTree, Message}; +pub struct Context { + keys: Vec, + func: String, +} + +impl Context { + pub fn new(func: String) -> Self { + Self { + keys: Vec::new(), + func, + } + } + + pub fn record(&mut self, key: Key) { + self.keys.push(key); + } + + pub fn func(&self) -> &str { + &self.func + } + + pub fn add_missing(&self, translation_tree: &mut TranslationTree) { + for translatable in &self.keys { + let message = Message::from_literal(translatable.default_value()); + let key = translatable + .key + .split('.') + .chain(if translatable.kind == Kind::Plural { + Some("other") + } else { + None + }); + + translation_tree.set_if_not_defined(key, message); + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum KeyKind { +pub enum Kind { Message, Plural, } #[derive(Debug, Clone)] pub struct Key { - kind: KeyKind, + kind: Kind, key: String, } impl Key { - pub fn new(kind: KeyKind, key: String) -> Self { + pub fn new(kind: Kind, key: String) -> Self { Self { kind, key } } pub fn default_value(&self) -> String { match self.kind { - KeyKind::Message => self.key.clone(), - KeyKind::Plural => format!("%(count)d {}", self.key), + Kind::Message => self.key.clone(), + Kind::Plural => format!("%(count)d {}", self.key), } } } - -pub fn add_missing(translation_tree: &mut TranslationTree, keys: &[Key]) { - for translatable in keys { - let message = Message::from_literal(translatable.default_value()); - let key = translatable - .key - .split('.') - .chain(if translatable.kind == KeyKind::Plural { - Some("other") - } else { - None - }); - - translation_tree.set_if_not_defined(key, message); - } -} diff --git a/crates/i18n-scan/src/main.rs b/crates/i18n-scan/src/main.rs index 5eb0809a..d032873f 100644 --- a/crates/i18n-scan/src/main.rs +++ b/crates/i18n-scan/src/main.rs @@ -17,17 +17,13 @@ use std::fs::File; -use ::tera::Tera; use camino::Utf8PathBuf; use clap::Parser; -use key::add_missing; +use key::Context; use mas_i18n::translations::TranslationTree; -use crate::tera::find_keys; - mod key; mod minijinja; -mod tera; /// Scan a directory of templates for usage of the translation function and /// output a translation tree. @@ -39,12 +35,12 @@ struct Options { /// Path of the existing translation file existing: Option, - /// Whether to use minijinja instead of tera - #[clap(long)] - minijinja: bool, + /// The extensions of the templates + #[clap(long, default_value = "html,txt,subject")] + extensions: String, /// The name of the translation function - #[clap(long, default_value = "t")] + #[clap(long, default_value = "_")] function: String, } @@ -53,6 +49,7 @@ fn main() { let options = Options::parse(); + // Open the existing translation file if one was provided let mut tree = if let Some(path) = options.existing { let mut file = File::open(path).expect("Failed to open existing translation file"); serde_json::from_reader(&mut file).expect("Failed to parse existing translation file") @@ -60,36 +57,36 @@ fn main() { TranslationTree::default() }; - let keys = if options.minijinja { - let mut keys = Vec::new(); - for entry in walkdir::WalkDir::new(&options.templates) { - let entry = entry.unwrap(); - let filename = entry.file_name().to_str().expect("Invalid filename"); - if entry.file_type().is_file() - && (filename.ends_with(".html") - || filename.ends_with(".txt") - || filename.ends_with(".subject")) - { - let content = std::fs::read_to_string(entry.path()).unwrap(); - match minijinja::parse(&content, filename) { - Ok(ast) => { - keys.extend(minijinja::find_in_stmt(&ast).unwrap()); - } - Err(err) => { - tracing::error!("Failed to parse {}: {}", entry.path().display(), err); - } + let mut context = Context::new(options.function); + + for entry in walkdir::WalkDir::new(&options.templates) { + let entry = entry.unwrap(); + if !entry.file_type().is_file() { + continue; + } + + let path: Utf8PathBuf = entry.into_path().try_into().expect("Non-UTF8 path"); + let relative = path.strip_prefix(&options.templates).expect("Invalid path"); + + let Some(extension) = path.extension() else { + continue; + }; + + if options.extensions.split(',').any(|e| e == extension) { + tracing::debug!("Parsing {relative}"); + let template = std::fs::read_to_string(&path).expect("Failed to read template"); + match minijinja::parse(&template, relative.as_str()) { + Ok(ast) => { + minijinja::find_in_stmt(&mut context, &ast).unwrap(); + } + Err(err) => { + tracing::error!("Failed to parse {relative}: {}", err); } } } - keys - } else { - let glob = format!("{base}/**/*.{{html,txt,subject}}", base = options.templates); - tracing::debug!("Scanning templates in {}", glob); - let tera = Tera::new(&glob).expect("Failed to load templates"); + } - find_keys(&tera, &options.function).unwrap() - }; - add_missing(&mut tree, &keys); + context.add_missing(&mut tree); serde_json::to_writer_pretty(std::io::stdout(), &tree) .expect("Failed to write translation tree"); diff --git a/crates/i18n-scan/src/minijinja.rs b/crates/i18n-scan/src/minijinja.rs index b6d95ac6..e7c1cbd4 100644 --- a/crates/i18n-scan/src/minijinja.rs +++ b/crates/i18n-scan/src/minijinja.rs @@ -14,92 +14,88 @@ pub use minijinja::machinery::parse; use minijinja::{ - machinery::ast::{Call, Const, Expr, Stmt}, + machinery::ast::{Call, Const, Expr, Macro, Stmt}, ErrorKind, }; -use crate::key::{Key, KeyKind}; - -pub fn find_in_stmt<'a>(stmt: &'a Stmt<'a>) -> Result, minijinja::Error> { - let mut keys = Vec::new(); +use crate::key::{Context, Key}; +pub fn find_in_stmt<'a>(context: &mut Context, stmt: &'a Stmt<'a>) -> Result<(), minijinja::Error> { match stmt { - Stmt::Template(template) => keys.extend(find_in_stmts(&template.children)?), - Stmt::EmitExpr(emit_expr) => keys.extend(find_in_expr(&emit_expr.expr)?), + Stmt::Template(template) => find_in_stmts(context, &template.children)?, + Stmt::EmitExpr(emit_expr) => find_in_expr(context, &emit_expr.expr)?, Stmt::EmitRaw(_raw) => {} Stmt::ForLoop(for_loop) => { - keys.extend(find_in_expr(&for_loop.iter)?); - keys.extend(find_in_optional_expr(&for_loop.filter_expr)?); - keys.extend(find_in_expr(&for_loop.target)?); - keys.extend(find_in_stmts(&for_loop.body)?); - keys.extend(find_in_stmts(&for_loop.else_body)?); + find_in_expr(context, &for_loop.iter)?; + find_in_optional_expr(context, &for_loop.filter_expr)?; + find_in_expr(context, &for_loop.target)?; + find_in_stmts(context, &for_loop.body)?; + find_in_stmts(context, &for_loop.else_body)?; } Stmt::IfCond(if_cond) => { - keys.extend(find_in_expr(&if_cond.expr)?); - keys.extend(find_in_stmts(&if_cond.true_body)?); - keys.extend(find_in_stmts(&if_cond.false_body)?); + find_in_expr(context, &if_cond.expr)?; + find_in_stmts(context, &if_cond.true_body)?; + find_in_stmts(context, &if_cond.false_body)?; } Stmt::WithBlock(with_block) => { - keys.extend(find_in_stmts(&with_block.body)?); + find_in_stmts(context, &with_block.body)?; for (left, right) in &with_block.assignments { - keys.extend(find_in_expr(left)?); - keys.extend(find_in_expr(right)?); + find_in_expr(context, left)?; + find_in_expr(context, right)?; } } Stmt::Set(set) => { - keys.extend(find_in_expr(&set.target)?); - keys.extend(find_in_expr(&set.expr)?); + find_in_expr(context, &set.target)?; + find_in_expr(context, &set.expr)?; } Stmt::SetBlock(set_block) => { - keys.extend(find_in_expr(&set_block.target)?); - keys.extend(find_in_stmts(&set_block.body)?); + find_in_expr(context, &set_block.target)?; + find_in_stmts(context, &set_block.body)?; if let Some(expr) = &set_block.filter { - keys.extend(find_in_expr(expr)?); + find_in_expr(context, expr)?; } } Stmt::AutoEscape(auto_escape) => { - keys.extend(find_in_expr(&auto_escape.enabled)?); - keys.extend(find_in_stmts(&auto_escape.body)?); + find_in_expr(context, &auto_escape.enabled)?; + find_in_stmts(context, &auto_escape.body)?; } Stmt::FilterBlock(filter_block) => { - keys.extend(find_in_expr(&filter_block.filter)?); - keys.extend(find_in_stmts(&filter_block.body)?); + find_in_expr(context, &filter_block.filter)?; + find_in_stmts(context, &filter_block.body)?; } Stmt::Block(block) => { - keys.extend(find_in_stmts(&block.body)?); + find_in_stmts(context, &block.body)?; } Stmt::Import(import) => { - keys.extend(find_in_expr(&import.name)?); - keys.extend(find_in_expr(&import.expr)?); + find_in_expr(context, &import.name)?; + find_in_expr(context, &import.expr)?; } Stmt::FromImport(from_import) => { - keys.extend(find_in_expr(&from_import.expr)?); + find_in_expr(context, &from_import.expr)?; for (name, alias) in &from_import.names { - keys.extend(find_in_expr(name)?); - keys.extend(find_in_optional_expr(alias)?); + find_in_expr(context, name)?; + find_in_optional_expr(context, alias)?; } } Stmt::Extends(extends) => { - keys.extend(find_in_expr(&extends.name)?); + find_in_expr(context, &extends.name)?; } Stmt::Include(include) => { - keys.extend(find_in_expr(&include.name)?); + find_in_expr(context, &include.name)?; } Stmt::Macro(macro_) => { - keys.extend(find_in_stmts(¯o_.body)?); - keys.extend(find_in_exprs(¯o_.args)?); - keys.extend(find_in_exprs(¯o_.defaults)?); + find_in_macro(context, macro_)?; } Stmt::CallBlock(call_block) => { - keys.extend(find_in_call(&call_block.call)?); - // TODO: call_block.macro_decl + find_in_call(context, &call_block.call)?; + find_in_macro(context, &call_block.macro_decl)?; } Stmt::Do(do_) => { - keys.extend(find_in_call(&do_.call)?); + find_in_call(context, &do_.call)?; } } - Ok(keys) + Ok(()) } fn as_const<'a>(expr: &'a Expr<'a>) -> Option<&'a Const> { @@ -109,13 +105,17 @@ fn as_const<'a>(expr: &'a Expr<'a>) -> Option<&'a Const> { } } -fn find_in_call<'a>(call: &'a Call<'a>) -> Result, minijinja::Error> { - let mut keys = Vec::new(); +fn find_in_macro<'a>(context: &mut Context, macro_: &'a Macro<'a>) -> Result<(), minijinja::Error> { + find_in_stmts(context, ¯o_.body)?; + find_in_exprs(context, ¯o_.args)?; + find_in_exprs(context, ¯o_.defaults)?; + Ok(()) +} + +fn find_in_call<'a>(context: &mut Context, call: &'a Call<'a>) -> Result<(), minijinja::Error> { if let Expr::Var(var_) = &call.expr { - // TODO: pass the function name - if var_.id == "t" { - // TODO: don't unwrap + if var_.id == context.func() { let key = call .args .get(0) @@ -134,111 +134,220 @@ fn find_in_call<'a>(call: &'a Call<'a>) -> Result, minijinja::Error> { } }); - // TODO: detect plurals - keys.push(Key::new( + context.record(Key::new( if has_count { - KeyKind::Plural + crate::key::Kind::Plural } else { - KeyKind::Message + crate::key::Kind::Message }, key.to_owned(), )); } } - keys.extend(find_in_expr(&call.expr)?); + find_in_expr(context, &call.expr)?; for arg in &call.args { - keys.extend(find_in_expr(arg)?); + find_in_expr(context, arg)?; } - Ok(keys) + Ok(()) } -fn find_in_stmts<'a>(stmts: &'a [Stmt<'a>]) -> Result, minijinja::Error> { - let mut keys = Vec::new(); - +fn find_in_stmts<'a>(context: &mut Context, stmts: &'a [Stmt<'a>]) -> Result<(), minijinja::Error> { for stmt in stmts { - keys.extend(find_in_stmt(stmt)?); + find_in_stmt(context, stmt)?; } - Ok(keys) + Ok(()) } -fn find_in_expr<'a>(expr: &'a Expr<'a>) -> Result, minijinja::Error> { - let mut keys = Vec::new(); - +fn find_in_expr<'a>(context: &mut Context, expr: &'a Expr<'a>) -> Result<(), minijinja::Error> { match expr { Expr::Var(_var) => {} Expr::Const(_const) => {} Expr::Slice(slice) => { - keys.extend(find_in_expr(&slice.expr)?); - keys.extend(find_in_optional_expr(&slice.start)?); - keys.extend(find_in_optional_expr(&slice.stop)?); - keys.extend(find_in_optional_expr(&slice.step)?); + find_in_expr(context, &slice.expr)?; + find_in_optional_expr(context, &slice.start)?; + find_in_optional_expr(context, &slice.stop)?; + find_in_optional_expr(context, &slice.step)?; } Expr::UnaryOp(unary_op) => { - keys.extend(find_in_expr(&unary_op.expr)?); + find_in_expr(context, &unary_op.expr)?; } Expr::BinOp(bin_op) => { - keys.extend(find_in_expr(&bin_op.left)?); - keys.extend(find_in_expr(&bin_op.right)?); + find_in_expr(context, &bin_op.left)?; + find_in_expr(context, &bin_op.right)?; } Expr::IfExpr(if_expr) => { - keys.extend(find_in_expr(&if_expr.test_expr)?); - keys.extend(find_in_expr(&if_expr.true_expr)?); - keys.extend(find_in_optional_expr(&if_expr.false_expr)?); + find_in_expr(context, &if_expr.test_expr)?; + find_in_expr(context, &if_expr.true_expr)?; + find_in_optional_expr(context, &if_expr.false_expr)?; } Expr::Filter(filter) => { - keys.extend(find_in_optional_expr(&filter.expr)?); - keys.extend(find_in_exprs(&filter.args)?); + find_in_optional_expr(context, &filter.expr)?; + find_in_exprs(context, &filter.args)?; } Expr::Test(test) => { - keys.extend(find_in_expr(&test.expr)?); - keys.extend(find_in_exprs(&test.args)?); + find_in_expr(context, &test.expr)?; + find_in_exprs(context, &test.args)?; } Expr::GetAttr(get_attr) => { - keys.extend(find_in_expr(&get_attr.expr)?); + find_in_expr(context, &get_attr.expr)?; } Expr::GetItem(get_item) => { - keys.extend(find_in_expr(&get_item.expr)?); - keys.extend(find_in_expr(&get_item.subscript_expr)?); + find_in_expr(context, &get_item.expr)?; + find_in_expr(context, &get_item.subscript_expr)?; } Expr::Call(call) => { - keys.extend(find_in_call(call)?); + find_in_call(context, call)?; } Expr::List(list) => { - keys.extend(find_in_exprs(&list.items)?); + find_in_exprs(context, &list.items)?; } Expr::Map(map) => { - keys.extend(find_in_exprs(&map.keys)?); - keys.extend(find_in_exprs(&map.values)?); + find_in_exprs(context, &map.keys)?; + find_in_exprs(context, &map.values)?; } Expr::Kwargs(kwargs) => { for (_key, value) in &kwargs.pairs { - keys.extend(find_in_expr(value)?); + find_in_expr(context, value)?; } } } - Ok(keys) + Ok(()) } -fn find_in_exprs<'a>(exprs: &'a [Expr<'a>]) -> Result, minijinja::Error> { - let mut keys = Vec::new(); - +fn find_in_exprs<'a>(context: &mut Context, exprs: &'a [Expr<'a>]) -> Result<(), minijinja::Error> { for expr in exprs { - keys.extend(find_in_expr(expr)?); + find_in_expr(context, expr)?; } - Ok(keys) + Ok(()) } -fn find_in_optional_expr<'a>(expr: &'a Option>) -> Result, minijinja::Error> { - let mut keys = Vec::new(); - +fn find_in_optional_expr<'a>( + context: &mut Context, + expr: &'a Option>, +) -> Result<(), minijinja::Error> { if let Some(expr) = expr { - keys.extend(find_in_expr(expr)?); + find_in_expr(context, expr)?; } - Ok(keys) + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_keys() { + let mut context = Context::new("t".to_owned()); + let templates = [ + ("hello.txt", r#"Hello {{ t("world") }}"#), + ("existing.txt", r#"{{ t("hello") }}"#), + ("plural.txt", r#"{{ t("plural", count=4) }}"#), + // Kitchen sink to make sure we're going through the whole AST + ( + "macros.txt", + r#" + {% macro test(arg="foo") %} + {% if function() == foo is test(t("nested.1")) %} + {% set foo = t("nested.2", arg=5 + 2) ~ "foo" in test %} + {{ foo | bar }} + {% else %} + {% for i in [t("nested.3", extra=t("nested.4")), "foo"] %} + {{ i | foo }} + {% else %} + {{ t("nested.5") }} + {% endfor %} + {% endif %} + {% endmacro %} + "#, + ), + ( + "nested.txt", + r#" + {% import "macros.txt" as macros %} + {% block test %} + {% filter upper %} + {{ macros.test(arg=t("nested.6")) }} + {% endfilter %} + {% endblock test %} + "#, + ), + ]; + + for (name, content) in templates { + let ast = parse(content, name).unwrap(); + find_in_stmt(&mut context, &ast).unwrap(); + } + + let mut tree = serde_json::from_value(serde_json::json!({ + "hello": "Hello!", + })) + .unwrap(); + + context.add_missing(&mut tree); + let tree = serde_json::to_value(&tree).unwrap(); + assert_eq!( + tree, + serde_json::json!({ + "hello": "Hello!", + "world": "world", + "plural": { + "other": "%(count)d plural" + }, + "nested": { + "1": "nested.1", + "2": "nested.2", + "3": "nested.3", + "4": "nested.4", + "5": "nested.5", + "6": "nested.6", + }, + }) + ); + } + + #[test] + fn test_invalid_key_not_string() { + // This is invalid because the key is not a string + let mut context = Context::new("t".to_owned()); + let ast = parse(r#"{{ t(5) }}"#, "invalid.txt").unwrap(); + + let res = find_in_stmt(&mut context, &ast); + assert!(res.is_err()); + } + + #[test] + fn test_invalid_key_filtered() { + // This is invalid because the key argument has a filter + let mut context = Context::new("t".to_owned()); + let ast = parse(r#"{{ t("foo" | bar) }}"#, "invalid.txt").unwrap(); + + let res = find_in_stmt(&mut context, &ast); + assert!(res.is_err()); + } + + #[test] + fn test_invalid_key_missing() { + // This is invalid because the key argument is missing + let mut context = Context::new("t".to_owned()); + let ast = parse(r#"{{ t() }}"#, "invalid.txt").unwrap(); + + let res = find_in_stmt(&mut context, &ast); + assert!(res.is_err()); + } + + #[test] + fn test_invalid_key_negated() { + // This is invalid because the key argument is missing + let mut context = Context::new("t".to_owned()); + let ast = parse(r#"{{ t(not "foo") }}"#, "invalid.txt").unwrap(); + + let res = find_in_stmt(&mut context, &ast); + assert!(res.is_err()); + } } diff --git a/crates/i18n-scan/src/tera.rs b/crates/i18n-scan/src/tera.rs deleted file mode 100644 index 9d78f0ed..00000000 --- a/crates/i18n-scan/src/tera.rs +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2023 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use tera::{ - ast::{Block, Expr, ExprVal, FunctionCall, MacroDefinition, Node}, - Error, Template, Tera, -}; - -use crate::key::{Key, KeyKind}; - -/// Find all translatable strings in a Tera instance. -/// -/// This is not particularly efficient in terms of allocations, but as it is -/// only meant to be used in an utility, it should be fine. -/// -/// # Parameters -/// -/// * `tera` - The Tera instance to scan. -/// * `function_name` - The name of the translation function. Usually `t`. -/// -/// # Errors -/// -/// This function will return an error if it encounters an invalid template. -pub fn find_keys(tera: &Tera, function_name: &str) -> Result, tera::Error> { - let names = tera.get_template_names(); - let mut keys = Vec::new(); - - for name in names { - tracing::trace!("Scanning {}", name); - // This should never fail, but who knows. - let template = tera.get_template(name)?; - keys.extend(find_in_template(template, function_name)?); - } - - Ok(keys) -} - -fn find_in_template(template: &Template, function_name: &str) -> Result, tera::Error> { - let mut keys = Vec::new(); - - for node in &template.ast { - keys.extend(find_in_node(node, function_name)?); - } - - for block in template.blocks.values() { - keys.extend(find_in_block(block, function_name)?); - } - - for block_definition in template.blocks_definitions.values() { - for (_, block) in block_definition { - keys.extend(find_in_block(block, function_name)?); - } - } - - for macro_definition in template.macros.values() { - keys.extend(find_in_macro_definition(macro_definition, function_name)?); - } - - Ok(keys) -} - -fn find_in_block(block: &Block, function_name: &str) -> Result, tera::Error> { - let mut keys = Vec::new(); - - for node in &block.body { - keys.extend(find_in_node(node, function_name)?); - } - - Ok(keys) -} - -fn find_in_node(node: &Node, function_name: &str) -> Result, tera::Error> { - let mut keys = Vec::new(); - - match node { - Node::VariableBlock(_, expr) => keys.extend(find_in_expr(expr, function_name)?), - - Node::MacroDefinition(_, definition, _) => { - keys.extend(find_in_macro_definition(definition, function_name)?); - } - - Node::Set(_, set) => keys.extend(find_in_expr(&set.value, function_name)?), - - Node::FilterSection(_, filter_section, _) => { - keys.extend(find_in_function_call( - &filter_section.filter, - function_name, - )?); - - for node in &filter_section.body { - keys.extend(find_in_node(node, function_name)?); - } - } - - Node::Block(_, block, _) => keys.extend(find_in_block(block, function_name)?), - - Node::Forloop(_, for_loop, _) => { - keys.extend(find_in_expr(&for_loop.container, function_name)?); - - for node in &for_loop.body { - keys.extend(find_in_node(node, function_name)?); - } - - if let Some(empty_body) = &for_loop.empty_body { - for node in empty_body { - keys.extend(find_in_node(node, function_name)?); - } - } - } - Node::If(if_block, _) => { - for (_ws, condition, expr) in &if_block.conditions { - keys.extend(find_in_expr(condition, function_name)?); - - for node in expr { - keys.extend(find_in_node(node, function_name)?); - } - } - - if let Some((_ws, expr)) = &if_block.otherwise { - for node in expr { - keys.extend(find_in_node(node, function_name)?); - } - } - } - - Node::Super - | Node::Text(_) - | Node::Extends(_, _) - | Node::Include(_, _, _) - | Node::ImportMacro(_, _, _) - | Node::Raw(_, _, _) - | Node::Break(_) - | Node::Continue(_) - | Node::Comment(_, _) => {} - }; - - Ok(keys) -} - -fn find_in_macro_definition( - definition: &MacroDefinition, - function_name: &str, -) -> Result, Error> { - let mut keys = Vec::new(); - - // Walk through argument defaults - for expr in definition.args.values().flatten() { - keys.extend(find_in_expr(expr, function_name)?); - } - - // Walk through the macro body - for node in &definition.body { - keys.extend(find_in_node(node, function_name)?); - } - - Ok(keys) -} - -fn find_in_expr_val(expr_val: &ExprVal, function_name: &str) -> Result, tera::Error> { - let mut keys = Vec::new(); - - match expr_val { - ExprVal::String(_) - | ExprVal::Int(_) - | ExprVal::Float(_) - | ExprVal::Bool(_) - | ExprVal::Ident(_) => {} - - ExprVal::Math(math_expr) => { - keys.extend(find_in_expr(&math_expr.lhs, function_name)?); - keys.extend(find_in_expr(&math_expr.rhs, function_name)?); - } - - ExprVal::Logic(logic_expr) => { - keys.extend(find_in_expr(&logic_expr.lhs, function_name)?); - keys.extend(find_in_expr(&logic_expr.rhs, function_name)?); - } - - ExprVal::Test(test_expr) => { - for arg in &test_expr.args { - keys.extend(find_in_expr(arg, function_name)?); - } - } - - ExprVal::MacroCall(macro_call) => { - for arg in macro_call.args.values() { - keys.extend(find_in_expr(arg, function_name)?); - } - } - - ExprVal::FunctionCall(function_call) => { - keys.extend(find_in_function_call(function_call, function_name)?); - } - - ExprVal::Array(array) => { - for expr in array { - keys.extend(find_in_expr(expr, function_name)?); - } - } - - ExprVal::StringConcat(string_concat) => { - for value in &string_concat.values { - keys.extend(find_in_expr_val(value, function_name)?); - } - } - - ExprVal::In(in_expr) => { - keys.extend(find_in_expr(&in_expr.lhs, function_name)?); - keys.extend(find_in_expr(&in_expr.rhs, function_name)?); - } - } - - Ok(keys) -} - -fn find_in_expr(expr: &Expr, function_name: &str) -> Result, tera::Error> { - let mut keys = Vec::new(); - - keys.extend(find_in_expr_val(&expr.val, function_name)?); - - for filter in &expr.filters { - keys.extend(find_in_function_call(filter, function_name)?); - } - - Ok(keys) -} - -fn find_in_function_call( - function_call: &FunctionCall, - function_name: &str, -) -> Result, tera::Error> { - tracing::trace!("Checking function call: {:?}", function_call); - let mut keys = Vec::new(); - - // Regardless of if it is the function we are looking for, we still need to - // check the arguments - for expr in function_call.args.values() { - keys.extend(find_in_expr(expr, function_name)?); - } - - // If it is the function we are looking for, we need to extract the key - if function_call.name == function_name { - let key = function_call - .args - .get("key") - .ok_or(tera::Error::msg("Missing key argument"))?; - if !key.filters.is_empty() { - return Err(tera::Error::msg("Key argument must not have filters")); - } - - if key.negated { - return Err(tera::Error::msg("Key argument must not be negated")); - } - - let key = match &key.val { - tera::ast::ExprVal::String(s) => s.clone(), - _ => return Err(tera::Error::msg("Key argument must be a string")), - }; - - let kind = if function_call.args.contains_key("count") { - KeyKind::Plural - } else { - KeyKind::Message - }; - - keys.push(Key::new(kind, key)) - } - - Ok(keys) -} - -#[cfg(test)] -mod tests { - use tera::Tera; - - use super::*; - use crate::key::add_missing; - - #[test] - fn test_find_keys() { - let mut tera = Tera::default(); - tera.add_raw_templates([ - ("hello.txt", r#"Hello {{ t(key="world") }}"#), - ("existing.txt", r#"{{ t(key="hello") }}"#), - ("plural.txt", r#"{{ t(key="plural", count=4) }}"#), - // Kitchen sink to make sure we're going through the whole AST - ( - "macros.txt", - r#" - {% macro test(arg="foo") %} - {% if function() == foo is test(t(key="nested.1")) %} - {% set foo = t(key="nested.2", arg=5 + 2) ~ "foo" in test %} - {{ foo | bar }} - {% else %} - {% for i in [t(key="nested.3", extra=t(key="nested.4")), "foo"] %} - {{ i | foo }} - {% else %} - {{ t(key="nested.5") }} - {% endfor %} - {% endif %} - {% endmacro %} - "#, - ), - ( - "nested.txt", - r#" - {% import "macros.txt" as macros %} - {% block test %} - {% filter upper %} - {{ macros::test(arg=t(key="nested.6")) }} - {% endfilter %} - {% endblock test %} - "#, - ), - ]) - .unwrap(); - - let mut tree = serde_json::from_value(serde_json::json!({ - "hello": "Hello!", - })) - .unwrap(); - - let keys = find_keys(&tera, "t").unwrap(); - add_missing(&mut tree, &keys); - let tree = serde_json::to_value(&tree).unwrap(); - assert_eq!( - tree, - serde_json::json!({ - "hello": "Hello!", - "world": "world", - "plural": { - "other": "%(count)d plural" - }, - "nested": { - "1": "nested.1", - "2": "nested.2", - "3": "nested.3", - "4": "nested.4", - "5": "nested.5", - "6": "nested.6", - }, - }) - ); - } - - #[test] - fn test_invalid_key_not_string() { - let mut tera = Tera::default(); - // This is invalid because the key is not a string - tera.add_raw_template("invalid.txt", r#"{{ t(key=5) }}"#) - .unwrap(); - - let keys = find_keys(&tera, "t"); - assert!(keys.is_err()); - } - - #[test] - fn test_invalid_key_filtered() { - let mut tera = Tera::default(); - // This is invalid because the key argument has a filter - tera.add_raw_template("invalid.txt", r#"{{ t(key="foo" | bar) }}"#) - .unwrap(); - - let keys = find_keys(&tera, "t"); - assert!(keys.is_err()); - } - - #[test] - fn test_invalid_key_missing() { - let mut tera = Tera::default(); - // This is invalid because the key argument is missing - tera.add_raw_template("invalid.txt", r#"{{ t() }}"#) - .unwrap(); - - let keys = find_keys(&tera, "t"); - assert!(keys.is_err()); - } - - #[test] - fn test_invalid_key_negated() { - let mut tera = Tera::default(); - // This is invalid because the key argument is missing - tera.add_raw_template("invalid.txt", r#"{{ t(key=not "foo") }}"#) - .unwrap(); - - let keys = find_keys(&tera, "t"); - assert!(keys.is_err()); - } -} diff --git a/crates/templates/src/functions.rs b/crates/templates/src/functions.rs index 895f12eb..55cc8395 100644 --- a/crates/templates/src/functions.rs +++ b/crates/templates/src/functions.rs @@ -43,7 +43,6 @@ pub fn register( env.add_filter("add_slashes", filter_add_slashes); env.add_filter("split", filter_split); env.add_function("add_params_to_url", function_add_params_to_url); - //tera.register_function("merge", function_merge); env.add_global( "include_asset", Value::from_object(IncludeAsset { @@ -183,20 +182,6 @@ fn function_add_params_to_url( Ok(uri.to_string()) } -/* -fn function_merge(params: &HashMap) -> Result { - let mut ret = serde_json::Map::new(); - for (k, v) in params { - let v = v - .as_object() - .ok_or_else(|| tera::Error::msg(format!("Parameter {k:?} should be an object")))?; - ret.extend(v.clone()); - } - - Ok(Value::Object(ret)) -} - */ - #[derive(Debug)] struct IncludeAsset { url_builder: UrlBuilder,