i18n-scan: remove tera support & cleanup minijinja support

Cargo.lock

@@ -718,16 +718,6 @@ dependencies = [
  "cipher",
 ]
 
-[[package]]
-name = "bstr"
-version = "1.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a"
-dependencies = [
- "memchr",
- "serde",
-]
-
 [[package]]
 name = "bumpalo"
 version = "3.14.0"
@@ -1881,30 +1871,6 @@ dependencies = [
  "stable_deref_trait",
 ]
 
-[[package]]
-name = "globset"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d"
-dependencies = [
- "aho-corasick",
- "bstr",
- "fnv",
- "log",
- "regex",
-]
-
-[[package]]
-name = "globwalk"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
-dependencies = [
- "bitflags 1.3.2",
- "ignore",
- "walkdir",
-]
-
 [[package]]
 name = "gloo-timers"
 version = "0.2.6"
@@ -2381,23 +2347,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "ignore"
-version = "0.4.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492"
-dependencies = [
- "globset",
- "lazy_static",
- "log",
- "memchr",
- "regex",
- "same-file",
- "thread_local",
- "walkdir",
- "winapi-util",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -3033,7 +2982,6 @@ dependencies = [
  "mas-i18n",
  "minijinja",
  "serde_json",
- "tera",
  "tracing",
  "tracing-subscriber",
  "walkdir",
@@ -5666,22 +5614,6 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "tera"
-version = "1.19.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "970dff17c11e884a4a09bc76e3a17ef71e01bb13447a11e85226e254fe6d10b8"
-dependencies = [
- "globwalk",
- "lazy_static",
- "pest",
- "pest_derive",
- "regex",
- "serde",
- "serde_json",
- "unic-segment",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.48"
@@ -6149,56 +6081,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "unic-char-property"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
-dependencies = [
- "unic-char-range",
-]
-
-[[package]]
-name = "unic-char-range"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
-
-[[package]]
-name = "unic-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
-
-[[package]]
-name = "unic-segment"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23"
-dependencies = [
- "unic-ucd-segment",
-]
-
-[[package]]
-name = "unic-ucd-segment"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-version"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
-dependencies = [
- "unic-common",
-]
-
 [[package]]
 name = "unicase"
 version = "2.7.0"

Cargo.toml

@@ -53,11 +53,6 @@ features = ["derive"] # Most of the time, if we need serde, we need derive
 [workspace.dependencies.serde_json]
 version = "1.0.107"
 
-# Templates
-[workspace.dependencies.tera]
-version = "1.19.1"
-default-features = false
-
 # Custom error types
 [workspace.dependencies.thiserror]
 version = "1.0.48"

crates/i18n-scan/Cargo.toml

@@ -12,7 +12,6 @@ camino.workspace = true
 clap.workspace = true
 minijinja = { workspace = true, features = ["unstable_machinery"] }
 serde_json.workspace = true
-tera.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
 walkdir = "2.4.0"

crates/i18n-scan/src/key.rs

@@ -14,43 +14,65 @@
 
 use mas_i18n::{translations::TranslationTree, Message};
 
+pub struct Context {
+    keys: Vec<Key>,
+    func: String,
+}
+
+impl Context {
+    pub fn new(func: String) -> Self {
+        Self {
+            keys: Vec::new(),
+            func,
+        }
+    }
+
+    pub fn record(&mut self, key: Key) {
+        self.keys.push(key);
+    }
+
+    pub fn func(&self) -> &str {
+        &self.func
+    }
+
+    pub fn add_missing(&self, translation_tree: &mut TranslationTree) {
+        for translatable in &self.keys {
+            let message = Message::from_literal(translatable.default_value());
+            let key = translatable
+                .key
+                .split('.')
+                .chain(if translatable.kind == Kind::Plural {
+                    Some("other")
+                } else {
+                    None
+                });
+
+            translation_tree.set_if_not_defined(key, message);
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum KeyKind {
+pub enum Kind {
     Message,
     Plural,
 }
 
 #[derive(Debug, Clone)]
 pub struct Key {
-    kind: KeyKind,
+    kind: Kind,
     key: String,
 }
 
 impl Key {
-    pub fn new(kind: KeyKind, key: String) -> Self {
+    pub fn new(kind: Kind, key: String) -> Self {
         Self { kind, key }
     }
 
     pub fn default_value(&self) -> String {
         match self.kind {
-            KeyKind::Message => self.key.clone(),
+            Kind::Message => self.key.clone(),
-            KeyKind::Plural => format!("%(count)d {}", self.key),
+            Kind::Plural => format!("%(count)d {}", self.key),
         }
     }
 }
-
-pub fn add_missing(translation_tree: &mut TranslationTree, keys: &[Key]) {
-    for translatable in keys {
-        let message = Message::from_literal(translatable.default_value());
-        let key = translatable
-            .key
-            .split('.')
-            .chain(if translatable.kind == KeyKind::Plural {
-                Some("other")
-            } else {
-                None
-            });
-
-        translation_tree.set_if_not_defined(key, message);
-    }
-}

crates/i18n-scan/src/main.rs

@@ -17,17 +17,13 @@
 
 use std::fs::File;
 
-use ::tera::Tera;
 use camino::Utf8PathBuf;
 use clap::Parser;
-use key::add_missing;
+use key::Context;
 use mas_i18n::translations::TranslationTree;
 
-use crate::tera::find_keys;
-
 mod key;
 mod minijinja;
-mod tera;
 
 /// Scan a directory of templates for usage of the translation function and
 /// output a translation tree.
@@ -39,12 +35,12 @@ struct Options {
     /// Path of the existing translation file
     existing: Option<Utf8PathBuf>,
 
-    /// Whether to use minijinja instead of tera
+    /// The extensions of the templates
-    #[clap(long)]
+    #[clap(long, default_value = "html,txt,subject")]
-    minijinja: bool,
+    extensions: String,
 
     /// The name of the translation function
-    #[clap(long, default_value = "t")]
+    #[clap(long, default_value = "_")]
     function: String,
 }
 
@@ -53,6 +49,7 @@ fn main() {
 
     let options = Options::parse();
 
+    // Open the existing translation file if one was provided
     let mut tree = if let Some(path) = options.existing {
         let mut file = File::open(path).expect("Failed to open existing translation file");
         serde_json::from_reader(&mut file).expect("Failed to parse existing translation file")
@@ -60,36 +57,36 @@ fn main() {
         TranslationTree::default()
     };
 
-    let keys = if options.minijinja {
+    let mut context = Context::new(options.function);
-        let mut keys = Vec::new();
+
-        for entry in walkdir::WalkDir::new(&options.templates) {
+    for entry in walkdir::WalkDir::new(&options.templates) {
-            let entry = entry.unwrap();
+        let entry = entry.unwrap();
-            let filename = entry.file_name().to_str().expect("Invalid filename");
+        if !entry.file_type().is_file() {
-            if entry.file_type().is_file()
+            continue;
-                && (filename.ends_with(".html")
+        }
-                    || filename.ends_with(".txt")
+
-                    || filename.ends_with(".subject"))
+        let path: Utf8PathBuf = entry.into_path().try_into().expect("Non-UTF8 path");
-            {
+        let relative = path.strip_prefix(&options.templates).expect("Invalid path");
-                let content = std::fs::read_to_string(entry.path()).unwrap();
+
-                match minijinja::parse(&content, filename) {
+        let Some(extension) = path.extension() else {
-                    Ok(ast) => {
+            continue;
-                        keys.extend(minijinja::find_in_stmt(&ast).unwrap());
+        };
-                    }
+
-                    Err(err) => {
+        if options.extensions.split(',').any(|e| e == extension) {
-                        tracing::error!("Failed to parse {}: {}", entry.path().display(), err);
+            tracing::debug!("Parsing {relative}");
-                    }
+            let template = std::fs::read_to_string(&path).expect("Failed to read template");
+            match minijinja::parse(&template, relative.as_str()) {
+                Ok(ast) => {
+                    minijinja::find_in_stmt(&mut context, &ast).unwrap();
+                }
+                Err(err) => {
+                    tracing::error!("Failed to parse {relative}: {}", err);
                 }
             }
         }
-        keys
+    }
-    } else {
-        let glob = format!("{base}/**/*.{{html,txt,subject}}", base = options.templates);
-        tracing::debug!("Scanning templates in {}", glob);
-        let tera = Tera::new(&glob).expect("Failed to load templates");
 
-        find_keys(&tera, &options.function).unwrap()
+    context.add_missing(&mut tree);
-    };
-    add_missing(&mut tree, &keys);
 
     serde_json::to_writer_pretty(std::io::stdout(), &tree)
         .expect("Failed to write translation tree");

crates/i18n-scan/src/minijinja.rs

@@ -14,92 +14,88 @@
 
 pub use minijinja::machinery::parse;
 use minijinja::{
-    machinery::ast::{Call, Const, Expr, Stmt},
+    machinery::ast::{Call, Const, Expr, Macro, Stmt},
     ErrorKind,
 };
 
-use crate::key::{Key, KeyKind};
+use crate::key::{Context, Key};
-
-pub fn find_in_stmt<'a>(stmt: &'a Stmt<'a>) -> Result<Vec<Key>, minijinja::Error> {
-    let mut keys = Vec::new();
 
+pub fn find_in_stmt<'a>(context: &mut Context, stmt: &'a Stmt<'a>) -> Result<(), minijinja::Error> {
     match stmt {
-        Stmt::Template(template) => keys.extend(find_in_stmts(&template.children)?),
+        Stmt::Template(template) => find_in_stmts(context, &template.children)?,
-        Stmt::EmitExpr(emit_expr) => keys.extend(find_in_expr(&emit_expr.expr)?),
+        Stmt::EmitExpr(emit_expr) => find_in_expr(context, &emit_expr.expr)?,
         Stmt::EmitRaw(_raw) => {}
         Stmt::ForLoop(for_loop) => {
-            keys.extend(find_in_expr(&for_loop.iter)?);
+            find_in_expr(context, &for_loop.iter)?;
-            keys.extend(find_in_optional_expr(&for_loop.filter_expr)?);
+            find_in_optional_expr(context, &for_loop.filter_expr)?;
-            keys.extend(find_in_expr(&for_loop.target)?);
+            find_in_expr(context, &for_loop.target)?;
-            keys.extend(find_in_stmts(&for_loop.body)?);
+            find_in_stmts(context, &for_loop.body)?;
-            keys.extend(find_in_stmts(&for_loop.else_body)?);
+            find_in_stmts(context, &for_loop.else_body)?;
         }
         Stmt::IfCond(if_cond) => {
-            keys.extend(find_in_expr(&if_cond.expr)?);
+            find_in_expr(context, &if_cond.expr)?;
-            keys.extend(find_in_stmts(&if_cond.true_body)?);
+            find_in_stmts(context, &if_cond.true_body)?;
-            keys.extend(find_in_stmts(&if_cond.false_body)?);
+            find_in_stmts(context, &if_cond.false_body)?;
         }
         Stmt::WithBlock(with_block) => {
-            keys.extend(find_in_stmts(&with_block.body)?);
+            find_in_stmts(context, &with_block.body)?;
             for (left, right) in &with_block.assignments {
-                keys.extend(find_in_expr(left)?);
+                find_in_expr(context, left)?;
-                keys.extend(find_in_expr(right)?);
+                find_in_expr(context, right)?;
             }
         }
         Stmt::Set(set) => {
-            keys.extend(find_in_expr(&set.target)?);
+            find_in_expr(context, &set.target)?;
-            keys.extend(find_in_expr(&set.expr)?);
+            find_in_expr(context, &set.expr)?;
         }
         Stmt::SetBlock(set_block) => {
-            keys.extend(find_in_expr(&set_block.target)?);
+            find_in_expr(context, &set_block.target)?;
-            keys.extend(find_in_stmts(&set_block.body)?);
+            find_in_stmts(context, &set_block.body)?;
             if let Some(expr) = &set_block.filter {
-                keys.extend(find_in_expr(expr)?);
+                find_in_expr(context, expr)?;
             }
         }
         Stmt::AutoEscape(auto_escape) => {
-            keys.extend(find_in_expr(&auto_escape.enabled)?);
+            find_in_expr(context, &auto_escape.enabled)?;
-            keys.extend(find_in_stmts(&auto_escape.body)?);
+            find_in_stmts(context, &auto_escape.body)?;
         }
         Stmt::FilterBlock(filter_block) => {
-            keys.extend(find_in_expr(&filter_block.filter)?);
+            find_in_expr(context, &filter_block.filter)?;
-            keys.extend(find_in_stmts(&filter_block.body)?);
+            find_in_stmts(context, &filter_block.body)?;
         }
         Stmt::Block(block) => {
-            keys.extend(find_in_stmts(&block.body)?);
+            find_in_stmts(context, &block.body)?;
         }
         Stmt::Import(import) => {
-            keys.extend(find_in_expr(&import.name)?);
+            find_in_expr(context, &import.name)?;
-            keys.extend(find_in_expr(&import.expr)?);
+            find_in_expr(context, &import.expr)?;
         }
         Stmt::FromImport(from_import) => {
-            keys.extend(find_in_expr(&from_import.expr)?);
+            find_in_expr(context, &from_import.expr)?;
             for (name, alias) in &from_import.names {
-                keys.extend(find_in_expr(name)?);
+                find_in_expr(context, name)?;
-                keys.extend(find_in_optional_expr(alias)?);
+                find_in_optional_expr(context, alias)?;
             }
         }
         Stmt::Extends(extends) => {
-            keys.extend(find_in_expr(&extends.name)?);
+            find_in_expr(context, &extends.name)?;
         }
         Stmt::Include(include) => {
-            keys.extend(find_in_expr(&include.name)?);
+            find_in_expr(context, &include.name)?;
         }
         Stmt::Macro(macro_) => {
-            keys.extend(find_in_stmts(&macro_.body)?);
+            find_in_macro(context, macro_)?;
-            keys.extend(find_in_exprs(&macro_.args)?);
-            keys.extend(find_in_exprs(&macro_.defaults)?);
         }
         Stmt::CallBlock(call_block) => {
-            keys.extend(find_in_call(&call_block.call)?);
+            find_in_call(context, &call_block.call)?;
-            // TODO: call_block.macro_decl
+            find_in_macro(context, &call_block.macro_decl)?;
         }
         Stmt::Do(do_) => {
-            keys.extend(find_in_call(&do_.call)?);
+            find_in_call(context, &do_.call)?;
         }
     }
 
-    Ok(keys)
+    Ok(())
 }
 
 fn as_const<'a>(expr: &'a Expr<'a>) -> Option<&'a Const> {
@@ -109,13 +105,17 @@ fn as_const<'a>(expr: &'a Expr<'a>) -> Option<&'a Const> {
     }
 }
 
-fn find_in_call<'a>(call: &'a Call<'a>) -> Result<Vec<Key>, minijinja::Error> {
+fn find_in_macro<'a>(context: &mut Context, macro_: &'a Macro<'a>) -> Result<(), minijinja::Error> {
-    let mut keys = Vec::new();
+    find_in_stmts(context, &macro_.body)?;
+    find_in_exprs(context, &macro_.args)?;
+    find_in_exprs(context, &macro_.defaults)?;
 
+    Ok(())
+}
+
+fn find_in_call<'a>(context: &mut Context, call: &'a Call<'a>) -> Result<(), minijinja::Error> {
     if let Expr::Var(var_) = &call.expr {
-        // TODO: pass the function name
+        if var_.id == context.func() {
-        if var_.id == "t" {
-            // TODO: don't unwrap
             let key = call
                 .args
                 .get(0)
@@ -134,111 +134,220 @@ fn find_in_call<'a>(call: &'a Call<'a>) -> Result<Vec<Key>, minijinja::Error> {
                 }
             });
 
-            // TODO: detect plurals
+            context.record(Key::new(
-            keys.push(Key::new(
                 if has_count {
-                    KeyKind::Plural
+                    crate::key::Kind::Plural
                 } else {
-                    KeyKind::Message
+                    crate::key::Kind::Message
                 },
                 key.to_owned(),
             ));
         }
     }
 
-    keys.extend(find_in_expr(&call.expr)?);
+    find_in_expr(context, &call.expr)?;
     for arg in &call.args {
-        keys.extend(find_in_expr(arg)?);
+        find_in_expr(context, arg)?;
     }
 
-    Ok(keys)
+    Ok(())
 }
 
-fn find_in_stmts<'a>(stmts: &'a [Stmt<'a>]) -> Result<Vec<Key>, minijinja::Error> {
+fn find_in_stmts<'a>(context: &mut Context, stmts: &'a [Stmt<'a>]) -> Result<(), minijinja::Error> {
-    let mut keys = Vec::new();
-
     for stmt in stmts {
-        keys.extend(find_in_stmt(stmt)?);
+        find_in_stmt(context, stmt)?;
     }
 
-    Ok(keys)
+    Ok(())
 }
 
-fn find_in_expr<'a>(expr: &'a Expr<'a>) -> Result<Vec<Key>, minijinja::Error> {
+fn find_in_expr<'a>(context: &mut Context, expr: &'a Expr<'a>) -> Result<(), minijinja::Error> {
-    let mut keys = Vec::new();
-
     match expr {
         Expr::Var(_var) => {}
         Expr::Const(_const) => {}
         Expr::Slice(slice) => {
-            keys.extend(find_in_expr(&slice.expr)?);
+            find_in_expr(context, &slice.expr)?;
-            keys.extend(find_in_optional_expr(&slice.start)?);
+            find_in_optional_expr(context, &slice.start)?;
-            keys.extend(find_in_optional_expr(&slice.stop)?);
+            find_in_optional_expr(context, &slice.stop)?;
-            keys.extend(find_in_optional_expr(&slice.step)?);
+            find_in_optional_expr(context, &slice.step)?;
         }
         Expr::UnaryOp(unary_op) => {
-            keys.extend(find_in_expr(&unary_op.expr)?);
+            find_in_expr(context, &unary_op.expr)?;
         }
         Expr::BinOp(bin_op) => {
-            keys.extend(find_in_expr(&bin_op.left)?);
+            find_in_expr(context, &bin_op.left)?;
-            keys.extend(find_in_expr(&bin_op.right)?);
+            find_in_expr(context, &bin_op.right)?;
         }
         Expr::IfExpr(if_expr) => {
-            keys.extend(find_in_expr(&if_expr.test_expr)?);
+            find_in_expr(context, &if_expr.test_expr)?;
-            keys.extend(find_in_expr(&if_expr.true_expr)?);
+            find_in_expr(context, &if_expr.true_expr)?;
-            keys.extend(find_in_optional_expr(&if_expr.false_expr)?);
+            find_in_optional_expr(context, &if_expr.false_expr)?;
         }
         Expr::Filter(filter) => {
-            keys.extend(find_in_optional_expr(&filter.expr)?);
+            find_in_optional_expr(context, &filter.expr)?;
-            keys.extend(find_in_exprs(&filter.args)?);
+            find_in_exprs(context, &filter.args)?;
         }
         Expr::Test(test) => {
-            keys.extend(find_in_expr(&test.expr)?);
+            find_in_expr(context, &test.expr)?;
-            keys.extend(find_in_exprs(&test.args)?);
+            find_in_exprs(context, &test.args)?;
         }
         Expr::GetAttr(get_attr) => {
-            keys.extend(find_in_expr(&get_attr.expr)?);
+            find_in_expr(context, &get_attr.expr)?;
         }
         Expr::GetItem(get_item) => {
-            keys.extend(find_in_expr(&get_item.expr)?);
+            find_in_expr(context, &get_item.expr)?;
-            keys.extend(find_in_expr(&get_item.subscript_expr)?);
+            find_in_expr(context, &get_item.subscript_expr)?;
         }
         Expr::Call(call) => {
-            keys.extend(find_in_call(call)?);
+            find_in_call(context, call)?;
         }
         Expr::List(list) => {
-            keys.extend(find_in_exprs(&list.items)?);
+            find_in_exprs(context, &list.items)?;
         }
         Expr::Map(map) => {
-            keys.extend(find_in_exprs(&map.keys)?);
+            find_in_exprs(context, &map.keys)?;
-            keys.extend(find_in_exprs(&map.values)?);
+            find_in_exprs(context, &map.values)?;
         }
         Expr::Kwargs(kwargs) => {
             for (_key, value) in &kwargs.pairs {
-                keys.extend(find_in_expr(value)?);
+                find_in_expr(context, value)?;
             }
         }
     }
 
-    Ok(keys)
+    Ok(())
 }
 
-fn find_in_exprs<'a>(exprs: &'a [Expr<'a>]) -> Result<Vec<Key>, minijinja::Error> {
+fn find_in_exprs<'a>(context: &mut Context, exprs: &'a [Expr<'a>]) -> Result<(), minijinja::Error> {
-    let mut keys = Vec::new();
-
     for expr in exprs {
-        keys.extend(find_in_expr(expr)?);
+        find_in_expr(context, expr)?;
     }
 
-    Ok(keys)
+    Ok(())
 }
 
-fn find_in_optional_expr<'a>(expr: &'a Option<Expr<'a>>) -> Result<Vec<Key>, minijinja::Error> {
+fn find_in_optional_expr<'a>(
-    let mut keys = Vec::new();
+    context: &mut Context,
-
+    expr: &'a Option<Expr<'a>>,
+) -> Result<(), minijinja::Error> {
     if let Some(expr) = expr {
-        keys.extend(find_in_expr(expr)?);
+        find_in_expr(context, expr)?;
     }
 
-    Ok(keys)
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_find_keys() {
+        let mut context = Context::new("t".to_owned());
+        let templates = [
+            ("hello.txt", r#"Hello {{ t("world") }}"#),
+            ("existing.txt", r#"{{ t("hello") }}"#),
+            ("plural.txt", r#"{{ t("plural", count=4) }}"#),
+            // Kitchen sink to make sure we're going through the whole AST
+            (
+                "macros.txt",
+                r#"
+                    {% macro test(arg="foo") %}
+                        {% if function() == foo is test(t("nested.1")) %}
+                            {% set foo = t("nested.2", arg=5 + 2) ~ "foo" in test %}
+                            {{ foo | bar }}
+                        {% else %}
+                            {% for i in [t("nested.3", extra=t("nested.4")), "foo"] %}
+                                {{ i | foo }}
+                            {% else %}
+                                {{ t("nested.5") }}
+                            {% endfor %}
+                        {% endif %}
+                    {% endmacro %}
+                "#,
+            ),
+            (
+                "nested.txt",
+                r#"
+                    {% import "macros.txt" as macros %}
+                    {% block test %}
+                        {% filter upper %}
+                            {{ macros.test(arg=t("nested.6")) }}
+                        {% endfilter %}
+                    {% endblock test %}
+                "#,
+            ),
+        ];
+
+        for (name, content) in templates {
+            let ast = parse(content, name).unwrap();
+            find_in_stmt(&mut context, &ast).unwrap();
+        }
+
+        let mut tree = serde_json::from_value(serde_json::json!({
+            "hello": "Hello!",
+        }))
+        .unwrap();
+
+        context.add_missing(&mut tree);
+        let tree = serde_json::to_value(&tree).unwrap();
+        assert_eq!(
+            tree,
+            serde_json::json!({
+                "hello": "Hello!",
+                "world": "world",
+                "plural": {
+                    "other": "%(count)d plural"
+                },
+                "nested": {
+                    "1": "nested.1",
+                    "2": "nested.2",
+                    "3": "nested.3",
+                    "4": "nested.4",
+                    "5": "nested.5",
+                    "6": "nested.6",
+                },
+            })
+        );
+    }
+
+    #[test]
+    fn test_invalid_key_not_string() {
+        // This is invalid because the key is not a string
+        let mut context = Context::new("t".to_owned());
+        let ast = parse(r#"{{ t(5) }}"#, "invalid.txt").unwrap();
+
+        let res = find_in_stmt(&mut context, &ast);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_invalid_key_filtered() {
+        // This is invalid because the key argument has a filter
+        let mut context = Context::new("t".to_owned());
+        let ast = parse(r#"{{ t("foo" | bar) }}"#, "invalid.txt").unwrap();
+
+        let res = find_in_stmt(&mut context, &ast);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_invalid_key_missing() {
+        // This is invalid because the key argument is missing
+        let mut context = Context::new("t".to_owned());
+        let ast = parse(r#"{{ t() }}"#, "invalid.txt").unwrap();
+
+        let res = find_in_stmt(&mut context, &ast);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_invalid_key_negated() {
+        // This is invalid because the key argument is missing
+        let mut context = Context::new("t".to_owned());
+        let ast = parse(r#"{{ t(not "foo") }}"#, "invalid.txt").unwrap();
+
+        let res = find_in_stmt(&mut context, &ast);
+        assert!(res.is_err());
+    }
 }

crates/i18n-scan/src/tera.rs

@@ -1,400 +0,0 @@
-// Copyright 2023 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use tera::{
-    ast::{Block, Expr, ExprVal, FunctionCall, MacroDefinition, Node},
-    Error, Template, Tera,
-};
-
-use crate::key::{Key, KeyKind};
-
-/// Find all translatable strings in a Tera instance.
-///
-/// This is not particularly efficient in terms of allocations, but as it is
-/// only meant to be used in an utility, it should be fine.
-///
-/// # Parameters
-///
-/// * `tera` - The Tera instance to scan.
-/// * `function_name` - The name of the translation function. Usually `t`.
-///
-/// # Errors
-///
-/// This function will return an error if it encounters an invalid template.
-pub fn find_keys(tera: &Tera, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let names = tera.get_template_names();
-    let mut keys = Vec::new();
-
-    for name in names {
-        tracing::trace!("Scanning {}", name);
-        // This should never fail, but who knows.
-        let template = tera.get_template(name)?;
-        keys.extend(find_in_template(template, function_name)?);
-    }
-
-    Ok(keys)
-}
-
-fn find_in_template(template: &Template, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let mut keys = Vec::new();
-
-    for node in &template.ast {
-        keys.extend(find_in_node(node, function_name)?);
-    }
-
-    for block in template.blocks.values() {
-        keys.extend(find_in_block(block, function_name)?);
-    }
-
-    for block_definition in template.blocks_definitions.values() {
-        for (_, block) in block_definition {
-            keys.extend(find_in_block(block, function_name)?);
-        }
-    }
-
-    for macro_definition in template.macros.values() {
-        keys.extend(find_in_macro_definition(macro_definition, function_name)?);
-    }
-
-    Ok(keys)
-}
-
-fn find_in_block(block: &Block, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let mut keys = Vec::new();
-
-    for node in &block.body {
-        keys.extend(find_in_node(node, function_name)?);
-    }
-
-    Ok(keys)
-}
-
-fn find_in_node(node: &Node, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let mut keys = Vec::new();
-
-    match node {
-        Node::VariableBlock(_, expr) => keys.extend(find_in_expr(expr, function_name)?),
-
-        Node::MacroDefinition(_, definition, _) => {
-            keys.extend(find_in_macro_definition(definition, function_name)?);
-        }
-
-        Node::Set(_, set) => keys.extend(find_in_expr(&set.value, function_name)?),
-
-        Node::FilterSection(_, filter_section, _) => {
-            keys.extend(find_in_function_call(
-                &filter_section.filter,
-                function_name,
-            )?);
-
-            for node in &filter_section.body {
-                keys.extend(find_in_node(node, function_name)?);
-            }
-        }
-
-        Node::Block(_, block, _) => keys.extend(find_in_block(block, function_name)?),
-
-        Node::Forloop(_, for_loop, _) => {
-            keys.extend(find_in_expr(&for_loop.container, function_name)?);
-
-            for node in &for_loop.body {
-                keys.extend(find_in_node(node, function_name)?);
-            }
-
-            if let Some(empty_body) = &for_loop.empty_body {
-                for node in empty_body {
-                    keys.extend(find_in_node(node, function_name)?);
-                }
-            }
-        }
-        Node::If(if_block, _) => {
-            for (_ws, condition, expr) in &if_block.conditions {
-                keys.extend(find_in_expr(condition, function_name)?);
-
-                for node in expr {
-                    keys.extend(find_in_node(node, function_name)?);
-                }
-            }
-
-            if let Some((_ws, expr)) = &if_block.otherwise {
-                for node in expr {
-                    keys.extend(find_in_node(node, function_name)?);
-                }
-            }
-        }
-
-        Node::Super
-        | Node::Text(_)
-        | Node::Extends(_, _)
-        | Node::Include(_, _, _)
-        | Node::ImportMacro(_, _, _)
-        | Node::Raw(_, _, _)
-        | Node::Break(_)
-        | Node::Continue(_)
-        | Node::Comment(_, _) => {}
-    };
-
-    Ok(keys)
-}
-
-fn find_in_macro_definition(
-    definition: &MacroDefinition,
-    function_name: &str,
-) -> Result<Vec<Key>, Error> {
-    let mut keys = Vec::new();
-
-    // Walk through argument defaults
-    for expr in definition.args.values().flatten() {
-        keys.extend(find_in_expr(expr, function_name)?);
-    }
-
-    // Walk through the macro body
-    for node in &definition.body {
-        keys.extend(find_in_node(node, function_name)?);
-    }
-
-    Ok(keys)
-}
-
-fn find_in_expr_val(expr_val: &ExprVal, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let mut keys = Vec::new();
-
-    match expr_val {
-        ExprVal::String(_)
-        | ExprVal::Int(_)
-        | ExprVal::Float(_)
-        | ExprVal::Bool(_)
-        | ExprVal::Ident(_) => {}
-
-        ExprVal::Math(math_expr) => {
-            keys.extend(find_in_expr(&math_expr.lhs, function_name)?);
-            keys.extend(find_in_expr(&math_expr.rhs, function_name)?);
-        }
-
-        ExprVal::Logic(logic_expr) => {
-            keys.extend(find_in_expr(&logic_expr.lhs, function_name)?);
-            keys.extend(find_in_expr(&logic_expr.rhs, function_name)?);
-        }
-
-        ExprVal::Test(test_expr) => {
-            for arg in &test_expr.args {
-                keys.extend(find_in_expr(arg, function_name)?);
-            }
-        }
-
-        ExprVal::MacroCall(macro_call) => {
-            for arg in macro_call.args.values() {
-                keys.extend(find_in_expr(arg, function_name)?);
-            }
-        }
-
-        ExprVal::FunctionCall(function_call) => {
-            keys.extend(find_in_function_call(function_call, function_name)?);
-        }
-
-        ExprVal::Array(array) => {
-            for expr in array {
-                keys.extend(find_in_expr(expr, function_name)?);
-            }
-        }
-
-        ExprVal::StringConcat(string_concat) => {
-            for value in &string_concat.values {
-                keys.extend(find_in_expr_val(value, function_name)?);
-            }
-        }
-
-        ExprVal::In(in_expr) => {
-            keys.extend(find_in_expr(&in_expr.lhs, function_name)?);
-            keys.extend(find_in_expr(&in_expr.rhs, function_name)?);
-        }
-    }
-
-    Ok(keys)
-}
-
-fn find_in_expr(expr: &Expr, function_name: &str) -> Result<Vec<Key>, tera::Error> {
-    let mut keys = Vec::new();
-
-    keys.extend(find_in_expr_val(&expr.val, function_name)?);
-
-    for filter in &expr.filters {
-        keys.extend(find_in_function_call(filter, function_name)?);
-    }
-
-    Ok(keys)
-}
-
-fn find_in_function_call(
-    function_call: &FunctionCall,
-    function_name: &str,
-) -> Result<Vec<Key>, tera::Error> {
-    tracing::trace!("Checking function call: {:?}", function_call);
-    let mut keys = Vec::new();
-
-    // Regardless of if it is the function we are looking for, we still need to
-    // check the arguments
-    for expr in function_call.args.values() {
-        keys.extend(find_in_expr(expr, function_name)?);
-    }
-
-    // If it is the function we are looking for, we need to extract the key
-    if function_call.name == function_name {
-        let key = function_call
-            .args
-            .get("key")
-            .ok_or(tera::Error::msg("Missing key argument"))?;
-        if !key.filters.is_empty() {
-            return Err(tera::Error::msg("Key argument must not have filters"));
-        }
-
-        if key.negated {
-            return Err(tera::Error::msg("Key argument must not be negated"));
-        }
-
-        let key = match &key.val {
-            tera::ast::ExprVal::String(s) => s.clone(),
-            _ => return Err(tera::Error::msg("Key argument must be a string")),
-        };
-
-        let kind = if function_call.args.contains_key("count") {
-            KeyKind::Plural
-        } else {
-            KeyKind::Message
-        };
-
-        keys.push(Key::new(kind, key))
-    }
-
-    Ok(keys)
-}
-
-#[cfg(test)]
-mod tests {
-    use tera::Tera;
-
-    use super::*;
-    use crate::key::add_missing;
-
-    #[test]
-    fn test_find_keys() {
-        let mut tera = Tera::default();
-        tera.add_raw_templates([
-            ("hello.txt", r#"Hello {{ t(key="world") }}"#),
-            ("existing.txt", r#"{{ t(key="hello") }}"#),
-            ("plural.txt", r#"{{ t(key="plural", count=4) }}"#),
-            // Kitchen sink to make sure we're going through the whole AST
-            (
-                "macros.txt",
-                r#"
-                    {% macro test(arg="foo") %}
-                        {% if function() == foo is test(t(key="nested.1")) %}
-                            {% set foo = t(key="nested.2", arg=5 + 2) ~ "foo" in test %}
-                            {{ foo | bar }}
-                        {% else %}
-                                {% for i in [t(key="nested.3", extra=t(key="nested.4")), "foo"] %}
-                                    {{ i | foo }}
-                                {% else %}
-                                    {{ t(key="nested.5") }}
-                                {% endfor %}
-                        {% endif %}
-                    {% endmacro %}
-                "#,
-            ),
-            (
-                "nested.txt",
-                r#"
-                    {% import "macros.txt" as macros %}
-                    {% block test %}
-                        {% filter upper %}
-                            {{ macros::test(arg=t(key="nested.6")) }}
-                        {% endfilter %}
-                    {% endblock test %}
-                "#,
-            ),
-        ])
-        .unwrap();
-
-        let mut tree = serde_json::from_value(serde_json::json!({
-            "hello": "Hello!",
-        }))
-        .unwrap();
-
-        let keys = find_keys(&tera, "t").unwrap();
-        add_missing(&mut tree, &keys);
-        let tree = serde_json::to_value(&tree).unwrap();
-        assert_eq!(
-            tree,
-            serde_json::json!({
-                "hello": "Hello!",
-                "world": "world",
-                "plural": {
-                    "other": "%(count)d plural"
-                },
-                "nested": {
-                    "1": "nested.1",
-                    "2": "nested.2",
-                    "3": "nested.3",
-                    "4": "nested.4",
-                    "5": "nested.5",
-                    "6": "nested.6",
-                },
-            })
-        );
-    }
-
-    #[test]
-    fn test_invalid_key_not_string() {
-        let mut tera = Tera::default();
-        // This is invalid because the key is not a string
-        tera.add_raw_template("invalid.txt", r#"{{ t(key=5) }}"#)
-            .unwrap();
-
-        let keys = find_keys(&tera, "t");
-        assert!(keys.is_err());
-    }
-
-    #[test]
-    fn test_invalid_key_filtered() {
-        let mut tera = Tera::default();
-        // This is invalid because the key argument has a filter
-        tera.add_raw_template("invalid.txt", r#"{{ t(key="foo" | bar) }}"#)
-            .unwrap();
-
-        let keys = find_keys(&tera, "t");
-        assert!(keys.is_err());
-    }
-
-    #[test]
-    fn test_invalid_key_missing() {
-        let mut tera = Tera::default();
-        // This is invalid because the key argument is missing
-        tera.add_raw_template("invalid.txt", r#"{{ t() }}"#)
-            .unwrap();
-
-        let keys = find_keys(&tera, "t");
-        assert!(keys.is_err());
-    }
-
-    #[test]
-    fn test_invalid_key_negated() {
-        let mut tera = Tera::default();
-        // This is invalid because the key argument is missing
-        tera.add_raw_template("invalid.txt", r#"{{ t(key=not "foo") }}"#)
-            .unwrap();
-
-        let keys = find_keys(&tera, "t");
-        assert!(keys.is_err());
-    }
-}

crates/templates/src/functions.rs

@@ -43,7 +43,6 @@ pub fn register(
     env.add_filter("add_slashes", filter_add_slashes);
     env.add_filter("split", filter_split);
     env.add_function("add_params_to_url", function_add_params_to_url);
-    //tera.register_function("merge", function_merge);
     env.add_global(
         "include_asset",
         Value::from_object(IncludeAsset {
@@ -183,20 +182,6 @@ fn function_add_params_to_url(
     Ok(uri.to_string())
 }
 
-/*
-fn function_merge(params: &HashMap<String, Value>) -> Result<Value, tera::Error> {
-    let mut ret = serde_json::Map::new();
-    for (k, v) in params {
-        let v = v
-            .as_object()
-            .ok_or_else(|| tera::Error::msg(format!("Parameter {k:?} should be an object")))?;
-        ret.extend(v.clone());
-    }
-
-    Ok(Value::Object(ret))
-}
- */
-
 #[derive(Debug)]
 struct IncludeAsset {
     url_builder: UrlBuilder,