new: dev: Support jinja2 templates in pytest runner

Configuration files in system tests which require some variables (e.g.
port numbers) filled in during test setup, can now use jinja2 templates
when `jinja2` python package is available.

Any `*.j2` file found within the system test directory will be
automatically rendered with the environment variables into a file
without the `.j2` extension by the pytest runner. E.g.
`ns1/named.conf.j2` will become `ns1/named.conf` during test setup. To
avoid automatic rendering, use `.j2.manual` extension and render the
files manually at test time.

New `templates` pytest fixture has been added. Its `render()` function
can be used to render a template with custom test variables. This can be
useful to fill in different config options during the test. With
advanced jinja2 template syntax, it can also be used to include/omit
entire sections of the config file rather than using `named1.conf.in`,
`named2.conf.in` etc.

Closes #4938

Merge branch '4938-use-jinja2-templates-in-system-tests' into 'main'

See merge request isc-projects/bind9!9587

bin/tests/system/README.md

@@ -51,6 +51,7 @@ To run system tests, make sure you have the following dependencies installed:
 - perl
 - dnspython
 - pytest-xdist (for parallel execution)
+- python-jinja2 (for tests which use jinja templates)
 
 Individual system tests might also require additional dependencies. If those
 are missing, the affected tests will be skipped and should produce a message
@@ -154,9 +155,17 @@ system test directories may contain the following standard files:
 - `tests_*.py`: These python files are picked up by pytest as modules. If they
   contain any test functions, they're added to the test suite.
 
-- `setup.sh`: This sets up the preconditions for the tests. Although optional,
-  virtually all tests will require such a file to set up the ports they should
-  use for the test.
+- `*.j2`: These jinja2 templates can be used for configuration files or any
+  other files which require certain variables filled in, e.g. ports from the
+  environment variables. During test setup, the pytest runner will automatically
+  fill those in and strip the filename extension .j2, e.g. `ns1/named.conf.j2`
+  becomes `ns1/named.conf`. When using advanced templating to conditionally
+  include/omit entire sections or when filling in custom variables used for the
+  test, ensure the templates always include the defaults. If you don't need the
+  file to be auto-templated during test setup, use `.j2.manual` instead and then
+  no defaults are needed.
+
+- `setup.sh`: This sets up the preconditions for the tests.
 
 - `tests.sh`: Any shell-based tests are located within this file. Runs the
   actual tests.

bin/tests/system/conftest.py

@@ -406,6 +406,11 @@ def system_test_dir(request, system_test_name):
             unlink(symlink_dst)
 
 
+@pytest.fixture(scope="module")
+def templates(system_test_dir: Path):
+    return isctest.template.TemplateEngine(system_test_dir)
+
+
 def _run_script(
     system_test_dir: Path,
     interpreter: str,
@@ -481,6 +486,7 @@ def run_tests_sh(system_test_dir, shell):
 def system_test(
     request,
     system_test_dir,
+    templates,
     shell,
     perl,
 ):
@@ -522,6 +528,7 @@ def system_test(
             pytest.skip("Prerequisites missing.")
 
     def setup_test():
+        templates.render_auto()
         try:
             shell(f"{system_test_dir}/setup.sh")
         except FileNotFoundError:

bin/tests/system/isctest/__init__.py

@@ -16,6 +16,7 @@ from . import kasp
 from . import name
 from . import rndc
 from . import run
+from . import template
 from . import log
 from . import vars  # pylint: disable=redefined-builtin
 from . import hypothesis

bin/tests/system/isctest/template.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import pytest
+
+from .log import debug
+from .vars import ALL
+
+
+class TemplateEngine:
+    """
+    Engine for rendering jinja2 templates in system test directories.
+    """
+
+    def __init__(self, directory: Union[str, Path], env_vars=ALL):
+        """
+        Initialize the template engine for `directory`, optionally overriding
+        the `env_vars` that will be used when rendering the templates (defaults
+        to the environment variables set by the pytest runner).
+        """
+        self.directory = Path(directory)
+        self._j2env = None
+        self.env_vars = dict(env_vars)
+
+    @property
+    def j2env(self):
+        """
+        Jinja2 engine that is initialized when first requested. In case the
+        jinja2 package in unavailable, the current test will be skipped.
+        """
+        if self._j2env is None:
+            try:
+                import jinja2  # pylint: disable=import-outside-toplevel
+            except ImportError:
+                pytest.skip("jinja2 not found")
+
+            loader = jinja2.FileSystemLoader(str(self.directory))
+            return jinja2.Environment(
+                loader=loader,
+                undefined=jinja2.StrictUndefined,
+                variable_start_string="@",
+                variable_end_string="@",
+            )
+        return self._j2env
+
+    def render(
+        self,
+        output: str,
+        data: Optional[Dict[str, Any]] = None,
+        template: Optional[str] = None,
+    ) -> None:
+        """
+        Render `output` file from jinja `template` and fill in the `data`. The
+        `template` defaults to *.j2.manual or *.j2 file. The environment
+        variables which the engine was initialized with are also filled in. In
+        case of a variable name clash, `data` has precedence.
+        """
+        if template is None:
+            template = f"{output}.j2.manual"
+            if not Path(template).is_file():
+                template = f"{output}.j2"
+        if not Path(template).is_file():
+            raise RuntimeError('No jinja2 template found for "{output}"')
+
+        if data is None:
+            data = self.env_vars
+        else:
+            data = {**self.env_vars, **data}
+
+        debug("rendering template `%s` to file `%s`", template, output)
+        stream = self.j2env.get_template(template).stream(data)
+        stream.dump(output, encoding="utf-8")
+
+    def render_auto(self):
+        """
+        Render all *.j2 templates with default values and write the output to
+        files without the .j2 extensions.
+        """
+        templates = [
+            str(filepath.relative_to(self.directory))
+            for filepath in self.directory.rglob("*.j2")
+        ]
+        for template in templates:
+            self.render(template[:-3])