Allow any PostgreSQL version installed on a vm to be tested.

The db-test list spreads the supported versions over the default vms so that a full run covers every version exactly once, but it also limited what --pg-version was able to select. That made it impossible to test a version on a vm that installs it but does not test it by default, e.g. --vm=u24 --pg-version=18 reported that no tests were selected even though u24 has PostgreSQL 18 installed.

Select from the versions the vm installs when a version is named and make --pg-version=all mean every version installed rather than an alias for the default selection. A version the project does not support is now an error rather than a selection of nothing, since that is a typo just as an unknown module name is.

test/container.yaml

@@ -11,7 +11,8 @@
 # - 'default' marks a vm that --vm=all selects
 # - 'image' is the image the container is built from, and a vm without one has no container, i.e. it runs on the host
 # - 'os-base' is the os the container is built on, which decides its package manager and where PostgreSQL is installed
-# - 'db' is the PostgreSQL versions installed and 'db-test' the ones integration tests run against, which defaults to all of them
+# - 'db' is the PostgreSQL versions installed and 'db-test' the ones integration tests run against by default, all of them unless
+#   declared. Any installed version can be tested by naming it with --pg-version, e.g. --vm=u24 --pg-version=18.
 # - the rest is what the container build installs or configures, and each defaults to false or unset
 #
 # A vm with no container declares only 'default' and 'coverage-c'. Everything else says how a container is built, and the os the

test/lib/command/test/list.py

@@ -78,6 +78,12 @@ def _check(module_list, config):
         "--test requires a single --module naming a test module",
     )
 
+    # A version the project does not support is a typo, which is worth reporting for the same reason a module name is
+    check(
+        config.pg_version in (PG_VERSION_ALL, PG_VERSION_MINIMAL) + PG_VERSION_LIST,
+        "'%s' does not match a supported PostgreSQL version" % config.pg_version,
+    )
+
 
 ####################################################################################################################################
 def test_list_get(module_list, config):
@@ -123,12 +129,24 @@ def test_list_get(module_list, config):
                 continue
 
             # PostgreSQL versions to test against, most recent first. Only an integration module declares that it needs them.
-            pg_version_list = list(reversed(vm.db_test_list)) if module.pg_required else [None]
+            #
+            # A run that does not name a version tests the versions the vm declares, which is the minimal set that covers every
+            # supported version across the default vms. A named version runs on every vm that installs it and --pg-version=all
+            # runs every version a vm installs, since what a vm declares only spreads the versions over the default vms rather
+            # than limiting what it is able to test.
+            pg_version_list = [None]
+
+            if module.pg_required:
+                if config.pg_version == PG_VERSION_MINIMAL:
+                    pg_version_list = vm.db_test_list
+                elif config.pg_version == PG_VERSION_ALL:
+                    pg_version_list = vm.db_list
+                else:
+                    pg_version_list = [version for version in vm.db_list if version == config.pg_version]
+
+                pg_version_list = list(reversed(pg_version_list))
 
             for pg_version in pg_version_list:
-                if pg_version is not None and config.pg_version not in ("all", "minimal", pg_version):
-                    continue
-
                 # An integration module runs each of its tests in its own container, so each one is a separate run here. Every
                 # other module runs all of its tests at once and passes on whatever --test selected.
                 if not integration:

test/lib/common/vm.py

@@ -56,6 +56,10 @@ _PG_LIST = bld_pg_version_list(_PATH_REPO)
 # PostgreSQL versions the project supports. Each one must be tested on a default vm, which vm_check() enforces.
 PG_VERSION_LIST = tuple(pg.version for pg in _PG_LIST)
 
+# Versions --pg-version selects rather than naming one of them, i.e. every version a vm installs and the versions it declares
+PG_VERSION_ALL = "all"
+PG_VERSION_MINIMAL = "minimal"
+
 # Versions that have not been released yet, which are only in the beta repository
 _PG_BETA_LIST = tuple(pg.version for pg in _PG_LIST if not pg.release)
 
@@ -93,7 +97,7 @@ class Vm:
         self.os_base = None  # Os the container is built on, which determines its package manager and where PostgreSQL is installed
         self.image = None  # Image the container is built from, None when the vm has no container
         self.db_list = []  # PostgreSQL versions installed
-        self.db_test_list = []  # Versions integration tests run against, which is every version installed unless declared
+        self.db_test_list = []  # Versions integration tests run against by default, every version installed unless declared
         self.pg_beta = None  # Unreleased version installed, which is only in the beta repository
         self.pg_repo = True  # Install PostgreSQL from the PGDG repo rather than from the distribution?
         self.pg_repo_key = True  # Import the PGDG signing key, which not every rpm backend accepts?

test/lib/config/cli.py

@@ -73,7 +73,12 @@ def _parser_test(parent):
     # Test selection
     result.add_argument("--module", action="append", default=[], metavar="MODULE", help="module or path of modules to test")
     result.add_argument("--test", action="append", type=int, default=[], metavar="TEST", help="test to run in a module")
-    result.add_argument("--pg-version", default="minimal", metavar="VERSION", help="pg version integration tests run against")
+    result.add_argument(
+        "--pg-version",
+        default="minimal",
+        metavar="VERSION",
+        help="pg version integration tests run against, or all of them installed on the vm",
+    )
     result.add_argument("--c-only", action="store_true", help="only run C tests")
     result.add_argument("--container-only", action="store_true", help="only run tests that require a container")
     result.add_argument("--coverage-only", action="store_true", help="only run tests that provide coverage")

test/src/module/test/command/test/list_test.py

@@ -139,27 +139,44 @@ def test_list_filter():
 
 ####################################################################################################################################
 def test_list_pg_version():
-    """An integration test runs against the PostgreSQL versions the vm tests, most recent first."""
+    """An integration test runs against the PostgreSQL versions the vm tests, most recent first.
 
-    version_list = vm_get("u24").db_test_list
+    A version named on the command line runs wherever it is installed, whether or not the vm tests it by default."""
+
+    vm = vm_get("u24")
 
     assert_equal(
         [run.pg_version for run in test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"]))],
-        [version for version in reversed(version_list) for _ in range(3)],
+        [version for version in reversed(vm.db_test_list) for _ in range(3)],
     )
 
-    # A single version can be selected, which is how a version specific problem is chased down
+    # A single version can be selected, which is how a version specific problem is chased down. Any version the vm installs may be
+    # named, not just the ones it tests by default, since a development vm installs every version.
+    version = next(version for version in vm.db_list if version not in vm.db_test_list)
+
     assert_equal(
-        [run.pg_version for run in test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"], pg_version="12"))],
-        ["12"] * 3,
+        [run.pg_version for run in test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"], pg_version=version))],
+        [version] * 3,
     )
 
-    # A version that the vm does not test selects nothing
+    # Every version the vm installs can be selected at once
     assert_equal(
-        test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"], pg_version="8.0")),
-        [],
+        [
+            run.pg_version
+            for run in test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"], pg_version=PG_VERSION_ALL))
+        ],
+        [version for version in reversed(vm.db_list) for _ in range(3)],
     )
 
+    # A version the vm does not install selects nothing, since another vm in the same run may well install it
+    assert_equal(test_list_get(MODULE_LIST, _Config(vm="d12", module=["integration"], pg_version="9.6")), [])
+
+    # A version the project does not support is a typo rather than a selection of nothing
+    with assert_raises(ToolError) as error:
+        test_list_get(MODULE_LIST, _Config(vm="u24", module=["integration"], pg_version="8.0"))
+
+    assert_equal(str(error.exception), "'8.0' does not match a supported PostgreSQL version")
+
 
 ####################################################################################################################################
 def test_list_run():