Merge branch 'Mbed-TLS:development' into sha3

scripts/abi_check.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-"""
-This script compares the interfaces of two versions of Mbed TLS, looking
+"""This script compares the interfaces of two versions of Mbed TLS, looking
 for backward incompatibilities between two different Git revisions within
 an Mbed TLS repository. It must be run from the root of a Git working tree.
 
+### How the script works ###
+
 For the source (API) and runtime (ABI) interface compatibility, this script
 is a small wrapper around the abi-compliance-checker and abi-dumper tools,
 applying them to compare the header and library files.
@@ -20,7 +21,66 @@ at a configurable location, or are given as a brief list of problems.
 Returns 0 on success, 1 on non-compliance, and 2 if there is an error
 while running the script.
 
-You must run this test from an Mbed TLS root.
+### How to interpret non-compliance ###
+
+This script has relatively common false positives. In many scenarios, it only
+reports a pass if there is a strict textual match between the old version and
+the new version, and it reports problems where there is a sufficient semantic
+match but not a textual match. This section lists some common false positives.
+This is not an exhaustive list: in the end what matters is whether we are
+breaking a backward compatibility goal.
+
+**API**: the goal is that if an application works with the old version of the
+library, it can be recompiled against the new version and will still work.
+This is normally validated by comparing the declarations in `include/*/*.h`.
+A failure is a declaration that has disappeared or that now has a different
+type.
+
+  * It's ok to change or remove macros and functions that are documented as
+    for internal use only or as experimental.
+  * It's ok to rename function or macro parameters as long as the semantics
+    has not changed.
+  * It's ok to change or remove structure fields that are documented as
+    private.
+  * It's ok to add fields to a structure that already had private fields
+    or was documented as extensible.
+
+**ABI**: the goal is that if an application was built against the old version
+of the library, the same binary will work when linked against the new version.
+This is normally validated by comparing the symbols exported by `libmbed*.so`.
+A failure is a symbol that is no longer exported by the same library or that
+now has a different type.
+
+  * All ABI changes are acceptable if the library version is bumped
+    (see `scripts/bump_version.sh`).
+  * ABI changes that concern functions which are declared only inside the
+    library directory, and not in `include/*/*.h`, are acceptable only if
+    the function was only ever used inside the same library (libmbedcrypto,
+    libmbedx509, libmbedtls). As a counter example, if the old version
+    of libmbedtls calls mbedtls_foo() from libmbedcrypto, and the new version
+    of libmbedcrypto no longer has a compatible mbedtls_foo(), this does
+    require a version bump for libmbedcrypto.
+
+**Storage format**: the goal is to check that persistent keys stored by the
+old version can be read by the new version. This is normally validated by
+comparing the `*read*` test cases in `test_suite*storage_format*.data`.
+A failure is a storage read test case that is no longer present with the same
+function name and parameter list.
+
+  * It's ok if the same test data is present, but its presentation has changed,
+    for example if a test function is renamed or has different parameters.
+  * It's ok if redundant tests are removed.
+
+**Generated test coverage**: the goal is to check that automatically
+generated tests have as much coverage as before. This is normally validated
+by comparing the test cases that are automatically generated by a script.
+A failure is a generated test case that is no longer present with the same
+function name and parameter list.
+
+  * It's ok if the same test data is present, but its presentation has changed,
+    for example if a test function is renamed or has different parameters.
+  * It's ok if redundant tests are removed.
+
 """
 
 # Copyright The Mbed TLS Contributors

scripts/assemble_changelog.py

@@ -122,7 +122,7 @@ class ChangelogFormat:
 class TextChangelogFormat(ChangelogFormat):
     """The traditional Mbed TLS changelog format."""
 
-    _unreleased_version_text = '= mbed TLS x.x.x branch released xxxx-xx-xx'
+    _unreleased_version_text = '= Mbed TLS x.x.x branch released xxxx-xx-xx'
     @classmethod
     def is_released_version(cls, title):
         # Look for an incomplete release date

scripts/bump_version.sh

@@ -96,7 +96,7 @@ then
   mv tmp library/CMakeLists.txt
 
   [ $VERBOSE ] && echo "Bumping SOVERSION for libmbedcrypto in library/Makefile"
-  sed -e "s/SOEXT_CRYPTO=so.[0-9]\{1,\}/SOEXT_CRYPTO=so.$SO_CRYPTO/g" < library/Makefile > tmp
+  sed -e "s/SOEXT_CRYPTO?=so.[0-9]\{1,\}/SOEXT_CRYPTO?=so.$SO_CRYPTO/g" < library/Makefile > tmp
   mv tmp library/Makefile
 fi
 
@@ -107,7 +107,7 @@ then
   mv tmp library/CMakeLists.txt
 
   [ $VERBOSE ] && echo "Bumping SOVERSION for libmbedx509 in library/Makefile"
-  sed -e "s/SOEXT_X509=so.[0-9]\{1,\}/SOEXT_X509=so.$SO_X509/g" < library/Makefile > tmp
+  sed -e "s/SOEXT_X509?=so.[0-9]\{1,\}/SOEXT_X509?=so.$SO_X509/g" < library/Makefile > tmp
   mv tmp library/Makefile
 fi
 
@@ -118,7 +118,7 @@ then
   mv tmp library/CMakeLists.txt
 
   [ $VERBOSE ] && echo "Bumping SOVERSION for libmbedtls in library/Makefile"
-  sed -e "s/SOEXT_TLS=so.[0-9]\{1,\}/SOEXT_TLS=so.$SO_TLS/g" < library/Makefile > tmp
+  sed -e "s/SOEXT_TLS?=so.[0-9]\{1,\}/SOEXT_TLS?=so.$SO_TLS/g" < library/Makefile > tmp
   mv tmp library/Makefile
 fi
 

scripts/code_size_compare.py

@@ -37,7 +37,7 @@ class CodeSizeComparison:
         """
         old_revision: revision to compare against
         new_revision:
-        result_dir: directory for comparision result
+        result_dir: directory for comparison result
         """
         self.repo_path = "."
         self.result_dir = os.path.abspath(result_dir)
@@ -140,7 +140,7 @@ class CodeSizeComparison:
                                      + "-" + self.new_rev + ".csv"), "w")
 
         res_file.write("file_name, this_size, old_size, change, change %\n")
-        print("Generating comparision results.")
+        print("Generating comparison results.")
 
         old_ds = {}
         for line in old_file.readlines()[1:]:
@@ -199,7 +199,7 @@ def main():
     parser.add_argument(
         "-n", "--new-rev", type=str, default=None,
         help="new revision for comparison, default is the current work \
-              directory, including uncommited changes."
+              directory, including uncommitted changes."
     )
     comp_args = parser.parse_args()
 

scripts/config.py

@@ -7,6 +7,11 @@ Basic usage, to read the Mbed TLS or Mbed Crypto configuration:
     if 'MBEDTLS_RSA_C' in config: print('RSA is enabled')
 """
 
+# Note that as long as Mbed TLS 2.28 LTS is maintained, the version of
+# this script in the mbedtls-2.28 branch must remain compatible with
+# Python 3.4. The version in development may only use more recent features
+# in parts that are not backported to 2.28.
+
 ## Copyright The Mbed TLS Contributors
 ## SPDX-License-Identifier: Apache-2.0
 ##
@@ -324,6 +329,9 @@ def crypto_adapter(adapter):
         return adapter(name, active, section)
     return continuation
 
+DEPRECATED = frozenset([
+    'MBEDTLS_PSA_CRYPTO_SE_C',
+])
 def no_deprecated_adapter(adapter):
     """Modify an adapter to disable deprecated symbols.
 
@@ -334,6 +342,8 @@ def no_deprecated_adapter(adapter):
     def continuation(name, active, section):
         if name == 'MBEDTLS_DEPRECATED_REMOVED':
             return True
+        if name in DEPRECATED:
+            return False
         if adapter is None:
             return active
         return adapter(name, active, section)
@@ -418,7 +428,7 @@ class ConfigFile(Config):
         value = setting.value
         if value is None:
             value = ''
-        # Normally the whitespace to separte the symbol name from the
+        # Normally the whitespace to separate the symbol name from the
         # value is part of middle, and there's no whitespace for a symbol
         # with no value. But if a symbol has been changed from having a
         # value to not having one, the whitespace is wrong, so fix it.

scripts/data_files/error.fmt

@@ -150,7 +150,7 @@ void mbedtls_strerror( int ret, char *buf, size_t buflen )
 #else /* MBEDTLS_ERROR_C */
 
 /*
- * Provide an non-function in case MBEDTLS_ERROR_C is not defined
+ * Provide a dummy implementation when MBEDTLS_ERROR_C is not defined
  */
 void mbedtls_strerror( int ret, char *buf, size_t buflen )
 {

scripts/data_files/query_config.fmt

@@ -30,8 +30,12 @@
 
 /*
  * Include all the headers with public APIs in case they define a macro to its
- * default value when that configuration is not set in the mbedtls_config.h.
+ * default value when that configuration is not set in mbedtls_config.h, or
+ * for PSA_WANT macros, in case they're auto-defined based on mbedtls_config.h
+ * rather than defined directly in crypto_config.h.
  */
+#include "psa/crypto.h"
+
 #include "mbedtls/aes.h"
 #include "mbedtls/aria.h"
 #include "mbedtls/asn1.h"

scripts/driver.requirements.txt

@@ -10,7 +10,9 @@ markupsafe < 2.1
 # See https://github.com/Mbed-TLS/mbedtls/pull/5067#discussion_r738794607 .
 # Note that Jinja 3.0 drops support for Python 3.5, so we need to support
 # Jinja 2.x as long as we're still using Python 3.5 anywhere.
-Jinja2 >= 2.10.1
+# Jinja 2.10.1 doesn't support Python 3.10+
+Jinja2 >= 2.10.1; python_version <  '3.10'
+Jinja2 >= 2.10.3; python_version >= '3.10'
 # Jinja2 >=2.10, <3.0 needs a separate package for type annotations
 types-Jinja2
 

scripts/generate_driver_wrappers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Generate library/psa_crypto_driver_wrappers.c
 
-   This module is invoked by the build sripts to auto generate the
+   This module is invoked by the build scripts to auto generate the
    psa_crypto_driver_wrappers.c based on template files in
    script/data_files/driver_templates/.
 """

scripts/generate_query_config.pl

@@ -15,7 +15,7 @@
 # function by using the template in scripts/data_files/query_config.fmt.
 #
 # Usage: scripts/generate_query_config.pl without arguments, or
-# generate_query_config.pl config_file template_file output_file
+# generate_query_config.pl mbedtls_config_file template_file output_file [psa_crypto_config_file]
 #
 # Copyright The Mbed TLS Contributors
 # SPDX-License-Identifier: Apache-2.0
@@ -34,22 +34,33 @@
 
 use strict;
 
-my ($config_file, $query_config_format_file, $query_config_file);
+my ($mbedtls_config_file, $query_config_format_file, $query_config_file, $psa_crypto_config_file);
+
+my $default_mbedtls_config_file = "./include/mbedtls/mbedtls_config.h";
+my $default_query_config_format_file = "./scripts/data_files/query_config.fmt";
+my $default_query_config_file = "./programs/test/query_config.c";
+my $default_psa_crypto_config_file = "./include/psa/crypto_config.h";
 
 if( @ARGV ) {
     die "Invalid number of arguments - usage: $0 [CONFIG_FILE TEMPLATE_FILE OUTPUT_FILE]" if scalar @ARGV != 3;
-    ($config_file, $query_config_format_file, $query_config_file) = @ARGV;
+    ($mbedtls_config_file, $query_config_format_file, $query_config_file) = @ARGV;
 
-    -f $config_file or die "No such file: $config_file";
+    -f $mbedtls_config_file or die "No such file: $mbedtls_config_file";
     -f $query_config_format_file or die "No such file: $query_config_format_file";
+    if (defined($psa_crypto_config_file) && length($psa_crypto_config_file)) {
+        -f $psa_crypto_config_file or die "No such file: $psa_crypto_config_file";
+    } else {
+        $psa_crypto_config_file = (-f $default_psa_crypto_config_file) ? $default_psa_crypto_config_file : undef;
+    }
 } else {
-    $config_file = "./include/mbedtls/mbedtls_config.h";
-    $query_config_format_file = "./scripts/data_files/query_config.fmt";
-    $query_config_file = "./programs/test/query_config.c";
+    $mbedtls_config_file = $default_mbedtls_config_file;
+    $query_config_format_file = $default_query_config_format_file;
+    $query_config_file = $default_query_config_file;
+    $psa_crypto_config_file = $default_psa_crypto_config_file;
 
-    unless( -f $config_file && -f $query_config_format_file ) {
+    unless(-f $mbedtls_config_file && -f $query_config_format_file  && -f $psa_crypto_config_file) {
         chdir '..' or die;
-        -f $config_file && -f $query_config_format_file
+        -f $mbedtls_config_file && -f $query_config_format_file && -f $psa_crypto_config_file
           or die "No arguments supplied, must be run from project root or a first-level subdirectory\n";
     }
 }
@@ -63,39 +74,50 @@ MBEDTLS_SSL_CIPHERSUITES
 );
 my $excluded_re = join '|', @excluded;
 
-open(CONFIG_FILE, "$config_file") or die "Opening config file '$config_file': $!";
-
 # This variable will contain the string to replace in the CHECK_CONFIG of the
 # format file
 my $config_check = "";
 my $list_config = "";
 
-while (my $line = <CONFIG_FILE>) {
-    if ($line =~ /^(\/\/)?\s*#\s*define\s+(MBEDTLS_\w+).*/) {
-        my $name = $2;
+for my $config_file ($mbedtls_config_file, $psa_crypto_config_file) {
 
-        # Skip over the macro if it is in the ecluded list
-        next if $name =~ /$excluded_re/;
+    next unless defined($config_file);  # we might not have been given a PSA crypto config file
 
-        $config_check .= "#if defined($name)\n";
-        $config_check .= "    if( strcmp( \"$name\", config ) == 0 )\n";
-        $config_check .= "    {\n";
-        $config_check .= "        MACRO_EXPANSION_TO_STR( $name );\n";
-        $config_check .= "        return( 0 );\n";
-        $config_check .= "    }\n";
-        $config_check .= "#endif /* $name */\n";
-        $config_check .= "\n";
+    open(CONFIG_FILE, "<", $config_file) or die "Opening config file '$config_file': $!";
 
-        $list_config .= "#if defined($name)\n";
-        $list_config .= "    OUTPUT_MACRO_NAME_VALUE($name);\n";
-        $list_config .= "#endif /* $name */\n";
-        $list_config .= "\n";
+    while (my $line = <CONFIG_FILE>) {
+        if ($line =~ /^(\/\/)?\s*#\s*define\s+(MBEDTLS_\w+|PSA_WANT_\w+).*/) {
+            my $name = $2;
+
+            # Skip over the macro if it is in the excluded list
+            next if $name =~ /$excluded_re/;
+
+            $config_check .= <<EOT;
+#if defined($name)
+    if( strcmp( "$name", config ) == 0 )
+    {
+        MACRO_EXPANSION_TO_STR( $name );
+        return( 0 );
     }
+#endif /* $name */
+
+EOT
+
+            $list_config .= <<EOT;
+#if defined($name)
+    OUTPUT_MACRO_NAME_VALUE($name);
+#endif /* $name */
+
+EOT
+        }
+    }
+
+    close(CONFIG_FILE);
 }
 
 # Read the full format file into a string
 local $/;
-open(FORMAT_FILE, "$query_config_format_file") or die "Opening query config format file '$query_config_format_file': $!";
+open(FORMAT_FILE, "<", $query_config_format_file) or die "Opening query config format file '$query_config_format_file': $!";
 my $query_config_format = <FORMAT_FILE>;
 close(FORMAT_FILE);
 
@@ -104,6 +126,6 @@ $query_config_format =~ s/CHECK_CONFIG/$config_check/g;
 $query_config_format =~ s/LIST_CONFIG/$list_config/g;
 
 # Rewrite the query_config.c file
-open(QUERY_CONFIG_FILE, ">$query_config_file") or die "Opening destination file '$query_config_file': $!";
+open(QUERY_CONFIG_FILE, ">", $query_config_file) or die "Opening destination file '$query_config_file': $!";
 print QUERY_CONFIG_FILE $query_config_format;
 close(QUERY_CONFIG_FILE);

scripts/generate_ssl_debug_helpers.py

@@ -53,7 +53,7 @@ def preprocess_c_source_code(source, *classes):
     """
         Simple preprocessor for C source code.
 
-        Only processses condition directives without expanding them.
+        Only processes condition directives without expanding them.
         Yield object according to the classes input. Most match firstly
 
         If the directive pair does not match , raise CondDirectiveNotMatch.
@@ -234,6 +234,7 @@ class EnumDefinition:
                            prototype=self._prototype)
         return body
 
+
 class SignatureAlgorithmDefinition:
     """
         Generate helper functions for signature algorithms.
@@ -267,6 +268,7 @@ class SignatureAlgorithmDefinition:
 
     def span(self):
         return self._definitions[0].span()
+
     def __str__(self):
         """
             Generate function for translating value to string
@@ -274,10 +276,9 @@ class SignatureAlgorithmDefinition:
         translation_table = []
         for m in self._definitions:
             name = m.groupdict()['name']
+            return_val = name[len('MBEDTLS_TLS1_3_SIG_'):].lower()
             translation_table.append(
-                '\tcase {}:\n\t    return "{}";'.format(name,
-                                                        name[len('MBEDTLS_TLS1_3_SIG_'):].lower())
-                )
+                '    case {}:\n        return "{}";'.format(name, return_val))
 
         body = textwrap.dedent('''\
             const char *mbedtls_ssl_sig_alg_to_str( uint16_t in )
@@ -287,11 +288,70 @@ class SignatureAlgorithmDefinition:
             {translation_table}
                 }};
 
+                return "UNKNOWN";
+            }}''')
+        body = body.format(translation_table='\n'.join(translation_table))
+        return body
+
+
+class NamedGroupDefinition:
+    """
+        Generate helper functions for named group
+
+        It generates translation function from named group define to string.
+        Named group definition looks like:
+        #define MBEDTLS_SSL_IANA_TLS_GROUP_[ upper case named group ] [ value(hex) ]
+
+        Known limitation:
+        - the definitions SHOULD exist in same macro blocks.
+    """
+
+    @classmethod
+    def extract(cls, source_code, start=0, end=-1):
+        named_group_pattern = re.compile(r'#define\s+(?P<name>MBEDTLS_SSL_IANA_TLS_GROUP_\w+)\s+' +
+                                         r'(?P<value>0[xX][0-9a-fA-F]+)$',
+                                         re.MULTILINE | re.DOTALL)
+        matches = list(named_group_pattern.finditer(source_code, start, end))
+        if matches:
+            yield NamedGroupDefinition(source_code, definitions=matches)
+
+    def __init__(self, source_code, definitions=None):
+        if definitions is None:
+            definitions = []
+        assert isinstance(definitions, list) and definitions
+        self._definitions = definitions
+        self._source = source_code
+
+    def __repr__(self):
+        return 'NamedGroup({})'.format(self._definitions[0].span())
+
+    def span(self):
+        return self._definitions[0].span()
+
+    def __str__(self):
+        """
+            Generate function for translating value to string
+        """
+        translation_table = []
+        for m in self._definitions:
+            name = m.groupdict()['name']
+            iana_name = name[len('MBEDTLS_SSL_IANA_TLS_GROUP_'):].lower()
+            translation_table.append('    case {}:\n        return "{}";'.format(name, iana_name))
+
+        body = textwrap.dedent('''\
+            const char *mbedtls_ssl_named_group_to_str( uint16_t in )
+            {{
+                switch( in )
+                {{
+            {translation_table}
+                }};
+
                 return "UNKOWN";
             }}''')
         body = body.format(translation_table='\n'.join(translation_table))
         return body
 
+
 OUTPUT_C_TEMPLATE = '''\
 /* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */
 
@@ -335,14 +395,16 @@ def generate_ssl_debug_helpers(output_directory, mbedtls_root):
     """
         Generate functions of debug helps
     """
-    mbedtls_root = os.path.abspath(mbedtls_root or build_tree.guess_mbedtls_root())
+    mbedtls_root = os.path.abspath(
+        mbedtls_root or build_tree.guess_mbedtls_root())
     with open(os.path.join(mbedtls_root, 'include/mbedtls/ssl.h')) as f:
         source_code = remove_c_comments(f.read())
 
     definitions = dict()
     for start, instance in preprocess_c_source_code(source_code,
                                                     EnumDefinition,
-                                                    SignatureAlgorithmDefinition):
+                                                    SignatureAlgorithmDefinition,
+                                                    NamedGroupDefinition):
         if start in definitions:
             continue
         if isinstance(instance, EnumDefinition):

scripts/make_generated_files.bat

@@ -10,4 +10,5 @@ perl scripts\generate_features.pl || exit /b 1
 python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
+python tests\scripts\generate_bignum_tests.py || exit /b 1
 python tests\scripts\generate_psa_tests.py || exit /b 1

scripts/mbedtls_dev/crypto_knowledge.py

@@ -357,6 +357,7 @@ class Algorithm:
         'HKDF': AlgorithmCategory.KEY_DERIVATION,
         'TLS12_PRF': AlgorithmCategory.KEY_DERIVATION,
         'TLS12_PSK_TO_MS': AlgorithmCategory.KEY_DERIVATION,
+        'TLS12_ECJPAKE_TO_PMS': AlgorithmCategory.KEY_DERIVATION,
         'PBKDF': AlgorithmCategory.KEY_DERIVATION,
         'ECDH': AlgorithmCategory.KEY_AGREEMENT,
         'FFDH': AlgorithmCategory.KEY_AGREEMENT,

scripts/mbedtls_dev/psa_storage.py

@@ -1,4 +1,9 @@
 """Knowledge about the PSA key store as implemented in Mbed TLS.
+
+Note that if you need to make a change that affects how keys are
+stored, this may indicate that the key store is changing in a
+backward-incompatible way! Think carefully about backward compatibility
+before changing how test data is constructed or validated.
 """
 
 # Copyright The Mbed TLS Contributors
@@ -146,6 +151,11 @@ class Key:
         This is the content of the PSA storage file. When PSA storage is
         implemented over stdio files, this does not include any wrapping made
         by the PSA-storage-over-stdio-file implementation.
+
+        Note that if you need to make a change in this function,
+        this may indicate that the key store is changing in a
+        backward-incompatible way! Think carefully about backward
+        compatibility before making any change here.
         """
         header = self.MAGIC + self.pack('L', self.version)
         if self.version == 0:

scripts/mbedtls_dev/test_case.py

@@ -92,9 +92,11 @@ def write_data_file(filename: str,
     """
     if caller is None:
         caller = os.path.basename(sys.argv[0])
-    with open(filename, 'w') as out:
+    tempfile = filename + '.new'
+    with open(tempfile, 'w') as out:
         out.write('# Automatically generated by {}. Do not edit!\n'
                   .format(caller))
         for tc in test_cases:
             tc.write(out)
         out.write('\n# End of automatically generated file.\n')
+    os.replace(tempfile, filename)

scripts/mbedtls_dev/test_generation.py

@@ -0,0 +1,219 @@
+"""Common test generation classes and main function.
+
+These are used both by generate_psa_tests.py and generate_bignum_tests.py.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import posixpath
+import re
+
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
+
+from mbedtls_dev import build_tree
+from mbedtls_dev import test_case
+
+T = TypeVar('T') #pylint: disable=invalid-name
+
+
+class BaseTarget(metaclass=ABCMeta):
+    """Base target for test case generation.
+
+    Child classes of this class represent an output file, and can be referred
+    to as file targets. These indicate where test cases will be written to for
+    all subclasses of the file target, which is set by `target_basename`.
+
+    Attributes:
+        count: Counter for test cases from this class.
+        case_description: Short description of the test case. This may be
+            automatically generated using the class, or manually set.
+        dependencies: A list of dependencies required for the test case.
+        show_test_count: Toggle for inclusion of `count` in the test description.
+        target_basename: Basename of file to write generated tests to. This
+            should be specified in a child class of BaseTarget.
+        test_function: Test function which the class generates cases for.
+        test_name: A common name or description of the test function. This can
+            be `test_function`, a clearer equivalent, or a short summary of the
+            test function's purpose.
+    """
+    count = 0
+    case_description = ""
+    dependencies = [] # type: List[str]
+    show_test_count = True
+    target_basename = ""
+    test_function = ""
+    test_name = ""
+
+    def __new__(cls, *args, **kwargs):
+        # pylint: disable=unused-argument
+        cls.count += 1
+        return super().__new__(cls)
+
+    @abstractmethod
+    def arguments(self) -> List[str]:
+        """Get the list of arguments for the test case.
+
+        Override this method to provide the list of arguments required for
+        the `test_function`.
+
+        Returns:
+            List of arguments required for the test function.
+        """
+        raise NotImplementedError
+
+    def description(self) -> str:
+        """Create a test case description.
+
+        Creates a description of the test case, including a name for the test
+        function, an optional case count, and a description of the specific
+        test case. This should inform a reader what is being tested, and
+        provide context for the test case.
+
+        Returns:
+            Description for the test case.
+        """
+        if self.show_test_count:
+            return "{} #{} {}".format(
+                self.test_name, self.count, self.case_description
+                ).strip()
+        else:
+            return "{} {}".format(self.test_name, self.case_description).strip()
+
+
+    def create_test_case(self) -> test_case.TestCase:
+        """Generate TestCase from the instance."""
+        tc = test_case.TestCase()
+        tc.set_description(self.description())
+        tc.set_function(self.test_function)
+        tc.set_arguments(self.arguments())
+        tc.set_dependencies(self.dependencies)
+
+        return tc
+
+    @classmethod
+    @abstractmethod
+    def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
+        """Generate test cases for the class test function.
+
+        This will be called in classes where `test_function` is set.
+        Implementations should yield TestCase objects, by creating instances
+        of the class with appropriate input data, and then calling
+        `create_test_case()` on each.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def generate_tests(cls) -> Iterator[test_case.TestCase]:
+        """Generate test cases for the class and its subclasses.
+
+        In classes with `test_function` set, `generate_function_tests()` is
+        called to generate test cases first.
+
+        In all classes, this method will iterate over its subclasses, and
+        yield from `generate_tests()` in each. Calling this method on a class X
+        will yield test cases from all classes derived from X.
+        """
+        if cls.test_function:
+            yield from cls.generate_function_tests()
+        for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
+            yield from subclass.generate_tests()
+
+
+class TestGenerator:
+    """Generate test cases and write to data files."""
+    def __init__(self, options) -> None:
+        self.test_suite_directory = self.get_option(options, 'directory',
+                                                    'tests/suites')
+        # Update `targets` with an entry for each child class of BaseTarget.
+        # Each entry represents a file generated by the BaseTarget framework,
+        # and enables generating the .data files using the CLI.
+        self.targets.update({
+            subclass.target_basename: subclass.generate_tests
+            for subclass in BaseTarget.__subclasses__()
+        })
+
+    @staticmethod
+    def get_option(options, name: str, default: T) -> T:
+        value = getattr(options, name, None)
+        return default if value is None else value
+
+    def filename_for(self, basename: str) -> str:
+        """The location of the data file with the specified base name."""
+        return posixpath.join(self.test_suite_directory, basename + '.data')
+
+    def write_test_data_file(self, basename: str,
+                             test_cases: Iterable[test_case.TestCase]) -> None:
+        """Write the test cases to a .data file.
+
+        The output file is ``basename + '.data'`` in the test suite directory.
+        """
+        filename = self.filename_for(basename)
+        test_case.write_data_file(filename, test_cases)
+
+    # Note that targets whose names contain 'test_format' have their content
+    # validated by `abi_check.py`.
+    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
+
+    def generate_target(self, name: str, *target_args) -> None:
+        """Generate cases and write to data file for a target.
+
+        For target callables which require arguments, override this function
+        and pass these arguments using super() (see PSATestGenerator).
+        """
+        test_cases = self.targets[name](*target_args)
+        self.write_test_data_file(name, test_cases)
+
+def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
+    """Command line entry point."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--list', action='store_true',
+                        help='List available targets and exit')
+    parser.add_argument('--list-for-cmake', action='store_true',
+                        help='Print \';\'-separated list of available targets and exit')
+    parser.add_argument('--directory', metavar='DIR',
+                        help='Output directory (default: tests/suites)')
+    # The `--directory` option is interpreted relative to the directory from
+    # which the script is invoked, but the default is relative to the root of
+    # the mbedtls tree. The default should not be set above, but instead after
+    # `build_tree.chdir_to_root()` is called.
+    parser.add_argument('targets', nargs='*', metavar='TARGET',
+                        help='Target file to generate (default: all; "-": none)')
+    options = parser.parse_args(args)
+    build_tree.chdir_to_root()
+    generator = generator_class(options)
+    if options.list:
+        for name in sorted(generator.targets):
+            print(generator.filename_for(name))
+        return
+    # List in a cmake list format (i.e. ';'-separated)
+    if options.list_for_cmake:
+        print(';'.join(generator.filename_for(name)
+                       for name in sorted(generator.targets)), end='')
+        return
+    if options.targets:
+        # Allow "-" as a special case so you can run
+        # ``generate_xxx_tests.py - $targets`` and it works uniformly whether
+        # ``$targets`` is empty or not.
+        options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
+                           for target in options.targets
+                           if target != '-']
+    else:
+        options.targets = sorted(generator.targets)
+    for target in options.targets:
+        generator.generate_target(target)

scripts/min_requirements.py

@@ -44,8 +44,9 @@ class Requirements:
         """Adjust a requirement to the minimum specified version."""
         # allow inheritance #pylint: disable=no-self-use
         # If a requirement specifies a minimum version, impose that version.
-        req = re.sub(r'>=|~=', r'==', req)
-        return req
+        split_req = req.split(';', 1)
+        split_req[0] = re.sub(r'>=|~=', r'==', split_req[0])
+        return ';'.join(split_req)
 
     def add_file(self, filename: str) -> None:
         """Add requirements from the specified file.