Merge pull request #8088 from tgonzalezorlandoarm/tg/check_test_cases-new

Make check_test_cases.py recognize test case name templates in ssl-opt.sh

tests/compat.sh

@@ -127,7 +127,7 @@ print_usage() {
     printf "            \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
-    printf "     --list-test-case\tList all potential test cases (No Execution)\n"
+    printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --preserve-logs\tPreserve logs of successful tests as well\n"
@@ -141,8 +141,8 @@ print_test_case() {
     done
 }
 
-# list_test_case lists all potential test cases in compat.sh without execution
+# list_test_cases lists all potential test cases in compat.sh without execution
-list_test_case() {
+list_test_cases() {
     reset_ciphersuites
     for TYPE in $TYPES; do
         add_common_ciphersuites
@@ -191,9 +191,9 @@ get_options() {
                 MEMCHECK=1
                 ;;
             # Please check scripts/check_test_cases.py correspondingly
-            # if you have to modify option, --list-test-case
+            # if you have to modify option, --list-test-cases
-            --list-test-case)
+            --list-test-cases)
-                list_test_case
+                list_test_cases
                 exit $?
                 ;;
             --outcome-file)
@@ -869,7 +869,7 @@ wait_client_done() {
 }
 
 # uniform_title <CLIENT> <SERVER> <STANDARD_CIPHER_SUITE>
-# $TITLE is considered as test case description for both --list-test-case and
+# $TITLE is considered as test case description for both --list-test-cases and
 # MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of
 # each test case description.
 uniform_title() {

tests/scripts/check_test_cases.py

@@ -28,6 +28,7 @@ import re
 import subprocess
 import sys
 
+
 class Results:
     """Store file and line information about errors or warnings in test suites."""
 
@@ -97,33 +98,21 @@ state may override this method.
                                            data_file_name, line_number, line)
                 in_paragraph = True
 
-    def walk_ssl_opt_sh(self, file_name):
+    def collect_from_script(self, file_name):
-        """Iterate over the test cases in ssl-opt.sh or a file with a similar format."""
+        """Collect the test cases in a script by calling its listing test cases
+option"""
         descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
-        with open(file_name, 'rb') as file_contents:
+        listed = subprocess.check_output(['sh', file_name, '--list-test-cases'])
-            for line_number, line in enumerate(file_contents, 1):
+        # Assume test file is responsible for printing identical format of
-                # Assume that all run_test calls have the same simple form
+        # test case description between --list-test-cases and its OUTCOME.CSV
-                # with the test description entirely on the same line as the
+        #
-                # function name.
-                m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
-                if not m:
-                    continue
-                description = m.group(1)
-                self.process_test_case(descriptions,
-                                       file_name, line_number, description)
-
-    def walk_compat_sh(self, file_name):
-        """Iterate over the test cases compat.sh with a similar format."""
-        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
-        compat_cmd = ['sh', file_name, '--list-test-case']
-        compat_output = subprocess.check_output(compat_cmd)
-        # Assume compat.sh is responsible for printing identical format of
-        # test case description between --list-test-case and its OUTCOME.CSV
-        description = compat_output.strip().split(b'\n')
         # idx indicates the number of test case since there is no line number
-        # in `compat.sh` for each test case.
+        # in the script for each test case.
-        for idx, descrip in enumerate(description):
+        for idx, description in enumerate(listed.splitlines()):
-            self.process_test_case(descriptions, file_name, idx, descrip)
+            self.process_test_case(descriptions,
+                                   file_name,
+                                   idx,
+                                   description.rstrip())
 
     @staticmethod
     def collect_test_directories():
@@ -144,15 +133,11 @@ state may override this method.
             for data_file_name in glob.glob(os.path.join(directory, 'suites',
                                                          '*.data')):
                 self.walk_test_suite(data_file_name)
-            ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
+
-            if os.path.exists(ssl_opt_sh):
+            for sh_file in ['ssl-opt.sh', 'compat.sh']:
-                self.walk_ssl_opt_sh(ssl_opt_sh)
+                sh_file = os.path.join(directory, sh_file)
-            for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases',
+                if os.path.exists(sh_file):
-                                                            '*.sh')):
+                    self.collect_from_script(sh_file)
-                self.walk_ssl_opt_sh(ssl_opt_file_name)
-            compat_sh = os.path.join(directory, 'compat.sh')
-            if os.path.exists(compat_sh):
-                self.walk_compat_sh(compat_sh)
 
 class TestDescriptions(TestDescriptionExplorer):
     """Collect the available test cases."""

tests/ssl-opt.sh

@@ -123,6 +123,7 @@ FILTER='.*'
 EXCLUDE='^$'
 
 SHOW_TEST_NUMBER=0
+LIST_TESTS=0
 RUN_TEST_NUMBER=''
 
 PRESERVE_LOGS=0
@@ -142,6 +143,7 @@ print_usage() {
     printf "  -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n"
     printf "  -s|--show-numbers\tShow test numbers in front of test names\n"
     printf "  -p|--preserve-logs\tPreserve logs of successful tests as well\n"
+    printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --port     \tTCP/UDP port (default: randomish 1xxxx)\n"
@@ -167,6 +169,9 @@ get_options() {
             -s|--show-numbers)
                 SHOW_TEST_NUMBER=1
                 ;;
+            -l|--list-test-cases)
+                LIST_TESTS=1
+                ;;
             -p|--preserve-logs)
                 PRESERVE_LOGS=1
                 ;;
@@ -196,11 +201,18 @@ get_options() {
     done
 }
 
+get_options "$@"
+
 # Read boolean configuration options from mbedtls_config.h for easy and quick
 # testing. Skip non-boolean options (with something other than spaces
 # and a comment after "#define SYMBOL"). The variable contains a
 # space-separated list of symbols.
+if [ "$LIST_TESTS" -eq 0 ];then
     CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )"
+else
+    P_QUERY=":"
+    CONFIGS_ENABLED=""
+fi
 # Skip next test; use this macro to skip tests which are legitimate
 # in theory and expected to be re-introduced at some point, but
 # aren't expected to succeed at the moment due to problems outside
@@ -296,7 +308,12 @@ get_config_value_or_default() {
     #
     # Note that if the configuration is not defined or is defined to nothing,
     # the output of this function will be an empty string.
+    if [ "$LIST_TESTS" -eq 0 ];then
         ${P_SRV} "query_config=${1}"
+    else
+        echo "1"
+    fi
+
 }
 
 requires_config_value_at_least() {
@@ -807,10 +824,11 @@ requires_not_i686() {
     fi
 }
 
-# Calculate the input & output maximum content lengths set in the config
 MAX_CONTENT_LEN=16384
 MAX_IN_LEN=$( get_config_value_or_default "MBEDTLS_SSL_IN_CONTENT_LEN" )
 MAX_OUT_LEN=$( get_config_value_or_default "MBEDTLS_SSL_OUT_CONTENT_LEN" )
+if [ "$LIST_TESTS" -eq 0 ];then
+    # Calculate the input & output maximum content lengths set in the config
 
     # Calculate the maximum content length that fits both
     if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then
@@ -819,7 +837,7 @@ fi
     if [ "$MAX_OUT_LEN" -lt "$MAX_CONTENT_LEN" ]; then
         MAX_CONTENT_LEN="$MAX_OUT_LEN"
     fi
-
+fi
 # skip the next test if the SSL output buffer is less than 16KB
 requires_full_size_output_buffer() {
     if [ "$MAX_OUT_LEN" -ne 16384 ]; then
@@ -861,6 +879,7 @@ print_name() {
     fi
 
     LINE="$LINE$1"
+
     printf "%s " "$LINE"
     LEN=$(( 72 - `echo "$LINE" | wc -c` ))
     for i in `seq 1 $LEN`; do printf '.'; done
@@ -876,7 +895,7 @@ record_outcome() {
     if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
         printf '%s;%s;%s;%s;%s;%s\n' \
                "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
-               "${TEST_SUITE_NAME:-ssl-opt}" "$NAME" \
+               "ssl-opt" "$NAME" \
                "$1" "${2-}" \
                >>"$MBEDTLS_TEST_OUTCOME_FILE"
     fi
@@ -1578,6 +1597,11 @@ run_test() {
         return
     fi
 
+    if [ "$LIST_TESTS" -gt 0 ]; then
+        printf "%s\n" "$NAME"
+        return
+    fi
+
     print_name "$NAME"
 
     # Do we only run numbered tests?
@@ -1775,8 +1799,6 @@ cleanup() {
 # MAIN
 #
 
-get_options "$@"
-
 # Make the outcome file path relative to the original directory, not
 # to .../tests
 case "$MBEDTLS_TEST_OUTCOME_FILE" in
@@ -1827,6 +1849,8 @@ else
     }
 fi
 
+if [ "$LIST_TESTS" -eq 0 ];then
+
     # sanity checks, avoid an avalanche of errors
     P_SRV_BIN="${P_SRV%%[  ]*}"
     P_CLI_BIN="${P_CLI%%[  ]*}"
@@ -1930,6 +1954,7 @@ fi
     P_SRV="$P_SRV allow_sha1=1"
     P_CLI="$P_CLI allow_sha1=1"
 
+fi
 # Also pick a unique name for intermediate files
 SRV_OUT="srv_out.$$"
 CLI_OUT="cli_out.$$"
@@ -13375,6 +13400,8 @@ requires_config_enabled MBEDTLS_SSL_MAX_FRAGMENT_LENGTH
 requires_max_content_len 16384
 run_tests_memory_after_hanshake
 
+if [ "$LIST_TESTS" -eq 0 ]; then
+
     # Final report
 
     echo "------------------------------------------------------------------------"
@@ -13387,6 +13414,8 @@ fi
     PASSES=$(( $TESTS - $FAILS ))
     echo " ($PASSES / $TESTS tests ($SKIPS skipped))"
 
+fi
+
 if [ $FAILS -gt 255 ]; then
     # Clamp at 255 as caller gets exit code & 0xFF
     # (so 256 would be 0, or success, etc)