diff --git a/tests/compat.sh b/tests/compat.sh index 75d5461599..952997537f 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -128,10 +128,40 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" + printf " --list-test-case\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } +# print_test_case +print_test_case() { + for i in $3; do + uniform_title $1 $2 $i + echo $TITLE + done +} + +# list_test_case lists all potential test cases in compat.sh without execution +list_test_case() { + for MODE in $MODES; do + for TYPE in $TYPES; do + for VERIFY in $VERIFIES; do + VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') + reset_ciphersuites + add_common_ciphersuites + add_openssl_ciphersuites + add_gnutls_ciphersuites + add_mbedtls_ciphersuites + print_test_case m O "$O_CIPHERS" + print_test_case O m "$O_CIPHERS" + print_test_case m G "$G_CIPHERS" + print_test_case G m "$G_CIPHERS" + print_test_case m m "$M_CIPHERS" + done + done + done +} + get_options() { while [ $# -gt 0 ]; do case "$1" in @@ -159,6 +189,12 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + # Please check scripts/check_test_cases.py correspondingly + # if you have to modify option, --list-test-case + --list-test-case) + list_test_case + exit $? + ;; --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE=$1 ;; @@ -1199,15 +1235,21 @@ report_fail() { fi } +# uniform_title +# $TITLE is considered as test case description for both --list-test-case and +# MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of +# each test case description. +uniform_title() { + TITLE="$1->$2 $MODE,$VERIF $3" +} + # run_client run_client() { # announce what we're going to do TESTS=$(( $TESTS + 1 )) - TITLE="`echo $1 | head -c1`->`echo $SERVER_NAME | head -c1`" - TITLE="$TITLE $MODE,$VERIF $2" - printf "%s " "$TITLE" - LEN=$(( 72 - `echo "$TITLE" | wc -c` )) - for i in `seq 1 $LEN`; do printf '.'; done; printf ' ' + uniform_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2 + DOTS72="........................................................................" + printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index c9f5e11dd8..213708b624 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -25,6 +25,7 @@ import argparse import glob import os import re +import subprocess import sys class Results: @@ -111,6 +112,19 @@ state may override this method. self.process_test_case(descriptions, file_name, line_number, description) + def walk_compat_sh(self, file_name): + """Iterate over the test cases compat.sh with a similar format.""" + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none + compat_cmd = ['sh', file_name, '--list-test-case'] + compat_output = subprocess.check_output(compat_cmd) + # Assume compat.sh is responsible for printing identical format of + # test case description between --list-test-case and its OUTCOME.CSV + description = compat_output.strip().split(b'\n') + # idx indicates the number of test case since there is no line number + # in `compat.sh` for each test case. + for idx, descrip in enumerate(description): + self.process_test_case(descriptions, file_name, idx, descrip) + @staticmethod def collect_test_directories(): """Get the relative path for the TLS and Crypto test directories.""" @@ -133,6 +147,9 @@ state may override this method. ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') if os.path.exists(ssl_opt_sh): self.walk_ssl_opt_sh(ssl_opt_sh) + compat_sh = os.path.join(directory, 'compat.sh') + if os.path.exists(compat_sh): + self.walk_compat_sh(compat_sh) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases."""