From 24552ff84edddb1a341b828cd092ad46333f1aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 17 Aug 2023 15:10:03 +0100 Subject: [PATCH 01/20] ssl-opt/run_test: Introduce -l option to list test case names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add an option in ssl-opt test case to list all the run_test calls and their names. This allows to show the parameters used and can make us avoid having to parse ssl-opt to look for extra parameters in the future. Signed-off-by: Tomás González --- tests/ssl-opt.sh | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 0dd7fe6d36..b9380bad9b 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -123,6 +123,7 @@ FILTER='.*' EXCLUDE='^$' SHOW_TEST_NUMBER=0 +LIST_TESTS=0 RUN_TEST_NUMBER='' PRESERVE_LOGS=0 @@ -140,6 +141,7 @@ print_usage() { printf " -f|--filter\tOnly matching tests are executed (substring or BRE)\n" printf " -e|--exclude\tMatching tests are excluded (substring or BRE)\n" printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n" + printf " -l|--list-tests\tList test names and exit\n" printf " -s|--show-numbers\tShow test numbers in front of test names\n" printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n" printf " --outcome-file\tFile where test outcomes are written\n" @@ -167,6 +169,9 @@ get_options() { -s|--show-numbers) SHOW_TEST_NUMBER=1 ;; + -l|--list-tests) + LIST_TESTS=1 + ;; -p|--preserve-logs) PRESERVE_LOGS=1 ;; @@ -862,9 +867,13 @@ print_name() { LINE="$LINE$1" printf "%s " "$LINE" - LEN=$(( 72 - `echo "$LINE" | wc -c` )) - for i in `seq 1 $LEN`; do printf '.'; done - printf ' ' + if [ "$LIST_TESTS" -gt 0 ]; then + printf "\n" + else + LEN=$(( 72 - `echo "$LINE" | wc -c` )) + for i in `seq 1 $LEN`; do printf '.'; done + printf ' ' + fi } @@ -1580,6 +1589,10 @@ run_test() { print_name "$NAME" + if [ "$LIST_TESTS" -gt 0 ]; then + return + fi + # Do we only run numbered tests? if [ -n "$RUN_TEST_NUMBER" ]; then case ",$RUN_TEST_NUMBER," in @@ -13375,17 +13388,21 @@ requires_config_enabled MBEDTLS_SSL_MAX_FRAGMENT_LENGTH requires_max_content_len 16384 run_tests_memory_after_hanshake -# Final report +if [ "$LIST_TESTS" -eq 0 ]; then -echo "------------------------------------------------------------------------" + # Final report + + echo "------------------------------------------------------------------------" + + if [ $FAILS = 0 ]; then + printf "PASSED" + else + printf "FAILED" + fi + PASSES=$(( $TESTS - $FAILS )) + echo " ($PASSES / $TESTS tests ($SKIPS skipped))" -if [ $FAILS = 0 ]; then - printf "PASSED" -else - printf "FAILED" fi -PASSES=$(( $TESTS - $FAILS )) -echo " ($PASSES / $TESTS tests ($SKIPS skipped))" if [ $FAILS -gt 255 ]; then # Clamp at 255 as caller gets exit code & 0xFF From 754f8cd9594245ba8c3787d2a3870ca7178c4e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 17 Aug 2023 15:11:10 +0100 Subject: [PATCH 02/20] tests/check_test_cases: Use ssl-opt.sh -l option instead of parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use the newly added ssl-opt.sh -l option to list all the tests cases and their used parameters instead of having to parse the file to discover them. This avoids having to add further parsing complexity in the future as discussed in https://github.com/Mbed-TLS/mbedtls/pull/8080#issuecomment-1681064743 Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 1395d4d901..be5b834b04 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -27,6 +27,8 @@ import os import re import subprocess import sys +import subprocess + class Results: """Store file and line information about errors or warnings in test suites.""" @@ -100,17 +102,11 @@ state may override this method. def walk_ssl_opt_sh(self, file_name): """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - with open(file_name, 'rb') as file_contents: - for line_number, line in enumerate(file_contents, 1): - # Assume that all run_test calls have the same simple form - # with the test description entirely on the same line as the - # function name. - m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) - if not m: - continue - description = m.group(1) - self.process_test_case(descriptions, - file_name, line_number, description) + listed = subprocess.run(['tests/ssl-opt.sh', '-l'], + capture_output=True) + listed = set(map(lambda x: x.rstrip(), listed.stdout.splitlines())) + for description in listed: + self.process_test_case(descriptions, file_name, None, description) def walk_compat_sh(self, file_name): """Iterate over the test cases compat.sh with a similar format.""" From 546fc9ce9e0dc1cd04b48fa31fd821a128377d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 17 Aug 2023 16:56:42 +0100 Subject: [PATCH 03/20] Revert "Add opt-testcases into check list" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ssl-opt.sh now takes care of looking at the files in the opt-testcases subdirectory, so use ssl-opt.sh directly in walk_ssl_opt_sh(). * Revert commit f17a60f147ec3ba6a9f76da38d3802a7f9f05227. Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index be5b834b04..96e3839776 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -143,9 +143,6 @@ state may override this method. ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') if os.path.exists(ssl_opt_sh): self.walk_ssl_opt_sh(ssl_opt_sh) - for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases', - '*.sh')): - self.walk_ssl_opt_sh(ssl_opt_file_name) compat_sh = os.path.join(directory, 'compat.sh') if os.path.exists(compat_sh): self.walk_compat_sh(compat_sh) From 079eaee8ca54918f0b851692c6c57e522c97f689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 17 Aug 2023 17:02:04 +0100 Subject: [PATCH 04/20] Use file_name parameter in walk_ssl_opt_sh() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove hardcoded file_name and use the parameter provided in the function. Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 96e3839776..0ca0defde4 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -102,7 +102,7 @@ state may override this method. def walk_ssl_opt_sh(self, file_name): """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.run(['tests/ssl-opt.sh', '-l'], + listed = subprocess.run([f'{file_name}', '-l'], capture_output=True) listed = set(map(lambda x: x.rstrip(), listed.stdout.splitlines())) for description in listed: From 970b39fb38d01dbed08bd6abd7d6224e821fb6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 17 Aug 2023 17:07:53 +0100 Subject: [PATCH 05/20] tests/check_test_cases: Use subprocess.check_output instead of run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 0ca0defde4..8d7e8baf30 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -102,9 +102,8 @@ state may override this method. def walk_ssl_opt_sh(self, file_name): """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.run([f'{file_name}', '-l'], - capture_output=True) - listed = set(map(lambda x: x.rstrip(), listed.stdout.splitlines())) + listed = subprocess.check_output([f'{file_name}', '-l']) + listed = set(map(lambda x: x.rstrip(), listed.splitlines())) for description in listed: self.process_test_case(descriptions, file_name, None, description) From 3a65d6368a492234352fcb707ab00b69641e99fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 18 Aug 2023 09:45:19 +0100 Subject: [PATCH 06/20] Remove formatted string to make pylint happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 8d7e8baf30..4301b3210f 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -102,7 +102,7 @@ state may override this method. def walk_ssl_opt_sh(self, file_name): """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.check_output([f'{file_name}', '-l']) + listed = subprocess.check_output([file_name, '-l']) listed = set(map(lambda x: x.rstrip(), listed.splitlines())) for description in listed: self.process_test_case(descriptions, file_name, None, description) From 787428a08c94662a0c6440f029a2d06f46ab3ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Wed, 23 Aug 2023 15:27:19 +0100 Subject: [PATCH 07/20] Avoid skipping test when printing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/ssl-opt.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index b9380bad9b..7c5db293a8 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -1581,18 +1581,18 @@ run_test() { NAME="$1" shift 1 - if is_excluded "$NAME"; then - SKIP_NEXT="NO" - # There was no request to run the test, so don't record its outcome. - return - fi - print_name "$NAME" if [ "$LIST_TESTS" -gt 0 ]; then return fi + if is_excluded "$NAME"; then + SKIP_NEXT="NO" + # There was no request to run the test, so don't record its outcome. + return + fi + # Do we only run numbered tests? if [ -n "$RUN_TEST_NUMBER" ]; then case ",$RUN_TEST_NUMBER," in From 0e8a08a1f7fbe5c7591a9083e32908e863d674fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Wed, 23 Aug 2023 15:29:57 +0100 Subject: [PATCH 08/20] Get options at beginning of program MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/ssl-opt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 7c5db293a8..62dcbacb35 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -201,6 +201,8 @@ get_options() { done } +get_options "$@" + # Read boolean configuration options from mbedtls_config.h for easy and quick # testing. Skip non-boolean options (with something other than spaces # and a comment after "#define SYMBOL"). The variable contains a @@ -1788,8 +1790,6 @@ cleanup() { # MAIN # -get_options "$@" - # Make the outcome file path relative to the original directory, not # to .../tests case "$MBEDTLS_TEST_OUTCOME_FILE" in From f162b4f497685fe0740152c8010c83ae48d17a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Wed, 23 Aug 2023 15:31:12 +0100 Subject: [PATCH 09/20] Only use CONFIGS_ENABLED when not listing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/ssl-opt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 62dcbacb35..05eba3d408 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -207,7 +207,11 @@ get_options "$@" # testing. Skip non-boolean options (with something other than spaces # and a comment after "#define SYMBOL"). The variable contains a # space-separated list of symbols. -CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )" +if [ "$LIST_TESTS" -eq 0 ];then + CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )" +else + CONFIGS_ENABLED="" +fi # Skip next test; use this macro to skip tests which are legitimate # in theory and expected to be re-introduced at some point, but # aren't expected to succeed at the moment due to problems outside From 06956a12aa83c22690004496e082d9813f6bbd2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Wed, 23 Aug 2023 15:46:20 +0100 Subject: [PATCH 10/20] Skip unnecessary logic when -l option is used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/ssl-opt.sh | 263 +++++++++++++++++++++++++++-------------------- 1 file changed, 154 insertions(+), 109 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 05eba3d408..31114c94a0 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -223,6 +223,9 @@ skip_next_test() { # Check if the required configuration ($1) is enabled is_config_enabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return 0; + fi case $CONFIGS_ENABLED in *" $1"[\ =]*) return 0;; *) return 1;; @@ -231,6 +234,9 @@ is_config_enabled() # skip next test if the flag is not enabled in mbedtls_config.h requires_config_enabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi case $CONFIGS_ENABLED in *" $1"[\ =]*) :;; *) SKIP_NEXT="YES";; @@ -239,12 +245,18 @@ requires_config_enabled() { # skip next test if the flag is enabled in mbedtls_config.h requires_config_disabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi case $CONFIGS_ENABLED in *" $1"[\ =]*) SKIP_NEXT="YES";; esac } requires_all_configs_enabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi if ! $P_QUERY -all $* then SKIP_NEXT="YES" @@ -252,6 +264,9 @@ requires_all_configs_enabled() { } requires_all_configs_disabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi if $P_QUERY -any $* then SKIP_NEXT="YES" @@ -259,6 +274,9 @@ requires_all_configs_disabled() { } requires_any_configs_enabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi if ! $P_QUERY -any $* then SKIP_NEXT="YES" @@ -266,6 +284,9 @@ requires_any_configs_enabled() { } requires_any_configs_disabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi if $P_QUERY -all $* then SKIP_NEXT="YES" @@ -290,6 +311,9 @@ TLS1_2_KEY_EXCHANGES_WITH_CERT_WO_ECDH="MBEDTLS_KEY_EXCHANGE_RSA_ENABLED \ MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED" requires_key_exchange_with_cert_in_tls12_or_tls13_enabled() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi if $P_QUERY -all MBEDTLS_SSL_PROTO_TLS1_2 then requires_any_configs_enabled $TLS1_2_KEY_EXCHANGES_WITH_CERT @@ -307,10 +331,18 @@ get_config_value_or_default() { # # Note that if the configuration is not defined or is defined to nothing, # the output of this function will be an empty string. - ${P_SRV} "query_config=${1}" + if [ "$LIST_TESTS" -eq 0 ];then + ${P_SRV} "query_config=${1}" + else + echo "1" + fi + } requires_config_value_at_least() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi VAL="$( get_config_value_or_default "$1" )" if [ -z "$VAL" ]; then # Should never happen @@ -322,6 +354,9 @@ requires_config_value_at_least() { } requires_config_value_at_most() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi VAL=$( get_config_value_or_default "$1" ) if [ -z "$VAL" ]; then # Should never happen @@ -333,6 +368,9 @@ requires_config_value_at_most() { } requires_config_value_equals() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi VAL=$( get_config_value_or_default "$1" ) if [ -z "$VAL" ]; then # Should never happen @@ -348,6 +386,9 @@ requires_config_value_equals() { # Inputs: # * $1: protocol version in mbedtls syntax (argument to force_version=) requires_protocol_version() { + if [ "$LIST_TESTS" -gt 0 ];then + return; + fi # Support for DTLS is detected separately in detect_dtls(). case "$1" in tls12|dtls12) requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2;; @@ -818,19 +859,20 @@ requires_not_i686() { fi } -# Calculate the input & output maximum content lengths set in the config MAX_CONTENT_LEN=16384 MAX_IN_LEN=$( get_config_value_or_default "MBEDTLS_SSL_IN_CONTENT_LEN" ) MAX_OUT_LEN=$( get_config_value_or_default "MBEDTLS_SSL_OUT_CONTENT_LEN" ) +if [ "$LIST_TESTS" -eq 0 ];then + # Calculate the input & output maximum content lengths set in the config -# Calculate the maximum content length that fits both -if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then - MAX_CONTENT_LEN="$MAX_IN_LEN" + # Calculate the maximum content length that fits both + if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then + MAX_CONTENT_LEN="$MAX_IN_LEN" + fi + if [ "$MAX_OUT_LEN" -lt "$MAX_CONTENT_LEN" ]; then + MAX_CONTENT_LEN="$MAX_OUT_LEN" + fi fi -if [ "$MAX_OUT_LEN" -lt "$MAX_CONTENT_LEN" ]; then - MAX_CONTENT_LEN="$MAX_OUT_LEN" -fi - # skip the next test if the SSL output buffer is less than 16KB requires_full_size_output_buffer() { if [ "$MAX_OUT_LEN" -ne 16384 ]; then @@ -1844,109 +1886,112 @@ else } fi -# sanity checks, avoid an avalanche of errors -P_SRV_BIN="${P_SRV%%[ ]*}" -P_CLI_BIN="${P_CLI%%[ ]*}" -P_PXY_BIN="${P_PXY%%[ ]*}" -if [ ! -x "$P_SRV_BIN" ]; then - echo "Command '$P_SRV_BIN' is not an executable file" - exit 1 -fi -if [ ! -x "$P_CLI_BIN" ]; then - echo "Command '$P_CLI_BIN' is not an executable file" - exit 1 -fi -if [ ! -x "$P_PXY_BIN" ]; then - echo "Command '$P_PXY_BIN' is not an executable file" - exit 1 -fi -if [ "$MEMCHECK" -gt 0 ]; then - if which valgrind >/dev/null 2>&1; then :; else - echo "Memcheck not possible. Valgrind not found" +if [ "$LIST_TESTS" -eq 0 ];then + + # sanity checks, avoid an avalanche of errors + P_SRV_BIN="${P_SRV%%[ ]*}" + P_CLI_BIN="${P_CLI%%[ ]*}" + P_PXY_BIN="${P_PXY%%[ ]*}" + if [ ! -x "$P_SRV_BIN" ]; then + echo "Command '$P_SRV_BIN' is not an executable file" exit 1 fi + if [ ! -x "$P_CLI_BIN" ]; then + echo "Command '$P_CLI_BIN' is not an executable file" + exit 1 + fi + if [ ! -x "$P_PXY_BIN" ]; then + echo "Command '$P_PXY_BIN' is not an executable file" + exit 1 + fi + if [ "$MEMCHECK" -gt 0 ]; then + if which valgrind >/dev/null 2>&1; then :; else + echo "Memcheck not possible. Valgrind not found" + exit 1 + fi + fi + if which $OPENSSL >/dev/null 2>&1; then :; else + echo "Command '$OPENSSL' not found" + exit 1 + fi + + # used by watchdog + MAIN_PID="$$" + + # We use somewhat arbitrary delays for tests: + # - how long do we wait for the server to start (when lsof not available)? + # - how long do we allow for the client to finish? + # (not to check performance, just to avoid waiting indefinitely) + # Things are slower with valgrind, so give extra time here. + # + # Note: without lsof, there is a trade-off between the running time of this + # script and the risk of spurious errors because we didn't wait long enough. + # The watchdog delay on the other hand doesn't affect normal running time of + # the script, only the case where a client or server gets stuck. + if [ "$MEMCHECK" -gt 0 ]; then + START_DELAY=6 + DOG_DELAY=60 + else + START_DELAY=2 + DOG_DELAY=20 + fi + + # some particular tests need more time: + # - for the client, we multiply the usual watchdog limit by a factor + # - for the server, we sleep for a number of seconds after the client exits + # see client_need_more_time() and server_needs_more_time() + CLI_DELAY_FACTOR=1 + SRV_DELAY_SECONDS=0 + + # fix commands to use this port, force IPv4 while at it + # +SRV_PORT will be replaced by either $SRV_PORT or $PXY_PORT later + # Note: Using 'localhost' rather than 127.0.0.1 here is unwise, as on many + # machines that will resolve to ::1, and we don't want ipv6 here. + P_SRV="$P_SRV server_addr=127.0.0.1 server_port=$SRV_PORT" + P_CLI="$P_CLI server_addr=127.0.0.1 server_port=+SRV_PORT" + P_PXY="$P_PXY server_addr=127.0.0.1 server_port=$SRV_PORT listen_addr=127.0.0.1 listen_port=$PXY_PORT ${SEED:+"seed=$SEED"}" + O_SRV="$O_SRV -accept $SRV_PORT" + O_CLI="$O_CLI -connect 127.0.0.1:+SRV_PORT" + G_SRV="$G_SRV -p $SRV_PORT" + G_CLI="$G_CLI -p +SRV_PORT" + + # Newer versions of OpenSSL have a syntax to enable all "ciphers", even + # low-security ones. This covers not just cipher suites but also protocol + # versions. It is necessary, for example, to use (D)TLS 1.0/1.1 on + # OpenSSL 1.1.1f from Ubuntu 20.04. The syntax was only introduced in + # OpenSSL 1.1.0 (21e0c1d23afff48601eb93135defddae51f7e2e3) and I can't find + # a way to discover it from -help, so check the openssl version. + case $($OPENSSL version) in + "OpenSSL 0"*|"OpenSSL 1.0"*) :;; + *) + O_CLI="$O_CLI -cipher ALL@SECLEVEL=0" + O_SRV="$O_SRV -cipher ALL@SECLEVEL=0" + ;; + esac + + if [ -n "${OPENSSL_NEXT:-}" ]; then + O_NEXT_SRV="$O_NEXT_SRV -accept $SRV_PORT" + O_NEXT_SRV_NO_CERT="$O_NEXT_SRV_NO_CERT -accept $SRV_PORT" + O_NEXT_SRV_EARLY_DATA="$O_NEXT_SRV_EARLY_DATA -accept $SRV_PORT" + O_NEXT_CLI="$O_NEXT_CLI -connect 127.0.0.1:+SRV_PORT" + O_NEXT_CLI_NO_CERT="$O_NEXT_CLI_NO_CERT -connect 127.0.0.1:+SRV_PORT" + fi + + if [ -n "${GNUTLS_NEXT_SERV:-}" ]; then + G_NEXT_SRV="$G_NEXT_SRV -p $SRV_PORT" + G_NEXT_SRV_NO_CERT="$G_NEXT_SRV_NO_CERT -p $SRV_PORT" + fi + + if [ -n "${GNUTLS_NEXT_CLI:-}" ]; then + G_NEXT_CLI="$G_NEXT_CLI -p +SRV_PORT" + G_NEXT_CLI_NO_CERT="$G_NEXT_CLI_NO_CERT -p +SRV_PORT localhost" + fi + + # Allow SHA-1, because many of our test certificates use it + P_SRV="$P_SRV allow_sha1=1" + P_CLI="$P_CLI allow_sha1=1" + fi -if which $OPENSSL >/dev/null 2>&1; then :; else - echo "Command '$OPENSSL' not found" - exit 1 -fi - -# used by watchdog -MAIN_PID="$$" - -# We use somewhat arbitrary delays for tests: -# - how long do we wait for the server to start (when lsof not available)? -# - how long do we allow for the client to finish? -# (not to check performance, just to avoid waiting indefinitely) -# Things are slower with valgrind, so give extra time here. -# -# Note: without lsof, there is a trade-off between the running time of this -# script and the risk of spurious errors because we didn't wait long enough. -# The watchdog delay on the other hand doesn't affect normal running time of -# the script, only the case where a client or server gets stuck. -if [ "$MEMCHECK" -gt 0 ]; then - START_DELAY=6 - DOG_DELAY=60 -else - START_DELAY=2 - DOG_DELAY=20 -fi - -# some particular tests need more time: -# - for the client, we multiply the usual watchdog limit by a factor -# - for the server, we sleep for a number of seconds after the client exits -# see client_need_more_time() and server_needs_more_time() -CLI_DELAY_FACTOR=1 -SRV_DELAY_SECONDS=0 - -# fix commands to use this port, force IPv4 while at it -# +SRV_PORT will be replaced by either $SRV_PORT or $PXY_PORT later -# Note: Using 'localhost' rather than 127.0.0.1 here is unwise, as on many -# machines that will resolve to ::1, and we don't want ipv6 here. -P_SRV="$P_SRV server_addr=127.0.0.1 server_port=$SRV_PORT" -P_CLI="$P_CLI server_addr=127.0.0.1 server_port=+SRV_PORT" -P_PXY="$P_PXY server_addr=127.0.0.1 server_port=$SRV_PORT listen_addr=127.0.0.1 listen_port=$PXY_PORT ${SEED:+"seed=$SEED"}" -O_SRV="$O_SRV -accept $SRV_PORT" -O_CLI="$O_CLI -connect 127.0.0.1:+SRV_PORT" -G_SRV="$G_SRV -p $SRV_PORT" -G_CLI="$G_CLI -p +SRV_PORT" - -# Newer versions of OpenSSL have a syntax to enable all "ciphers", even -# low-security ones. This covers not just cipher suites but also protocol -# versions. It is necessary, for example, to use (D)TLS 1.0/1.1 on -# OpenSSL 1.1.1f from Ubuntu 20.04. The syntax was only introduced in -# OpenSSL 1.1.0 (21e0c1d23afff48601eb93135defddae51f7e2e3) and I can't find -# a way to discover it from -help, so check the openssl version. -case $($OPENSSL version) in - "OpenSSL 0"*|"OpenSSL 1.0"*) :;; - *) - O_CLI="$O_CLI -cipher ALL@SECLEVEL=0" - O_SRV="$O_SRV -cipher ALL@SECLEVEL=0" - ;; -esac - -if [ -n "${OPENSSL_NEXT:-}" ]; then - O_NEXT_SRV="$O_NEXT_SRV -accept $SRV_PORT" - O_NEXT_SRV_NO_CERT="$O_NEXT_SRV_NO_CERT -accept $SRV_PORT" - O_NEXT_SRV_EARLY_DATA="$O_NEXT_SRV_EARLY_DATA -accept $SRV_PORT" - O_NEXT_CLI="$O_NEXT_CLI -connect 127.0.0.1:+SRV_PORT" - O_NEXT_CLI_NO_CERT="$O_NEXT_CLI_NO_CERT -connect 127.0.0.1:+SRV_PORT" -fi - -if [ -n "${GNUTLS_NEXT_SERV:-}" ]; then - G_NEXT_SRV="$G_NEXT_SRV -p $SRV_PORT" - G_NEXT_SRV_NO_CERT="$G_NEXT_SRV_NO_CERT -p $SRV_PORT" -fi - -if [ -n "${GNUTLS_NEXT_CLI:-}" ]; then - G_NEXT_CLI="$G_NEXT_CLI -p +SRV_PORT" - G_NEXT_CLI_NO_CERT="$G_NEXT_CLI_NO_CERT -p +SRV_PORT localhost" -fi - -# Allow SHA-1, because many of our test certificates use it -P_SRV="$P_SRV allow_sha1=1" -P_CLI="$P_CLI allow_sha1=1" - # Also pick a unique name for intermediate files SRV_OUT="srv_out.$$" CLI_OUT="cli_out.$$" From be2c66e548db0c95be7f74b38c203c48c9a84856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 1 Sep 2023 10:34:49 +0100 Subject: [PATCH 11/20] ssl-opt.sh: Simplify the implementation of the -l option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of verifying if the LIST_TESTS variable has been set in every function to avoid using the P_QUERY variable and avoid calling a program that has not necessarily been compiled yet: * Define P_QUERY=":" when LIST_TESTS has been set. Signed-off-by: Tomás González --- tests/ssl-opt.sh | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 31114c94a0..b925a0133d 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -210,6 +210,7 @@ get_options "$@" if [ "$LIST_TESTS" -eq 0 ];then CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )" else + P_QUERY=":" CONFIGS_ENABLED="" fi # Skip next test; use this macro to skip tests which are legitimate @@ -223,9 +224,6 @@ skip_next_test() { # Check if the required configuration ($1) is enabled is_config_enabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return 0; - fi case $CONFIGS_ENABLED in *" $1"[\ =]*) return 0;; *) return 1;; @@ -234,9 +232,6 @@ is_config_enabled() # skip next test if the flag is not enabled in mbedtls_config.h requires_config_enabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi case $CONFIGS_ENABLED in *" $1"[\ =]*) :;; *) SKIP_NEXT="YES";; @@ -245,18 +240,12 @@ requires_config_enabled() { # skip next test if the flag is enabled in mbedtls_config.h requires_config_disabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi case $CONFIGS_ENABLED in *" $1"[\ =]*) SKIP_NEXT="YES";; esac } requires_all_configs_enabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi if ! $P_QUERY -all $* then SKIP_NEXT="YES" @@ -264,9 +253,6 @@ requires_all_configs_enabled() { } requires_all_configs_disabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi if $P_QUERY -any $* then SKIP_NEXT="YES" @@ -274,9 +260,6 @@ requires_all_configs_disabled() { } requires_any_configs_enabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi if ! $P_QUERY -any $* then SKIP_NEXT="YES" @@ -284,9 +267,6 @@ requires_any_configs_enabled() { } requires_any_configs_disabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi if $P_QUERY -all $* then SKIP_NEXT="YES" @@ -311,9 +291,6 @@ TLS1_2_KEY_EXCHANGES_WITH_CERT_WO_ECDH="MBEDTLS_KEY_EXCHANGE_RSA_ENABLED \ MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED" requires_key_exchange_with_cert_in_tls12_or_tls13_enabled() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi if $P_QUERY -all MBEDTLS_SSL_PROTO_TLS1_2 then requires_any_configs_enabled $TLS1_2_KEY_EXCHANGES_WITH_CERT @@ -340,9 +317,6 @@ get_config_value_or_default() { } requires_config_value_at_least() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi VAL="$( get_config_value_or_default "$1" )" if [ -z "$VAL" ]; then # Should never happen @@ -354,9 +328,6 @@ requires_config_value_at_least() { } requires_config_value_at_most() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi VAL=$( get_config_value_or_default "$1" ) if [ -z "$VAL" ]; then # Should never happen @@ -368,9 +339,6 @@ requires_config_value_at_most() { } requires_config_value_equals() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi VAL=$( get_config_value_or_default "$1" ) if [ -z "$VAL" ]; then # Should never happen @@ -386,9 +354,6 @@ requires_config_value_equals() { # Inputs: # * $1: protocol version in mbedtls syntax (argument to force_version=) requires_protocol_version() { - if [ "$LIST_TESTS" -gt 0 ];then - return; - fi # Support for DTLS is detected separately in detect_dtls(). case "$1" in tls12|dtls12) requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2;; From 37a8739e4d6c4d9a65691fdfd94f1deb9fa8991d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 1 Sep 2023 11:25:44 +0100 Subject: [PATCH 12/20] ssl-opt.sh: Don't affect the order at which functions are printed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When adding the LIST_TESTS option, print_name can be called before checking if the test case should be excluded or not. Change this back to its previous state while still taking into account the LIST_TESTS option. Signed-off-by: Tomás González --- tests/ssl-opt.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index b925a0133d..de4c83a114 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -1594,18 +1594,18 @@ run_test() { NAME="$1" shift 1 - print_name "$NAME" - - if [ "$LIST_TESTS" -gt 0 ]; then - return - fi - if is_excluded "$NAME"; then SKIP_NEXT="NO" # There was no request to run the test, so don't record its outcome. return fi + print_name "$NAME" + + if [ "$LIST_TESTS" -gt 0 ]; then + return + fi + # Do we only run numbered tests? if [ -n "$RUN_TEST_NUMBER" ]; then case ",$RUN_TEST_NUMBER," in From 4a86da2460fba520bfd22547675af29faacd84a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 1 Sep 2023 17:41:16 +0100 Subject: [PATCH 13/20] check_test_cases: Unify walk_compat_sh and walk_opt_sh into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit walk_compat_sh and walk_opt_sh are basically the same now, so: * Merge them into one function. * Use the --list-test-cases option for both of them. * Rename this merged function as collect_from_script which seems more appropriate as since it isn't iterating the script but calling it. Signed-off-by: Tomás González --- tests/compat.sh | 8 +++---- tests/scripts/check_test_cases.py | 36 +++++++++++-------------------- tests/ssl-opt.sh | 4 ++-- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 6506e6c09d..570c8e83d7 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -127,7 +127,7 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" - printf " --list-test-case\tList all potential test cases (No Execution)\n" + printf " -l|--list-test-cases\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" printf " --preserve-logs\tPreserve logs of successful tests as well\n" @@ -191,8 +191,8 @@ get_options() { MEMCHECK=1 ;; # Please check scripts/check_test_cases.py correspondingly - # if you have to modify option, --list-test-case - --list-test-case) + # if you have to modify option, --list-test-cases + --list-test-cases) list_test_case exit $? ;; @@ -869,7 +869,7 @@ wait_client_done() { } # uniform_title -# $TITLE is considered as test case description for both --list-test-case and +# $TITLE is considered as test case description for both --list-test-cases and # MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of # each test case description. uniform_title() { diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 4301b3210f..2dddf7a16a 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -27,7 +27,6 @@ import os import re import subprocess import sys -import subprocess class Results: @@ -99,26 +98,18 @@ state may override this method. data_file_name, line_number, line) in_paragraph = True - def walk_ssl_opt_sh(self, file_name): - """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" + def collect_from_script(self, file_name): + """Collect the test cases in a script by calling its listing test cases +option""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.check_output([file_name, '-l']) + listed = subprocess.check_output(['sh', file_name, '--list-test-cases']) + # Assume test file is responsible for printing identical format of + # test case description between --list-test-cases and its OUTCOME.CSV listed = set(map(lambda x: x.rstrip(), listed.splitlines())) - for description in listed: - self.process_test_case(descriptions, file_name, None, description) - - def walk_compat_sh(self, file_name): - """Iterate over the test cases compat.sh with a similar format.""" - descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - compat_cmd = ['sh', file_name, '--list-test-case'] - compat_output = subprocess.check_output(compat_cmd) - # Assume compat.sh is responsible for printing identical format of - # test case description between --list-test-case and its OUTCOME.CSV - description = compat_output.strip().split(b'\n') # idx indicates the number of test case since there is no line number # in `compat.sh` for each test case. - for idx, descrip in enumerate(description): - self.process_test_case(descriptions, file_name, idx, descrip) + for idx, description in enumerate(listed): + self.process_test_case(descriptions, file_name, idx, description) @staticmethod def collect_test_directories(): @@ -139,12 +130,11 @@ state may override this method. for data_file_name in glob.glob(os.path.join(directory, 'suites', '*.data')): self.walk_test_suite(data_file_name) - ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') - if os.path.exists(ssl_opt_sh): - self.walk_ssl_opt_sh(ssl_opt_sh) - compat_sh = os.path.join(directory, 'compat.sh') - if os.path.exists(compat_sh): - self.walk_compat_sh(compat_sh) + + for sh_file in ['ssl-opt.sh', 'compat.sh']: + sh_file = os.path.join(directory, sh_file) + if os.path.exists(sh_file): + self.collect_from_script(sh_file) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases.""" diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index de4c83a114..04be26f2e9 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -141,7 +141,7 @@ print_usage() { printf " -f|--filter\tOnly matching tests are executed (substring or BRE)\n" printf " -e|--exclude\tMatching tests are excluded (substring or BRE)\n" printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n" - printf " -l|--list-tests\tList test names and exit\n" + printf " -l|--list-test-cases\tList all potential test cases (No Execution)\n" printf " -s|--show-numbers\tShow test numbers in front of test names\n" printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n" printf " --outcome-file\tFile where test outcomes are written\n" @@ -169,7 +169,7 @@ get_options() { -s|--show-numbers) SHOW_TEST_NUMBER=1 ;; - -l|--list-tests) + -l|--list-test-cases) LIST_TESTS=1 ;; -p|--preserve-logs) From 38ecf9fa1e6151656c542419d2a0cb8bca182673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Mon, 4 Sep 2023 10:23:04 +0100 Subject: [PATCH 14/20] check_test_cases: Avoid removing duplicated test cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One of the jobs of check_test_cases is to check for duplicate test descriptions and to have them ordered: * Stop using a set to collect the different test cases from the test scripts. Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 2dddf7a16a..4a3ecef30c 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -105,11 +105,14 @@ option""" listed = subprocess.check_output(['sh', file_name, '--list-test-cases']) # Assume test file is responsible for printing identical format of # test case description between --list-test-cases and its OUTCOME.CSV - listed = set(map(lambda x: x.rstrip(), listed.splitlines())) + # # idx indicates the number of test case since there is no line number # in `compat.sh` for each test case. - for idx, description in enumerate(listed): - self.process_test_case(descriptions, file_name, idx, description) + for idx, description in enumerate(listed.splitlines()): + self.process_test_case(descriptions, + file_name, + idx, + description.rstrip()) @staticmethod def collect_test_directories(): From 12787c9ba5f573f0ccc051ead6d2347c6366119b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Mon, 4 Sep 2023 10:26:00 +0100 Subject: [PATCH 15/20] Remove invalid -l option from test scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The -l option mentioned in previous commits for both ssl-opt.sh and compat.sh scripts should only be a --list-test-cases option. Remove -l option from the help list. Signed-off-by: Tomás González --- tests/compat.sh | 2 +- tests/ssl-opt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 570c8e83d7..f55bb0e051 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -127,7 +127,7 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" - printf " -l|--list-test-cases\tList all potential test cases (No Execution)\n" + printf " --list-test-cases\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" printf " --preserve-logs\tPreserve logs of successful tests as well\n" diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 04be26f2e9..156e94406e 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -141,9 +141,9 @@ print_usage() { printf " -f|--filter\tOnly matching tests are executed (substring or BRE)\n" printf " -e|--exclude\tMatching tests are excluded (substring or BRE)\n" printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n" - printf " -l|--list-test-cases\tList all potential test cases (No Execution)\n" printf " -s|--show-numbers\tShow test numbers in front of test names\n" printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n" + printf " --list-test-cases\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" printf " --port \tTCP/UDP port (default: randomish 1xxxx)\n" From 378e364c3c6bc1638fad25e8f0e78db90b7371bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Mon, 4 Sep 2023 10:41:37 +0100 Subject: [PATCH 16/20] ssl-opt.sh: Correct print format for test cases' names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid printing an extra space when using the --list-test-cases option. Signed-off-by: Tomás González --- tests/ssl-opt.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 156e94406e..6644b0de07 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -879,15 +879,16 @@ print_name() { fi LINE="$LINE$1" - printf "%s " "$LINE" if [ "$LIST_TESTS" -gt 0 ]; then - printf "\n" - else - LEN=$(( 72 - `echo "$LINE" | wc -c` )) - for i in `seq 1 $LEN`; do printf '.'; done - printf ' ' + printf "%s\n" "$LINE" + return fi + printf "%s " "$LINE" + LEN=$(( 72 - `echo "$LINE" | wc -c` )) + for i in `seq 1 $LEN`; do printf '.'; done + printf ' ' + } # record_outcome [] From 51cb70434258b656a34742e9810a6b17511c5148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 7 Sep 2023 10:21:19 +0100 Subject: [PATCH 17/20] Avoid using print_name when --list-test-cases is used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/ssl-opt.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 6644b0de07..b93d439de3 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -879,10 +879,6 @@ print_name() { fi LINE="$LINE$1" - if [ "$LIST_TESTS" -gt 0 ]; then - printf "%s\n" "$LINE" - return - fi printf "%s " "$LINE" LEN=$(( 72 - `echo "$LINE" | wc -c` )) @@ -1601,12 +1597,13 @@ run_test() { return fi - print_name "$NAME" - if [ "$LIST_TESTS" -gt 0 ]; then + printf "%s\n" "$NAME" return fi + print_name "$NAME" + # Do we only run numbered tests? if [ -n "$RUN_TEST_NUMBER" ]; then case ",$RUN_TEST_NUMBER," in From cfe68a0cb6f5ba882c6528034a161d7ff45d0ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 7 Sep 2023 10:37:50 +0100 Subject: [PATCH 18/20] ssl-opt.sh: Make record_outcome record the ssl-opt.sh file only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ignore the test suite name as file from opt-testcases cannot actually be called separately. Signed-off-by: Tomás González --- tests/ssl-opt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index b93d439de3..36753af8a8 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -895,7 +895,7 @@ record_outcome() { if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then printf '%s;%s;%s;%s;%s;%s\n' \ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ - "${TEST_SUITE_NAME:-ssl-opt}" "$NAME" \ + "ssl-opt" "$NAME" \ "$1" "${2-}" \ >>"$MBEDTLS_TEST_OUTCOME_FILE" fi From 4fc582461ba4c09f2fb19dded82aa45bba83b325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Wed, 20 Sep 2023 22:14:06 +0100 Subject: [PATCH 19/20] compat.sh: Rename list_test_case to list_test_cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- tests/compat.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index f55bb0e051..a98ed0eec5 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -141,8 +141,8 @@ print_test_case() { done } -# list_test_case lists all potential test cases in compat.sh without execution -list_test_case() { +# list_test_cases lists all potential test cases in compat.sh without execution +list_test_cases() { reset_ciphersuites for TYPE in $TYPES; do add_common_ciphersuites @@ -193,7 +193,7 @@ get_options() { # Please check scripts/check_test_cases.py correspondingly # if you have to modify option, --list-test-cases --list-test-cases) - list_test_case + list_test_cases exit $? ;; --outcome-file) From 7f2cddb1aeb939972a634820243c0a243dcaadca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 27 Oct 2023 11:45:26 +0100 Subject: [PATCH 20/20] check_test_cases: Minor documentation change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make an iteration comment generic to every file it may affect instead of making it specific a particular file. Signed-off-by: Tomás González --- tests/scripts/check_test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 4a3ecef30c..30879d7635 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -107,7 +107,7 @@ option""" # test case description between --list-test-cases and its OUTCOME.CSV # # idx indicates the number of test case since there is no line number - # in `compat.sh` for each test case. + # in the script for each test case. for idx, description in enumerate(listed.splitlines()): self.process_test_case(descriptions, file_name,