From dd21fc97baf54e8797ecb8372069bbc5a42f4d72 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 23 Feb 2023 10:04:58 +0800 Subject: [PATCH 01/11] compat.sh: add --list-test-case The option --list-test-case lists all potential test cases without executing them. The test case description is identical with $TITLE during test case execution. Signed-off-by: Yanray Wang --- tests/compat.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/compat.sh b/tests/compat.sh index 75d5461599..a2283039a4 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -128,10 +128,40 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" + printf " --list-test-case\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } +# print_test_title +print_test_title() { + for i in $3; do + TITLE="$1->$2 $MODE,$VERIF $i" + echo "$TITLE" + done +} + +list_test_case() { + reset_ciphersuites + for TYPE in $TYPES; do + add_common_ciphersuites + add_openssl_ciphersuites + add_gnutls_ciphersuites + add_mbedtls_ciphersuites + done + + for VERIFY in $VERIFIES; do + VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') + for MODE in $MODES; do + print_test_title m o "$O_CIPHERS" + print_test_title o m "$O_CIPHERS" + print_test_title m g "$G_CIPHERS" + print_test_title g m "$G_CIPHERS" + print_test_title m m "$M_CIPHERS" + done + done +} + get_options() { while [ $# -gt 0 ]; do case "$1" in @@ -159,6 +189,10 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + --list-test-case) + list_test_case + exit 0 + ;; --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE=$1 ;; From baced97929ebb083e664d0edd33c2570c1890c42 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 24 Feb 2023 14:53:29 +0800 Subject: [PATCH 02/11] check_test_cases.py: support checking test coverage in compat.sh Test case description in compat.sh is in format of [ogm]->[ogm] TLSmode, VERIFY CIPHERSUITE_NAME This program calls compat.sh to list all potential test case descriptions then checks test case duplication. Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index c9f5e11dd8..86506f5e59 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -25,6 +25,7 @@ import argparse import glob import os import re +import subprocess import sys class Results: @@ -111,6 +112,24 @@ state may override this method. self.process_test_case(descriptions, file_name, line_number, description) + def walk_compat_sh(self, file_name): + """Iterate over the test cases compat.sh with a similar format.""" + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none + compat_cmd = ['sh', file_name, '--list-test-case'] + result = subprocess.run(compat_cmd, + stdout=subprocess.PIPE, + check=False) + if result.returncode != 0: + print(*compat_cmd, 'returned', str(result.returncode)) + return + else: + # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA\n + m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\n', + result.stdout) + if m: + for i in m: + self.process_test_case(descriptions, file_name, 1, i) + @staticmethod def collect_test_directories(): """Get the relative path for the TLS and Crypto test directories.""" @@ -133,6 +152,9 @@ state may override this method. ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') if os.path.exists(ssl_opt_sh): self.walk_ssl_opt_sh(ssl_opt_sh) + compat_sh = os.path.join(directory, 'compat.sh') + if os.path.exists(compat_sh): + self.walk_compat_sh(compat_sh) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases.""" From 7e1c0c7f79f6993a2857d18d45e92cbfbc6f9893 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 24 Feb 2023 17:07:47 +0800 Subject: [PATCH 03/11] compat.sh: uniform test description Test case description is printed by different block of code. This causes code maintenance harder since we need to maintain two parts of code with same functionality. print_test_title is used to control test case description in compat.sh Signed-off-by: Yanray Wang --- tests/compat.sh | 26 +++++++++++++------------- tests/scripts/check_test_cases.py | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index a2283039a4..7e58cac2b3 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -40,6 +40,7 @@ TESTS=0 FAILED=0 SKIPPED=0 SRVMEM=0 +LIST_TEST_CASE=0 # default commands, can be overridden by the environment : ${M_SRV:=../programs/ssl/ssl_server2} @@ -133,14 +134,6 @@ print_usage() { printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } -# print_test_title -print_test_title() { - for i in $3; do - TITLE="$1->$2 $MODE,$VERIF $i" - echo "$TITLE" - done -} - list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -190,6 +183,7 @@ get_options() { MEMCHECK=1 ;; --list-test-case) + LIST_TEST_CASE=1 list_test_case exit 0 ;; @@ -1233,15 +1227,21 @@ report_fail() { fi } +# print_test_title +print_test_title() { + for i in $3; do + TITLE="$1->$2 $MODE,$VERIF $i" + DOTS72="........................................................................" + printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" + [ $LIST_TEST_CASE -eq 1 ] && printf "\n" + done +} + # run_client run_client() { # announce what we're going to do TESTS=$(( $TESTS + 1 )) - TITLE="`echo $1 | head -c1`->`echo $SERVER_NAME | head -c1`" - TITLE="$TITLE $MODE,$VERIF $2" - printf "%s " "$TITLE" - LEN=$(( 72 - `echo "$TITLE" | wc -c` )) - for i in `seq 1 $LEN`; do printf '.'; done; printf ' ' + print_test_title ${1%"${1#?}"} ${SERVER_NAME%"${SERVER_NAME#?}"} $2 # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 86506f5e59..28f0f0a5fe 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -123,8 +123,8 @@ state may override this method. print(*compat_cmd, 'returned', str(result.returncode)) return else: - # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA\n - m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\n', + # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA .......... \n + m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\s*\.*\s*\n', result.stdout) if m: for i in m: From a81131d358d72f702b831637274f06bec6662c84 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 11:41:26 +0800 Subject: [PATCH 04/11] compat.sh: fix uncompatiable name of peers in --list-test-case Signed-off-by: Yanray Wang --- tests/compat.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 7e58cac2b3..cedf232eb4 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -146,10 +146,10 @@ list_test_case() { for VERIFY in $VERIFIES; do VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') for MODE in $MODES; do - print_test_title m o "$O_CIPHERS" - print_test_title o m "$O_CIPHERS" - print_test_title m g "$G_CIPHERS" - print_test_title g m "$G_CIPHERS" + print_test_title m O "$O_CIPHERS" + print_test_title O m "$O_CIPHERS" + print_test_title m G "$G_CIPHERS" + print_test_title G m "$G_CIPHERS" print_test_title m m "$M_CIPHERS" done done From f0dbde1bdc402a39db3340767236c8f71f2bc655 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 11:55:55 +0800 Subject: [PATCH 05/11] compat.sh: uniform TITLE format for --list-test-case and run_client uniform_title is used to print identical format of $TITLE between --list-test-case and run_client. In such way, no matter how $TITLE is developed, --list-test-case will in the same format of test case description as stored in OUTCOME.CSV. Signed-off-by: Yanray Wang --- tests/compat.sh | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index cedf232eb4..3724bda6a7 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -40,7 +40,6 @@ TESTS=0 FAILED=0 SKIPPED=0 SRVMEM=0 -LIST_TEST_CASE=0 # default commands, can be overridden by the environment : ${M_SRV:=../programs/ssl/ssl_server2} @@ -134,6 +133,14 @@ print_usage() { printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } +# print_test_case +print_test_case() { + for i in $3; do + uniform_title $1 $2 $i + echo $TITLE + done +} + list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -146,11 +153,11 @@ list_test_case() { for VERIFY in $VERIFIES; do VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') for MODE in $MODES; do - print_test_title m O "$O_CIPHERS" - print_test_title O m "$O_CIPHERS" - print_test_title m G "$G_CIPHERS" - print_test_title G m "$G_CIPHERS" - print_test_title m m "$M_CIPHERS" + print_test_case m O "$O_CIPHERS" + print_test_case O m "$O_CIPHERS" + print_test_case m G "$G_CIPHERS" + print_test_case G m "$G_CIPHERS" + print_test_case m m "$M_CIPHERS" done done } @@ -183,7 +190,6 @@ get_options() { MEMCHECK=1 ;; --list-test-case) - LIST_TEST_CASE=1 list_test_case exit 0 ;; @@ -1227,21 +1233,21 @@ report_fail() { fi } -# print_test_title -print_test_title() { - for i in $3; do - TITLE="$1->$2 $MODE,$VERIF $i" - DOTS72="........................................................................" - printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" - [ $LIST_TEST_CASE -eq 1 ] && printf "\n" - done +# uniform_title +# $TITLE is considered as test case description for both --list-test-case and +# MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of +# each test case description. +uniform_title() { + TITLE="$1->$2 $MODE,$VERIF $3" } # run_client run_client() { # announce what we're going to do TESTS=$(( $TESTS + 1 )) - print_test_title ${1%"${1#?}"} ${SERVER_NAME%"${SERVER_NAME#?}"} $2 + uniform_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2 + DOTS72="........................................................................" + printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then From 9412a46ab68d738f4183be2cf47109a228949e35 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 14:45:01 +0800 Subject: [PATCH 06/11] check_test_cases.py: simplify how to store test case description Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 28f0f0a5fe..5f7ffb8612 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -123,12 +123,11 @@ state may override this method. print(*compat_cmd, 'returned', str(result.returncode)) return else: - # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA .......... \n - m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\s*\.*\s*\n', - result.stdout) - if m: - for i in m: - self.process_test_case(descriptions, file_name, 1, i) + # Assume compat.sh is responsible for printing identical format of + # test case description between --list-test-case and its OUTCOME.CSV + description = result.stdout.strip().split(b'\n') + for idx, descrip in enumerate(description): + self.process_test_case(descriptions, file_name, idx, descrip) @staticmethod def collect_test_directories(): From 88448445823df8c4a60c42bcbf63add1e3a9a762 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 3 Mar 2023 17:12:29 +0800 Subject: [PATCH 07/11] check_test_cases.py: use check_output to capture error and return This commit includes: - use subprocess.check_output to report error and capture return value - add comment as a reminder for option --list-test-case Signed-off-by: Yanray Wang --- tests/compat.sh | 3 +++ tests/scripts/check_test_cases.py | 19 +++++++------------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 3724bda6a7..44c3480102 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -141,6 +141,7 @@ print_test_case() { done } +# list_test_case lists all potential test cases in compat.sh without execution list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -189,6 +190,8 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + # Please check scripts/check_test_cases.py correspondingly + # if you have to modify option, --list-test-case --list-test-case) list_test_case exit 0 diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 5f7ffb8612..76465169af 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -116,18 +116,13 @@ state may override this method. """Iterate over the test cases compat.sh with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none compat_cmd = ['sh', file_name, '--list-test-case'] - result = subprocess.run(compat_cmd, - stdout=subprocess.PIPE, - check=False) - if result.returncode != 0: - print(*compat_cmd, 'returned', str(result.returncode)) - return - else: - # Assume compat.sh is responsible for printing identical format of - # test case description between --list-test-case and its OUTCOME.CSV - description = result.stdout.strip().split(b'\n') - for idx, descrip in enumerate(description): - self.process_test_case(descriptions, file_name, idx, descrip) + compat_output = subprocess.check_output(compat_cmd, + stderr=subprocess.STDOUT) + # Assume compat.sh is responsible for printing identical format of + # test case description between --list-test-case and its OUTCOME.CSV + description = compat_output.strip().split(b'\n') + for idx, descrip in enumerate(description): + self.process_test_case(descriptions, file_name, idx, descrip) @staticmethod def collect_test_directories(): From 67fe2644ae596eabb458b906b35d85452fe07f3d Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 6 Mar 2023 19:35:04 +0800 Subject: [PATCH 08/11] check_test_cases.py: do not redirect stderr to stdout Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 76465169af..6069a666aa 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -116,8 +116,7 @@ state may override this method. """Iterate over the test cases compat.sh with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none compat_cmd = ['sh', file_name, '--list-test-case'] - compat_output = subprocess.check_output(compat_cmd, - stderr=subprocess.STDOUT) + compat_output = subprocess.check_output(compat_cmd) # Assume compat.sh is responsible for printing identical format of # test case description between --list-test-case and its OUTCOME.CSV description = compat_output.strip().split(b'\n') From 8aba83bf22cdc3d7cf77b97e88f6bd0cc16253aa Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 6 Mar 2023 19:37:07 +0800 Subject: [PATCH 09/11] compat.sh: return $? in option --list-test-case to handle error case Signed-off-by: Yanray Wang --- tests/compat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compat.sh b/tests/compat.sh index 44c3480102..dff693b80b 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -194,7 +194,7 @@ get_options() { # if you have to modify option, --list-test-case --list-test-case) list_test_case - exit 0 + exit $? ;; --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE=$1 From b2cd07ce09f6b4c7376cd372dfbb8c963baa4688 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 30 Aug 2023 12:07:26 +0800 Subject: [PATCH 10/11] compat: list all test cases properly When calling `add_xxx_ciphersuites`, we have to set MODE properly. This commit adjusts order to address this issue in list_test_case which matches what we do in a normal execution. Signed-off-by: Yanray Wang --- tests/compat.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index dff693b80b..952997537f 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -143,22 +143,21 @@ print_test_case() { # list_test_case lists all potential test cases in compat.sh without execution list_test_case() { - reset_ciphersuites - for TYPE in $TYPES; do - add_common_ciphersuites - add_openssl_ciphersuites - add_gnutls_ciphersuites - add_mbedtls_ciphersuites - done - - for VERIFY in $VERIFIES; do - VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') - for MODE in $MODES; do - print_test_case m O "$O_CIPHERS" - print_test_case O m "$O_CIPHERS" - print_test_case m G "$G_CIPHERS" - print_test_case G m "$G_CIPHERS" - print_test_case m m "$M_CIPHERS" + for MODE in $MODES; do + for TYPE in $TYPES; do + for VERIFY in $VERIFIES; do + VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') + reset_ciphersuites + add_common_ciphersuites + add_openssl_ciphersuites + add_gnutls_ciphersuites + add_mbedtls_ciphersuites + print_test_case m O "$O_CIPHERS" + print_test_case O m "$O_CIPHERS" + print_test_case m G "$G_CIPHERS" + print_test_case G m "$G_CIPHERS" + print_test_case m m "$M_CIPHERS" + done done done } From 930cbeeb5bcbed4cb673012581b24ce6da30f276 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 30 Aug 2023 18:31:35 +0800 Subject: [PATCH 11/11] check_test_cases: add a comment to explain idx in walk_compat_sh Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 6069a666aa..213708b624 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -120,6 +120,8 @@ state may override this method. # Assume compat.sh is responsible for printing identical format of # test case description between --list-test-case and its OUTCOME.CSV description = compat_output.strip().split(b'\n') + # idx indicates the number of test case since there is no line number + # in `compat.sh` for each test case. for idx, descrip in enumerate(description): self.process_test_case(descriptions, file_name, idx, descrip)