Merge pull request #9019 from mpg/compat-not-executed-2.28

[2.28] Fix compat.sh tests (reported as) not executed

tests/compat.sh

@@ -133,6 +133,14 @@ print_test_case() {
 
 # list_test_case lists all potential test cases in compat.sh without execution
 list_test_cases() {
+    # We want to call filter_ciphersuites to apply standard-defined exclusions
+    # (like "no RC4 with DTLS") but without user-defined exludes/filters.
+    EXCLUDE='^$'
+    FILTER=""
+
+    # ssl3 is excluded by default, but it's still available
+    MODES="ssl3 $MODES"
+
     for MODE in $MODES; do
         for TYPE in $TYPES; do
             # PSK cipher suites do not allow client certificate verification.
@@ -142,16 +150,31 @@ list_test_cases() {
             fi
             for VERIFY in $SUB_VERIFIES; do
                 VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]')
-                reset_ciphersuites
+                for PEER in $PEERS; do
-                add_common_ciphersuites
+                    reset_ciphersuites
-                add_openssl_ciphersuites
+                    add_common_ciphersuites
-                add_gnutls_ciphersuites
+                    case "$PEER" in
-                add_mbedtls_ciphersuites
+                        [Oo]pen*)
-                print_test_case m O "$O_CIPHERS"
+                            add_openssl_ciphersuites
-                print_test_case O m "$O_CIPHERS"
+                            filter_ciphersuites
-                print_test_case m G "$G_CIPHERS"
+                            print_test_case m O "$M_CIPHERS"
-                print_test_case G m "$G_CIPHERS"
+                            print_test_case O m "$O_CIPHERS"
-                print_test_case m m "$M_CIPHERS"
+                            ;;
+                        [Gg]nu*)
+                            add_gnutls_ciphersuites
+                            filter_ciphersuites
+                            print_test_case m G "$M_CIPHERS"
+                            print_test_case G m "$G_CIPHERS"
+                            ;;
+                        mbed*)
+                            add_openssl_ciphersuites
+                            add_gnutls_ciphersuites
+                            add_mbedtls_ciphersuites
+                            filter_ciphersuites
+                            print_test_case m m "$M_CIPHERS"
+                            ;;
+                    esac
+                done
             done
         done
     done
@@ -272,17 +295,9 @@ filter()
 
 filter_ciphersuites()
 {
-    if [ "X" != "X$FILTER" -o "X" != "X$EXCLUDE" ];
+    M_CIPHERS=$( filter "$M_CIPHERS" )
-    then
+    O_CIPHERS=$( filter "$O_CIPHERS" )
-        # Ciphersuite for Mbed TLS
+    G_CIPHERS=$( filter "$G_CIPHERS" )
-        M_CIPHERS=$( filter "$M_CIPHERS" )
-
-        # Ciphersuite for OpenSSL
-        O_CIPHERS=$( filter "$O_CIPHERS" )
-
-        # Ciphersuite for GnuTLS
-        G_CIPHERS=$( filter "$G_CIPHERS" )
-    fi
 }
 
 reset_ciphersuites()
@@ -640,14 +655,18 @@ add_gnutls_ciphersuites()
             ;;
 
         "RSA")
-            # Not actually supported with all GnuTLS versions. See
+            if [ `minor_ver "$MODE"` -ge 1 ]
-            # GNUTLS_HAS_TLS1_RSA_NULL_SHA256= below.
+            then
-            M_CIPHERS="$M_CIPHERS                               \
+                # Not actually supported with all GnuTLS versions. See
-                    TLS-RSA-WITH-NULL-SHA256                    \
+                # GNUTLS_HAS_TLS1_RSA_NULL_SHA256= below.
-                    "
+                M_CIPHERS="$M_CIPHERS                               \
-            G_CIPHERS="$G_CIPHERS                               \
+                        TLS-RSA-WITH-NULL-SHA256                    \
-                    +RSA:+NULL:+SHA256                          \
+                        "
-                    "
+                G_CIPHERS="$G_CIPHERS                               \
+                        +RSA:+NULL:+SHA256                          \
+                        "
+            fi
+
             if [ `minor_ver "$MODE"` -ge 3 ]
             then
                 M_CIPHERS="$M_CIPHERS                           \
@@ -912,7 +931,26 @@ add_mbedtls_ciphersuites()
 # o_check_ciphersuite CIPHER_SUITE_NAME
 o_check_ciphersuite()
 {
-    if [ "${O_SUPPORT_ECDH}" = "NO" ]; then
+    # skip DTLS when lack of support was declared
+    if test "$OSSL_NO_DTLS" -gt 0 && is_dtls "$MODE"; then
+        SKIP_NEXT_="YES"
+    fi
+
+    # skip DTLS 1.2 is support was not detected
+    if [ "$O_SUPPORT_DTLS12" = "NO" -a "$MODE" = "dtls12" ]; then
+        SKIP_NEXT="YES"
+    fi
+
+    # skip single-DES ciphersuite if no longer supported
+    if [ "$O_SUPPORT_SINGLE_DES" = "NO" ]; then
+        case "$1" in
+            # note: 3DES is DES-CBC3 for OpenSSL, 3DES for Mbed TLS
+            *-DES-CBC-*|DES-CBC-*) SKIP_NEXT="YES"
+        esac
+    fi
+
+    # skip static ECDH when OpenSSL doesn't support it
+    if [ "${O_SUPPORT_STATIC_ECDH}" = "NO" ]; then
         case "$1" in
             *ECDH-*) SKIP_NEXT="YES"
         esac
@@ -1021,10 +1059,25 @@ setup_arguments()
     esac
 
     case $($OPENSSL ciphers ALL) in
-        *ECDH-ECDSA*|*ECDH-RSA*) O_SUPPORT_ECDH="YES";;
+        *ECDH-ECDSA*|*ECDH-RSA*) O_SUPPORT_STATIC_ECDH="YES";;
-        *) O_SUPPORT_ECDH="NO";;
+        *) O_SUPPORT_STATIC_ECDH="NO";;
     esac
 
+    case $($OPENSSL ciphers ALL) in
+        *DES-CBC-*) O_SUPPORT_SINGLE_DES="YES";;
+        *) O_SUPPORT_SINGLE_DES="NO";;
+    esac
+
+    # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL
+    # supports -dtls1_2 from the s_server help. (The s_client
+    # help isn't accurate as of 1.0.2g: it supports DTLS 1.2
+    # but doesn't list it. But the s_server help seems to be
+    # accurate.)
+    O_SUPPORT_DTLS12="NO"
+    if $OPENSSL s_server -help 2>&1 | grep -q "^ *-dtls1_2 "; then
+        O_SUPPORT_DTLS12="YES"
+    fi
+
     if [ "X$VERIFY" = "XYES" ];
     then
         M_SERVER_ARGS="$M_SERVER_ARGS ca_file=data_files/test-ca_cat12.crt auth_mode=required"
@@ -1474,19 +1527,6 @@ for MODE in $MODES; do
 
                 [Oo]pen*)
 
-                    if test "$OSSL_NO_DTLS" -gt 0 && is_dtls "$MODE"; then
-                        continue;
-                    fi
-
-                    # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL
-                    # supports $O_MODE from the s_server help. (The s_client
-                    # help isn't accurate as of 1.0.2g: it supports DTLS 1.2
-                    # but doesn't list it. But the s_server help seems to be
-                    # accurate.)
-                    if ! $OPENSSL s_server -help 2>&1 | grep -q "^ *-$O_MODE "; then
-                        continue;
-                    fi
-
                     reset_ciphersuites
                     add_common_ciphersuites
                     add_openssl_ciphersuites

tests/scripts/all.sh

@@ -928,8 +928,18 @@ component_test_full_cmake_gcc_asan () {
     msg "test: ssl-opt.sh (full config, ASan build)"
     tests/ssl-opt.sh
 
-    msg "test: compat.sh (full config, ASan build)"
+    msg "test: compat.sh all except legacy/next (full config, ASan build)"
-    tests/compat.sh
+    tests/compat.sh -e '^DES-CBC-\|-DES-CBC-\|ARIA\|CHACHA' \
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
+
+    msg "test: compat.sh single-DES (full config, ASan build)"
+    env OPENSSL="$OPENSSL_LEGACY" tests/compat.sh -e '^$' -f '^DES-CBC\|-DES-CBC-' \
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
+
+    # ARIA and ChachaPoly are both (D)TLS 1.2 only
+    msg "test: compat.sh ARIA + ChachaPoly (full config, ASan build)"
+    env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' \
+        -m 'tls12 dtls12'
 
     msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec
     tests/context-info.sh
@@ -1628,15 +1638,6 @@ component_test_full_cmake_clang () {
 
     msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s
     tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private'
-
-    msg "test: compat.sh RC4, 3DES & NULL (full config)" # ~ 2min
-    tests/compat.sh -e '^$' -f 'NULL\|3DES\|DES-CBC3\|RC4\|ARCFOUR'
-
-    msg "test: compat.sh single-DES (full config)" # ~ 30s
-    env OPENSSL="$OPENSSL_LEGACY" tests/compat.sh -e '3DES\|DES-CBC3' -f 'DES'
-
-    msg "test: compat.sh ARIA + ChachaPoly"
-    env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
 }
 
 skip_suites_without_constant_flow () {
@@ -1925,17 +1926,18 @@ component_test_no_use_psa_crypto_full_cmake_asan() {
     msg "test: ssl-opt.sh (full minus MBEDTLS_USE_PSA_CRYPTO)"
     tests/ssl-opt.sh
 
-    msg "test: compat.sh default (full minus MBEDTLS_USE_PSA_CRYPTO)"
+    msg "test: compat.sh all except legacy/next (full minus MBEDTLS_USE_PSA_CRYPTO)"
-    tests/compat.sh
+    tests/compat.sh -e '^DES-CBC-\|-DES-CBC-\|ARIA\|CHACHA' \
-
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
-    msg "test: compat.sh RC4, 3DES & NULL (full minus MBEDTLS_USE_PSA_CRYPTO)"
-    tests/compat.sh -e '^$' -f 'NULL\|3DES\|DES-CBC3\|RC4\|ARCFOUR'
 
     msg "test: compat.sh single-DES (full minus MBEDTLS_USE_PSA_CRYPTO)"
-    env OPENSSL="$OPENSSL_LEGACY" tests/compat.sh -e '3DES\|DES-CBC3' -f 'DES'
+    env OPENSSL="$OPENSSL_LEGACY" tests/compat.sh -e '^$' -f '^DES-CBC\|-DES-CBC-' \
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
 
+    # ARIA and ChachaPoly are both (D)TLS 1.2 only
     msg "test: compat.sh ARIA + ChachaPoly (full minus MBEDTLS_USE_PSA_CRYPTO)"
-    env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
+    env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' \
+        -m 'tls12 dtls12'
 }
 
 component_test_psa_crypto_config_accel_ecdsa () {

tests/scripts/analyze_outcomes.py

@@ -113,6 +113,18 @@ TASKS = {
                 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
                 # Algorithm not supported yet
                 'test_suite_psa_crypto_metadata;Cipher: XTS',
+                # compat.sh tests with OpenSSL, DTLS 1.2 and singled-DES:
+                # we have no version of OpenSSL on the CI that supports both
+                # DTLS 1.2 and single-DES (1.0.2g is too recent for single-DES
+                # and 1.0.1j is too old for DTLS 1.2).
+                'compat;O->m dtls12,no DES-CBC-SHA',
+                'compat;O->m dtls12,no EDH-RSA-DES-CBC-SHA',
+                'compat;O->m dtls12,yes DES-CBC-SHA',
+                'compat;O->m dtls12,yes EDH-RSA-DES-CBC-SHA',
+                'compat;m->O dtls12,no TLS-DHE-RSA-WITH-DES-CBC-SHA',
+                'compat;m->O dtls12,no TLS-RSA-WITH-DES-CBC-SHA',
+                'compat;m->O dtls12,yes TLS-DHE-RSA-WITH-DES-CBC-SHA',
+                'compat;m->O dtls12,yes TLS-RSA-WITH-DES-CBC-SHA',
             ],
             'full_coverage': False,
         }

tests/scripts/basic-build-test.sh

@@ -104,17 +104,20 @@ echo
 # Step 2c - Compatibility tests (keep going even if some tests fail)
 echo '################ compat.sh ################'
 {
-    echo '#### compat.sh: Default ciphers'
+    echo '#### compat.sh: all except legacy/next'
-    sh compat.sh -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
+    sh compat.sh -e '^DES-CBC-\|-DES-CBC-\|ARIA\|CHACHA' \
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
     echo
 
-    echo '#### compat.sh: legacy (null, DES, RC4)'
+    echo '#### compat.sh: legacy (single-DES)'
-    OPENSSL="$OPENSSL_LEGACY" \
+    OPENSSL="$OPENSSL_LEGACY" sh compat.sh -e '^$' -f '^DES-CBC\|-DES-CBC-' \
-    sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR'
+        -m 'ssl3 tls1 tls1_1 tls12 dtls1 dtls12'
     echo
 
+    # ARIA and ChachaPoly are both (D)TLS 1.2 only
     echo '#### compat.sh: next (ARIA, ChaCha)'
-    OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA'
+    OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' \
+        -m 'tls12 dtls12'
     echo
 } | tee compat-test-$TEST_OUTPUT
 echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^'