Merge pull request #6946 from valeriosetti/issue6856

driver-only ECDSA: fix testing disparities in ecp, random, se_driver_hal

tests/scripts/analyze_outcomes.py

@@ -61,24 +61,32 @@ def analyze_coverage(results, outcomes):
             # fixed this branch to have full coverage of test cases.
             results.warning('Test case not executed: {}', key)
 
-def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests):
+def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
+                                ignored_suites, ignored_test=None):
     """Check that all tests executed in the reference component are also
     executed in the corresponding driver component.
-    Skip test suites provided in ignored_tests list.
+    Skip:
+    - full test suites provided in ignored_suites list
+    - only some specific test inside a test suite, for which the corresponding
+      output string is provided
     """
     available = check_test_cases.collect_available_test_cases()
     result = True
 
     for key in available:
-        # Skip ignored test suites
-        test_suite = key.split(';')[0] # retrieve test suit name
-        test_suite = test_suite.split('.')[0] # retrieve main part of test suit name
-        if test_suite in ignored_tests:
-            continue
         # Continue if test was not executed by any component
         hits = outcomes[key].hits() if key in outcomes else 0
         if hits == 0:
             continue
+        # Skip ignored test suites
+        full_test_suite = key.split(';')[0] # retrieve full test suite name
+        test_string = key.split(';')[1] # retrieve the text string of this test
+        test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
+        if test_suite in ignored_suites:
+            continue
+        if ((full_test_suite in ignored_test) and
+                (test_string in ignored_test[full_test_suite])):
+            continue
         # Search for tests that run in reference component and not in driver component
         driver_test_passed = False
         reference_test_passed = False
@@ -129,13 +137,14 @@ def do_analyze_coverage(outcome_file, args):
 
 def do_analyze_driver_vs_reference(outcome_file, args):
     """Perform driver vs reference analyze."""
-    ignored_tests = ['test_suite_' + x for x in args['ignored_suites']]
+    ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
 
     outcomes = read_outcome_file(outcome_file)
     print("\n*** Analyze driver {} vs reference {} ***\n".format(
         args['component_driver'], args['component_ref']))
     return analyze_driver_vs_reference(outcomes, args['component_ref'],
-                                       args['component_driver'], ignored_tests)
+                                       args['component_driver'], ignored_suites,
+                                       args['ignored_tests'])
 
 # List of tasks with a function that can handle this task and additional arguments if required
 TASKS = {
@@ -154,7 +163,11 @@ TASKS = {
             'ignored_suites': [
                 'shax', 'mdx', # the software implementations that are being excluded
                 'md',  # the legacy abstraction layer that's being excluded
-            ]}},
+            ],
+            'ignored_tests': {
+            }
+        }
+    },
     'analyze_driver_vs_reference_ecdsa': {
         'test_function': do_analyze_driver_vs_reference,
         'args': {
@@ -164,15 +177,19 @@ TASKS = {
                 'ecdsa', # the software implementation that's excluded
                 # the following lines should not be needed,
                 # they will be removed by upcoming work
-                'psa_crypto_se_driver_hal', # #6856
-                'random', # #6856
-                'ecp', # #6856
                 'pk', # #6857
                 'x509parse', # #6858
                 'x509write', # #6858
                 'debug', # #6860
                 'ssl', # #6860
-            ]}},
+            ],
+            'ignored_tests': {
+                'test_suite_random': [
+                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
+                ],
+            }
+        }
+    },
 }
 
 def main():

tests/suites/test_suite_ecp.function

@@ -811,7 +811,7 @@ exit:
 }
 /* END_CASE */
 
-/* BEGIN_CASE depends_on:MBEDTLS_ECDH_C:MBEDTLS_ECDSA_C */
+/* BEGIN_CASE */
 void mbedtls_ecp_group_metadata(int id, int bit_size, int crv_type,
                                 char *P, char *A, char *B,
                                 char *G_x, char *G_y, char *N,
@@ -903,9 +903,13 @@ void mbedtls_ecp_group_metadata(int id, int bit_size, int crv_type,
 
     // Check curve type, and if it can be used for ECDH/ECDSA
     TEST_EQUAL(mbedtls_ecp_get_type(&grp), crv_type);
+#if defined(MBEDTLS_ECDH_C)
     TEST_EQUAL(mbedtls_ecdh_can_do(id), 1);
+#endif
+#if defined(MBEDTLS_ECDSA_C)
     TEST_EQUAL(mbedtls_ecdsa_can_do(id),
                crv_type == MBEDTLS_ECP_TYPE_SHORT_WEIERSTRASS);
+#endif
 
     // Copy group and compare with original
     TEST_EQUAL(mbedtls_ecp_group_copy(&grp_cpy, &grp), 0);

tests/suites/test_suite_psa_crypto_se_driver_hal.data

@@ -178,25 +178,25 @@ Key registration: key id max volatile
 register_key_smoke_test:TEST_SE_PERSISTENT_LIFETIME:7:PSA_KEY_ID_VOLATILE_MAX:1:PSA_ERROR_INVALID_ARGUMENT
 
 Import-sign-verify: sign in driver, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_DRIVER_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"
 
 Import-sign-verify: sign in driver then export_public, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_DRIVER_THEN_EXPORT_PUBLIC:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"
 
 Import-sign-verify: sign in software, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_SOFTWARE_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"
 
 Generate-sign-verify: sign in driver, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_DRIVER_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"
 
 Generate-sign-verify: sign in driver then export_public, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_DRIVER_THEN_EXPORT_PUBLIC:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"
 
 Generate-sign-verify: sign in software, ECDSA
-depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256
+depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256
 sign_verify:SIGN_IN_SOFTWARE_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e"