From 01df9ddda758536f9020eb7c4dfaba9600e7b22b Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 20 Oct 2022 14:21:21 +0200 Subject: [PATCH 01/23] Add test component: component_test_psa_crypto_config_reference_hash_use_psa Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 61d675f4fc..140166b8fe 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2110,6 +2110,34 @@ component_test_psa_crypto_config_accel_hash_use_psa () { fi } +component_test_psa_crypto_config_reference_hash_use_psa() { + msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + # start with full + scripts/config.py full + # use PSA config and disable driver-less algs as in the component + scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING + # disable options as in the component + # (no need to disable whole modules, we'll just skip their test suite) + scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA + + msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + make test + + # hidden option: when running outcome-analysis.sh, we can skip this + if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then + msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + tests/ssl-opt.sh + + msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + tests/compat.sh + else + echo "skip ssl-opt.sh and compat.sh" + fi +} + component_test_psa_crypto_config_accel_cipher () { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated cipher" From 4e95590ae79ba9c026eec992f3e307f4cf16f8a4 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Fri, 21 Oct 2022 13:42:08 +0200 Subject: [PATCH 02/23] analyze_outcomes.py: Add test coverage regresion analyze for driver only builds Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 77 ++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index d06a0596f3..f5d2ac1313 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -60,6 +60,41 @@ def analyze_coverage(results, outcomes): # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) +def analyze_driver_vs_reference(outcomes, components, ignored_tests): + """Check that all tests executed in the reference component are also + executed in the corresponding driver component. + Skip test suits provided in ignored_tests list. + """ + driver_component = components[0] + reference_component = components[1] + available = check_test_cases.collect_available_test_cases() + result = True + + for key in available: + # Skip ignored test suites + test_suit = key.split(';')[0] # retrieve test suit name + test_suit = test_suit.split('.')[0] # retrieve main part of test suit name + if(test_suit in ignored_tests): + continue + # Continue if test was not executed by any component + hits = outcomes[key].hits() if key in outcomes else 0 + if(hits == 0): + continue + # Search for tests that run in reference component and not in driver component + driver_test_passed = False + reference_test_passed = False + for entry in outcomes[key].successes: + if(driver_component in entry): + driver_test_passed = True + if(reference_component in entry): + reference_test_passed = True + #if(driver_test_passed == True and reference_test_passed == False): + # print('{}: driver: passed; reference: skipped'.format(key)) + if(driver_test_passed == False and reference_test_passed == True): + print('{}: driver: skipped/failed; reference: passed'.format(key)) + result = False + return result + def analyze_outcomes(outcomes): """Run all analyses on the given outcome collection.""" results = Results() @@ -87,20 +122,50 @@ by a semicolon. outcomes[key].failures.append(setup) return outcomes -def analyze_outcome_file(outcome_file): - """Analyze the given outcome file.""" +def do_analyze_coverage(outcome_file): + """Perform coverage analyze.""" outcomes = read_outcome_file(outcome_file) - return analyze_outcomes(outcomes) + results = analyze_outcomes(outcomes) + return (True if results.error_count == 0 else False) + +def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests): + """Perform driver vs reference analyze.""" + # We need exactly 2 components to analyze (first driver and second reference) + if(len(components) != 2 or "accel" not in components[0] or "reference" not in components[1]): + print('Error: Wrong component list. Exactly 2 components are required (driver,reference). ') + return False + outcomes = read_outcome_file(outcome_file) + return analyze_driver_vs_reference(outcomes, components, ignored_tests) def main(): try: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('outcomes', metavar='OUTCOMES.CSV', + parser.add_argument('--outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') + parser.add_argument('--task', + help='Analyze to be done: analyze_coverage or analyze_driver_vs_reference') + parser.add_argument('--components', + help='List of test components to compare. Must be exactly 2 in valid order: driver,reference. ' + 'Apply only for analyze_driver_vs_reference task.') + parser.add_argument('--ignore', + help='List of test suits to ignore. Apply only for analyze_driver_vs_reference task.') options = parser.parse_args() - results = analyze_outcome_file(options.outcomes) - if results.error_count > 0: + + result = False + + if(options.task == 'analyze_coverage'): + result = do_analyze_coverage(options.outcomes) + elif(options.task == 'analyze_driver_vs_reference'): + components_list = options.components.split(',') + ignored_tests_list = options.ignore.split(',') + ignored_tests_list = ['test_suite_' + x for x in ignored_tests_list] + result = do_analyze_driver_vs_reference(options.outcomes, components_list, ignored_tests_list) + else: + print('Error: Unknown task: {}'.format(options.task)) + + if(result == False): sys.exit(1) + print("SUCCESS :-)") except Exception: # pylint: disable=broad-except # Print the backtrace and exit explicitly with our chosen status. traceback.print_exc() From 58bbc23ca30794e109609948f4f64051615e9c39 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 24 Oct 2022 08:10:10 +0200 Subject: [PATCH 03/23] Use coverage analyze as default task Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index f5d2ac1313..1100086c1b 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -140,9 +140,9 @@ def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests): def main(): try: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--outcomes', metavar='OUTCOMES.CSV', + parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') - parser.add_argument('--task', + parser.add_argument('--task', default='analyze_coverage', help='Analyze to be done: analyze_coverage or analyze_driver_vs_reference') parser.add_argument('--components', help='List of test components to compare. Must be exactly 2 in valid order: driver,reference. ' From c86dedfdc183eb0a7681887416085be182e08101 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 24 Oct 2022 09:16:04 +0200 Subject: [PATCH 04/23] Fix code style Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 1100086c1b..3e95997305 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -74,23 +74,23 @@ def analyze_driver_vs_reference(outcomes, components, ignored_tests): # Skip ignored test suites test_suit = key.split(';')[0] # retrieve test suit name test_suit = test_suit.split('.')[0] # retrieve main part of test suit name - if(test_suit in ignored_tests): + if test_suit in ignored_tests: continue # Continue if test was not executed by any component hits = outcomes[key].hits() if key in outcomes else 0 - if(hits == 0): + if hits == 0: continue # Search for tests that run in reference component and not in driver component driver_test_passed = False reference_test_passed = False for entry in outcomes[key].successes: - if(driver_component in entry): + if driver_component in entry: driver_test_passed = True - if(reference_component in entry): + if reference_component in entry: reference_test_passed = True - #if(driver_test_passed == True and reference_test_passed == False): + #if(driver_test_passed is True and reference_test_passed is False): # print('{}: driver: passed; reference: skipped'.format(key)) - if(driver_test_passed == False and reference_test_passed == True): + if(driver_test_passed is False and reference_test_passed is True): print('{}: driver: skipped/failed; reference: passed'.format(key)) result = False return result @@ -126,7 +126,7 @@ def do_analyze_coverage(outcome_file): """Perform coverage analyze.""" outcomes = read_outcome_file(outcome_file) results = analyze_outcomes(outcomes) - return (True if results.error_count == 0 else False) + return results.error_count == 0 def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests): """Perform driver vs reference analyze.""" @@ -143,27 +143,31 @@ def main(): parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') parser.add_argument('--task', default='analyze_coverage', - help='Analyze to be done: analyze_coverage or analyze_driver_vs_reference') + help='Analyze to be done: analyze_coverage or ' + 'analyze_driver_vs_reference') parser.add_argument('--components', - help='List of test components to compare. Must be exactly 2 in valid order: driver,reference. ' + help='List of test components to compare. ' + 'Must be exactly 2 in valid order: driver,reference. ' 'Apply only for analyze_driver_vs_reference task.') parser.add_argument('--ignore', - help='List of test suits to ignore. Apply only for analyze_driver_vs_reference task.') + help='List of test suits to ignore. ' + 'Apply only for analyze_driver_vs_reference task.') options = parser.parse_args() result = False - if(options.task == 'analyze_coverage'): + if options.task == 'analyze_coverage': result = do_analyze_coverage(options.outcomes) - elif(options.task == 'analyze_driver_vs_reference'): + elif options.task == 'analyze_driver_vs_reference': components_list = options.components.split(',') ignored_tests_list = options.ignore.split(',') ignored_tests_list = ['test_suite_' + x for x in ignored_tests_list] - result = do_analyze_driver_vs_reference(options.outcomes, components_list, ignored_tests_list) + result = do_analyze_driver_vs_reference(options.outcomes, + components_list, ignored_tests_list) else: print('Error: Unknown task: {}'.format(options.task)) - if(result == False): + if result is False: sys.exit(1) print("SUCCESS :-)") except Exception: # pylint: disable=broad-except From ab0451bc2c28185608243273327d10aa92c34e90 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 24 Oct 2022 11:29:35 +0200 Subject: [PATCH 05/23] Fix build command in test_psa_crypto_config_reference_hash_use_psa Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 140166b8fe..d84ad85314 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2123,6 +2123,8 @@ component_test_psa_crypto_config_reference_hash_use_psa() { scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA + make + msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" make test From 4d13c833dad616601b5b1c8d78ee470fc6b28224 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 26 Oct 2022 16:11:26 +0200 Subject: [PATCH 06/23] analyze_outcomes.py: remove components and ignore parameters Use a dictionary to specify optional parameters for each task. If the task is not specified then all tasks are executed. Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 54 +++++++++++++++++++------------ 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 3e95997305..de52d776b4 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -122,14 +122,18 @@ by a semicolon. outcomes[key].failures.append(setup) return outcomes -def do_analyze_coverage(outcome_file): +def do_analyze_coverage(outcome_file, args): """Perform coverage analyze.""" + del args # unused outcomes = read_outcome_file(outcome_file) results = analyze_outcomes(outcomes) return results.error_count == 0 -def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests): +def do_analyze_driver_vs_reference(outcome_file, args): """Perform driver vs reference analyze.""" + components = args['components'].split(',') + ignored_tests = args['ignored'].split(',') + ignored_tests = ['test_suite_' + x for x in ignored_tests] # We need exactly 2 components to analyze (first driver and second reference) if(len(components) != 2 or "accel" not in components[0] or "reference" not in components[1]): print('Error: Wrong component list. Exactly 2 components are required (driver,reference). ') @@ -137,35 +141,43 @@ def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests): outcomes = read_outcome_file(outcome_file) return analyze_driver_vs_reference(outcomes, components, ignored_tests) +# List of tasks with function that can handle this task and additional arguments if required +# pylint: disable=line-too-long +TASKS = { + 'analyze_coverage': { + 'test_function': do_analyze_coverage, + 'args': {}}, + 'analyze_driver_vs_reference_hash': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'components': 'test_psa_crypto_config_accel_hash_use_psa,test_psa_crypto_config_reference_hash_use_psa', + 'ignored': 'md,mdx,shax,entropy,hmac_drbg,random,psa_crypto_init,hkdf'}} +} +# pylint: enable=line-too-long + def main(): try: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') - parser.add_argument('--task', default='analyze_coverage', - help='Analyze to be done: analyze_coverage or ' - 'analyze_driver_vs_reference') - parser.add_argument('--components', - help='List of test components to compare. ' - 'Must be exactly 2 in valid order: driver,reference. ' - 'Apply only for analyze_driver_vs_reference task.') - parser.add_argument('--ignore', - help='List of test suits to ignore. ' - 'Apply only for analyze_driver_vs_reference task.') + parser.add_argument('--task', default='all', + help='Analyze to be done: all or analyze_coverage or ' + 'analyze_driver_vs_reference_hash') options = parser.parse_args() - result = False + result = True - if options.task == 'analyze_coverage': - result = do_analyze_coverage(options.outcomes) - elif options.task == 'analyze_driver_vs_reference': - components_list = options.components.split(',') - ignored_tests_list = options.ignore.split(',') - ignored_tests_list = ['test_suite_' + x for x in ignored_tests_list] - result = do_analyze_driver_vs_reference(options.outcomes, - components_list, ignored_tests_list) + if options.task == 'all': + for task in TASKS: + if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): + result = False + elif options.task in TASKS: + if not TASKS[options.task]['test_function'](options.outcomes, + TASKS[options.task]['args']): + result = False else: print('Error: Unknown task: {}'.format(options.task)) + result = False if result is False: sys.exit(1) From 5f6f32a0addcfb3b70197f216a745da58d2cd92e Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 27 Oct 2022 08:24:43 +0200 Subject: [PATCH 07/23] Remove hidden option to skip ssl-opt and compat tests Also remove compat tests from reference component as results from this run are not included in outcome file. Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index d84ad85314..cecfb5d61e 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2098,16 +2098,11 @@ component_test_psa_crypto_config_accel_hash_use_psa () { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" make test - # hidden option: when running outcome-analysis.sh, we can skip this - if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then - msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/ssl-opt.sh + msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" + tests/ssl-opt.sh - msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/compat.sh - else - echo "skip ssl-opt.sh and compat.sh" - fi + msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + tests/compat.sh } component_test_psa_crypto_config_reference_hash_use_psa() { @@ -2128,16 +2123,8 @@ component_test_psa_crypto_config_reference_hash_use_psa() { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" make test - # hidden option: when running outcome-analysis.sh, we can skip this - if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then - msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" - tests/ssl-opt.sh - - msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" - tests/compat.sh - else - echo "skip ssl-opt.sh and compat.sh" - fi + msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + tests/ssl-opt.sh } component_test_psa_crypto_config_accel_cipher () { From 120ed8f8faf0128a9632917710059d27816001ec Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 27 Oct 2022 10:29:15 +0200 Subject: [PATCH 08/23] Add comments to explan the purpose of the reference component Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index cecfb5d61e..6a7501a978 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2041,6 +2041,9 @@ component_test_psa_crypto_config_accel_hash () { make test } +# Note that component_test_psa_crypto_config_reference_hash_use_psa +# is related to this component and both components need to be kept in sync. +# For details please see comments for component_test_psa_crypto_config_reference_hash_use_psa. component_test_psa_crypto_config_accel_hash_use_psa () { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" @@ -2105,6 +2108,10 @@ component_test_psa_crypto_config_accel_hash_use_psa () { tests/compat.sh } +# This component provides reference configuration for test_psa_crypto_config_accel_hash_use_psa +# without accelerated hash. The outcome from both components are used by the analyze_outcomes.py +# script to find regression in test coverage when accelerated hash is used (tests and ssl-opt). +# Both components need to be kept in sync. component_test_psa_crypto_config_reference_hash_use_psa() { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" # start with full From a380b06c26086e695345a04163c1d174c3eb7d20 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 27 Oct 2022 14:15:26 +0200 Subject: [PATCH 09/23] Add fake dependency to test CI Signed-off-by: Przemek Stekiel --- tests/suites/test_suite_error.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/test_suite_error.data b/tests/suites/test_suite_error.data index dec5639ee0..65f0daa847 100644 --- a/tests/suites/test_suite_error.data +++ b/tests/suites/test_suite_error.data @@ -3,7 +3,7 @@ depends_on:MBEDTLS_AES_C error_strerror:-0x0020:"AES - Invalid key length" Single high error -depends_on:MBEDTLS_RSA_C +depends_on:MBEDTLS_RSA_C:MBEDTLS_ENTROPY_C error_strerror:-0x4080:"RSA - Bad input parameters to function" Low and high error From 6856f4c70d2a0e3fb16e180ae45aa19db36772d7 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 9 Nov 2022 10:50:29 +0100 Subject: [PATCH 10/23] Fix typos and comments Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index de52d776b4..74b3184d0a 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -63,7 +63,7 @@ def analyze_coverage(results, outcomes): def analyze_driver_vs_reference(outcomes, components, ignored_tests): """Check that all tests executed in the reference component are also executed in the corresponding driver component. - Skip test suits provided in ignored_tests list. + Skip test suites provided in ignored_tests list. """ driver_component = components[0] reference_component = components[1] @@ -72,9 +72,9 @@ def analyze_driver_vs_reference(outcomes, components, ignored_tests): for key in available: # Skip ignored test suites - test_suit = key.split(';')[0] # retrieve test suit name - test_suit = test_suit.split('.')[0] # retrieve main part of test suit name - if test_suit in ignored_tests: + test_suite = key.split(';')[0] # retrieve test suit name + test_suite = test_suite.split('.')[0] # retrieve main part of test suit name + if test_suite in ignored_tests: continue # Continue if test was not executed by any component hits = outcomes[key].hits() if key in outcomes else 0 @@ -88,8 +88,6 @@ def analyze_driver_vs_reference(outcomes, components, ignored_tests): driver_test_passed = True if reference_component in entry: reference_test_passed = True - #if(driver_test_passed is True and reference_test_passed is False): - # print('{}: driver: passed; reference: skipped'.format(key)) if(driver_test_passed is False and reference_test_passed is True): print('{}: driver: skipped/failed; reference: passed'.format(key)) result = False @@ -123,7 +121,7 @@ by a semicolon. return outcomes def do_analyze_coverage(outcome_file, args): - """Perform coverage analyze.""" + """Perform coverage analysis.""" del args # unused outcomes = read_outcome_file(outcome_file) results = analyze_outcomes(outcomes) @@ -141,7 +139,7 @@ def do_analyze_driver_vs_reference(outcome_file, args): outcomes = read_outcome_file(outcome_file) return analyze_driver_vs_reference(outcomes, components, ignored_tests) -# List of tasks with function that can handle this task and additional arguments if required +# List of tasks with a function that can handle this task and additional arguments if required # pylint: disable=line-too-long TASKS = { 'analyze_coverage': { @@ -161,7 +159,7 @@ def main(): parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') parser.add_argument('--task', default='all', - help='Analyze to be done: all or analyze_coverage or ' + help='Analysis to be done: all or analyze_coverage or ' 'analyze_driver_vs_reference_hash') options = parser.parse_args() From 51f30ff6e687ccd669611a4d9190cc98d67886a9 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 9 Nov 2022 12:07:29 +0100 Subject: [PATCH 11/23] Make separate components for ref and driver in TASKS Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 74b3184d0a..031e16132d 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -60,13 +60,11 @@ def analyze_coverage(results, outcomes): # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) -def analyze_driver_vs_reference(outcomes, components, ignored_tests): +def analyze_driver_vs_reference(outcomes, component_ref,component_driver, ignored_tests): """Check that all tests executed in the reference component are also executed in the corresponding driver component. Skip test suites provided in ignored_tests list. """ - driver_component = components[0] - reference_component = components[1] available = check_test_cases.collect_available_test_cases() result = True @@ -84,9 +82,9 @@ def analyze_driver_vs_reference(outcomes, components, ignored_tests): driver_test_passed = False reference_test_passed = False for entry in outcomes[key].successes: - if driver_component in entry: + if component_driver in entry: driver_test_passed = True - if reference_component in entry: + if component_ref in entry: reference_test_passed = True if(driver_test_passed is False and reference_test_passed is True): print('{}: driver: skipped/failed; reference: passed'.format(key)) @@ -129,18 +127,14 @@ def do_analyze_coverage(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args): """Perform driver vs reference analyze.""" - components = args['components'].split(',') ignored_tests = args['ignored'].split(',') ignored_tests = ['test_suite_' + x for x in ignored_tests] - # We need exactly 2 components to analyze (first driver and second reference) - if(len(components) != 2 or "accel" not in components[0] or "reference" not in components[1]): - print('Error: Wrong component list. Exactly 2 components are required (driver,reference). ') - return False + outcomes = read_outcome_file(outcome_file) - return analyze_driver_vs_reference(outcomes, components, ignored_tests) + return analyze_driver_vs_reference(outcomes, args['component_ref'], + args['component_driver'], ignored_tests) # List of tasks with a function that can handle this task and additional arguments if required -# pylint: disable=line-too-long TASKS = { 'analyze_coverage': { 'test_function': do_analyze_coverage, @@ -148,10 +142,10 @@ TASKS = { 'analyze_driver_vs_reference_hash': { 'test_function': do_analyze_driver_vs_reference, 'args': { - 'components': 'test_psa_crypto_config_accel_hash_use_psa,test_psa_crypto_config_reference_hash_use_psa', + 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', + 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 'ignored': 'md,mdx,shax,entropy,hmac_drbg,random,psa_crypto_init,hkdf'}} } -# pylint: enable=line-too-long def main(): try: From be279c7bcc6f5b33a704ff925960d81fdc72b3c1 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 9 Nov 2022 12:17:08 +0100 Subject: [PATCH 12/23] Make a list from ignored tests in TASKS Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 031e16132d..85ec97c164 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -127,8 +127,7 @@ def do_analyze_coverage(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args): """Perform driver vs reference analyze.""" - ignored_tests = args['ignored'].split(',') - ignored_tests = ['test_suite_' + x for x in ignored_tests] + ignored_tests = ['test_suite_' + x for x in args['ignored_suites']] outcomes = read_outcome_file(outcome_file) return analyze_driver_vs_reference(outcomes, args['component_ref'], @@ -144,7 +143,13 @@ TASKS = { 'args': { 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', - 'ignored': 'md,mdx,shax,entropy,hmac_drbg,random,psa_crypto_init,hkdf'}} + 'ignored_suites': ['shax','mdx', # the software implementations that are being excluded + 'md' # the legacy abstraction layer that's being excluded + 'entropy','hmac_drbg','random', # temporary limitation (see RNG EPIC) + 'psa_crypto_init', # doesn't work with external RNG + 'hkdf', # legacy still depends on MD, but there's a PSA interface that doesn't + 'pkcs7 ' # recent addition, will be addressed later + ]}} } def main(): From 992de3c56284bf48758a33d189a511a73c7c727e Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 9 Nov 2022 13:54:49 +0100 Subject: [PATCH 13/23] Make TASK parameter positional and allow more than one task Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 36 +++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 85ec97c164..f78af68f3c 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -144,7 +144,7 @@ TASKS = { 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 'ignored_suites': ['shax','mdx', # the software implementations that are being excluded - 'md' # the legacy abstraction layer that's being excluded + 'md', # the legacy abstraction layer that's being excluded 'entropy','hmac_drbg','random', # temporary limitation (see RNG EPIC) 'psa_crypto_init', # doesn't work with external RNG 'hkdf', # legacy still depends on MD, but there's a PSA interface that doesn't @@ -157,24 +157,38 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') - parser.add_argument('--task', default='all', - help='Analysis to be done: all or analyze_coverage or ' - 'analyze_driver_vs_reference_hash') + parser.add_argument('task', default='all', + help='Analysis to be done. By default, run all tasks. ' + 'With one or more TASK, run only those. ' + 'TASK can be the name of a single task or ' + 'coma-separated list of tasks. ') + parser.add_argument('--list', action='store_true', + help='List all available tasks and exit.') options = parser.parse_args() + if options.list: + for task in TASKS: + print(task) + sys.exit(0) + result = True + tasks = [] if options.task == 'all': for task in TASKS: + tasks.append(task) + else: + tasks = options.task.split(',') + + for task in tasks: + if task not in TASKS: + print('Error: invalid task: {}'.format(task)) + sys.exit(1) + + for task in TASKS: + if task in tasks: if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): result = False - elif options.task in TASKS: - if not TASKS[options.task]['test_function'](options.outcomes, - TASKS[options.task]['args']): - result = False - else: - print('Error: Unknown task: {}'.format(options.task)) - result = False if result is False: sys.exit(1) From 93986645d8a6f6e157d04c892386ddc6fa5a7de5 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 9 Nov 2022 15:06:44 +0100 Subject: [PATCH 14/23] Remove reference vs drivers test from outcome-analysis.sh Signed-off-by: Przemek Stekiel --- .../psa-migration/outcome-analysis.sh | 71 ++++++------------- 1 file changed, 20 insertions(+), 51 deletions(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index 81ab69183c..9084685482 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -13,6 +13,7 @@ # - the set of tests skipped in the driver-only build is the same as in an # equivalent software-based configuration, or the difference is small enough, # justified, and a github issue is created to track it. +# This part is verified by tests/scripts/analyze_outcomes.py # # WARNING: this script checks out a commit other than the head of the current # branch; it checks out the current branch again when running successfully, @@ -26,30 +27,12 @@ # re-running this script (for example "get numbers before this PR"). # ----- BEGIN edit this ----- -# The component in all.sh that builds and tests with drivers. -DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa -# A similar configuration to that of the component, except without drivers, -# for comparison. -reference_config () { - # start with full - scripts/config.py full - # use PSA config and disable driver-less algs as in the component - scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING - # disable options as in the component - # (no need to disable whole modules, we'll just skip their test suite) - scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA -} # Space-separated list of test suites to ignore: # if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored. IGNORE="md mdx shax" # accelerated IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG) IGNORE="$IGNORE psa_crypto_init" # needs internal RNG IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested -# Compare only "reference vs driver" or also "before vs after"? -BEFORE_AFTER=1 # 0 or 1 # ----- END edit this ----- set -eu @@ -65,38 +48,27 @@ record() { make check } -if [ "$BEFORE_AFTER" -eq 1 ]; then - # save current HEAD - HEAD=$(git branch --show-current) +# save current HEAD +HEAD=$(git branch --show-current) - # get the numbers before this PR for default and full - cleanup - git checkout $(git merge-base HEAD development) - record "before-default" - - cleanup - scripts/config.py full - record "before-full" - - # get the numbers now for default and full - cleanup - git checkout $HEAD - record "after-default" - - cleanup - scripts/config.py full - record "after-full" -fi - -# get the numbers now for driver-only and reference +# get the numbers before this PR for default and full cleanup -reference_config -record "reference" +git checkout $(git merge-base HEAD development) +record "before-default" cleanup -export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-drivers.csv" -export SKIP_SSL_OPT_COMPAT_SH=1 -tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa +scripts/config.py full +record "before-full" + +# get the numbers now for default and full +cleanup +git checkout $HEAD +record "after-default" + +cleanup +scripts/config.py full +record "after-full" + # analysis @@ -156,8 +128,5 @@ compare_builds () { } populate_suites -if [ "$BEFORE_AFTER" -eq 1 ]; then - compare_builds before-default after-default - compare_builds before-full after-full -fi -compare_builds reference drivers +compare_builds before-default after-default +compare_builds before-full after-full From 733c76e08a32b81905fb0a50a486c203d3699776 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 14 Nov 2022 08:33:21 +0100 Subject: [PATCH 15/23] Fix style issues pointed by pylint Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index f78af68f3c..2cdad692b9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -60,7 +60,7 @@ def analyze_coverage(results, outcomes): # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) -def analyze_driver_vs_reference(outcomes, component_ref,component_driver, ignored_tests): +def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests): """Check that all tests executed in the reference component are also executed in the corresponding driver component. Skip test suites provided in ignored_tests list. @@ -143,12 +143,14 @@ TASKS = { 'args': { 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', - 'ignored_suites': ['shax','mdx', # the software implementations that are being excluded + 'ignored_suites': ['shax', 'mdx', # the software implementations that are being excluded 'md', # the legacy abstraction layer that's being excluded - 'entropy','hmac_drbg','random', # temporary limitation (see RNG EPIC) + 'entropy', 'hmac_drbg', 'random', # temporary limitation + # (see RNG EPIC) 'psa_crypto_init', # doesn't work with external RNG - 'hkdf', # legacy still depends on MD, but there's a PSA interface that doesn't - 'pkcs7 ' # recent addition, will be addressed later + 'hkdf', # legacy still depends on MD, + # but there's a PSA interface that doesn't + 'pkcs7' # recent addition, will be addressed later ]}} } From 8b6826d309fcf90d6f7bde72e2fad184b56a6d8b Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 14 Nov 2022 08:33:47 +0100 Subject: [PATCH 16/23] Revert "Add fake dependency to test CI" This reverts commit a380b06c26086e695345a04163c1d174c3eb7d20. Signed-off-by: Przemek Stekiel --- tests/suites/test_suite_error.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/test_suite_error.data b/tests/suites/test_suite_error.data index 65f0daa847..dec5639ee0 100644 --- a/tests/suites/test_suite_error.data +++ b/tests/suites/test_suite_error.data @@ -3,7 +3,7 @@ depends_on:MBEDTLS_AES_C error_strerror:-0x0020:"AES - Invalid key length" Single high error -depends_on:MBEDTLS_RSA_C:MBEDTLS_ENTROPY_C +depends_on:MBEDTLS_RSA_C error_strerror:-0x4080:"RSA - Bad input parameters to function" Low and high error From d3068af2a8b6067e0c8a5574fa984eecd8027782 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Mon, 14 Nov 2022 16:15:19 +0100 Subject: [PATCH 17/23] Optimize code (tasks list initialization, task verification) Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 2cdad692b9..ba38ec2808 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -175,17 +175,15 @@ def main(): result = True - tasks = [] if options.task == 'all': - for task in TASKS: - tasks.append(task) + tasks = TASKS.keys() else: tasks = options.task.split(',') - for task in tasks: - if task not in TASKS: - print('Error: invalid task: {}'.format(task)) - sys.exit(1) + for task in tasks: + if task not in TASKS: + print('Error: invalid task: {}'.format(task)) + sys.exit(1) for task in TASKS: if task in tasks: From aa88e0b86b5df3d70b9b6fb80d8116a61e984450 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Tue, 15 Nov 2022 13:21:14 +0100 Subject: [PATCH 18/23] Make configurations (driver, reference) as close as possible Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 83 +++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 6a7501a978..9fba034cde 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2041,6 +2041,45 @@ component_test_psa_crypto_config_accel_hash () { make test } +# Auxiliary function to build config for hashes with and without drivers +config_psa_crypto_hash_use_psa () { + DRIVER_ONLY="$1" + # start with config full for maximum coverage (also enables USE_PSA) + scripts/config.py full + # enable support for configuring PSA-only algorithms + scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + if [ "$DRIVER_ONLY" -eq 1 ]; then + # enable support for drivers + scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS + # disable the built-in implementation of hashes + scripts/config.py unset MBEDTLS_MD5_C + scripts/config.py unset MBEDTLS_RIPEMD160_C + scripts/config.py unset MBEDTLS_SHA1_C + scripts/config.py unset MBEDTLS_SHA224_C + scripts/config.py unset MBEDTLS_SHA256_C # see external RNG below + scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT + scripts/config.py unset MBEDTLS_SHA384_C + scripts/config.py unset MBEDTLS_SHA512_C + scripts/config.py unset MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT + fi + # Use an external RNG as currently internal RNGs depend on entropy.c + # which in turn hard-depends on SHA256_C (or SHA512_C). + # See component_test_psa_external_rng_no_drbg_use_psa. + scripts/config.py set MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG + scripts/config.py unset MBEDTLS_ENTROPY_C + scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED # depends on ENTROPY_C + scripts/config.py unset MBEDTLS_PLATFORM_NV_SEED_ALT # depends on former + # Also unset MD_C and things that depend on it; + # see component_test_crypto_full_no_md. + if [ "$DRIVER_ONLY" -eq 1 ]; then + scripts/config.py unset MBEDTLS_MD_C + fi + scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation + scripts/config.py unset MBEDTLS_HMAC_DRBG_C + scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA +} + # Note that component_test_psa_crypto_config_reference_hash_use_psa # is related to this component and both components need to be kept in sync. # For details please see comments for component_test_psa_crypto_config_reference_hash_use_psa. @@ -2056,35 +2095,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { loc_accel_flags=$( echo "$loc_accel_list" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) make -C tests libtestdriver1.a CFLAGS="$ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS" - # start with config full for maximum coverage (also enables USE_PSA) - scripts/config.py full - # enable support for drivers and configuring PSA-only algorithms - scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS - scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG - # disable the built-in implementation of hashes - scripts/config.py unset MBEDTLS_MD5_C - scripts/config.py unset MBEDTLS_RIPEMD160_C - scripts/config.py unset MBEDTLS_SHA1_C - scripts/config.py unset MBEDTLS_SHA224_C - scripts/config.py unset MBEDTLS_SHA256_C # see external RNG below - scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT - scripts/config.py unset MBEDTLS_SHA384_C - scripts/config.py unset MBEDTLS_SHA512_C - scripts/config.py unset MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT - # Use an external RNG as currently internal RNGs depend on entropy.c - # which in turn hard-depends on SHA256_C (or SHA512_C). - # See component_test_psa_external_rng_no_drbg_use_psa. - scripts/config.py set MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG - scripts/config.py unset MBEDTLS_ENTROPY_C - scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED # depends on ENTROPY_C - scripts/config.py unset MBEDTLS_PLATFORM_NV_SEED_ALT # depends on former - # Also unset MD_C and things that depend on it; - # see component_test_crypto_full_no_md. - scripts/config.py unset MBEDTLS_MD_C - scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation - scripts/config.py unset MBEDTLS_HMAC_DRBG_C - scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA + config_psa_crypto_hash_use_psa 1 loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )" make CFLAGS="$ASAN_CFLAGS -Werror -I../tests/include -I../tests -I../../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS" all @@ -2105,7 +2116,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { tests/ssl-opt.sh msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" - tests/compat.sh + #tests/compat.sh } # This component provides reference configuration for test_psa_crypto_config_accel_hash_use_psa @@ -2114,16 +2125,8 @@ component_test_psa_crypto_config_accel_hash_use_psa () { # Both components need to be kept in sync. component_test_psa_crypto_config_reference_hash_use_psa() { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" - # start with full - scripts/config.py full - # use PSA config and disable driver-less algs as in the component - scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING - # disable options as in the component - # (no need to disable whole modules, we'll just skip their test suite) - scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC - scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA + + config_psa_crypto_hash_use_psa 0 make From f3be7ccadebd605ad23c4b46bfce1aa2af02666a Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 16 Nov 2022 12:53:20 +0100 Subject: [PATCH 19/23] Keep drivers enabled also in reference build Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 9fba034cde..c76a94fcf4 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2046,11 +2046,10 @@ config_psa_crypto_hash_use_psa () { DRIVER_ONLY="$1" # start with config full for maximum coverage (also enables USE_PSA) scripts/config.py full - # enable support for configuring PSA-only algorithms + # enable support for drivers and configuring PSA-only algorithms scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS if [ "$DRIVER_ONLY" -eq 1 ]; then - # enable support for drivers - scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS # disable the built-in implementation of hashes scripts/config.py unset MBEDTLS_MD5_C scripts/config.py unset MBEDTLS_RIPEMD160_C From 52d8e96ff6f729c9f64222c56f1088458f023dce Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 16 Nov 2022 12:55:27 +0100 Subject: [PATCH 20/23] Disable PSA_WANT_ALG_STREAM_CIPHER, PSA_WANT_ALG_ECB_NO_PADDING also in reference config Signed-off-by: Przemek Stekiel Signed-off-by: Przemek Stekiel --- tests/scripts/all.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index c76a94fcf4..22c3f760ed 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2115,7 +2115,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { tests/ssl-opt.sh msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" - #tests/compat.sh + tests/compat.sh } # This component provides reference configuration for test_psa_crypto_config_accel_hash_use_psa @@ -2125,6 +2125,9 @@ component_test_psa_crypto_config_accel_hash_use_psa () { component_test_psa_crypto_config_reference_hash_use_psa() { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA" + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING + config_psa_crypto_hash_use_psa 0 make From 6419ab5299a723684f7ada8d69f0e8046ecc7d26 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Wed, 16 Nov 2022 12:57:06 +0100 Subject: [PATCH 21/23] Reduce number of skipped suites (after making configs more similar) Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ba38ec2808..508933718d 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -145,12 +145,6 @@ TASKS = { 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 'ignored_suites': ['shax', 'mdx', # the software implementations that are being excluded 'md', # the legacy abstraction layer that's being excluded - 'entropy', 'hmac_drbg', 'random', # temporary limitation - # (see RNG EPIC) - 'psa_crypto_init', # doesn't work with external RNG - 'hkdf', # legacy still depends on MD, - # but there's a PSA interface that doesn't - 'pkcs7' # recent addition, will be addressed later ]}} } From 542d9323521434bb5b4533eec244e95f79e73eae Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 17 Nov 2022 09:43:34 +0100 Subject: [PATCH 22/23] Fix handling of default value for task argument Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 508933718d..4ebd5f6304 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -153,7 +153,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') - parser.add_argument('task', default='all', + parser.add_argument('task', default='all', nargs='?', help='Analysis to be done. By default, run all tasks. ' 'With one or more TASK, run only those. ' 'TASK can be the name of a single task or ' From 85c54ea3613f44c0f8c4c52d265128dcf478aaf4 Mon Sep 17 00:00:00 2001 From: Przemek Stekiel Date: Thu, 17 Nov 2022 11:50:23 +0100 Subject: [PATCH 23/23] Allow providing space sepatated tasks Signed-off-by: Przemek Stekiel --- tests/scripts/analyze_outcomes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 4ebd5f6304..bb44396534 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -9,6 +9,7 @@ less likely to be useful. import argparse import sys import traceback +import re import check_test_cases @@ -157,7 +158,7 @@ def main(): help='Analysis to be done. By default, run all tasks. ' 'With one or more TASK, run only those. ' 'TASK can be the name of a single task or ' - 'coma-separated list of tasks. ') + 'comma/space-separated list of tasks. ') parser.add_argument('--list', action='store_true', help='List all available tasks and exit.') options = parser.parse_args() @@ -172,7 +173,7 @@ def main(): if options.task == 'all': tasks = TASKS.keys() else: - tasks = options.task.split(',') + tasks = re.split(r'[, ]+', options.task) for task in tasks: if task not in TASKS: