From 4da369f74136d975c8d01eaf68d7261638faecb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 18 Oct 2023 09:40:32 +0200 Subject: [PATCH 01/11] analyze_outcomes: minor code cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 706421f6c4..ae3450d39a 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -112,15 +112,18 @@ def analyze_driver_vs_reference(results: Results, outcomes, hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0: continue - # Skip ignored test suites - full_test_suite = key.split(';')[0] # retrieve full test suite name - test_string = key.split(';')[1] # retrieve the text string of this test + + # key is like "test_suite_foo.bar;Description of test case" + (full_test_suite, test_string) = key.split(';') test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + # Skip fully-ignored test suites if test_suite in ignored_suites or full_test_suite in ignored_suites: continue + # Skip ignored test cases inside test suites if ((full_test_suite in ignored_test) and (test_string in ignored_test[full_test_suite])): continue + # Search for tests that run in reference component and not in driver component driver_test_passed = False reference_test_passed = False From 881ce01db3ffc62989ff0641a1dd71b7581f5807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 18 Oct 2023 10:22:07 +0200 Subject: [PATCH 02/11] analyze_outcomes: add regex match for ignored tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ae3450d39a..60717442f3 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -95,9 +95,22 @@ def analyze_coverage(results, outcomes, allow_list, full_coverage): else: results.warning('Allow listed test case was executed: {}', key) +def name_matches_pattern(name, str_or_re): + """Check if name matches a pattern, that may be a string or regex. + - If the pattern is a string, name must be equal to match. + - If the pattern is a regex, name must fully match. + """ + if isinstance(str_or_re, re.Pattern): + if str_or_re.fullmatch(name): + return True + else: + if str_or_re == name: + return True + return False + def analyze_driver_vs_reference(results: Results, outcomes, component_ref, component_driver, - ignored_suites, ignored_test=None): + ignored_suites, ignored_tests=None): """Check that all tests executed in the reference component are also executed in the corresponding driver component. Skip: @@ -120,9 +133,10 @@ def analyze_driver_vs_reference(results: Results, outcomes, if test_suite in ignored_suites or full_test_suite in ignored_suites: continue # Skip ignored test cases inside test suites - if ((full_test_suite in ignored_test) and - (test_string in ignored_test[full_test_suite])): - continue + if full_test_suite in ignored_tests: + for str_or_re in ignored_tests[full_test_suite]: + if name_matches_pattern(test_string, str_or_re): + continue # Search for tests that run in reference component and not in driver component driver_test_passed = False @@ -921,21 +935,7 @@ KNOWN_TASKS = { ], 'test_suite_asn1write': [ # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile('ASN.1 Write mpi.*'), ], } } From 371165aec0cc4ea2570a77ec763b989ffc815c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 18 Oct 2023 12:44:54 +0200 Subject: [PATCH 03/11] analyze_outcomes: useless ignores are now errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change from iterating on all available tests to iterating on tests with an outcome: initially we were iterating on available tests but immediately ignoring those that were always skipped. That last part played poorly with the new error (we want to know if the test ignored due to a pattern was skipped in the reference component), but when removing it, we were left with iterating on all available tests then skipping those that don't have outcomes entries: this is equivalent to iterating on tests with an outcome entry, which is more readable. Also, give an error if the outcome file contains no passing test from the reference component: probably means we're using the wrong outcome file. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 60717442f3..6611373fed 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -118,25 +118,24 @@ def analyze_driver_vs_reference(results: Results, outcomes, - only some specific test inside a test suite, for which the corresponding output string is provided """ - available = check_test_cases.collect_available_test_cases() - - for key in available: - # Continue if test was not executed by any component - hits = outcomes[key].hits() if key in outcomes else 0 - if hits == 0: - continue - + seen_reference_passing = False + for key in outcomes: # key is like "test_suite_foo.bar;Description of test case" (full_test_suite, test_string) = key.split(';') test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name - # Skip fully-ignored test suites + + # Immediately skip fully-ignored test suites if test_suite in ignored_suites or full_test_suite in ignored_suites: continue - # Skip ignored test cases inside test suites + + # For ignored test cases inside test suites, just remember and: + # don't issue an error if they're skipped with drivers, + # but issue an error if they're not (means we have a bad entry). + ignored = False if full_test_suite in ignored_tests: for str_or_re in ignored_tests[full_test_suite]: if name_matches_pattern(test_string, str_or_re): - continue + ignored = True # Search for tests that run in reference component and not in driver component driver_test_passed = False @@ -146,8 +145,16 @@ def analyze_driver_vs_reference(results: Results, outcomes, driver_test_passed = True if component_ref in entry: reference_test_passed = True + seen_reference_passing = True if(reference_test_passed and not driver_test_passed): - results.error("Did not pass with driver: {}", key) + if not ignored: + results.error("PASS -> SKIP/FAIL: {}", key) + else: + if ignored: + results.error("uselessly ignored: {}", key) + + if not seen_reference_passing: + results.error("no passing test in reference component: bad outcome file?") def analyze_outcomes(results: Results, outcomes, args): """Run all analyses on the given outcome collection.""" From b4558bd6e46af48eb83181e84f233d3dae745e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 18 Oct 2023 13:00:49 +0200 Subject: [PATCH 04/11] analyze_outcomes: remove useless ignore entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 99 ------------------------------- 1 file changed, 99 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 6611373fed..7c6b47d208 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -587,14 +587,6 @@ KNOWN_TASKS = { 'ECP test vectors secp384r1 rfc 5114', 'ECP test vectors secp521r1 rfc 5114', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - ], 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -618,21 +610,6 @@ KNOWN_TASKS = { 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], 'test_suite_pkparse': [ # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED # is automatically enabled in build_info.h (backward compatibility) @@ -686,21 +663,6 @@ KNOWN_TASKS = { 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], 'test_suite_pkparse': [ # See the description provided above in the # analyze_driver_vs_reference_no_ecp_at_all component. @@ -783,21 +745,6 @@ KNOWN_TASKS = { 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], 'test_suite_pkparse': [ # See the description provided above in the # analyze_driver_vs_reference_no_ecp_at_all component. @@ -887,55 +834,9 @@ KNOWN_TASKS = { 'ignored_tests': { # Ignore all tests that require DERIVE support which is disabled # in the driver version - 'test_suite_psa_crypto': [ - 'PSA key agreement setup: ECDH + HKDF-SHA-256: good', - ('PSA key agreement setup: ECDH + HKDF-SHA-256: good, key algorithm broader ' - 'than required'), - 'PSA key agreement setup: ECDH + HKDF-SHA-256: public key not on curve', - 'PSA key agreement setup: KDF instead of a key agreement algorithm', - 'PSA key agreement setup: bad key agreement algorithm', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: capacity=8160', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 0+32', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 1+31', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 31+1', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+0', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+32', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 64+0', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, info first', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, key output', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, missing info', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, omitted salt', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, raw output', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, salt after secret', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, good case', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label and secret', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, no inputs', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 0+48, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 24+24, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 48+0, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #1, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #3, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #4, ka', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC MONTGOMERY (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - 'PSA raw key agreement: ECDH SECP256R1 (RFC 5903)', - ], 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto_pake': [ - 'PSA PAKE: ecjpake size macros', - ], 'test_suite_asn1parse': [ # This test depends on BIGNUM_C 'INTEGER too large for mpi', From 4fd5a6ac9ed8e7873cca7c5514b584378865516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Fri, 20 Oct 2023 10:21:09 +0200 Subject: [PATCH 05/11] analyze_outcomes: use regexes with ECC components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 231 ++++++------------------------ 1 file changed, 45 insertions(+), 186 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 7c6b47d208..641d114c1f 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -541,11 +541,13 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 'ignored_suites': [ - 'ecdsa', - 'ecdh', - 'ecjpake', + # Modules replaced by drivers + 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + # This test wants a legacy function that takes f_rng, p_rng + # arguments, and uses legacy ECDSA for that. The test is + # really about the wrapper around the PSA RNG, not ECDSA. 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], @@ -553,41 +555,14 @@ KNOWN_TASKS = { # so we must ignore disparities in the tests for which ECP_C # is required. 'test_suite_ecp': [ - 'ECP check public-private #1 (OK)', - 'ECP check public-private #2 (group none)', - 'ECP check public-private #3 (group mismatch)', - 'ECP check public-private #4 (Qx mismatch)', - 'ECP check public-private #5 (Qy mismatch)', - 'ECP check public-private #6 (wrong Qx)', - 'ECP check public-private #7 (wrong Qy)', - 'ECP gen keypair [#1]', - 'ECP gen keypair [#2]', - 'ECP gen keypair [#3]', - 'ECP gen keypair wrapper', - 'ECP point muladd secp256r1 #1', - 'ECP point muladd secp256r1 #2', - 'ECP point multiplication Curve25519 (element of order 2: origin) #3', - 'ECP point multiplication Curve25519 (element of order 4: 1) #4', - 'ECP point multiplication Curve25519 (element of order 8) #5', - 'ECP point multiplication Curve25519 (normalized) #1', - 'ECP point multiplication Curve25519 (not normalized) #2', - 'ECP point multiplication rng fail Curve25519', - 'ECP point multiplication rng fail secp256r1', - 'ECP test vectors Curve25519', - 'ECP test vectors Curve448 (RFC 7748 6.2, after decodeUCoordinate)', - 'ECP test vectors brainpoolP256r1 rfc 7027', - 'ECP test vectors brainpoolP384r1 rfc 7027', - 'ECP test vectors brainpoolP512r1 rfc 7027', - 'ECP test vectors secp192k1', - 'ECP test vectors secp192r1 rfc 5114', - 'ECP test vectors secp224k1', - 'ECP test vectors secp224r1 rfc 5114', - 'ECP test vectors secp256k1', - 'ECP test vectors secp256r1 rfc 5114', - 'ECP test vectors secp384r1 rfc 5114', - 'ECP test vectors secp521r1 rfc 5114', + re.compile(r'ECP check public-private .*'), + re.compile(r'ECP gen keypair .*'), + re.compile(r'ECP point muladd .*'), + re.compile(r'ECP point multiplication .*'), + re.compile(r'ECP test vectors .*'), ], 'test_suite_ssl': [ + # This deprecated function is only present when ECP_C is On. 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } @@ -599,14 +574,11 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], @@ -617,23 +589,10 @@ KNOWN_TASKS = { # consequence compressed points are supported in the reference # component but not in the accelerated one, so they should be skipped # while checking driver's coverage. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -646,75 +605,31 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], + # See no_ecp_at_all 'test_suite_pkparse': [ - # See the description provided above in the - # analyze_driver_vs_reference_no_ecp_at_all component. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ - # Following tests depends on BIGNUM_C - 'Debug print mbedtls_mpi #2: 3 bits', - 'Debug print mbedtls_mpi: 0 (empty representation)', - 'Debug print mbedtls_mpi: 0 (non-empty representation)', - 'Debug print mbedtls_mpi: 49 bits', - 'Debug print mbedtls_mpi: 759 bits', - 'Debug print mbedtls_mpi: 764 bits #1', - 'Debug print mbedtls_mpi: 764 bits #2', + re.compile(r'Debug print mbedtls_mpi.*'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -727,76 +642,31 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', - 'dhm', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], + # See no_ecp_at_all 'test_suite_pkparse': [ - # See the description provided above in the - # analyze_driver_vs_reference_no_ecp_at_all component. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ - # Following tests depends on BIGNUM_C - 'Debug print mbedtls_mpi #2: 3 bits', - 'Debug print mbedtls_mpi: 0 (empty representation)', - 'Debug print mbedtls_mpi: 0 (non-empty representation)', - 'Debug print mbedtls_mpi: 49 bits', - 'Debug print mbedtls_mpi: 759 bits', - 'Debug print mbedtls_mpi: 764 bits #1', - 'Debug print mbedtls_mpi: 764 bits #2', + re.compile(r'Debug print mbedtls_mpi.*'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -818,32 +688,21 @@ KNOWN_TASKS = { 'component_ref': 'test_tfm_config', 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { - # Ignore all tests that require DERIVE support which is disabled - # in the driver version + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - re.compile('ASN.1 Write mpi.*'), + re.compile(r'ASN.1 Write mpi.*'), ], } } From 62d6131e5e78f5dabee836264c77862cb1b1d745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Fri, 20 Oct 2023 10:51:57 +0200 Subject: [PATCH 06/11] analyze_outcomes: minor output fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 641d114c1f..a5340bd62c 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -741,7 +741,7 @@ def main(): tasks_list = re.split(r'[, ]+', options.specified_tasks) for task in tasks_list: if task not in KNOWN_TASKS: - sys.stderr.write('invalid task: {}'.format(task)) + sys.stderr.write('invalid task: {}\n'.format(task)) sys.exit(2) KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage From b26954375ff3026ef071caaac8ca023b9172f472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 23 Oct 2023 09:30:40 +0200 Subject: [PATCH 07/11] analyze_outcome: work around old Python in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index a5340bd62c..a0127be863 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -100,7 +100,9 @@ def name_matches_pattern(name, str_or_re): - If the pattern is a string, name must be equal to match. - If the pattern is a regex, name must fully match. """ - if isinstance(str_or_re, re.Pattern): + # The CI's python is too old for re.Pattern + #if isinstance(str_or_re, re.Pattern): + if not isinstance(str_or_re, str): if str_or_re.fullmatch(name): return True else: From 9d9c2344ea526312e6178fc379ec1906cf76d1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Thu, 26 Oct 2023 09:37:40 +0200 Subject: [PATCH 08/11] analyze_outcome: Simplify some code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index a0127be863..11a8cbf792 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -103,12 +103,9 @@ def name_matches_pattern(name, str_or_re): # The CI's python is too old for re.Pattern #if isinstance(str_or_re, re.Pattern): if not isinstance(str_or_re, str): - if str_or_re.fullmatch(name): - return True + return str_or_re.fullmatch(name) else: - if str_or_re == name: - return True - return False + return str_or_re == name def analyze_driver_vs_reference(results: Results, outcomes, component_ref, component_driver, From d36a37f0deecc7a309038855c1148807e9bf7fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Thu, 26 Oct 2023 09:41:59 +0200 Subject: [PATCH 09/11] analyze_outcomes: ignore patterns apply to whole suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This may come in handy when ignoring patterns in test_suite_cipher for example which is split in several .data files where we'll want to ignore the same patterns. Currently none of the entries had a '.' in the test suite name, so this doesn't change anything for existing entries. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 11a8cbf792..8f45dae787 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -132,7 +132,7 @@ def analyze_driver_vs_reference(results: Results, outcomes, # but issue an error if they're not (means we have a bad entry). ignored = False if full_test_suite in ignored_tests: - for str_or_re in ignored_tests[full_test_suite]: + for str_or_re in ignored_tests[test_suite]: if name_matches_pattern(test_string, str_or_re): ignored = True From cd84a290a9f47e236d9ef2033d26c03075316cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Fri, 27 Oct 2023 09:24:44 +0200 Subject: [PATCH 10/11] analyze_outcomes: use regexes for cipher/aead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 257 +----------------------------- 1 file changed, 6 insertions(+), 251 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 8f45dae787..81e3ca37fb 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -255,15 +255,7 @@ KNOWN_TASKS = { # PEM decryption is not supported so far. # The rest of PEM (write, unencrypted read) works though. 'test_suite_pem': [ - 'PEM read (AES-128-CBC + invalid iv)' - 'PEM read (DES-CBC + invalid iv)', - 'PEM read (DES-EDE3-CBC + invalid iv)', - 'PEM read (malformed PEM AES-128-CBC)', - 'PEM read (malformed PEM DES-CBC)', - 'PEM read (malformed PEM DES-EDE3-CBC)', - 'PEM read (unknown encryption algorithm)', - 'PEM read (AES-128-CBC + invalid iv)', - 'PEM read (DES-CBC + invalid iv)', + re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), ], # Following tests depend on AES_C/DES_C but are not about # them really, just need to know some error code is there. @@ -278,258 +270,21 @@ KNOWN_TASKS = { # The en/decryption part of PKCS#12 is not supported so far. # The rest of PKCS#12 (key derivation) works though. 'test_suite_pkcs12': [ - 'PBE Decrypt, (Invalid padding & PKCS7 padding enabled)', - 'PBE Decrypt, pad = 7 (OK)', - 'PBE Decrypt, pad = 8 (Invalid output size)', - 'PBE Decrypt, pad = 8 (OK)', - 'PBE Encrypt, pad = 7 (OK)', - 'PBE Encrypt, pad = 8 (Invalid output size)', - 'PBE Encrypt, pad = 8 (OK)', + re.compile(r'PBE Encrypt, .*'), + re.compile(r'PBE Decrypt, .*'), ], # The en/decryption part of PKCS#5 is not supported so far. # The rest of PKCS#5 (PBKDF2) works though. 'test_suite_pkcs5': [ - 'PBES2 Decrypt (Invalid output size)', - 'PBES2 Decrypt (Invalid padding & PKCS7 padding enabled)', - 'PBES2 Decrypt (KDF != PBKDF2)', - 'PBES2 Decrypt (OK)', - 'PBES2 Decrypt (OK, PBKDF2 params explicit keylen)', - 'PBES2 Decrypt (OK, PBKDF2 params explicit prf_alg)', - 'PBES2 Decrypt (bad KDF AlgId: not a sequence)', - 'PBES2 Decrypt (bad KDF AlgId: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params explicit keylen: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params iter: not an int)', - 'PBES2 Decrypt (bad PBKDF2 params iter: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params salt: not an octet string)', - 'PBES2 Decrypt (bad PBKDF2 params salt: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params: not a sequence)', - 'PBES2 Decrypt (bad PBKDF2 params: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg params: len != iv_len)', - 'PBES2 Decrypt (bad enc_scheme_alg params: not an octet string)', - 'PBES2 Decrypt (bad enc_scheme_alg params: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg: not a sequence)', - 'PBES2 Decrypt (bad enc_scheme_alg: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg: unknown oid)', - 'PBES2 Decrypt (bad iter value)', - 'PBES2 Decrypt (bad params tag)', - 'PBES2 Decrypt (bad password)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg != HMAC-SHA*)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg not a sequence)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg overlong)', - 'PBES2 Decrypt (bad, PBKDF2 params extra data)', - 'PBES2 Encrypt, pad=6 (OK)', - 'PBES2 Encrypt, pad=8 (Invalid output size)', - 'PBES2 Encrypt, pad=8 (OK)', + re.compile(r'PBES2 Encrypt, .*'), + re.compile(r'PBES2 Decrypt .*'), ], # Encrypted keys are not supported so far. # pylint: disable=line-too-long 'test_suite_pkparse': [ 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', - 'Parse RSA Key #20 (PKCS#8 encrypted SHA1-3DES)', - 'Parse RSA Key #20.1 (PKCS#8 encrypted SHA1-3DES, wrong PW)', - 'Parse RSA Key #20.2 (PKCS#8 encrypted SHA1-3DES, no PW)', - 'Parse RSA Key #21 (PKCS#8 encrypted SHA1-3DES, 2048-bit)', - 'Parse RSA Key #21.1 (PKCS#8 encrypted SHA1-3DES, 2048-bit, wrong PW)', - 'Parse RSA Key #21.2 (PKCS#8 encrypted SHA1-3DES, 2048-bit, no PW)', - 'Parse RSA Key #22 (PKCS#8 encrypted SHA1-3DES, 4096-bit)', - 'Parse RSA Key #22.1 (PKCS#8 encrypted SHA1-3DES, 4096-bit, wrong PW)', - 'Parse RSA Key #22.2 (PKCS#8 encrypted SHA1-3DES, 4096-bit, no PW)', - 'Parse RSA Key #23 (PKCS#8 encrypted SHA1-3DES DER)', - 'Parse RSA Key #24 (PKCS#8 encrypted SHA1-3DES DER, 2048-bit)', - 'Parse RSA Key #25 (PKCS#8 encrypted SHA1-3DES DER, 4096-bit)', - 'Parse RSA Key #26 (PKCS#8 encrypted SHA1-2DES)', - 'Parse RSA Key #26.1 (PKCS#8 encrypted SHA1-2DES, wrong PW)', - 'Parse RSA Key #26.2 (PKCS#8 encrypted SHA1-2DES, no PW)', - 'Parse RSA Key #27 (PKCS#8 encrypted SHA1-2DES, 2048-bit)', - 'Parse RSA Key #27.1 (PKCS#8 encrypted SHA1-2DES, 2048-bit, wrong PW)', - 'Parse RSA Key #27.2 (PKCS#8 encrypted SHA1-2DES, 2048-bit no PW)', - 'Parse RSA Key #28 (PKCS#8 encrypted SHA1-2DES, 4096-bit)', - 'Parse RSA Key #28.1 (PKCS#8 encrypted SHA1-2DES, 4096-bit, wrong PW)', - 'Parse RSA Key #28.2 (PKCS#8 encrypted SHA1-2DES, 4096-bit, no PW)', - 'Parse RSA Key #29 (PKCS#8 encrypted SHA1-2DES DER)', - 'Parse RSA Key #30 (PKCS#8 encrypted SHA1-2DES DER, 2048-bit)', - 'Parse RSA Key #31 (PKCS#8 encrypted SHA1-2DES DER, 4096-bit)', - 'Parse RSA Key #38 (PKCS#8 encrypted v2 PBKDF2 3DES)', - 'Parse RSA Key #38.1 (PKCS#8 encrypted v2 PBKDF2 3DES, wrong PW)', - 'Parse RSA Key #38.2 (PKCS#8 encrypted v2 PBKDF2 3DES, no PW)', - 'Parse RSA Key #39 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit)', - 'Parse RSA Key #39.1 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit, wrong PW)', - 'Parse RSA Key #39.2 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit, no PW)', - 'Parse RSA Key #40 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit)', - 'Parse RSA Key #40.1 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit, wrong PW)', - 'Parse RSA Key #40.2 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit, no PW)', - 'Parse RSA Key #41 (PKCS#8 encrypted v2 PBKDF2 3DES DER)', - 'Parse RSA Key #41.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, wrong PW)', - 'Parse RSA Key #41.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, no PW)', - 'Parse RSA Key #42 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit)', - 'Parse RSA Key #42.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit, wrong PW)', - 'Parse RSA Key #42.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit, no PW)', - 'Parse RSA Key #43 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit)', - 'Parse RSA Key #43.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit, wrong PW)', - 'Parse RSA Key #43.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit, no PW)', - 'Parse RSA Key #44 (PKCS#8 encrypted v2 PBKDF2 DES)', - 'Parse RSA Key #44.1 (PKCS#8 encrypted v2 PBKDF2 DES, wrong PW)', - 'Parse RSA Key #44.2 (PKCS#8 encrypted v2 PBKDF2 DES, no PW)', - 'Parse RSA Key #45 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit)', - 'Parse RSA Key #45.1 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit, wrong PW)', - 'Parse RSA Key #45.2 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit, no PW)', - 'Parse RSA Key #46 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit)', - 'Parse RSA Key #46.1 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit, wrong PW)', - 'Parse RSA Key #46.2 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit, no PW)', - 'Parse RSA Key #47 (PKCS#8 encrypted v2 PBKDF2 DES DER)', - 'Parse RSA Key #47.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, wrong PW)', - 'Parse RSA Key #47.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, no PW)', - 'Parse RSA Key #48 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit)', - 'Parse RSA Key #48.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit, wrong PW)', - 'Parse RSA Key #48.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit, no PW)', - 'Parse RSA Key #49 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit)', - 'Parse RSA Key #49.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit, wrong PW)', - 'Parse RSA Key #49.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit, no PW)', - 'Parse RSA Key #50 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224)', - 'Parse RSA Key #50.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, wrong PW)', - 'Parse RSA Key #50.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, no PW)', - 'Parse RSA Key #51 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit)', - 'Parse RSA Key #51.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit, wrong PW)', - 'Parse RSA Key #51.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit, no PW)', - 'Parse RSA Key #52 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit)', - 'Parse RSA Key #52.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit, wrong PW)', - 'Parse RSA Key #52.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit, no PW)', - 'Parse RSA Key #53 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER)', - 'Parse RSA Key #53.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, wrong PW)', - 'Parse RSA Key #53.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, no PW)', - 'Parse RSA Key #54 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit)', - 'Parse RSA Key #54.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #54.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit, no PW)', - 'Parse RSA Key #55 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit)', - 'Parse RSA Key #55.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #55.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit, no PW)', - 'Parse RSA Key #56 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224)', - 'Parse RSA Key #56.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, wrong PW)', - 'Parse RSA Key #56.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, no PW)', - 'Parse RSA Key #57 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit)', - 'Parse RSA Key #57.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit, wrong PW)', - 'Parse RSA Key #57.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit, no PW)', - 'Parse RSA Key #58 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit)', - 'Parse RSA Key #58.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit, wrong PW)', - 'Parse RSA Key #58.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit, no PW)', - 'Parse RSA Key #59 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER)', - 'Parse RSA Key #59.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, wrong PW)', - 'Parse RSA Key #59.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, no PW)', - 'Parse RSA Key #60 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit)', - 'Parse RSA Key #60.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #60.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit, no PW)', - 'Parse RSA Key #61 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit)', - 'Parse RSA Key #61.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #61.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit, no PW)', - 'Parse RSA Key #62 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256)', - 'Parse RSA Key #62.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, wrong PW)', - 'Parse RSA Key #62.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, no PW)', - 'Parse RSA Key #63 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit)', - 'Parse RSA Key #63.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit, wrong PW)', - 'Parse RSA Key #63.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit, no PW)', - 'Parse RSA Key #64 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit)', - 'Parse RSA Key #64.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit, wrong PW)', - 'Parse RSA Key #64.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit, no PW)', - 'Parse RSA Key #65 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER)', - 'Parse RSA Key #65.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, wrong PW)', - 'Parse RSA Key #65.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, no PW)', - 'Parse RSA Key #66 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit)', - 'Parse RSA Key #66.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #66.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit, no PW)', - 'Parse RSA Key #67 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit)', - 'Parse RSA Key #68.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #68.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit, no PW)', - 'Parse RSA Key #69 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256)', - 'Parse RSA Key #69.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, wrong PW)', - 'Parse RSA Key #69.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, no PW)', - 'Parse RSA Key #70 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit)', - 'Parse RSA Key #70.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit, wrong PW)', - 'Parse RSA Key #70.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit, no PW)', - 'Parse RSA Key #71 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit)', - 'Parse RSA Key #71.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit, wrong PW)', - 'Parse RSA Key #71.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit, no PW)', - 'Parse RSA Key #72 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER)', - 'Parse RSA Key #72.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, wrong PW)', - 'Parse RSA Key #72.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, no PW)', - 'Parse RSA Key #73 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit)', - 'Parse RSA Key #73.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #73.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit, no PW)', - 'Parse RSA Key #74 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit)', - 'Parse RSA Key #74.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #74.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit, no PW)', - 'Parse RSA Key #75 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384)', - 'Parse RSA Key #75.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, wrong PW)', - 'Parse RSA Key #75.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, no PW)', - 'Parse RSA Key #76 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit)', - 'Parse RSA Key #76.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit, wrong PW)', - 'Parse RSA Key #76.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit, no PW)', - 'Parse RSA Key #77 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit)', - 'Parse RSA Key #77.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit, wrong PW)', - 'Parse RSA Key #77.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit, no PW)', - 'Parse RSA Key #78 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER)', - 'Parse RSA Key #78.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, wrong PW)', - 'Parse RSA Key #78.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, no PW)', - 'Parse RSA Key #79 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit)', - 'Parse RSA Key #79.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #79.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit, no PW)', - 'Parse RSA Key #80 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit)', - 'Parse RSA Key #80.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #80.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit, no PW)', - 'Parse RSA Key #81 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384)', - 'Parse RSA Key #81.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, wrong PW)', - 'Parse RSA Key #81.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, no PW)', - 'Parse RSA Key #82 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit)', - 'Parse RSA Key #82.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit, wrong PW)', - 'Parse RSA Key #82.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit, no PW)', - 'Parse RSA Key #83 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit)', - 'Parse RSA Key #83.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit, wrong PW)', - 'Parse RSA Key #83.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit, no PW)', - 'Parse RSA Key #84 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER)', - 'Parse RSA Key #84.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, wrong PW)', - 'Parse RSA Key #85.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, no PW)', - 'Parse RSA Key #86 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit)', - 'Parse RSA Key #86.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #86.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit, no PW)', - 'Parse RSA Key #87 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit)', - 'Parse RSA Key #87.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #87.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit, no PW)', - 'Parse RSA Key #88 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512)', - 'Parse RSA Key #88.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, wrong PW)', - 'Parse RSA Key #88.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, no PW)', - 'Parse RSA Key #89 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit)', - 'Parse RSA Key #89.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit, wrong PW)', - 'Parse RSA Key #89.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit, no PW)', - 'Parse RSA Key #90 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit)', - 'Parse RSA Key #90.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit, wrong PW)', - 'Parse RSA Key #90.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit, no PW)', - 'Parse RSA Key #91 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER)', - 'Parse RSA Key #91.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, wrong PW)', - 'Parse RSA Key #91.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, no PW)', - 'Parse RSA Key #92 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit)', - 'Parse RSA Key #92.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #92.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit, no PW)', - 'Parse RSA Key #93 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit)', - 'Parse RSA Key #93.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #93.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit, no PW)', - 'Parse RSA Key #94 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512)', - 'Parse RSA Key #94.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, wrong PW)', - 'Parse RSA Key #94.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, no PW)', - 'Parse RSA Key #95 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit)', - 'Parse RSA Key #95.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit, wrong PW)', - 'Parse RSA Key #95.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit, no PW)', - 'Parse RSA Key #96 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit)', - 'Parse RSA Key #96.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit, wrong PW)', - 'Parse RSA Key #96.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit, no PW)', - 'Parse RSA Key #97 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER)', - 'Parse RSA Key #97.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, wrong PW)', - 'Parse RSA Key #97.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, no PW)', - 'Parse RSA Key #98 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit)', - 'Parse RSA Key #98.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #98.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit, no PW)', - 'Parse RSA Key #99 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit)', - 'Parse RSA Key #99.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #99.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit, no PW)', + re.compile(r'Parse RSA Key .*\(PKCS#8 encrypted .*\)'), ], } } From c51c411cc1228a6916c2ba44ac1135cfdca37725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 30 Oct 2023 10:21:22 +0100 Subject: [PATCH 11/11] analyze_outcome: only warn on ignored tests that pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous check also warned when on tests that were already skipped in the reference config, which are not really a problem. The purpose of this "uselessly ignored" check is to make sure that the ignore list (together with the config common to driver and reference in all.sh) always correct reflects what works or doesn't in driver-only builds. For this it's enough to warn when a test is ignored but passing. The previous, stricter check, was causing issues like: Error: uselessly ignored: test_suite_pkcs12;PBE Encrypt, pad = 8 (PKCS7 padding disabled) Error: uselessly ignored: test_suite_pkcs12;PBE Decrypt, (Invalid padding & PKCS7 padding disabled) Error: uselessly ignored: test_suite_pkcs5;PBES2 Decrypt (Invalid padding & PKCS7 padding disabled) These are skipped in the reference config because is has PKCS7 padding enabled, and that's OK. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 81e3ca37fb..80b6459cd7 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -145,12 +145,10 @@ def analyze_driver_vs_reference(results: Results, outcomes, if component_ref in entry: reference_test_passed = True seen_reference_passing = True - if(reference_test_passed and not driver_test_passed): - if not ignored: - results.error("PASS -> SKIP/FAIL: {}", key) - else: - if ignored: - results.error("uselessly ignored: {}", key) + if reference_test_passed and not driver_test_passed and not ignored: + results.error("PASS -> SKIP/FAIL: {}", key) + if ignored and driver_test_passed: + results.error("uselessly ignored: {}", key) if not seen_reference_passing: results.error("no passing test in reference component: bad outcome file?")