diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index 67510c1447..81ab69183c 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -31,13 +31,25 @@ DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa # A similar configuration to that of the component, except without drivers, # for comparison. reference_config () { - scripts/config.py set MBEDTLS_USE_PSA_CRYPTO - scripts/config.py unset MBEDTLS_PKCS1_V21 - scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT + # start with full + scripts/config.py full + # use PSA config and disable driver-less algs as in the component + scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING + # disable options as in the component + # (no need to disable whole modules, we'll just skip their test suite) scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA } -# Space-separated list of test suites of interest. -SUITES="rsa pkcs1_v15 pk pkparse pkwrite" +# Space-separated list of test suites to ignore: +# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored. +IGNORE="md mdx shax" # accelerated +IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG) +IGNORE="$IGNORE psa_crypto_init" # needs internal RNG +IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested +# Compare only "reference vs driver" or also "before vs after"? +BEFORE_AFTER=1 # 0 or 1 # ----- END edit this ----- set -eu @@ -53,26 +65,28 @@ record() { make check } -# save current HEAD -HEAD=$(git branch --show-current) +if [ "$BEFORE_AFTER" -eq 1 ]; then + # save current HEAD + HEAD=$(git branch --show-current) -# get the numbers before this PR for default and full -cleanup -git checkout $(git merge-base HEAD development) -record "before-default" + # get the numbers before this PR for default and full + cleanup + git checkout $(git merge-base HEAD development) + record "before-default" -cleanup -scripts/config.py full -record "before-full" + cleanup + scripts/config.py full + record "before-full" -# get the numbers now for default and full -cleanup -git checkout $HEAD -record "after-default" + # get the numbers now for default and full + cleanup + git checkout $HEAD + record "after-default" -cleanup -scripts/config.py full -record "after-full" + cleanup + scripts/config.py full + record "after-full" +fi # get the numbers now for driver-only and reference cleanup @@ -81,10 +95,27 @@ record "reference" cleanup export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-drivers.csv" +export SKIP_SSL_OPT_COMPAT_SH=1 tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa # analysis +populate_suites () { + SUITES='' + make generated_files >/dev/null + data_files=$(cd tests/suites && echo *.data) + for data in $data_files; do + suite=${data#test_suite_} + suite=${suite%.data} + suite_base=${suite%%.*} + case " $IGNORE " in + *" $suite_base "*) :;; + *) SUITES="$SUITES $suite";; + esac + done + make neat +} + compare_suite () { ref="outcome-$1.csv" new="outcome-$2.csv" @@ -98,19 +129,35 @@ compare_suite () { nb_ref=$(wc -l %3d\n" \ + printf "%36s: total %4d; skipped %4d -> %4d\n" \ $suite $total $nb_ref $nb_new - diff skipped-ref skipped-new | grep '^> ' || true + if diff skipped-ref skipped-new | grep '^> '; then + ret=1 + else + ret=0 + fi rm skipped-ref skipped-new + return $ret } compare_builds () { printf "\n*** Comparing $1 -> $2 ***\n" + failed='' for suite in $SUITES; do - compare_suite "$1" "$2" "$suite" + if compare_suite "$1" "$2" "$suite"; then :; else + failed="$failed $suite" + fi done + if [ -z "$failed" ]; then + printf "No coverage gap found.\n" + else + printf "Suites with less coverage:%s\n" "$failed" + fi } -compare_builds before-default after-default -compare_builds before-full after-full +populate_suites +if [ "$BEFORE_AFTER" -eq 1 ]; then + compare_builds before-default after-default + compare_builds before-full after-full +fi compare_builds reference drivers diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index f1b2f0e29f..7b93fce681 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2043,7 +2043,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { # Also unset MD_C and things that depend on it; # see component_test_crypto_full_no_md. scripts/config.py unset MBEDTLS_MD_C - scripts/config.py unset MBEDTLS_HKDF_C + scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation scripts/config.py unset MBEDTLS_HMAC_DRBG_C scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA @@ -2063,11 +2063,16 @@ component_test_psa_crypto_config_accel_hash_use_psa () { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" make test - msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/ssl-opt.sh + # hidden option: when running outcome-analysis.sh, we can skip this + if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then + msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" + tests/ssl-opt.sh - msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/compat.sh + msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" + tests/compat.sh + else + echo "skip ssl-opt.sh and compat.sh" + fi } component_test_psa_crypto_config_accel_cipher () {