From 2bb2f15772b93d5878fb6e239d3e248d697d4ae0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?=
 <manuel.pegourie-gonnard@arm.com>
Date: Wed, 12 Oct 2022 10:57:31 +0200
Subject: [PATCH] Improve driver-only outcome-analysis script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Instead of having a list of test suites of interest, have a list of
suites to ignore and look at all the others.

In order for this to only yield interesting results, we need to tune the
reference configuration a bit, in order to exclude STREAM and ECB to
more closely match the driver-based configuration.

Signed-off-by: Manuel Pégourié-Gonnard <manuel.pegourie-gonnard@arm.com>
---
 .../psa-migration/outcome-analysis.sh         | 52 ++++++++++++++++---
 1 file changed, 44 insertions(+), 8 deletions(-)

diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh
index 67510c1447..f3566b20a7 100755
--- a/docs/architecture/psa-migration/outcome-analysis.sh
+++ b/docs/architecture/psa-migration/outcome-analysis.sh
@@ -31,13 +31,23 @@ DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa
 # A similar configuration to that of the component, except without drivers,
 # for comparison.
 reference_config () {
-    scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
-    scripts/config.py unset MBEDTLS_PKCS1_V21
-    scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT
+    # start with full
+    scripts/config.py full
+    # use PSA config and disable driver-less algs as in the component
+    scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING
+    # disable options as in the component
+    # (no need to disable whole modules, we'll just skip their test suite)
     scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
 }
-# Space-separated list of test suites of interest.
-SUITES="rsa pkcs1_v15 pk pkparse pkwrite"
+# Space-separated list of test suites to ignore:
+# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored.
+IGNORE="md mdx shax" # accelerated
+IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG)
+IGNORE="$IGNORE psa_crypto_init" # needs internal RNG
+IGNORE="$IGNORE hkdf" # disabled
 # ----- END edit this -----
 
 set -eu
@@ -85,6 +95,22 @@ tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa
 
 # analysis
 
+populate_suites () {
+    SUITES=''
+    make generated_files >/dev/null
+    data_files=$(cd tests/suites && echo *.data)
+    for data in $data_files; do
+        suite=${data#test_suite_}
+        suite=${suite%.data}
+        suite_base=${suite%%.*}
+        case " $IGNORE " in
+            *" $suite_base "*) :;;
+            *) SUITES="$SUITES $suite";;
+        esac
+    done
+    make neat
+}
+
 compare_suite () {
     ref="outcome-$1.csv"
     new="outcome-$2.csv"
@@ -98,19 +124,29 @@ compare_suite () {
     nb_ref=$(wc -l <skipped-ref)
     nb_new=$(wc -l <skipped-new)
 
-    printf "%12s: total %3d; skipped %3d -> %3d\n" \
+    printf "%36s: total %4d; skipped %4d -> %4d\n" \
             $suite      $total       $nb_ref $nb_new
-    diff skipped-ref skipped-new | grep '^> ' || true
+    if diff skipped-ref skipped-new | grep '^> '; then
+        ret=1
+    else
+        ret=0
+    fi
     rm skipped-ref skipped-new
+    return $ret
 }
 
 compare_builds () {
     printf "\n*** Comparing $1 -> $2 ***\n"
+    failed=''
     for suite in $SUITES; do
-        compare_suite "$1" "$2" "$suite"
+        if compare_suite "$1" "$2" "$suite"; then :; else
+            failed="$failed $suite"
+        fi
     done
+    printf "suites with less coverage: %s\n" "$failed"
 }
 
+populate_suites
 compare_builds before-default after-default
 compare_builds before-full after-full
 compare_builds reference drivers