Merge pull request #6855 from mpg/driver-only-ecdsa-starter

Driver-only ECDSA starter

docs/architecture/psa-migration/outcome-analysis.sh

@@ -1,42 +1,30 @@
 #!/bin/sh
 
-# This script runs tests in various revisions and configurations and analyses
-# the results in order to highlight any difference in the set of tests skipped
-# in the test suites of interest.
+# This script runs tests before and after a PR and analyzes the results in
+# order to highlight any difference in the set of tests skipped.
 #
-# It can be used to ensure the testing criteria mentioned in strategy.md,
+# It can be used to check the first testing criterion mentioned in strategy.md,
 # end of section "Supporting builds with drivers without the software
-# implementation" are met, namely:
+# implementation", namely: the sets of tests skipped in the default config and
+# the full config must be the same before and after the PR.
 #
-# - the sets of tests skipped in the default config and the full config must be
-#   the same before and after the PR that implements step 3;
-# - the set of tests skipped in the driver-only build is the same as in an
-#   equivalent software-based configuration, or the difference is small enough,
-#   justified, and a github issue is created to track it.
-#   This part is verified by tests/scripts/analyze_outcomes.py
+# USAGE:
+# - First, commit any uncommited changes. (Also, see warning below.)
+# - Then launch --> [SKIP_SSL_OPT=1] docs/architecture/psa-migration/outcome-analysis.sh
+#     - SKIP_SSL_OPT=1 can optionally be set to skip ssl-opt.sh tests
 #
 # WARNING: this script checks out a commit other than the head of the current
 # branch; it checks out the current branch again when running successfully,
 # but while the script is running, or if it terminates early in error, you
 # should be aware that you might be at a different commit than expected.
 #
-# NOTE: This is only an example/template script, you should make a copy and
-# edit it to suit your needs. The part that needs editing is at the top.
-#
-# Also, you can comment out parts that don't need to be re-done when
+# NOTE: you can comment out parts that don't need to be re-done when
 # re-running this script (for example "get numbers before this PR").
 
-# ----- BEGIN edit this -----
-# Space-separated list of test suites to ignore:
-# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored.
-IGNORE="md mdx shax" # accelerated
-IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG)
-IGNORE="$IGNORE psa_crypto_init" # needs internal RNG
-IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested
-# ----- END edit this -----
-
 set -eu
 
+: ${SKIP_SSL_OPT:=0}
+
 cleanup() {
     make clean
     git checkout -- include/mbedtls/mbedtls_config.h include/psa/crypto_config.h
@@ -45,7 +33,14 @@ cleanup() {
 record() {
     export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv"
     rm -f $MBEDTLS_TEST_OUTCOME_FILE
+
     make check
+
+    if [ $SKIP_SSL_OPT -eq 0 ]; then
+        make -C programs ssl/ssl_server2 ssl/ssl_client2 \
+            test/udp_proxy test/query_compile_time_config
+        tests/ssl-opt.sh
+    fi
 }
 
 # save current HEAD
@@ -54,21 +49,26 @@ HEAD=$(git branch --show-current)
 # get the numbers before this PR for default and full
 cleanup
 git checkout $(git merge-base HEAD development)
+
 record "before-default"
 
 cleanup
+
 scripts/config.py full
 record "before-full"
 
 # get the numbers now for default and full
 cleanup
 git checkout $HEAD
+
 record "after-default"
 
 cleanup
+
 scripts/config.py full
 record "after-full"
 
+cleanup
 
 # analysis
 
@@ -77,15 +77,19 @@ populate_suites () {
     make generated_files >/dev/null
     data_files=$(cd tests/suites && echo *.data)
     for data in $data_files; do
-        suite=${data#test_suite_}
-        suite=${suite%.data}
-        suite_base=${suite%%.*}
-        case " $IGNORE " in
-            *" $suite_base "*) :;;
-            *) SUITES="$SUITES $suite";;
-        esac
+        suite=${data%.data}
+        SUITES="$SUITES $suite"
     done
     make neat
+
+    if [ $SKIP_SSL_OPT -eq 0 ]; then
+        SUITES="$SUITES ssl-opt"
+        extra_files=$(cd tests/opt-testcases && echo *.sh)
+        for extra in $extra_files; do
+            suite=${extra%.sh}
+            SUITES="$SUITES $suite"
+        done
+    fi
 }
 
 compare_suite () {
@@ -93,7 +97,7 @@ compare_suite () {
     new="outcome-$2.csv"
     suite="$3"
 
-    pattern_suite=";test_suite_$suite;"
+    pattern_suite=";$suite;"
     total=$(grep -c "$pattern_suite" "$ref")
     sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p"
     sed -n "$sed_cmd" "$ref" > skipped-ref
@@ -101,8 +105,9 @@ compare_suite () {
     nb_ref=$(wc -l <skipped-ref)
     nb_new=$(wc -l <skipped-new)
 
-    printf "%36s: total %4d; skipped %4d -> %4d\n" \
-            $suite      $total       $nb_ref $nb_new
+    name=${suite#test_suite_}
+    printf "%40s: total %4d; skipped %4d -> %4d\n" \
+            $name       $total       $nb_ref $nb_new
     if diff skipped-ref skipped-new | grep '^> '; then
         ret=1
     else

docs/architecture/psa-migration/strategy.md

@@ -386,15 +386,16 @@ are expressed (sometimes in bulk), to get things wrong in a way that would
 result in more tests being skipped, which is easy to miss. Care must be
 taken to ensure this does not happen. The following criteria can be used:
 
-- the sets of tests skipped in the default config and the full config must be
-  the same before and after the PR that implements step 3;
-- the set of tests skipped in the driver-only build is the same as in an
-  equivalent software-based configuration, or the difference is small enough,
-  justified, and a github issue is created to track it.
-
-Note that the favourable case is when the number of tests skipped is 0 in the
-driver-only build. In other cases, analysis of the outcome files is needed,
-see the example script `outcome-analysis.sh` in the same directory.
+1. The sets of tests skipped in the default config and the full config must be
+  the same before and after the PR that implements step 3. This is tested
+manually for each PR that changes dependency declarations by using the script
+`outcome-analysis.sh` in the present directory.
+2. The set of tests skipped in the driver-only build is the same as in an
+  equivalent software-based configuration. This is tested automatically by the
+CI in the "Results analysis" stage, by running
+`tests/scripts/analyze_outcomes.py`. See the
+`analyze_driver_vs_reference_xxx` actions in the script and the comments above
+their declaration for how to do that locally.
 
 
 Migrating away from the legacy API