diff --git a/programs/.gitignore b/programs/.gitignore index 353a92d4d6..5d3c5bdc1e 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -53,6 +53,7 @@ ssl/ssl_server ssl/ssl_server2 test/benchmark test/cpp_dummy_build +test/cpp_dummy_build.cpp test/ecp-bench test/query_compile_time_config test/selftest diff --git a/programs/Makefile b/programs/Makefile index 35ef7adb8c..550887a125 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -310,6 +310,10 @@ test/benchmark$(EXEXT): test/benchmark.c $(DEP) echo " CC test/benchmark.c" $(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/benchmark.c $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@ +test/cpp_dummy_build.cpp: test/generate_cpp_dummy_build.sh + echo " Gen test/cpp_dummy_build.cpp" + test/generate_cpp_dummy_build.sh + test/cpp_dummy_build$(EXEXT): test/cpp_dummy_build.cpp $(DEP) echo " CXX test/cpp_dummy_build.cpp" $(CXX) $(LOCAL_CXXFLAGS) $(CXXFLAGS) test/cpp_dummy_build.cpp $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@ @@ -374,10 +378,11 @@ clean: ifndef WINDOWS rm -f $(APPS) -rm -f ssl/ssl_pthread_server$(EXEXT) - -rm -f test/cpp_dummy_build$(EXEXT) + -rm -f test/cpp_dummy_build.cpp test/cpp_dummy_build$(EXEXT) else if exist *.o del /Q /F *.o if exist *.exe del /Q /F *.exe + if exist test\cpp_dummy_build.cpp del /Q /F test\cpp_dummy_build.cpp endif $(MAKE) -C fuzz clean diff --git a/programs/test/CMakeLists.txt b/programs/test/CMakeLists.txt index 2b1e61ebf2..04ec7fcac6 100644 --- a/programs/test/CMakeLists.txt +++ b/programs/test/CMakeLists.txt @@ -22,7 +22,17 @@ set(executables_mbedcrypto ) if(TEST_CPP) - list(APPEND executables_mbedcrypto cpp_dummy_build) + set(cpp_dummy_build_cpp "${CMAKE_CURRENT_BINARY_DIR}/cpp_dummy_build.cpp") + set(generate_cpp_dummy_build "${CMAKE_CURRENT_SOURCE_DIR}/generate_cpp_dummy_build.sh") + add_custom_command( + OUTPUT "${cpp_dummy_build_cpp}" + COMMAND "${generate_cpp_dummy_build}" "${cpp_dummy_build_cpp}" + DEPENDS "${generate_cpp_dummy_build}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + add_executable(cpp_dummy_build "${cpp_dummy_build_cpp}") + target_include_directories(cpp_dummy_build PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include) + target_link_libraries(cpp_dummy_build ${mbedcrypto_target}) endif() foreach(exe IN LISTS executables_libs executables_mbedcrypto) diff --git a/programs/test/cpp_dummy_build.cpp b/programs/test/cpp_dummy_build.cpp deleted file mode 100644 index d0526820ce..0000000000 --- a/programs/test/cpp_dummy_build.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* - * This program is a dummy C++ program to ensure Mbed TLS library header files - * can be included and built with a C++ compiler. - * - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#if !defined(MBEDTLS_CONFIG_FILE) -#include "mbedtls/config.h" -#else -#include MBEDTLS_CONFIG_FILE -#endif - -#include "mbedtls/aes.h" -#include "mbedtls/aesni.h" -#include "mbedtls/arc4.h" -#include "mbedtls/aria.h" -#include "mbedtls/asn1.h" -#include "mbedtls/asn1write.h" -#include "mbedtls/base64.h" -#include "mbedtls/bignum.h" -#include "mbedtls/blowfish.h" -#include "mbedtls/bn_mul.h" -#include "mbedtls/camellia.h" -#include "mbedtls/ccm.h" -#include "mbedtls/certs.h" -#include "mbedtls/chacha20.h" -#include "mbedtls/chachapoly.h" -#include "mbedtls/check_config.h" -#include "mbedtls/cipher.h" -#include "mbedtls/cipher_internal.h" -#include "mbedtls/cmac.h" -#include "mbedtls/compat-1.3.h" -#include "mbedtls/config_psa.h" -#include "mbedtls/ctr_drbg.h" -#include "mbedtls/debug.h" -#include "mbedtls/des.h" -#include "mbedtls/dhm.h" -#include "mbedtls/ecdh.h" -#include "mbedtls/ecdsa.h" -#include "mbedtls/ecjpake.h" -#include "mbedtls/ecp.h" -#include "mbedtls/ecp_internal.h" -#include "mbedtls/entropy.h" -#include "mbedtls/entropy_poll.h" -#include "mbedtls/error.h" -#include "mbedtls/gcm.h" -#include "mbedtls/havege.h" -#include "mbedtls/hkdf.h" -#include "mbedtls/hmac_drbg.h" -#include "mbedtls/md.h" -#include "mbedtls/md2.h" -#include "mbedtls/md4.h" -#include "mbedtls/md5.h" -#include "mbedtls/md_internal.h" -#include "mbedtls/net.h" -#include "mbedtls/net_sockets.h" -#include "mbedtls/nist_kw.h" -#include "mbedtls/oid.h" -#include "mbedtls/padlock.h" -#include "mbedtls/pem.h" -#include "mbedtls/pk.h" -#include "mbedtls/pk_internal.h" -#include "mbedtls/pkcs11.h" -#include "mbedtls/pkcs12.h" -#include "mbedtls/pkcs5.h" -#include "mbedtls/platform_time.h" -#include "mbedtls/platform_util.h" -#include "mbedtls/poly1305.h" -#include "mbedtls/psa_util.h" -#include "mbedtls/ripemd160.h" -#include "mbedtls/rsa.h" -#include "mbedtls/rsa_internal.h" -#include "mbedtls/sha1.h" -#include "mbedtls/sha256.h" -#include "mbedtls/sha512.h" -#include "mbedtls/ssl.h" -#include "mbedtls/ssl_cache.h" -#include "mbedtls/ssl_ciphersuites.h" -#include "mbedtls/ssl_cookie.h" -#include "mbedtls/ssl_internal.h" -#include "mbedtls/ssl_ticket.h" -#include "mbedtls/threading.h" -#include "mbedtls/timing.h" -#include "mbedtls/version.h" -#include "mbedtls/x509.h" -#include "mbedtls/x509_crl.h" -#include "mbedtls/x509_crt.h" -#include "mbedtls/x509_csr.h" -#include "mbedtls/xtea.h" - -#if defined(MBEDTLS_PLATFORM_C) -#include "mbedtls/platform.h" -#endif - -#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) -#include "mbedtls/memory_buffer_alloc.h" -#endif - -#include "psa/crypto.h" -#include "psa/crypto_se_driver.h" -#include "../library/psa_crypto_its.h" - -int main() -{ - mbedtls_platform_context *ctx = NULL; - mbedtls_platform_setup(ctx); - mbedtls_printf("CPP Build test\n"); - mbedtls_platform_teardown(ctx); -} diff --git a/programs/test/generate_cpp_dummy_build.sh b/programs/test/generate_cpp_dummy_build.sh new file mode 100755 index 0000000000..90a181d994 --- /dev/null +++ b/programs/test/generate_cpp_dummy_build.sh @@ -0,0 +1,97 @@ +#!/bin/sh + +DEFAULT_OUTPUT_FILE=programs/test/cpp_dummy_build.cpp + +if [ "$1" = "--help" ]; then + cat <&2 "This script must be run from an Mbed TLS source tree." + exit 3 +fi + +print_cpp >"${1:-$DEFAULT_OUTPUT_FILE}" diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 98950c06a6..07503a2ccd 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env sh +#! /usr/bin/env bash # all.sh # @@ -59,6 +59,15 @@ # This script must be invoked from the toplevel directory of a git # working copy of Mbed TLS. # +# The behavior on an error depends on whether --keep-going (alias -k) +# is in effect. +# * Without --keep-going: the script stops on the first error without +# cleaning up. This lets you work in the configuration of the failing +# component. +# * With --keep-going: the script runs all requested components and +# reports failures at the end. In particular the script always cleans +# up on exit. +# # Note that the output is not saved. You may want to run # script -c tests/scripts/all.sh # or @@ -81,6 +90,12 @@ # # Each component must start by invoking `msg` with a short informative message. # +# Warning: due to the way bash detects errors, the failure of a command +# inside 'if' or '!' is not detected. Use the 'not' function instead of '!'. +# +# Each component is executed in a separate shell process. The component +# fails if any command in it returns a non-zero status. +# # The framework performs some cleanup tasks after each component. This # means that components can assume that the working directory is in a # cleaned-up state, and don't need to perform the cleanup themselves. @@ -91,19 +106,6 @@ # `tests/Makefile` and `programs/fuzz/Makefile` from git. # This cleans up after an in-tree use of CMake. # -# Any command that is expected to fail must be protected so that the -# script keeps running in --keep-going mode despite `set -e`. In keep-going -# mode, if a protected command fails, this is logged as a failure and the -# script will exit with a failure status once it has run all components. -# Commands can be protected in any of the following ways: -# * `make` is a function which runs the `make` command with protection. -# Note that you must write `make VAR=value`, not `VAR=value make`, -# because the `VAR=value make` syntax doesn't work with functions. -# * Put `report_status` before the command to protect it. -# * Put `if_build_successful` before a command. This protects it, and -# additionally skips it if a prior invocation of `make` in the same -# component failed. -# # The tests are roughly in order from fastest to slowest. This doesn't # have to be exact, but in general you should add slower tests towards # the end and fast checks near the beginning. @@ -114,8 +116,9 @@ #### Initialization and command line parsing ################################################################ -# Abort on errors (and uninitialised variables) -set -eu +# Abort on errors (even on the left-hand side of a pipe). +# Treat uninitialised variables as errors. +set -e -o pipefail -u pre_check_environment () { if [ -d library -a -d include -a -d tests ]; then :; else @@ -126,9 +129,16 @@ pre_check_environment () { pre_initialize_variables () { CONFIG_H='include/mbedtls/config.h' - CONFIG_BAK="$CONFIG_H.bak" CRYPTO_CONFIG_H='include/psa/crypto_config.h' - CRYPTO_CONFIG_BAK="$CRYPTO_CONFIG_H.bak" + + # Files that are clobbered by some jobs will be backed up. Use a different + # suffix from auxiliary scripts so that all.sh and auxiliary scripts can + # independently decide when to remove the backup file. + backup_suffix='.all.bak' + # Files clobbered by config.py + files_to_back_up="$CONFIG_H $CRYPTO_CONFIG_H" + # Files clobbered by in-tree cmake + files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile" append_outcome=0 MEMORY=0 @@ -176,8 +186,8 @@ pre_initialize_variables () { # Gather the list of available components. These are the functions # defined in this script whose name starts with "component_". - # Parse the script with sed, because in sh there is no way to list - # defined functions. + # Parse the script with sed. This way we get the functions in the order + # they are defined. ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") # Exclude components that are not supported on this platform. @@ -194,6 +204,8 @@ pre_initialize_variables () { # Test whether the component $1 is included in the command line patterns. is_component_included() { + # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS + # only does word splitting. set -f for pattern in $COMMAND_LINE_COMPONENTS; do set +f @@ -235,6 +247,11 @@ General options: Prefix for a cross-compiler for arm-linux-gnueabi (default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}") --armcc Run ARM Compiler builds (on by default). + --restore First clean up the build tree, restoring backed up + files. Do not run any components unless they are + explicitly specified. + --error-test Error test mode: run a failing function in addition + to any specified component. May be repeated. --except Exclude the COMPONENTs listed on the command line, instead of running only those. --no-append-outcome Write a new outcome file and analyze it (default). @@ -263,13 +280,11 @@ Tool path options: EOF } -# remove built files as well as the cmake cache/config +# Cleanup before/after running a component. +# Remove built files as well as the cmake cache/config. +# Does not remove generated source files. cleanup() { - if [ -n "${MBEDTLS_ROOT_DIR+set}" ]; then - cd "$MBEDTLS_ROOT_DIR" - fi - command make clean # Remove CMake artefacts @@ -280,21 +295,26 @@ cleanup() -iname CMakeCache.txt \) -exec rm {} \+ # Recover files overwritten by in-tree CMake builds rm -f include/Makefile include/mbedtls/Makefile programs/*/Makefile - git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile - git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile # Remove any artifacts from the component_test_cmake_as_subdirectory test. rm -rf programs/test/cmake_subproject/build rm -f programs/test/cmake_subproject/Makefile rm -f programs/test/cmake_subproject/cmake_subproject - if [ -f "$CONFIG_BAK" ]; then - mv "$CONFIG_BAK" "$CONFIG_H" - fi + # Restore files that may have been clobbered by the job + for x in $files_to_back_up; do + cp -p "$x$backup_suffix" "$x" + done +} - if [ -f "$CRYPTO_CONFIG_BAK" ]; then - mv "$CRYPTO_CONFIG_BAK" "$CRYPTO_CONFIG_H" - fi +# Final cleanup when this script exits (except when exiting on a failure +# in non-keep-going mode). +final_cleanup () { + cleanup + + for x in $files_to_back_up; do + rm -f "$x$backup_suffix" + done } # Executed on exit. May be redefined depending on command line options. @@ -303,7 +323,7 @@ final_report () { } fatal_signal () { - cleanup + final_cleanup final_report $1 trap - $1 kill -$1 $$ @@ -373,17 +393,11 @@ check_tools() done } -check_headers_in_cpp () { - ls include/mbedtls | grep "\.h$" >headers.txt - &2 "Unknown option: $1" @@ -434,7 +450,7 @@ pre_parse_command_line () { done # With no list of components, run everything. - if [ -z "$COMMAND_LINE_COMPONENTS" ]; then + if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then all_except=1 fi @@ -444,6 +460,32 @@ pre_parse_command_line () { COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" fi + # Error out if an explicitly requested component doesn't exist. + if [ $all_except -eq 0 ]; then + unsupported=0 + # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS + # only does word splitting. + set -f + for component in $COMMAND_LINE_COMPONENTS; do + set +f + # If the requested name includes a wildcard character, don't + # check it. Accept wildcard patterns that don't match anything. + case $component in + *[*?\[]*) continue;; + esac + case " $SUPPORTED_COMPONENTS " in + *" $component "*) :;; + *) + echo >&2 "Component $component was explicitly requested, but is not known or not supported." + unsupported=$((unsupported + 1));; + esac + done + set +f + if [ $unsupported -ne 0 ]; then + exit 2 + fi + fi + # Build the list of components to run. RUN_COMPONENTS= for component in $SUPPORTED_COMPONENTS; do @@ -479,9 +521,36 @@ pre_check_git () { fi } +pre_restore_files () { + # If the makefiles have been generated by a framework such as cmake, + # restore them from git. If the makefiles look like modifications from + # the ones checked into git, take care not to modify them. Whatever + # this function leaves behind is what the script will restore before + # each component. + case "$(head -n1 Makefile)" in + *[Gg]enerated*) + git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile + git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile + ;; + esac +} + +pre_back_up () { + for x in $files_to_back_up; do + cp -p "$x" "$x$backup_suffix" + done +} + pre_setup_keep_going () { - failure_summary= - failure_count=0 + failure_count=0 # Number of failed components + last_failure_status=0 # Last failure status in this component + + # See err_trap + previous_failure_status=0 + previous_failed_command= + previous_failure_funcall_depth=0 + unset report_failed_command + start_red= end_color= if [ -t 1 ]; then @@ -492,76 +561,106 @@ pre_setup_keep_going () { ;; esac fi - record_status () { - if "$@"; then - last_status=0 - else - last_status=$? - text="$current_section: $* -> $last_status" - failure_summary="$failure_summary -$text" - failure_count=$((failure_count + 1)) - echo "${start_red}^^^^$text^^^^${end_color}" >&2 - fi - } - make () { - case "$*" in - *test|*check) - if [ $build_status -eq 0 ]; then - record_status command make "$@" - else - echo "(skipped because the build failed)" - fi - ;; - *) - record_status command make "$@" - build_status=$last_status - ;; + + # Keep a summary of failures in a file. We'll print it out at the end. + failure_summary_file=$PWD/all-sh-failures-$$.log + : >"$failure_summary_file" + + # Whether it makes sense to keep a component going after the specified + # command fails (test command) or not (configure or build). + # This function normally receives the failing simple command + # ($BASH_COMMAND) as an argument, but if $report_failed_command is set, + # this is passed instead. + # This doesn't have to be 100% accurate: all failures are recorded anyway. + # False positives result in running things that can't be expected to + # work. False negatives result in things not running after something else + # failed even though they might have given useful feedback. + can_keep_going_after_failure () { + case "$1" in + "msg "*) false;; + "cd "*) false;; + *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ... + *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ... + *make*check*) true;; + "grep "*) true;; + "[ "*) true;; + "! "*) true;; + *) false;; esac } + + # This function runs if there is any error in a component. + # It must either exit with a nonzero status, or set + # last_failure_status to a nonzero value. + err_trap () { + # Save $? (status of the failing command). This must be the very + # first thing, before $? is overridden. + last_failure_status=$? + failed_command=${report_failed_command-$BASH_COMMAND} + + if [[ $last_failure_status -eq $previous_failure_status && + "$failed_command" == "$previous_failed_command" && + ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]] + then + # The same command failed twice in a row, but this time one level + # less deep in the function call stack. This happens when the last + # command of a function returns a nonzero status, and the function + # returns that same status. Ignore the second failure. + previous_failure_funcall_depth=${#FUNCNAME[@]} + return + fi + previous_failure_status=$last_failure_status + previous_failed_command=$failed_command + previous_failure_funcall_depth=${#FUNCNAME[@]} + + text="$current_section: $failed_command -> $last_failure_status" + echo "${start_red}^^^^$text^^^^${end_color}" >&2 + echo "$text" >>"$failure_summary_file" + + # If the command is fatal (configure or build command), stop this + # component. Otherwise (test command) keep the component running + # (run more tests from the same build). + if ! can_keep_going_after_failure "$failed_command"; then + exit $last_failure_status + fi + } + final_report () { if [ $failure_count -gt 0 ]; then echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "${start_red}FAILED: $failure_count${end_color}$failure_summary" + echo "${start_red}FAILED: $failure_count components${end_color}" + cat "$failure_summary_file" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - exit 1 elif [ -z "${1-}" ]; then echo "SUCCESS :)" fi if [ -n "${1-}" ]; then echo "Killed by SIG$1." fi + rm -f "$failure_summary_file" + if [ $failure_count -gt 0 ]; then + exit 1 + fi } } +# record_status() and if_build_succeeded() are kept temporarily for backward +# compatibility. Don't use them in new components. +record_status () { + "$@" +} if_build_succeeded () { - if [ $build_status -eq 0 ]; then - record_status "$@" - fi + "$@" } -# to be used instead of ! for commands run with -# record_status or if_build_succeeded -not() { - ! "$@" -} - -pre_setup_quiet_redirect () { - if [ $QUIET -ne 1 ]; then - redirect_out () { - "$@" - } - redirect_err () { - "$@" - } - else - redirect_out () { - "$@" >/dev/null - } - redirect_err () { - "$@" 2>/dev/null - } +# '! true' does not trigger the ERR trap. Arrange to trigger it, with +# a reasonably informative error message (not just "$@"). +not () { + if "$@"; then + report_failed_command="! $*" + false + unset report_failed_command fi } @@ -686,28 +785,28 @@ pre_check_tools () { component_check_recursion () { msg "Check: recursion.pl" # < 1s - record_status tests/scripts/recursion.pl library/*.c + tests/scripts/recursion.pl library/*.c } component_check_generated_files () { msg "Check: freshness of generated source files" # < 1s - record_status tests/scripts/check-generated-files.sh + tests/scripts/check-generated-files.sh } component_check_doxy_blocks () { msg "Check: doxygen markup outside doxygen blocks" # < 1s - record_status tests/scripts/check-doxy-blocks.pl + tests/scripts/check-doxy-blocks.pl } component_check_files () { msg "Check: file sanity checks (permissions, encodings)" # < 1s - record_status tests/scripts/check_files.py + tests/scripts/check_files.py } component_check_changelog () { msg "Check: changelog entries" # < 1s rm -f ChangeLog.new - record_status scripts/assemble_changelog.py -o ChangeLog.new + scripts/assemble_changelog.py -o ChangeLog.new if [ -e ChangeLog.new ]; then # Show the diff for information. It isn't an error if the diff is # non-empty. @@ -718,7 +817,7 @@ component_check_changelog () { component_check_names () { msg "Check: declared and exported names (builds the library)" # < 3s - record_status tests/scripts/check_names.py -v + tests/scripts/check_names.py -v } component_check_test_cases () { @@ -728,13 +827,13 @@ component_check_test_cases () { else opt='' fi - record_status tests/scripts/check_test_cases.py $opt + tests/scripts/check_test_cases.py $opt unset opt } component_check_doxygen_warnings () { msg "Check: doxygen warnings (builds the documentation)" # ~ 3s - record_status tests/scripts/doxygen.sh + tests/scripts/doxygen.sh } @@ -754,7 +853,7 @@ component_test_default_out_of_box () { make test msg "selftest: make, default config (out-of-box)" # ~10s - if_build_succeeded programs/test/selftest + programs/test/selftest export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE" unset SAVE_MBEDTLS_TEST_OUTCOME_FILE @@ -769,16 +868,16 @@ component_test_default_cmake_gcc_asan () { make test msg "test: selftest (ASan build)" # ~ 10s - if_build_succeeded programs/test/selftest + programs/test/selftest msg "test: ssl-opt.sh (ASan build)" # ~ 1 min - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh (ASan build)" # ~ 6 min - if_build_succeeded tests/compat.sh + tests/compat.sh msg "test: context-info.sh (ASan build)" # ~ 15 sec - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_full_cmake_gcc_asan () { @@ -791,16 +890,16 @@ component_test_full_cmake_gcc_asan () { make test msg "test: selftest (ASan build)" # ~ 10s - if_build_succeeded programs/test/selftest + programs/test/selftest msg "test: ssl-opt.sh (full config, ASan build)" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh (full config, ASan build)" - if_build_succeeded tests/compat.sh + tests/compat.sh msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_psa_crypto_key_id_encodes_owner () { @@ -838,7 +937,7 @@ component_build_psa_crypto_spm () { # Check that if a symbol is renamed by crypto_spe.h, the non-renamed # version is not present. echo "Checking for renamed symbols in the library" - if_build_succeeded check_renamed_symbols tests/include/spe/crypto_spe.h library/libmbedcrypto.a + check_renamed_symbols tests/include/spe/crypto_spe.h library/libmbedcrypto.a } component_test_psa_crypto_client () { @@ -861,7 +960,7 @@ component_test_zlib_make() { make test msg "test: ssl-opt.sh (zlib, make)" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh } support_test_zlib_make () { base=support_test_zlib_$$ @@ -885,7 +984,7 @@ component_test_zlib_cmake() { make test msg "test: ssl-opt.sh (zlib, cmake)" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh } support_test_zlib_cmake () { support_test_zlib_make "$@" @@ -903,7 +1002,7 @@ component_test_psa_crypto_rsa_no_genprime() { component_test_ref_configs () { msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . - record_status tests/scripts/test-ref-configs.pl + tests/scripts/test-ref-configs.pl } component_test_sslv3 () { @@ -916,14 +1015,14 @@ component_test_sslv3 () { make test msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min - if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' - if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' + tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' + env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "build: SSLv3 - context-info.sh (ASan build)" # ~ 15 sec - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_no_renegotiation () { @@ -936,7 +1035,7 @@ component_test_no_renegotiation () { make test msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh } component_test_no_pem_no_fs () { @@ -952,7 +1051,7 @@ component_test_no_pem_no_fs () { make test msg "test: !MBEDTLS_PEM_PARSE_C !MBEDTLS_FS_IO - ssl-opt.sh (ASan build)" # ~ 6 min - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh } component_test_rsa_no_crt () { @@ -965,13 +1064,13 @@ component_test_rsa_no_crt () { make test msg "test: RSA_NO_CRT - RSA-related part of ssl-opt.sh (ASan build)" # ~ 5s - if_build_succeeded tests/ssl-opt.sh -f RSA + tests/ssl-opt.sh -f RSA msg "test: RSA_NO_CRT - RSA-related part of compat.sh (ASan build)" # ~ 3 min - if_build_succeeded tests/compat.sh -t RSA + tests/compat.sh -t RSA msg "test: RSA_NO_CRT - RSA-related part of context-info.sh (ASan build)" # ~ 15 sec - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_no_ctr_drbg_classic () { @@ -990,10 +1089,10 @@ component_test_no_ctr_drbg_classic () { # The SSL tests are slow, so run a small subset, just enough to get # confidence that the SSL code copes with HMAC_DRBG. msg "test: Full minus CTR_DRBG, classic crypto - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server' + tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server' msg "test: Full minus CTR_DRBG, classic crypto - compat.sh (subset)" - if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL + tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL } component_test_no_ctr_drbg_use_psa () { @@ -1012,10 +1111,10 @@ component_test_no_ctr_drbg_use_psa () { # The SSL tests are slow, so run a small subset, just enough to get # confidence that the SSL code copes with HMAC_DRBG. msg "test: Full minus CTR_DRBG, USE_PSA_CRYPTO - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server' + tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server' msg "test: Full minus CTR_DRBG, USE_PSA_CRYPTO - compat.sh (subset)" - if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL + tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL } component_test_no_hmac_drbg_classic () { @@ -1037,12 +1136,12 @@ component_test_no_hmac_drbg_classic () { # Test SSL with non-deterministic ECDSA. Only test features that # might be affected by how ECDSA signature is performed. msg "test: Full minus HMAC_DRBG, classic crypto - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private: sign' + tests/ssl-opt.sh -f 'Default\|SSL async private: sign' # To save time, only test one protocol version, since this part of # the protocol is identical in (D)TLS up to 1.2. msg "test: Full minus HMAC_DRBG, classic crypto - compat.sh (ECDSA)" - if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA' + tests/compat.sh -m tls1_2 -t 'ECDSA' } component_test_no_hmac_drbg_use_psa () { @@ -1064,12 +1163,12 @@ component_test_no_hmac_drbg_use_psa () { # Test SSL with non-deterministic ECDSA. Only test features that # might be affected by how ECDSA signature is performed. msg "test: Full minus HMAC_DRBG, USE_PSA_CRYPTO - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private: sign' + tests/ssl-opt.sh -f 'Default\|SSL async private: sign' # To save time, only test one protocol version, since this part of # the protocol is identical in (D)TLS up to 1.2. msg "test: Full minus HMAC_DRBG, USE_PSA_CRYPTO - compat.sh (ECDSA)" - if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA' + tests/compat.sh -m tls1_2 -t 'ECDSA' } component_test_psa_external_rng_no_drbg_classic () { @@ -1093,7 +1192,7 @@ component_test_psa_external_rng_no_drbg_classic () { make test msg "test: PSA_CRYPTO_EXTERNAL_RNG minus *_DRBG, classic crypto - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default' + tests/ssl-opt.sh -f 'Default' } component_test_psa_external_rng_no_drbg_use_psa () { @@ -1113,7 +1212,7 @@ component_test_psa_external_rng_no_drbg_use_psa () { make test msg "test: PSA_CRYPTO_EXTERNAL_RNG minus *_DRBG, PSA crypto - ssl-opt.sh (subset)" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|opaque' + tests/ssl-opt.sh -f 'Default\|opaque' } component_test_psa_external_rng_use_psa_crypto () { @@ -1128,7 +1227,7 @@ component_test_psa_external_rng_use_psa_crypto () { make test msg "test: full + PSA_CRYPTO_EXTERNAL_RNG + USE_PSA_CRYPTO minus CTR_DRBG" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|opaque' + tests/ssl-opt.sh -f 'Default\|opaque' } component_test_ecp_no_internal_rng () { @@ -1178,11 +1277,11 @@ component_test_new_ecdh_context () { make test msg "test: new ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s - if_build_succeeded tests/ssl-opt.sh -f ECDH + tests/ssl-opt.sh -f ECDH msg "test: new ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min # Exclude some symmetric ciphers that are redundant here to gain time. - if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' + tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' } component_test_everest () { @@ -1196,11 +1295,11 @@ component_test_everest () { make test msg "test: Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s - if_build_succeeded tests/ssl-opt.sh -f ECDH + tests/ssl-opt.sh -f ECDH msg "test: Everest ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min # Exclude some symmetric ciphers that are redundant here to gain time. - if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' + tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' } component_test_everest_curve25519_only () { @@ -1230,7 +1329,7 @@ component_test_small_ssl_out_content_len () { make msg "test: small SSL_OUT_CONTENT_LEN - ssl-opt.sh MFL and large packet tests" - if_build_succeeded tests/ssl-opt.sh -f "Max fragment\|Large packet" + tests/ssl-opt.sh -f "Max fragment\|Large packet" } component_test_small_ssl_in_content_len () { @@ -1241,7 +1340,7 @@ component_test_small_ssl_in_content_len () { make msg "test: small SSL_IN_CONTENT_LEN - ssl-opt.sh MFL tests" - if_build_succeeded tests/ssl-opt.sh -f "Max fragment" + tests/ssl-opt.sh -f "Max fragment" } component_test_small_ssl_dtls_max_buffering () { @@ -1251,7 +1350,7 @@ component_test_small_ssl_dtls_max_buffering () { make msg "test: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #0 - ssl-opt.sh specific reordering test" - if_build_succeeded tests/ssl-opt.sh -f "DTLS reordering: Buffer out-of-order hs msg before reassembling next, free buffered msg" + tests/ssl-opt.sh -f "DTLS reordering: Buffer out-of-order hs msg before reassembling next, free buffered msg" } component_test_small_mbedtls_ssl_dtls_max_buffering () { @@ -1261,15 +1360,15 @@ component_test_small_mbedtls_ssl_dtls_max_buffering () { make msg "test: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #1 - ssl-opt.sh specific reordering test" - if_build_succeeded tests/ssl-opt.sh -f "DTLS reordering: Buffer encrypted Finished message, drop for fragmented NewSessionTicket" + tests/ssl-opt.sh -f "DTLS reordering: Buffer encrypted Finished message, drop for fragmented NewSessionTicket" } component_test_psa_collect_statuses () { msg "build+test: psa_collect_statuses" # ~30s scripts/config.py full - record_status tests/scripts/psa_collect_statuses.py + tests/scripts/psa_collect_statuses.py # Check that psa_crypto_init() succeeded at least once - record_status grep -q '^0:psa_crypto_init:' tests/statuses.log + grep -q '^0:psa_crypto_init:' tests/statuses.log rm -f tests/statuses.log } @@ -1283,16 +1382,16 @@ component_test_full_cmake_clang () { make test msg "test: psa_constant_names (full config, clang)" # ~ 1s - record_status tests/scripts/test_psa_constant_names.py + tests/scripts/test_psa_constant_names.py msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s - if_build_succeeded tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private' + tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private' msg "test: compat.sh RC4, DES, 3DES & NULL (full config)" # ~ 2 min - if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' + env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' msg "test: compat.sh ARIA + ChachaPoly" - if_build_succeeded env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' + env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' } component_test_memsan_constant_flow () { @@ -1399,70 +1498,68 @@ component_build_crypto_default () { msg "build: make, crypto only" scripts/config.py crypto make CFLAGS='-O1 -Werror' - if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.* + are_empty_libraries library/libmbedx509.* library/libmbedtls.* } component_build_crypto_full () { msg "build: make, crypto only, full config" scripts/config.py crypto_full make CFLAGS='-O1 -Werror' - if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.* + are_empty_libraries library/libmbedx509.* library/libmbedtls.* } component_build_crypto_baremetal () { msg "build: make, crypto only, baremetal config" scripts/config.py crypto_baremetal make CFLAGS='-O1 -Werror' - if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.* + are_empty_libraries library/libmbedx509.* library/libmbedtls.* } component_test_depends_curves () { msg "test/build: curves.pl (gcc)" # ~ 4 min - record_status tests/scripts/curves.pl + tests/scripts/curves.pl } component_test_depends_curves_psa () { msg "test/build: curves.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)" scripts/config.py set MBEDTLS_USE_PSA_CRYPTO - record_status tests/scripts/curves.pl + tests/scripts/curves.pl } component_test_depends_hashes () { msg "test/build: depends-hashes.pl (gcc)" # ~ 2 min - record_status tests/scripts/depends-hashes.pl + tests/scripts/depends-hashes.pl } component_test_depends_hashes_psa () { msg "test/build: depends-hashes.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)" scripts/config.py set MBEDTLS_USE_PSA_CRYPTO - record_status tests/scripts/depends-hashes.pl + tests/scripts/depends-hashes.pl } component_test_depends_pkalgs () { msg "test/build: depends-pkalgs.pl (gcc)" # ~ 2 min - record_status tests/scripts/depends-pkalgs.pl + tests/scripts/depends-pkalgs.pl } component_test_depends_pkalgs_psa () { msg "test/build: depends-pkalgs.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)" scripts/config.py set MBEDTLS_USE_PSA_CRYPTO - record_status tests/scripts/depends-pkalgs.pl + tests/scripts/depends-pkalgs.pl } component_build_key_exchanges () { msg "test/build: key-exchanges (gcc)" # ~ 1 min - record_status tests/scripts/key-exchanges.pl + tests/scripts/key-exchanges.pl } -component_build_default_make_gcc_and_cxx () { - msg "build: Unix make, -Os (gcc)" # ~ 30s - make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' +component_test_make_cxx () { + msg "build: Unix make, full, gcc + g++" + scripts/config.py full + make TEST_CPP=1 lib programs - msg "test: verify header list in cpp_dummy_build.cpp" - record_status check_headers_in_cpp - - msg "build: Unix make, incremental g++" - make TEST_CPP=1 + msg "test: cpp_dummy_build" + programs/test/cpp_dummy_build } component_test_no_use_psa_crypto_full_cmake_asan() { @@ -1482,16 +1579,16 @@ component_test_no_use_psa_crypto_full_cmake_asan() { make test msg "test: ssl-opt.sh (full minus MBEDTLS_USE_PSA_CRYPTO)" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh default (full minus MBEDTLS_USE_PSA_CRYPTO)" - if_build_succeeded tests/compat.sh + tests/compat.sh msg "test: compat.sh RC4, DES & NULL (full minus MBEDTLS_USE_PSA_CRYPTO)" - if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' + env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' msg "test: compat.sh ARIA + ChachaPoly (full minus MBEDTLS_USE_PSA_CRYPTO)" - if_build_succeeded env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' + env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' } component_test_psa_crypto_config_basic() { @@ -2040,7 +2137,7 @@ component_test_memory_buffer_allocator () { msg "test: ssl-opt.sh, MBEDTLS_MEMORY_BUFFER_ALLOC_C" # MBEDTLS_MEMORY_BUFFER_ALLOC is slow. Skip tests that tend to time out. - if_build_succeeded tests/ssl-opt.sh -e '^DTLS proxy' + tests/ssl-opt.sh -e '^DTLS proxy' } component_test_no_max_fragment_length () { @@ -2051,7 +2148,7 @@ component_test_no_max_fragment_length () { make msg "test: ssl-opt.sh, MFL-related tests" - if_build_succeeded tests/ssl-opt.sh -f "Max fragment length" + tests/ssl-opt.sh -f "Max fragment length" } component_test_asan_remove_peer_certificate () { @@ -2064,13 +2161,13 @@ component_test_asan_remove_peer_certificate () { make test msg "test: ssl-opt.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE" - if_build_succeeded tests/compat.sh + tests/compat.sh msg "test: context-info.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE" - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_no_max_fragment_length_small_ssl_out_content_len () { @@ -2082,10 +2179,10 @@ component_test_no_max_fragment_length_small_ssl_out_content_len () { make msg "test: MFL tests (disabled MFL extension case) & large packet tests" - if_build_succeeded tests/ssl-opt.sh -f "Max fragment length\|Large buffer" + tests/ssl-opt.sh -f "Max fragment length\|Large buffer" msg "test: context-info.sh (disabled MFL extension case)" - if_build_succeeded tests/context-info.sh + tests/context-info.sh } component_test_variable_ssl_in_out_buffer_len () { @@ -2098,10 +2195,10 @@ component_test_variable_ssl_in_out_buffer_len () { make test msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH enabled" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH enabled" - if_build_succeeded tests/compat.sh + tests/compat.sh } component_test_variable_ssl_in_out_buffer_len_CID () { @@ -2116,10 +2213,10 @@ component_test_variable_ssl_in_out_buffer_len_CID () { make test msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_DTLS_CONNECTION_ID enabled" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_DTLS_CONNECTION_ID enabled" - if_build_succeeded tests/compat.sh + tests/compat.sh } component_test_variable_ssl_in_out_buffer_len_record_splitting () { @@ -2134,10 +2231,10 @@ component_test_variable_ssl_in_out_buffer_len_record_splitting () { make test msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_CBC_RECORD_SPLITTING enabled" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_CBC_RECORD_SPLITTING enabled" - if_build_succeeded tests/compat.sh + tests/compat.sh } component_test_ssl_alloc_buffer_and_mfl () { @@ -2154,7 +2251,7 @@ component_test_ssl_alloc_buffer_and_mfl () { make test msg "test: MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH, MBEDTLS_MEMORY_BUFFER_ALLOC_C, MBEDTLS_MEMORY_DEBUG and MBEDTLS_SSL_MAX_FRAGMENT_LENGTH" - if_build_succeeded tests/ssl-opt.sh -f "Handshake memory usage" + tests/ssl-opt.sh -f "Handshake memory usage" } component_test_when_no_ciphersuites_have_mac () { @@ -2168,7 +2265,7 @@ component_test_when_no_ciphersuites_have_mac () { make test msg "test ssl-opt.sh: !MBEDTLS_SSL_SOME_MODES_USE_MAC" - if_build_succeeded tests/ssl-opt.sh -f 'Default\|EtM' -e 'without EtM' + tests/ssl-opt.sh -f 'Default\|EtM' -e 'without EtM' } component_test_null_entropy () { @@ -2220,7 +2317,7 @@ component_test_malloc_0_null () { msg "selftest: malloc(0) returns NULL (ASan+UBSan build)" # Just the calloc selftest. "make test" ran the others as part of the # test suites. - if_build_succeeded programs/test/selftest calloc + programs/test/selftest calloc msg "test ssl-opt.sh: malloc(0) returns NULL (ASan+UBSan build)" # Run a subset of the tests. The choice is a balance between coverage @@ -2228,7 +2325,7 @@ component_test_malloc_0_null () { # The current choice is to skip tests whose description includes # "proxy", which is an approximation of skipping tests that use the # UDP proxy, which tend to be slower and flakier. - if_build_succeeded tests/ssl-opt.sh -e 'proxy' + tests/ssl-opt.sh -e 'proxy' } component_test_aes_fewer_tables () { @@ -2396,7 +2493,7 @@ component_test_m32_o1 () { make test msg "test ssl-opt.sh, i386, make, gcc-O1" - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh } support_test_m32_o1 () { support_test_m32_o0 "$@" @@ -2412,11 +2509,11 @@ component_test_m32_everest () { make test msg "test: i386, Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s - if_build_succeeded tests/ssl-opt.sh -f ECDH + tests/ssl-opt.sh -f ECDH msg "test: i386, Everest ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min # Exclude some symmetric ciphers that are redundant here to gain time. - if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' + tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4' } support_test_m32_everest () { support_test_m32_o0 "$@" @@ -2556,7 +2653,7 @@ component_build_arm_none_eabi_gcc_no_udbl_division () { scripts/config.py set MBEDTLS_NO_UDBL_DIVISION make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-std=c99 -Werror -Wall -Wextra' lib echo "Checking that software 64-bit division is not required" - if_build_succeeded not grep __aeabi_uldiv library/*.o + not grep __aeabi_uldiv library/*.o } component_build_arm_none_eabi_gcc_no_64bit_multiplication () { @@ -2565,7 +2662,7 @@ component_build_arm_none_eabi_gcc_no_64bit_multiplication () { scripts/config.py set MBEDTLS_NO_64BIT_MULTIPLICATION make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-std=c99 -Werror -O1 -march=armv6-m -mthumb' lib echo "Checking that software 64-bit multiplication is not required" - if_build_succeeded not grep __aeabi_lmul library/*.o + not grep __aeabi_lmul library/*.o } component_build_armcc () { @@ -2639,13 +2736,13 @@ component_test_memsan () { make test msg "test: ssl-opt.sh (MSan)" # ~ 1 min - if_build_succeeded tests/ssl-opt.sh + tests/ssl-opt.sh # Optional part(s) if [ "$MEMORY" -gt 0 ]; then msg "test: compat.sh (MSan)" # ~ 6 min 20s - if_build_succeeded tests/compat.sh + tests/compat.sh fi } @@ -2661,17 +2758,17 @@ component_test_valgrind () { # seem to receive signals under valgrind on OS X). if [ "$MEMORY" -gt 0 ]; then msg "test: ssl-opt.sh --memcheck (Release)" - if_build_succeeded tests/ssl-opt.sh --memcheck + tests/ssl-opt.sh --memcheck fi if [ "$MEMORY" -gt 1 ]; then msg "test: compat.sh --memcheck (Release)" - if_build_succeeded tests/compat.sh --memcheck + tests/compat.sh --memcheck fi if [ "$MEMORY" -gt 0 ]; then msg "test: context-info.sh --memcheck (Release)" - if_build_succeeded tests/context-info.sh --memcheck + tests/context-info.sh --memcheck fi } @@ -2690,15 +2787,13 @@ component_test_cmake_out_of_source () { # "No such file or directory", which would indicate that some required # file is missing (ssl-opt.sh tolerates the absence of some files so # may exit with status 0 but emit errors). - if_build_succeeded ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err - if [ -s ssl-opt.err ]; then - cat ssl-opt.err >&2 - record_status [ ! -s ssl-opt.err ] - rm ssl-opt.err - fi + ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err + cat ssl-opt.err >&2 + # If ssl-opt.err is non-empty, record an error and keep going. + [ ! -s ssl-opt.err ] + rm ssl-opt.err cd "$MBEDTLS_ROOT_DIR" rm -rf "$OUT_OF_SOURCE_DIR" - unset MBEDTLS_ROOT_DIR } component_test_cmake_as_subdirectory () { @@ -2708,7 +2803,7 @@ component_test_cmake_as_subdirectory () { cd programs/test/cmake_subproject cmake . make - if_build_succeeded ./cmake_subproject + ./cmake_subproject cd "$MBEDTLS_ROOT_DIR" unset MBEDTLS_ROOT_DIR @@ -2733,9 +2828,9 @@ component_test_zeroize () { for compiler in clang gcc; do msg "test: $compiler $optimization_flag, mbedtls_platform_zeroize()" make programs CC="$compiler" DEBUG=1 CFLAGS="$optimization_flag" - if_build_succeeded gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log - if_build_succeeded grep "The buffer was correctly zeroized" test_zeroize.log - if_build_succeeded not grep -i "error" test_zeroize.log + gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log + grep "The buffer was correctly zeroized" test_zeroize.log + not grep -i "error" test_zeroize.log rm -f test_zeroize.log make clean done @@ -2746,7 +2841,7 @@ component_test_zeroize () { component_check_python_files () { msg "Lint: Python scripts" - record_status tests/scripts/check-python-files.sh + tests/scripts/check-python-files.sh } component_check_generate_test_code () { @@ -2754,7 +2849,7 @@ component_check_generate_test_code () { # unittest writes out mundane stuff like number or tests run on stderr. # Our convention is to reserve stderr for actual errors, and write # harmless info on stdout so it can be suppress with --quiet. - record_status ./tests/scripts/test_generate_test_code.py 2>&1 + ./tests/scripts/test_generate_test_code.py 2>&1 } ################################################################ @@ -2763,7 +2858,7 @@ component_check_generate_test_code () { post_report () { msg "Done, cleaning up" - cleanup + final_cleanup final_report } @@ -2774,26 +2869,71 @@ post_report () { #### Run all the things ################################################################ +# Function invoked by --error-test to test error reporting. +pseudo_component_error_test () { + msg "Testing error reporting $error_test_i" + if [ $KEEP_GOING -ne 0 ]; then + echo "Expect three failing commands." + fi + # If the component doesn't run in a subshell, changing error_test_i to an + # invalid integer will cause an error in the loop that runs this function. + error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell + # Expected error: 'grep non_existent /dev/null -> 1' + grep non_existent /dev/null + # Expected error: '! grep -q . tests/scripts/all.sh -> 1' + not grep -q . "$0" + # Expected error: 'make unknown_target -> 2' + make unknown_target + false "this should not be executed" +} + # Run one component and clean up afterwards. run_component () { - # Back up the configuration in case the component modifies it. - # The cleanup function will restore it. - cp -p "$CONFIG_H" "$CONFIG_BAK" - cp -p "$CRYPTO_CONFIG_H" "$CRYPTO_CONFIG_BAK" current_component="$1" export MBEDTLS_TEST_CONFIGURATION="$current_component" # Unconditionally create a seedfile that's sufficiently long. # Do this before each component, because a previous component may # have messed it up or shortened it. - redirect_err dd if=/dev/urandom of=./tests/seedfile bs=64 count=1 + local dd_cmd + dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1) + case $OSTYPE in + linux*|freebsd*|openbsd*|darwin*) dd_cmd+=(status=none) + esac + "${dd_cmd[@]}" - # Run the component code. - if [ $QUIET -eq 1 ]; then - # msg() is silenced, so just print the component name here - echo "${current_component#component_}" + # Run the component in a subshell, with error trapping and output + # redirection set up based on the relevant options. + if [ $KEEP_GOING -eq 1 ]; then + # We want to keep running if the subshell fails, so 'set -e' must + # be off when the subshell runs. + set +e + fi + ( + if [ $QUIET -eq 1 ]; then + # msg() will be silenced, so just print the component name here. + echo "${current_component#component_}" + exec >/dev/null + fi + if [ $KEEP_GOING -eq 1 ]; then + # Keep "set -e" off, and run an ERR trap instead to record failures. + set -E + trap err_trap ERR + fi + # The next line is what runs the component + "$@" + if [ $KEEP_GOING -eq 1 ]; then + trap - ERR + exit $last_failure_status + fi + ) + component_status=$? + if [ $KEEP_GOING -eq 1 ]; then + set -e + if [ $component_status -ne 0 ]; then + failure_count=$((failure_count + 1)) + fi fi - redirect_out "$@" # Restore the build tree to a clean state. cleanup @@ -2806,22 +2946,23 @@ pre_initialize_variables pre_parse_command_line "$@" pre_check_git +pre_restore_files +pre_back_up build_status=0 if [ $KEEP_GOING -eq 1 ]; then pre_setup_keep_going -else - record_status () { - "$@" - } fi -pre_setup_quiet_redirect pre_prepare_outcome_file pre_print_configuration pre_check_tools cleanup # Run the requested tests. +for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do + run_component pseudo_component_error_test +done +unset error_test_i for component in $RUN_COMPONENTS; do run_component "component_$component" done