diff --git a/docs/architecture/.gitignore b/docs/architecture/.gitignore new file mode 100644 index 0000000000..23f832b734 --- /dev/null +++ b/docs/architecture/.gitignore @@ -0,0 +1,2 @@ +*.html +*.pdf diff --git a/docs/architecture/Makefile b/docs/architecture/Makefile new file mode 100644 index 0000000000..2b2a11b68d --- /dev/null +++ b/docs/architecture/Makefile @@ -0,0 +1,19 @@ +PANDOC = pandoc + +default: all + +all_markdown = \ + testing/test-framework.md \ + # This line is intentionally left blank + +html: $(all_markdown:.md=.html) +pdf: $(all_markdown:.md=.pdf) +all: html pdf + +.SUFFIXES: +.SUFFIXES: .md .html .pdf + +.md.html: + $(PANDOC) -o $@ $< +.md.pdf: + $(PANDOC) -o $@ $< diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md new file mode 100644 index 0000000000..e0e960f87c --- /dev/null +++ b/docs/architecture/testing/test-framework.md @@ -0,0 +1,58 @@ +# Mbed TLS test framework + +This document is an overview of the Mbed TLS test framework and test tools. + +This document is incomplete. You can help by expanding it. + +## Unit tests + +See + +### Unit test descriptions + +Each test case has a description which succinctly describes for a human audience what the test does. The first non-comment line of each paragraph in a `.data` file is the test description. The following rules and guidelines apply: + +* Test descriptions may not contain semicolons, line breaks and other control characters, or non-ASCII characters.
+ Rationale: keep the tools that process test descriptions (`generate_test_code.py`, [outcome file](#outcome-file) tools) simple. +* Test descriptions must be unique within a `.data` file. If you can't think of a better description, the convention is to append `#1`, `#2`, etc.
+ Rationale: make it easy to relate a failure log to the test data. Avoid confusion between cases in the [outcome file](#outcome-file). +* Test descriptions should be a maximum of **66 characters**.
+ Rationale: 66 characters is what our various tools assume (leaving room for 14 more characters on an 80-column line). Longer descriptions may be truncated or may break a visual alignment.
+ We have a lot of test cases with longer descriptions, but they should be avoided. At least please make sure that the first 66 characters describe the test uniquely. +* Make the description descriptive. “foo: x=2, y=4” is more descriptive than “foo #2”. “foo: 0 Directory used for CMake out-of-source build tests. + --outcome-file= File where test outcomes are written (not done if + empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). --random-seed Use a random seed value for randomized tests (default). -r|--release-test Run this script in release mode. This fixes the seed value to 1. -s|--seed Integer seed value to use for this test run. @@ -323,6 +333,7 @@ pre_parse_command_line () { while [ $# -gt 0 ]; do case "$1" in + --append-outcome) append_outcome=1;; --armcc) no_armcc=;; --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; @@ -337,6 +348,7 @@ pre_parse_command_line () { --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;; --memory|-m) MEMORY=1;; + --no-append-outcome) append_outcome=0;; --no-armcc) no_armcc=1;; --no-force) FORCE=0;; --no-keep-going) KEEP_GOING=0;; @@ -344,6 +356,7 @@ pre_parse_command_line () { --openssl) shift; OPENSSL="$1";; --openssl-legacy) shift; OPENSSL_LEGACY="$1";; --openssl-next) shift; OPENSSL_NEXT="$1";; + --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; --random-seed) unset SEED;; --release-test|-r) SEED=1;; @@ -485,11 +498,22 @@ not() { ! "$@" } +pre_prepare_outcome_file () { + case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; + esac + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then + rm -f "$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + pre_print_configuration () { msg "info: $0 configuration" echo "MEMORY: $MEMORY" echo "FORCE: $FORCE" + echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" echo "SEED: ${SEED-"UNSET"}" + echo echo "OPENSSL: $OPENSSL" echo "OPENSSL_LEGACY: $OPENSSL_LEGACY" echo "OPENSSL_NEXT: $OPENSSL_NEXT" @@ -582,32 +606,37 @@ pre_check_tools () { # Indicative running times are given for reference. component_check_recursion () { - msg "test: recursion.pl" # < 1s + msg "Check: recursion.pl" # < 1s record_status tests/scripts/recursion.pl library/*.c } component_check_generated_files () { - msg "test: freshness of generated source files" # < 1s + msg "Check: freshness of generated source files" # < 1s record_status tests/scripts/check-generated-files.sh } component_check_doxy_blocks () { - msg "test: doxygen markup outside doxygen blocks" # < 1s + msg "Check: doxygen markup outside doxygen blocks" # < 1s record_status tests/scripts/check-doxy-blocks.pl } component_check_files () { - msg "test: check-files.py" # < 1s + msg "Check: file sanity checks (permissions, encodings)" # < 1s record_status tests/scripts/check-files.py } component_check_names () { - msg "test/build: declared and exported names" # < 3s + msg "Check: declared and exported names (builds the library)" # < 3s record_status tests/scripts/check-names.sh -v } +component_check_test_cases () { + msg "Check: test case descriptions" # < 1s + record_status tests/scripts/check-test-cases.py +} + component_check_doxygen_warnings () { - msg "test: doxygen warnings" # ~ 3s + msg "Check: doxygen warnings (builds the documentation)" # ~ 3s record_status tests/scripts/doxygen.sh } @@ -637,12 +666,18 @@ component_test_large_ecdsa_key_signature () { component_test_default_out_of_box () { msg "build: make, default config (out-of-box)" # ~1min make + # Disable fancy stuff + SAVE_MBEDTLS_TEST_OUTCOME_FILE="$MBEDTLS_TEST_OUTCOME_FILE" + unset MBEDTLS_TEST_OUTCOME_FILE msg "test: main suites make, default config (out-of-box)" # ~10s make test msg "selftest: make, default config (out-of-box)" # ~10s programs/test/selftest + + export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE" + unset SAVE_MBEDTLS_TEST_OUTCOME_FILE } component_test_default_cmake_gcc_asan () { @@ -1424,6 +1459,7 @@ run_component () { # The cleanup function will restore it. cp -p "$CONFIG_H" "$CONFIG_BAK" current_component="$1" + export MBEDTLS_TEST_CONFIGURATION="$current_component" "$@" cleanup } @@ -1444,6 +1480,7 @@ else "$@" } fi +pre_prepare_outcome_file pre_print_configuration pre_check_tools cleanup diff --git a/tests/scripts/check-test-cases.py b/tests/scripts/check-test-cases.py new file mode 100755 index 0000000000..87a35e47ed --- /dev/null +++ b/tests/scripts/check-test-cases.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +"""Sanity checks for test data. +""" + +# Copyright (C) 2019, Arm Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of Mbed TLS (https://tls.mbed.org) + +import glob +import os +import re +import sys + +class Results: + def __init__(self): + self.errors = 0 + self.warnings = 0 + + def error(self, file_name, line_number, fmt, *args): + sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). + format(file_name, line_number, *args)) + self.errors += 1 + + def warning(self, file_name, line_number, fmt, *args): + sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') + .format(file_name, line_number, *args)) + self.warnings += 1 + +def collect_test_directories(): + if os.path.isdir('tests'): + tests_dir = 'tests' + elif os.path.isdir('suites'): + tests_dir = '.' + elif os.path.isdir('../suites'): + tests_dir = '..' + directories = [tests_dir] + crypto_tests_dir = os.path.normpath(os.path.join(tests_dir, + '../crypto/tests')) + if os.path.isdir(crypto_tests_dir): + directories.append(crypto_tests_dir) + return directories + +def check_description(results, seen, file_name, line_number, description): + if description in seen: + results.error(file_name, line_number, + 'Duplicate description (also line {})', + seen[description]) + return + if re.search(br'[\t;]', description): + results.error(file_name, line_number, + 'Forbidden character \'{}\' in description', + re.search(br'[\t;]', description).group(0).decode('ascii')) + if re.search(br'[^ -~]', description): + results.error(file_name, line_number, + 'Non-ASCII character in description') + if len(description) > 66: + results.warning(file_name, line_number, + 'Test description too long ({} > 66)', + len(description)) + seen[description] = line_number + +def check_test_suite(results, data_file_name): + in_paragraph = False + descriptions = {} + with open(data_file_name, 'rb') as data_file: + for line_number, line in enumerate(data_file, 1): + line = line.rstrip(b'\r\n') + if not line: + in_paragraph = False + continue + if line.startswith(b'#'): + continue + if not in_paragraph: + # This is a test case description line. + check_description(results, descriptions, + data_file_name, line_number, line) + in_paragraph = True + +def check_ssl_opt_sh(results, file_name): + descriptions = {} + with open(file_name, 'rb') as file_contents: + for line_number, line in enumerate(file_contents, 1): + # Assume that all run_test calls have the same simple form + # with the test description entirely on the same line as the + # function name. + m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) + if not m: + continue + description = m.group(1) + check_description(results, descriptions, + file_name, line_number, description) + +def main(): + test_directories = collect_test_directories() + results = Results() + for directory in test_directories: + for data_file_name in glob.glob(os.path.join(directory, 'suites', + '*.data')): + check_test_suite(results, data_file_name) + ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') + if os.path.exists(ssl_opt_sh): + check_ssl_opt_sh(results, ssl_opt_sh) + if results.warnings or results.errors: + sys.stderr.write('{}: {} errors, {} warnings\n' + .format(sys.argv[0], results.errors, results.warnings)) + sys.exit(1 if results.errors else 0) + +if __name__ == '__main__': + main() diff --git a/tests/scripts/curves.pl b/tests/scripts/curves.pl index 3e2255277a..8119a46e69 100755 --- a/tests/scripts/curves.pl +++ b/tests/scripts/curves.pl @@ -51,6 +51,7 @@ for my $curve (@curves) { print "\n******************************************\n"; print "* Testing without curve: $curve\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$curve"; system( "scripts/config.py unset $curve" ) and abort "Failed to disable $curve\n"; diff --git a/tests/scripts/depends-hashes.pl b/tests/scripts/depends-hashes.pl index 92bcceb821..7cb41b55cd 100755 --- a/tests/scripts/depends-hashes.pl +++ b/tests/scripts/depends-hashes.pl @@ -57,6 +57,7 @@ for my $hash (@hashes) { print "\n******************************************\n"; print "* Testing without hash: $hash\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$hash"; system( "scripts/config.py unset $hash" ) and abort "Failed to disable $hash\n"; diff --git a/tests/scripts/depends-pkalgs.pl b/tests/scripts/depends-pkalgs.pl index e3eac005d4..0cc01f241f 100755 --- a/tests/scripts/depends-pkalgs.pl +++ b/tests/scripts/depends-pkalgs.pl @@ -72,6 +72,7 @@ while( my ($alg, $extras) = each %algs ) { print "\n******************************************\n"; print "* Testing without alg: $alg\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$alg"; system( "scripts/config.py unset $alg" ) and abort "Failed to disable $alg\n"; diff --git a/tests/scripts/key-exchanges.pl b/tests/scripts/key-exchanges.pl index be029c7bd0..851de1b361 100755 --- a/tests/scripts/key-exchanges.pl +++ b/tests/scripts/key-exchanges.pl @@ -45,6 +45,7 @@ for my $kex (@kexes) { print "\n******************************************\n"; print "* Testing with key exchange: $kex\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = $kex; # full config with all key exchanges disabled except one system( "scripts/config.py full" ) and abort "Failed config full\n"; diff --git a/tests/scripts/test-ref-configs.pl b/tests/scripts/test-ref-configs.pl index 80d5f38751..1df84f74ff 100755 --- a/tests/scripts/test-ref-configs.pl +++ b/tests/scripts/test-ref-configs.pl @@ -18,7 +18,7 @@ use strict; my %configs = ( 'config-mini-tls1_1.h' => { - 'compat' => '-m tls1_1 -f \'^DES-CBC3-SHA$\|^TLS-RSA-WITH-3DES-EDE-CBC-SHA$\'', + 'compat' => '-m tls1_1 -f \'^DES-CBC3-SHA$\|^TLS-RSA-WITH-3DES-EDE-CBC-SHA$\'', #' }, 'config-suite-b.h' => { 'compat' => "-m tls1_2 -f 'ECDHE-ECDSA.*AES.*GCM' -p mbedTLS", @@ -65,6 +65,7 @@ while( my ($conf, $data) = each %configs ) { print "\n******************************************\n"; print "* Testing configuration: $conf\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = $conf; system( "cp configs/$conf $config_h" ) and abort "Failed to activate $conf\n"; diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 91fceeaec5..afaae69d8a 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -25,9 +25,9 @@ set -u # where it may output seemingly unlimited length error logs. ulimit -f 20971520 -if cd $( dirname $0 ); then :; else - echo "cd $( dirname $0 ) failed" >&2 - exit 1 +ORIGINAL_PWD=$PWD +if ! cd "$(dirname "$0")"; then + exit 125 fi # default values, can be overridden by the environment @@ -39,6 +39,17 @@ fi : ${GNUTLS_SERV:=gnutls-serv} : ${PERL:=perl} +guess_config_name() { + if git diff --quiet ../include/mbedtls/config.h 2>/dev/null; then + echo "default" + else + echo "unknown" + fi +} +: ${MBEDTLS_TEST_OUTCOME_FILE=} +: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"} +: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} + O_SRV="$OPENSSL_CMD s_server -www -cert data_files/server5.crt -key data_files/server5.key" O_CLI="echo 'GET / HTTP/1.0' | $OPENSSL_CMD s_client" G_SRV="$GNUTLS_SERV --x509certfile data_files/server5.crt --x509keyfile data_files/server5.key" @@ -97,9 +108,11 @@ print_usage() { printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n" printf " -s|--show-numbers\tShow test numbers in front of test names\n" printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n" - printf " --port\tTCP/UDP port (default: randomish 1xxxx)\n" + printf " --outcome-file\tFile where test outcomes are written\n" + printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" + printf " --port \tTCP/UDP port (default: randomish 1xxxx)\n" printf " --proxy-port\tTCP/UDP proxy port (default: randomish 2xxxx)\n" - printf " --seed\tInteger seed value to use for this test run\n" + printf " --seed \tInteger seed value to use for this test run\n" } get_options() { @@ -146,6 +159,14 @@ get_options() { done } +# Make the outcome file path relative to the original directory, not +# to .../tests +case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) + MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" + ;; +esac + # Skip next test; use this macro to skip tests which are legitimate # in theory and expected to be re-introduced at some point, but # aren't expected to succeed at the moment due to problems outside @@ -359,9 +380,22 @@ print_name() { } +# record_outcome [] +# The test name must be in $NAME. +record_outcome() { + echo "$1" + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then + printf '%s;%s;%s;%s;%s;%s\n' \ + "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ + "ssl-opt" "$NAME" \ + "$1" "${2-}" \ + >>"$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + # fail fail() { - echo "FAIL" + record_outcome "FAIL" "$1" echo " ! $1" mv $SRV_OUT o-srv-${TESTS}.log @@ -539,6 +573,7 @@ run_test() { if echo "$NAME" | grep "$FILTER" | grep -v "$EXCLUDE" >/dev/null; then : else SKIP_NEXT="NO" + # There was no request to run the test, so don't record its outcome. return fi @@ -586,7 +621,7 @@ run_test() { # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then SKIP_NEXT="NO" - echo "SKIP" + record_outcome "SKIP" SKIPS=$(( $SKIPS + 1 )) return fi @@ -772,7 +807,7 @@ run_test() { fi # if we're here, everything is ok - echo "PASS" + record_outcome "PASS" if [ "$PRESERVE_LOGS" -gt 0 ]; then mv $SRV_OUT o-srv-${TESTS}.log mv $CLI_OUT o-cli-${TESTS}.log @@ -1127,7 +1162,7 @@ run_test "SHA-1 forbidden by default in server certificate" \ -c "The certificate is signed with an unacceptable hash" requires_config_enabled MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES -run_test "SHA-1 forbidden by default in server certificate" \ +run_test "SHA-1 allowed by default in server certificate" \ "$P_SRV key_file=data_files/server2.key crt_file=data_files/server2.crt" \ "$P_CLI debug_level=2 allow_sha1=0" \ 0 @@ -1150,7 +1185,7 @@ run_test "SHA-1 forbidden by default in client certificate" \ -s "The certificate is signed with an unacceptable hash" requires_config_enabled MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES -run_test "SHA-1 forbidden by default in client certificate" \ +run_test "SHA-1 allowed by default in client certificate" \ "$P_SRV auth_mode=required allow_sha1=0" \ "$P_CLI key_file=data_files/cli-rsa.key crt_file=data_files/cli-rsa-sha1.crt" \ 0 @@ -6992,7 +7027,7 @@ run_test "SSL async private: sign, error in resume then fall back to transpar requires_config_enabled MBEDTLS_SSL_ASYNC_PRIVATE requires_config_enabled MBEDTLS_SSL_RENEGOTIATION -run_test "SSL async private: renegotiation: client-initiated; sign" \ +run_test "SSL async private: renegotiation: client-initiated, sign" \ "$P_SRV \ async_operations=s async_private_delay1=1 async_private_delay2=1 \ exchanges=2 renegotiation=1" \ @@ -7003,7 +7038,7 @@ run_test "SSL async private: renegotiation: client-initiated; sign" \ requires_config_enabled MBEDTLS_SSL_ASYNC_PRIVATE requires_config_enabled MBEDTLS_SSL_RENEGOTIATION -run_test "SSL async private: renegotiation: server-initiated; sign" \ +run_test "SSL async private: renegotiation: server-initiated, sign" \ "$P_SRV \ async_operations=s async_private_delay1=1 async_private_delay2=1 \ exchanges=2 renegotiation=1 renegotiate=1" \ @@ -7014,7 +7049,7 @@ run_test "SSL async private: renegotiation: server-initiated; sign" \ requires_config_enabled MBEDTLS_SSL_ASYNC_PRIVATE requires_config_enabled MBEDTLS_SSL_RENEGOTIATION -run_test "SSL async private: renegotiation: client-initiated; decrypt" \ +run_test "SSL async private: renegotiation: client-initiated, decrypt" \ "$P_SRV \ async_operations=d async_private_delay1=1 async_private_delay2=1 \ exchanges=2 renegotiation=1" \ @@ -7026,7 +7061,7 @@ run_test "SSL async private: renegotiation: client-initiated; decrypt" \ requires_config_enabled MBEDTLS_SSL_ASYNC_PRIVATE requires_config_enabled MBEDTLS_SSL_RENEGOTIATION -run_test "SSL async private: renegotiation: server-initiated; decrypt" \ +run_test "SSL async private: renegotiation: server-initiated, decrypt" \ "$P_SRV \ async_operations=d async_private_delay1=1 async_private_delay2=1 \ exchanges=2 renegotiation=1 renegotiate=1" \ @@ -7594,7 +7629,7 @@ requires_config_enabled MBEDTLS_ECDSA_C requires_config_enabled MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA requires_config_enabled MBEDTLS_AES_C requires_config_enabled MBEDTLS_GCM_C -run_test "DTLS fragmenting: proxy MTU: auto-reduction" \ +run_test "DTLS fragmenting: proxy MTU: auto-reduction (not valgrind)" \ -p "$P_PXY mtu=508" \ "$P_SRV dtls=1 debug_level=2 auth_mode=required \ crt_file=data_files/server7_int-ca.crt \ @@ -7618,7 +7653,7 @@ requires_config_enabled MBEDTLS_ECDSA_C requires_config_enabled MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA requires_config_enabled MBEDTLS_AES_C requires_config_enabled MBEDTLS_GCM_C -run_test "DTLS fragmenting: proxy MTU: auto-reduction" \ +run_test "DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)" \ -p "$P_PXY mtu=508" \ "$P_SRV dtls=1 debug_level=2 auth_mode=required \ crt_file=data_files/server7_int-ca.crt \ diff --git a/tests/suites/helpers.function b/tests/suites/helpers.function index b4b6d72863..5d0a3906d4 100644 --- a/tests/suites/helpers.function +++ b/tests/suites/helpers.function @@ -271,7 +271,7 @@ typedef enum TEST_RESULT_SKIPPED } test_result_t; -static struct +typedef struct { paramfail_test_state_t paramfail_test_state; test_result_t result; @@ -279,7 +279,8 @@ static struct const char *filename; int line_no; } -test_info; +test_info_t; +static test_info_t test_info; #if defined(MBEDTLS_PLATFORM_C) mbedtls_platform_context platform_ctx; diff --git a/tests/suites/host_test.function b/tests/suites/host_test.function index 0f98d23aa6..9e56ca3ed2 100644 --- a/tests/suites/host_test.function +++ b/tests/suites/host_test.function @@ -368,6 +368,118 @@ static int run_test_snprintf( void ) test_snprintf( 5, "123", 3 ) != 0 ); } +/** \brief Write the description of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param argv0 The test suite name. + * \param test_case The test case description. + */ +static void write_outcome_entry( FILE *outcome_file, + const char *argv0, + const char *test_case ) +{ + /* The non-varying fields are initialized on first use. */ + static const char *platform = NULL; + static const char *configuration = NULL; + static const char *test_suite = NULL; + + if( outcome_file == NULL ) + return; + + if( platform == NULL ) + { + platform = getenv( "MBEDTLS_TEST_PLATFORM" ); + if( platform == NULL ) + platform = "unknown"; + } + if( configuration == NULL ) + { + configuration = getenv( "MBEDTLS_TEST_CONFIGURATION" ); + if( configuration == NULL ) + configuration = "unknown"; + } + if( test_suite == NULL ) + { + test_suite = strrchr( argv0, '/' ); + if( test_suite != NULL ) + test_suite += 1; // skip the '/' + else + test_suite = argv0; + } + + /* Write the beginning of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + mbedtls_fprintf( outcome_file, "%s;%s;%s;%s;", + platform, configuration, test_suite, test_case ); +} + +/** \brief Write the result of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param unmet_dep_count The number of unmet dependencies. + * \param unmet_dependencies The array of unmet dependencies. + * \param ret The test dispatch status (DISPATCH_xxx). + * \param test_info A pointer to the test info structure. + */ +static void write_outcome_result( FILE *outcome_file, + size_t unmet_dep_count, + char *unmet_dependencies[], + int ret, + const test_info_t *info ) +{ + if( outcome_file == NULL ) + return; + + /* Write the end of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + switch( ret ) + { + case DISPATCH_TEST_SUCCESS: + if( unmet_dep_count > 0 ) + { + size_t i; + mbedtls_fprintf( outcome_file, "SKIP" ); + for( i = 0; i < unmet_dep_count; i++ ) + { + mbedtls_fprintf( outcome_file, "%c%s", + i == 0 ? ';' : ':', + unmet_dependencies[i] ); + } + break; + } + switch( info->result ) + { + case TEST_RESULT_SUCCESS: + mbedtls_fprintf( outcome_file, "PASS;" ); + break; + case TEST_RESULT_SKIPPED: + mbedtls_fprintf( outcome_file, "SKIP;Runtime skip" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;%s:%d:%s", + info->filename, info->line_no, + info->test ); + break; + } + break; + case DISPATCH_TEST_FN_NOT_FOUND: + mbedtls_fprintf( outcome_file, "FAIL;Test function not found" ); + break; + case DISPATCH_INVALID_TEST_DATA: + mbedtls_fprintf( outcome_file, "FAIL;Invalid test data" ); + break; + case DISPATCH_UNSUPPORTED_SUITE: + mbedtls_fprintf( outcome_file, "SKIP;Unsupported suite" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;Unknown cause" ); + break; + } + mbedtls_fprintf( outcome_file, "\n" ); + fflush( outcome_file ); +} /** * \brief Desktop implementation of execute_tests(). @@ -385,15 +497,16 @@ int execute_tests( int argc , const char ** argv ) const char *default_filename = "DATA_FILE"; const char *test_filename = NULL; const char **test_files = NULL; - int testfile_count = 0; + size_t testfile_count = 0; int option_verbose = 0; int function_id = 0; /* Other Local variables */ int arg_index = 1; const char *next_arg; - int testfile_index, ret, i, cnt; - int total_errors = 0, total_tests = 0, total_skipped = 0; + size_t testfile_index, i, cnt; + int ret; + unsigned total_errors = 0, total_tests = 0, total_skipped = 0; FILE *file; char buf[5000]; char *params[50]; @@ -403,6 +516,8 @@ int execute_tests( int argc , const char ** argv ) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) int stdout_fd = -1; #endif /* __unix__ || __APPLE__ __MACH__ */ + const char *outcome_file_name = getenv( "MBEDTLS_TEST_OUTCOME_FILE" ); + FILE *outcome_file = NULL; #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC) @@ -410,6 +525,15 @@ int execute_tests( int argc , const char ** argv ) mbedtls_memory_buffer_alloc_init( alloc_buf, sizeof( alloc_buf ) ); #endif + if( outcome_file_name != NULL ) + { + outcome_file = fopen( outcome_file_name, "a" ); + if( outcome_file == NULL ) + { + mbedtls_fprintf( stderr, "Unable to open outcome file. Continuing anyway.\n" ); + } + } + /* * The C standard doesn't guarantee that all-bits-0 is the representation * of a NULL pointer. We do however use that in our code for initializing @@ -473,7 +597,7 @@ int execute_tests( int argc , const char ** argv ) testfile_index < testfile_count; testfile_index++ ) { - int unmet_dep_count = 0; + size_t unmet_dep_count = 0; char *unmet_dependencies[20]; test_filename = test_files[ testfile_index ]; @@ -505,6 +629,7 @@ int execute_tests( int argc , const char ** argv ) mbedtls_fprintf( stdout, "." ); mbedtls_fprintf( stdout, " " ); fflush( stdout ); + write_outcome_entry( outcome_file, argv[0], buf ); total_tests++; @@ -584,6 +709,9 @@ int execute_tests( int argc , const char ** argv ) } + write_outcome_result( outcome_file, + unmet_dep_count, unmet_dependencies, + ret, &test_info ); if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE ) { total_skipped++; @@ -638,7 +766,7 @@ int execute_tests( int argc , const char ** argv ) } else if( ret == DISPATCH_TEST_FN_NOT_FOUND ) { - mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FUND\n" ); + mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FOUND\n" ); fclose( file ); mbedtls_exit( 2 ); } @@ -652,14 +780,17 @@ int execute_tests( int argc , const char ** argv ) free( unmet_dependencies[i] ); } + if( outcome_file != NULL ) + fclose( outcome_file ); + mbedtls_fprintf( stdout, "\n----------------------------------------------------------------------------\n\n"); if( total_errors == 0 ) mbedtls_fprintf( stdout, "PASSED" ); else mbedtls_fprintf( stdout, "FAILED" ); - mbedtls_fprintf( stdout, " (%d / %d tests (%d skipped))\n", - total_tests - total_errors, total_tests, total_skipped ); + mbedtls_fprintf( stdout, " (%u / %u tests (%u skipped))\n", + total_tests - total_errors, total_tests, total_skipped ); #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC) diff --git a/tests/suites/test_suite_x509parse.data b/tests/suites/test_suite_x509parse.data index ce49ff0fc1..775e17e09f 100644 --- a/tests/suites/test_suite_x509parse.data +++ b/tests/suites/test_suite_x509parse.data @@ -611,11 +611,11 @@ X509 CRT verification #26 (domain not matching multi certificate) depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_SHA1_C:MBEDTLS_SHA256_C:MBEDTLS_RSA_C:MBEDTLS_PKCS1_V15 x509_verify:"data_files/cert_example_multi.crt":"data_files/test-ca.crt":"data_files/crl.pem":"www.example.net":MBEDTLS_ERR_X509_CERT_VERIFY_FAILED:MBEDTLS_X509_BADCERT_CN_MISMATCH:"compat":"NULL" -X509 CRT verification #27 (domain not matching multi certificate) +X509 CRT verification #27.1 (domain not matching multi certificate: suffix) depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_SHA1_C:MBEDTLS_SHA256_C:MBEDTLS_RSA_C:MBEDTLS_PKCS1_V15 x509_verify:"data_files/cert_example_multi.crt":"data_files/test-ca.crt":"data_files/crl.pem":"xample.net":MBEDTLS_ERR_X509_CERT_VERIFY_FAILED:MBEDTLS_X509_BADCERT_CN_MISMATCH:"compat":"NULL" -X509 CRT verification #27 (domain not matching multi certificate) +X509 CRT verification #27.2 (domain not matching multi certificate: head junk) depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_SHA1_C:MBEDTLS_SHA256_C:MBEDTLS_RSA_C:MBEDTLS_PKCS1_V15 x509_verify:"data_files/cert_example_multi.crt":"data_files/test-ca.crt":"data_files/crl.pem":"bexample.net":MBEDTLS_ERR_X509_CERT_VERIFY_FAILED:MBEDTLS_X509_BADCERT_CN_MISMATCH:"compat":"NULL" @@ -1292,10 +1292,6 @@ X509 CRT ASN1 (TBS, inv Validity, notBefore length out of bounds) depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA256_C x509parse_crt:"307b3066a0030201028204deadbeef300d06092a864886f70d01010b0500300c310a300806000c045465737430021701300c310a30080600130454657374302a300d06092A864886F70D010101050003190030160210ffffffffffffffffffffffffffffffff0202ffff300d06092a864886f70d01010b0500030200ff":"":MBEDTLS_ERR_X509_INVALID_DATE + MBEDTLS_ERR_ASN1_OUT_OF_DATA -X509 CRT ASN1 (TBS, inv Validity, notBefore length out of bounds) -depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA256_C -x509parse_crt:"307b3066a0030201028204deadbeef300d06092a864886f70d01010b0500300c310a300806000c045465737430021701300c310a30080600130454657374302a300d06092A864886F70D010101050003190030160210ffffffffffffffffffffffffffffffff0202ffff300d06092a864886f70d01010b0500030200ff":"":MBEDTLS_ERR_X509_INVALID_DATE + MBEDTLS_ERR_ASN1_OUT_OF_DATA - X509 CRT ASN1 (TBS, inv Validity, notBefore empty) depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA256_C x509parse_crt:"3081893074a0030201008204deadbeef300d06092a864886f70d01010b0500300c310a3008060013045465737430101700170c303931323331323335393539300c310a30080600130454657374302a300d06092A864886F70D010101050003190030160210ffffffffffffffffffffffffffffffff0202ffff300d06092a864886f70d01010b0500030200ff":"":MBEDTLS_ERR_X509_INVALID_DATE