From 3d4ea5490f6004cbf1ad7a03280a6a5ff5c69351 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:35:44 +0100 Subject: [PATCH 1/9] Move lcov commands to a separate script Signed-off-by: Gilles Peskine --- Makefile | 9 +-------- scripts/lcov.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100755 scripts/lcov.sh diff --git a/Makefile b/Makefile index 5b2ad16cdb..85fa093aec 100644 --- a/Makefile +++ b/Makefile @@ -145,14 +145,7 @@ covtest: tests/ssl-opt.sh lcov: - rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions + scripts/lcov.sh apidoc: mkdir -p apidoc diff --git a/scripts/lcov.sh b/scripts/lcov.sh new file mode 100755 index 0000000000..ab86421f9e --- /dev/null +++ b/scripts/lcov.sh @@ -0,0 +1,10 @@ +#!/bin/sh + + rm -rf Coverage + lcov --capture --initial --directory library -o files.info + lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info + lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info + lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' + gendesc tests/Descriptions.txt -o descriptions + genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info + rm -f files.info tests.info all.info final.info descriptions From eff88034d4ac8bb6d56777e174db4ece606e5568 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:51:44 +0100 Subject: [PATCH 2/9] Add a bit of documentation Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 58 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index ab86421f9e..8c8c8468bd 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -1,10 +1,52 @@ #!/bin/sh - rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions +help () { + cat < Date: Wed, 30 Nov 2022 17:52:01 +0100 Subject: [PATCH 3/9] Call the new lcov script from CMake builds The code in CMakeLists.txt was an old copy of the code in Makefile. This brings in branch coverage, which had only been added to Makefile. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59a960a8b6..3a870a7585 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -312,14 +312,7 @@ if(ENABLE_TESTING) ) ADD_CUSTOM_TARGET(lcov - COMMAND rm -rf Coverage - COMMAND lcov --capture --initial --directory library/CMakeFiles/mbedtls.dir -o files.info - COMMAND lcov --capture --directory library/CMakeFiles/mbedtls.dir -o tests.info - COMMAND lcov --add-tracefile files.info --add-tracefile tests.info -o all.info - COMMAND lcov --remove all.info -o final.info '*.h' - COMMAND gendesc tests/Descriptions.txt -o descriptions - COMMAND genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --no-branch-coverage -o Coverage final.info - COMMAND rm -f files.info tests.info all.info final.info descriptions + COMMAND scripts/lcov.sh ) ADD_CUSTOM_TARGET(memcheck From d7db883b712448dfcbed989787b7931fa4d75823 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:52:54 +0100 Subject: [PATCH 4/9] Fix "make lcov" in CMake out-of-tree builds Signed-off-by: Gilles Peskine --- tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ae3d054592..965daa88fb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -242,6 +242,7 @@ if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/seedfile") link_to_source(seedfile) endif() + link_to_source(Descriptions.txt) link_to_source(compat.sh) link_to_source(context-info.sh) link_to_source(data_files) From e628f291ae6efe77383aae11dfcf564e359fb631 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:56:58 +0100 Subject: [PATCH 5/9] Put temporary files in a temporary directory Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 8c8c8468bd..80ebc79870 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -34,13 +34,14 @@ set -eu lcov_rebuild_stats () { rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions + mkdir Coverage Coverage/tmp + lcov --capture --initial --directory library -o Coverage/tmp/files.info + lcov --rc lcov_branch_coverage=1 --capture --directory library -o Coverage/tmp/tests.info + lcov --rc lcov_branch_coverage=1 --add-tracefile Coverage/tmp/files.info --add-tracefile Coverage/tmp/tests.info -o Coverage/tmp/all.info + lcov --rc lcov_branch_coverage=1 --remove Coverage/tmp/all.info -o Coverage/tmp/final.info '*.h' + gendesc tests/Descriptions.txt -o Coverage/tmp/descriptions + genhtml --title "mbed TLS" --description-file Coverage/tmp/descriptions --keep-descriptions --legend --branch-coverage -o Coverage Coverage/tmp/final.info + rm -f Coverage/tmp/*.info Coverage/tmp/descriptions echo "Coverage report in: Coverage/index.html" } From 749a0d7be8d2a70443e553c1d26f86cb8b8c66fe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 18:08:14 +0100 Subject: [PATCH 6/9] Add option to reset the traces, to use after rework Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 80ebc79870..718166ddb2 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -2,7 +2,7 @@ help () { cat <&1; exit 120;; + esac +done +shift $((OPTIND - 1)) + +"$main" "$@" From 202b1a07bacceb2c4b7041f3c95c0554f847c5d3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Dec 2022 17:41:36 +0100 Subject: [PATCH 7/9] You need --coverage when linking as well Signed-off-by: Gilles Peskine --- Makefile | 5 +++-- scripts/lcov.sh | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 85fa093aec..272252e919 100644 --- a/Makefile +++ b/Makefile @@ -136,8 +136,9 @@ check: lib tests test: check ifndef WINDOWS -# note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' +# Note: for coverage testing, build with: +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage -g3 -O0' +# For the reference coverage measurement, see tests/scripts/basic-build-test.sh covtest: $(MAKE) check programs/test/selftest diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 718166ddb2..8d141eedf7 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -6,8 +6,14 @@ Usage: $0 [-r] Collect coverage statistics of library code into an HTML report. General instructions: -1. Build the library with CFLAGS="--coverage -O0 -g3". +1. Build the library with CFLAGS="--coverage -O0 -g3" and link the test + programs with LDFLAGS="--coverage". This can be an out-of-tree build. + For example (in-tree): + make CFLAGS="--coverage -O0 -g3" LDFLAGS="--coverage" + Or (out-of-tree): + mkdir build-coverage && cd build-coverage && + cmake -D CMAKE_BUILD_TYPE=Coverage .. && make 2. Run whatever tests you want. 3. Run this script from the parent of the directory containing the library object files and coverage statistics files. From 2f66115e0e0287d0510ab9135f1c6e449ffcd9ba Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Dec 2022 18:05:31 +0100 Subject: [PATCH 8/9] No need for -g or -O in LDFLAGS Fix a mistake in the previous commit. Signed-off-by: Gilles Peskine --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 272252e919..af4430510b 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ test: check ifndef WINDOWS # Note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage -g3 -O0' +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' # For the reference coverage measurement, see tests/scripts/basic-build-test.sh covtest: $(MAKE) check From 0ab6c8acb7e604abe2570f94d913013b416fcedd Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 8 Dec 2022 22:18:31 +0100 Subject: [PATCH 9/9] Remove obsolete target covtest This target dates back from before basic-build-test.sh existed. Refer to that script instead. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 14 +++++++------- Makefile | 18 ++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a870a7585..6bf9fae09a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,13 +304,13 @@ if(ENABLE_TESTING) # additional convenience targets for Unix only if(UNIX) - ADD_CUSTOM_TARGET(covtest - COMMAND make test - COMMAND programs/test/selftest - COMMAND tests/compat.sh - COMMAND tests/ssl-opt.sh - ) - + # For coverage testing: + # 1. Build with: + # cmake -D CMAKE_BUILD_TYPE=Coverage /path/to/source && make + # 2. Run the relevant tests for the part of the code you're interested in. + # For the reference coverage measurement, see + # tests/scripts/basic-build-test.sh + # 3. Run scripts/lcov.sh to generate an HTML report. ADD_CUSTOM_TARGET(lcov COMMAND scripts/lcov.sh ) diff --git a/Makefile b/Makefile index af4430510b..2f1be65288 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PERL ?= perl .SILENT: -.PHONY: all no_test programs lib tests install uninstall clean test check covtest lcov apidoc apidoc_clean +.PHONY: all no_test programs lib tests install uninstall clean test check lcov apidoc apidoc_clean all: programs tests $(MAKE) post_build @@ -136,15 +136,13 @@ check: lib tests test: check ifndef WINDOWS -# Note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' -# For the reference coverage measurement, see tests/scripts/basic-build-test.sh -covtest: - $(MAKE) check - programs/test/selftest - tests/compat.sh - tests/ssl-opt.sh - +# For coverage testing: +# 1. Build with: +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' +# 2. Run the relevant tests for the part of the code you're interested in. +# For the reference coverage measurement, see +# tests/scripts/basic-build-test.sh +# 3. Run scripts/lcov.sh to generate an HTML report. lcov: scripts/lcov.sh