diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index c85d4865ed..2322a982a3 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -5176,11 +5176,16 @@ support_test_cmake_out_of_source () {
 }
 
 component_test_cmake_out_of_source () {
+    # remove existing generated files so that we use the ones cmake
+    # generates
+    make neat
+
     msg "build: cmake 'out-of-source' build"
     MBEDTLS_ROOT_DIR="$PWD"
     mkdir "$OUT_OF_SOURCE_DIR"
     cd "$OUT_OF_SOURCE_DIR"
-    cmake -D CMAKE_BUILD_TYPE:String=Check "$MBEDTLS_ROOT_DIR"
+    # Note: Explicitly generate files as these are turned off in releases
+    cmake -D CMAKE_BUILD_TYPE:String=Check -D GEN_FILES=ON "$MBEDTLS_ROOT_DIR"
     make
 
     msg "test: cmake 'out-of-source' build"
@@ -5201,9 +5206,14 @@ component_test_cmake_out_of_source () {
 }
 
 component_test_cmake_as_subdirectory () {
+    # Remove existing generated files so that we use the ones CMake
+    # generates
+    make neat
+
     msg "build: cmake 'as-subdirectory' build"
     cd programs/test/cmake_subproject
-    cmake .
+    # Note: Explicitly generate files as these are turned off in releases
+    cmake -D GEN_FILES=ON .
     make
     ./cmake_subproject
 }