From 986c22a6d0704e6b13005262a613dc21b25e9161 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 1 Feb 2023 20:14:59 +0000
Subject: [PATCH] Update testrunner.tcl to use a separate process for each test
 script. And to run some extra tests too.

FossilOrigin-Name: d090948a69a9c4b86693bd3caedba3d7e5883e4b2ad1f2d4bf7ae14c105ddea7
---
 ext/recover/recover1.test |  12 +-
 ext/rtree/rtreecheck.test |  15 +-
 manifest                  |  36 +-
 manifest.uuid             |   2 +-
 test/distinctagg.test     |   1 +
 test/pager2.test          |   2 +
 test/permutations.test    |   5 +-
 test/sort4.test           |   2 +
 test/tester.tcl           |  15 +-
 test/testrunner.tcl       | 986 +++++++++++++++++++-------------------
 test/testrunner_data.tcl  | 123 +++++
 test/vt02.c               |   6 +
 test/windowB.test         |   3 +-
 13 files changed, 674 insertions(+), 534 deletions(-)
 create mode 100644 test/testrunner_data.tcl

diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test
index 3e8a691492..fef1bf90f4 100644
--- a/ext/recover/recover1.test
+++ b/ext/recover/recover1.test
@@ -275,11 +275,13 @@ do_recover_test 15
 
 #-------------------------------------------------------------------------
 reset_db
-do_execsql_test 16.1 {
-  PRAGMA journal_mode = wal;
-  CREATE TABLE t1(x);
-  INSERT INTO t1 VALUES(1), (2), (3);
-} {wal}
+do_test 16.1 {
+  execsql { PRAGMA journal_mode = wal }
+  execsql {
+    CREATE TABLE t1(x);
+    INSERT INTO t1 VALUES(1), (2), (3);
+  }
+} {}
 do_test 16.2 {
   set R [sqlite3_recover_init db main test.db2]
   $R run
diff --git a/ext/rtree/rtreecheck.test b/ext/rtree/rtreecheck.test
index 1754ff5c3d..ff5397e1e1 100644
--- a/ext/rtree/rtreecheck.test
+++ b/ext/rtree/rtreecheck.test
@@ -167,9 +167,18 @@ do_execsql_test 6.0 {
 }
 db close
 sqlite3 db test.db
-do_catchsql_test 6.1 {
-  SELECT ( 'elvis' IN(SELECT rtreecheck('t1')) ) FROM (SELECT 1) GROUP BY 1;
-} {1 {database table is locked}}
+
+if {[permutation]=="inmemory_journal"} {
+  # This doesn't hit an SQLITE_LOCKED in this permutation as the schema
+  # has already been loaded. 
+  do_catchsql_test 6.1.inmemory_journal {
+    SELECT ( 'elvis' IN(SELECT rtreecheck('t1')) ) FROM (SELECT 1) GROUP BY 1;
+  } {0 0}
+} else {
+  do_catchsql_test 6.1 {
+    SELECT ( 'elvis' IN(SELECT rtreecheck('t1')) ) FROM (SELECT 1) GROUP BY 1;
+  } {1 {database table is locked}}
+}
 
 finish_test
 
diff --git a/manifest b/manifest
index 18a6046fe0..2558c1e997 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Just\sbecause\sa\sCTE\sis\sused\smore\sthan\sonce,\sdoes\snot\smean\sit\sshould\sbe\ntagged\swith\sM10d_Yes\sand\sthereby\sprohibited\sfrom\sparticipating\sin\sthe\nquery\sflattening\soptimization.\s\sSee\n[forum:/forumpost/1d571c02963355ed|forum\sthread\s1d571c02963].
-D 2023-02-01T15:41:07.645
+C Update\stestrunner.tcl\sto\suse\sa\sseparate\sprocess\sfor\seach\stest\sscript.\sAnd\sto\srun\ssome\sextra\stests\stoo.
+D 2023-02-01T20:14:59.965
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -365,7 +365,7 @@ F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac96
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
 F ext/recover/dbdata.c dc25628e405c86936c597e28f3e6f56a257029c3034c5ef7f6b10f7c02f41018
-F ext/recover/recover1.test 2a2df2943d6696f9487e75868feae4b1511c4a511b102854ba0d2af0326d9dfb
+F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e
 F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49
@@ -413,7 +413,7 @@ F ext/rtree/rtreeH.test 0885151ee8429242625600ae47142cca935332c70a06737f35af53a7
 F ext/rtree/rtreeI.test 608e77f7fde9be5a12eae316baef640fffaafcfa90a3d67443e78123e19c4ca4
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed
-F ext/rtree/rtreecheck.test e53fc47fb727d7ffa17ea79fecfe3e709815b65233c11464a0b8f1f4ac0cb50a
+F ext/rtree/rtreecheck.test 4e859a9cd49d2353ff10c122f72183ec37b400e35d2b0349b2e9696649b6a00e
 F ext/rtree/rtreecirc.test aec664eb21ae943aeb344191407afff5d392d3ae9d12b9a112ced0d9c5de298e
 F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d
 F ext/rtree/rtreedoc.test 27a5703cb1200f6f69051de68da546cef3dfdcf59be73afadfc50b9f9c9960d9
@@ -936,7 +936,7 @@ F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a199
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test a7687c2fb50c93f6a486936c51439a93221c6e1188f9bc7b27b3ec26f9c58b1e
 F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1
-F test/distinctagg.test d76ef2e91fe810630c176d6bd0a58c14d5851c3125f0a1d977db87ba76359639
+F test/distinctagg.test 14ec5026e684eddd414c61c08692b43773e224ac92efbed6ec08c6994bc39723
 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52
 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075
 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50
@@ -1360,7 +1360,7 @@ F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d7109671
 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17
 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f
 F test/pager1.test ffd885cdc98b986c9f746496508c0c4810ed0eaade3575ddf53c222e85880552
-F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71
+F test/pager2.test 57ce815e31a7509fcdf7c5474577fd2e9cfee1281d45601e0f7a3bd5534d70a4
 F test/pager3.test 4e9a83d6ca0838d7c602c9eb93d1357562d9059c1e02ffb138a8271020838370
 F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e
 F test/pagerfault.test 63c5da625562c66345ab4528790327ca63db2f6f9cbae2aba8cb7c51de3d1628
@@ -1373,7 +1373,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/pendingrace.test fb997b46d6c144a508fb8025d988a1e511b53d42d24143c57b51de3a405c7490
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 4705a032bbfef531bb3fd98b8c6ba4a739998949eae9ac0ea97c8696b331211d
+F test/permutations.test 3a202a870735688a85c03c880a0560ce5a39d7360d5e26469594f365bca9e2b9
 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f
 F test/pragma.test a74a9c9642e5d7e32f5a2aa77a2ed64ec5b69fecff39d52c4daf5945a2a4de65
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1508,7 +1508,7 @@ F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
 F test/sort.test f86751134159abb5e5fd4381a0d7038c91013638cd1e3fa1d7850901f6df6196
 F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0
 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5
-F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c
+F test/sort4.test cca6f4b0b5255882645bbbe346a6a9f4a5c7b6a18513a6a7bf4ac1c4761ddc19
 F test/sort5.test 6b43ae0e2169b5ceed441844492e55ba7f1ae0790528395ddf7888ab3094525d
 F test/sorterref.test 9a606c86a4c682db5eeaaefa0565b52102778db53e48ca7101cd4f9ebcc0ad94
 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66
@@ -1564,8 +1564,9 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl e72c337f01e47c2833c83288b60e0a1730165cc7de7b59724e925c4ce026c0a1
-F test/testrunner.tcl 407fc02be0c859ef7a85b9431de9c5aa79363cab822ecbee3500a0daeb6b82cd
+F test/tester.tcl 8d54d40a55e12554b4509bc12078b201b233c8e842a7543629094a21b1ba956d
+F test/testrunner.tcl de0f2922631fc1cd43903e6e28fb69d01f09685c4e52bceea9da4f051dc03dab
+F test/testrunner_data.tcl 7c9e97b0caea7420e29bbc731eef7e44a27781c192c4c9e4c605c74ed81a9d2f
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1801,7 +1802,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487
 F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679
 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456
-F test/vt02.c 33ecddc0832d4cd24e9e9fa83d868981b1e049462f4ec9080710353f6479a534
+F test/vt02.c 86253b57d6bc2170dfca33f45fd099b66d0bf874e95ecd7786dcbb134f179469
 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07
 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082
 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e
@@ -1911,7 +1912,7 @@ F test/window8.tcl 5e02e41d9d9a80f597063aed1a381eb19d1d0ef677a4f0df352c5365cf23f
 F test/window8.test 4ab16817414af0c904abe2ebdf88eb6c2b00058b84f9748c6174ff11fc45f1ed
 F test/window9.test 349c71eab4288a1ffc19e2f65872ec2c37e6cf8a1dda2ad300364b7450ae4836
 F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af23be
-F test/windowB.test 57f9fb931ed5e04f17db4fcb7059e76026079daf451aac27e54d41217078039b
+F test/windowB.test cf1278912c7a059763c49cbd6e1b3213b5e9b9d87a5b22714c63808fc33bc8d5
 F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2
 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b
 F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c
@@ -2046,8 +2047,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6b3546c871fe78a4e550e0144b48ac98325787cc8b192a9e7f5f2a2ffa57f76d
-R 70533f1fdb382e48eceffd4225281dd8
-U drh
-Z 915bf0e8238a05acb8b086524ea9263b
+P 66f29c403d28630bfaea9124bd63ee4a047b1fe4a7e27dc5d10d67d1601b15e0
+R 06a2d845016cccabda66a252a397ec86
+T *branch * testrunner
+T *sym-testrunner *
+T -sym-trunk *
+U dan
+Z a224100f94ae40bc55815239c2a30b96
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3fedf45517..feab0b951a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-66f29c403d28630bfaea9124bd63ee4a047b1fe4a7e27dc5d10d67d1601b15e0
\ No newline at end of file
+d090948a69a9c4b86693bd3caedba3d7e5883e4b2ad1f2d4bf7ae14c105ddea7
\ No newline at end of file
diff --git a/test/distinctagg.test b/test/distinctagg.test
index a34312ef98..6e46c88616 100644
--- a/test/distinctagg.test
+++ b/test/distinctagg.test
@@ -184,6 +184,7 @@ foreach {tn use_t3 sql res} {
  10 1 "SELECT count(DISTINCT x) FROM t1 LEFT JOIN t3 ON (t3.x=0)"  0
 
 } {
+  unset -nocomplain a
   do_test 5.$tn.1 {
     set bUse 0
     db eval "EXPLAIN $sql" a {
diff --git a/test/pager2.test b/test/pager2.test
index 0e2b33b833..1d78b30f5a 100644
--- a/test/pager2.test
+++ b/test/pager2.test
@@ -165,4 +165,6 @@ do_test pager2-3.1 {
   list [catch { db3 eval { INSERT INTO t1 VALUES(3, 4) } } msg] $msg
 } {1 {no such table: t1}}
 
+db1 close
+
 finish_test
diff --git a/test/permutations.test b/test/permutations.test
index 1b44ba52bf..79e549c138 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -99,6 +99,7 @@ foreach f [glob -nocomplain            \
 foreach f [glob -nocomplain $testdir/../ext/session/*.test] { 
   lappend alltests $f 
 }
+unset f
 
 if {$::tcl_platform(platform)!="unix"} {
   set alltests [test_set $alltests -exclude crash.test crash2.test]
@@ -115,7 +116,7 @@ set allquicktests [test_set $alltests -exclude {
   corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test
   crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test
   fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test
-  misc7.test mutex2.test notify2.test onefile.test pagerfault2.test 
+  misc7.test mutex2.test onefile.test pagerfault2.test 
   savepoint4.test savepoint6.test select9.test 
   speed1.test speed1p.test speed2.test speed3.test speed4.test 
   speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
@@ -915,6 +916,8 @@ ifcapable threadsafe {
     autoinstall_test_functions
   } -shutdown {
     catch {db close}
+    catch {db2 close}
+    catch {db3 close}
     sqlite3_shutdown
     install_mutex_counters 0
     sqlite3_initialize
diff --git a/test/sort4.test b/test/sort4.test
index 13d9a5999a..84125885ab 100644
--- a/test/sort4.test
+++ b/test/sort4.test
@@ -8,6 +8,8 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: superslow
+#
 # This file implements regression tests for SQLite library. 
 #
 # The tests in this file are brute force tests of the multi-threaded
diff --git a/test/tester.tcl b/test/tester.tcl
index 022dad900d..4658590cf4 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -552,6 +552,7 @@ if {[info exists cmdlinearg]==0} {
       }
     }
   }
+  unset -nocomplain a
   set testdir [file normalize $testdir]
   set cmdlinearg(TESTFIXTURE_HOME) [pwd]
   set cmdlinearg(INFO_SCRIPT) [file normalize [info script]]
@@ -1318,9 +1319,11 @@ proc finalize_testing {} {
   if {$::cmdlinearg(binarylog)} {
     vfslog finalize binarylog
   }
-  if {$sqlite_open_file_count} {
-    output2 "$sqlite_open_file_count files were left open"
-    incr nErr
+  if {[info exists ::run_thread_tests_called]==0} {
+    if {$sqlite_open_file_count} {
+      output2 "$sqlite_open_file_count files were left open"
+      incr nErr
+    }
   }
   if {[lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]>0 ||
               [sqlite3_memory_used]>0} {
@@ -2502,8 +2505,10 @@ proc test_restore_config_pagecache {} {
   catch {db3 close}
 
   sqlite3_shutdown
-  eval sqlite3_config_pagecache $::old_pagecache_config
-  unset ::old_pagecache_config 
+  if {[info exists ::old_pagecache_config]} {
+    eval sqlite3_config_pagecache $::old_pagecache_config
+    unset ::old_pagecache_config 
+  }
   sqlite3_initialize
   autoinstall_test_functions
   sqlite3 db test.db
diff --git a/test/testrunner.tcl b/test/testrunner.tcl
index 48afff47e6..2e3ad1193b 100644
--- a/test/testrunner.tcl
+++ b/test/testrunner.tcl
@@ -1,4 +1,6 @@
 
+source [file join [file dirname [info script]] testrunner_data.tcl]
+
 #-------------------------------------------------------------------------
 # Usage:
 #
@@ -47,38 +49,77 @@ proc usage {} {
 }
 #-------------------------------------------------------------------------
 
+
+# If this script is invoked using:
+#
+#   testrunner.tcl helper <directory> <permutation> <script>
+#
+if {[lindex $argv 0]=="helper"} {
+  if {[llength $argv]!=3} { error "BAD ARGS" }
+
+  set permutation [lindex $argv 1]
+  set script [file normalize [lindex $argv 2]]
+
+  set ::argv [list]
+
+  if {$permutation=="full"} {
+
+    set testdir [file dirname $argv0]
+    source $::testdir/tester.tcl
+    unset -nocomplain ::G(isquick)
+    reset_db
+
+  } elseif {$permutation!="default" && $permutation!=""} {
+    set testdir [file dirname $argv0]
+    source $::testdir/permutations.test
+
+    if {[info exists ::testspec($permutation)]==0} {
+      error "no such permutation: $permutation"
+    }
+
+    array set O $::testspec($permutation)
+    set ::G(perm:name)         $permutation
+    set ::G(perm:prefix)       $O(-prefix)
+    set ::G(isquick)           1
+    set ::G(perm:dbconfig)     $O(-dbconfig)
+    set ::G(perm:presql)       $O(-presql)
+
+    rename finish_test helper_finish_test
+    proc finish_test {} "
+      uplevel {
+        $O(-shutdown)
+      }
+      helper_finish_test
+    "
+
+    eval $O(-initialize)
+    reset_db
+  }
+
+  source $script
+  exit
+}
+
 #-------------------------------------------------------------------------
 # The database schema used by the testrunner.db database.
 #
 set R(schema) {
   DROP TABLE IF EXISTS script;
-  DROP TABLE IF EXISTS msg;
-  DROP TABLE IF EXISTS malloc;
 
   CREATE TABLE script(
+    build TEXT DEFAULT '',
     config TEXT,
     filename TEXT,                -- full path to test script
     slow BOOLEAN,                 -- true if script is "slow"
-    state TEXT CHECK( state IN ('ready', 'running', 'done') ),
-    testfixtureid,                -- Id of process that ran script
+    state TEXT CHECK( state IN ('ready', 'running', 'done', 'failed') ),
     time INTEGER,                 -- Time in ms
-    nerr INTEGER,                 -- if 'done', the number of errors
-    ntest INTEGER,                -- if 'done', the number of tests
     output TEXT,                  -- full output of test script
-    PRIMARY KEY(config, filename)
+    ismake AS (config='make'),
+    PRIMARY KEY(build, config, filename)
   );
 
-  CREATE TABLE malloc(
-    id INTEGER PRIMARY KEY,
-    nmalloc INTEGER,
-    nbyte INTEGER,
-    leaker TEXT
-  );
-
-  CREATE TABLE msg(
-    id INTEGER PRIMARY KEY,
-    msg TEXT
-  );
+  CREATE INDEX i1 ON script(state, ismake);
+  CREATE INDEX i2 ON script(state, slow DESC, ismake);
 }
 #-------------------------------------------------------------------------
 
@@ -116,6 +157,7 @@ proc default_njob {} {
 
 set R(dbname) [file normalize testrunner.db]
 set R(logname) [file normalize testrunner.log]
+set R(build.logname) [file normalize testrunner_build.log]
 set R(info_script) [file normalize [info script]]
 set R(timeout) 10000              ;# Default busy-timeout for testrunner.db 
 set R(nJob)    [default_njob]     ;# Default number of helper processes
@@ -125,128 +167,247 @@ set R(patternlist) [list]
 
 set testdir [file dirname $argv0]
 
-# Parse the command line options. There are two ways to invoke this
-# script - to create a helper or coordinator process. If there are
-# no helper processes, the coordinator runs test scripts.
-#
-# To create a helper process:
-#
-#    testrunner.tcl helper ID
-#
-# where ID is an integer greater than 0. The process will create and
-# run tests in the "testdir$ID" directory. Helper processes are only
-# created by coordinators - there is no need for a user to create
-# helper processes manually.
-#
-# If the first argument is anything other than "helper", then a coordinator
-# process is started. See the implementation of the [usage] proc above for
-# details.
-#
-switch -- [lindex $argv 0] {
-  helper {
-    set R(helper) 1
-    set R(helper_id) [lindex $argv 1]
-    set argv [list --testdir=testdir$R(helper_id)]
-  }
 
-  default {
-    set R(helper) 0
-    set R(helper_id) 0
 
+# Check that directory $dir exists. If it does not, create it. If 
+# it does, delete its contents.
+#
+proc create_or_clear_dir {dir} {
+  set dir [file normalize $dir]
+  catch { file mkdir $dir }
+  foreach f [glob -nocomplain [file join $dir *]] {
+    catch { file delete -force $f }
   }
 }
-if {$R(helper)==0} {
-  for {set ii 0} {$ii < [llength $argv]} {incr ii} {
-    set a [lindex $argv $ii]
-    set n [string length $a]
 
-    if {[string range $a 0 0]=="-"} {
-      if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} {
-        incr ii
-          set R(nJob) [lindex $argv $ii]
-      } else {
-        usage
-      }
+proc copy_dir {from to} {
+  foreach f [glob -nocomplain [file join $from *]] {
+    catch { file copy -force $f $to }
+  }
+}
+
+##########################################################################
+##########################################################################
+proc build_to_dirname {bname} {
+  set fold [string tolower [string map {- _} $bname]]
+  return "testrunner_build_$fold"
+}
+
+proc build_input_ready {fd build} {
+  global R
+  global O
+
+  if {[eof $fd]} {
+    foreach {dirname b} $build {}
+
+    fconfigure $fd -blocking 1
+    set rc [catch { close $fd } msg]
+    if {$rc} { 
+      puts "Build \"$b\" finished - FAILED"
+      lappend R(lBuildFail) $build
     } else {
-      lappend R(patternlist) [string map {% *} $a]
+      puts "Build \"$b\" finished - ok"
+    }
+
+    puts $R(log) "### Build \"$b\" in directory $dirname"
+    puts $R(log) $O($fd)
+
+    launch_another_build
+    incr R(nHelperRunning) -1
+    incr ::wakeup
+  } else {
+    if {[gets $fd line]>=0} {
+      append O($fd) "$line\n"
     }
   }
-
-  set argv [list]
+  global R
 }
+
+proc launch_another_build {} {
+  global R
+  if {[llength $R(lBuild)]>0} {
+    set build [lindex $R(lBuild) 0]
+    set R(lBuild) [lrange $R(lBuild) 1 end]
+    foreach {dirname b} $build {}
+
+    puts "Launching build \"$b\" in directory $dirname..."
+    set srcdir [file dirname [file dirname $R(info_script)]]
+
+    incr R(nHelperRunning)
+
+    set pwd [pwd]
+    cd $dirname
+    set fd [open "|bash build.sh $srcdir 2>@1"]
+    cd $pwd
+
+    set O($fd) ""
+    fconfigure $fd -blocking 0
+    fileevent $fd readable [list build_input_ready $fd $build]
+  }
+}
+
+if {[lindex $argv 0]=="build"} {
+
+  # Todo...
+  set platform linux
+
+  # Load configuration data.
+  source [file join [file dirname [info script]] testrunner_data.tcl]
+
+  foreach b [trd_builds $platform] {
+    set dirname [build_to_dirname $b]
+    create_or_clear_dir $dirname
+
+    set target "testprogs"
+    if {$b=="Device-One"} { set target coretestprogs }
+    if {$b=="User-Auth"}  { set target testfixture }
+
+    set cmd [list \
+        [info nameofexec] \
+        [file join [file dirname $R(info_script)] releasetest_data.tcl] \
+        script $b $target
+    ]
+    set script [exec {*}$cmd]
+
+    set fd [open [file join $dirname build.sh] w]
+    puts $fd $script
+    close $fd
+
+    lappend R(lBuild) [list $dirname $b]
+  }
+
+  set R(log) [open $R(build.logname) w]
+
+  set R(nHelperRunning) 0
+  set R(lBuildFail) [list]
+  for {set ii 0} {$ii < $R(nJob)} {incr ii} {
+    launch_another_build
+  }
+
+  while {$R(nHelperRunning)>0} {
+    vwait ::wakeup
+  }
+  close $R(log)
+
+  if {[llength $R(lBuildFail)]==0} {
+    puts "All builds succeeded!"
+  } else {
+    puts "Builds failed:"
+    foreach build $R(lBuildFail) {
+      foreach {dirname b} $build {}
+      puts "  $b ($dirname)"
+    }
+    exit 1
+  }
+
+  puts "Log file is $R(build.logname)"
+  exit
+}
+##########################################################################
+##########################################################################
+
+set R(helper) 0
+set R(helper_id) 0
+for {set ii 0} {$ii < [llength $argv]} {incr ii} {
+  set a [lindex $argv $ii]
+  set n [string length $a]
+
+  if {[string range $a 0 0]=="-"} {
+    if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} {
+      incr ii
+      set R(nJob) [lindex $argv $ii]
+    } else {
+      usage
+    }
+  } else {
+    lappend R(patternlist) [string map {% *} $a]
+  }
+}
+
+set argv [list]
+
+set dir [pwd]
 source $testdir/permutations.test
+cd $dir
 
 #-------------------------------------------------------------------------
 # Return a list of tests to run. Each element of the list is itself a
 # list of two elements - the name of a permuations.test configuration
 # followed by the full path to a test script. i.e.:
 #
-#    {CONFIG FILENAME} {CONFIG FILENAME} ...
+#    {BUILD CONFIG FILENAME} {BUILD CONFIG FILENAME} ...
 #
 proc testset_patternlist {patternlist} {
 
+  set testset [list]              ;# return value
+
   set first [lindex $patternlist 0]
-  if {$first=="all"} { set first "full" }
 
   if {$first=="release"} {
+    set platform linux
 
-    # The following mirrors the set of test suites invoked by "all.test".
-    #
-    set clist {
-      full
-      no_optimization memsubsys1 memsubsys2 singlethread 
-      multithread onefile utf16 exclusive persistent_journal 
-      persistent_journal_error no_journal no_journal_error
-      autovacuum_ioerr no_mutex_try fullmutex journaltest 
-      inmemory_journal pcache0 pcache10 pcache50 pcache90 
-      pcache100 prepare mmap
-    }
-    ifcapable rbu { lappend clist rbu }
-    if {$::tcl_platform(platform)=="unix"} {
-      ifcapable !default_autovacuum {
-        lappend clist autovacuum_crash 
+    set patternlist [lrange $patternlist 1 end]
+    foreach b [trd_builds $platform] {
+      foreach c [trd_configs $platform $b] {
+        testset_append testset $b $c $patternlist
       }
     }
+
+    if {[llength $patternlist]==0} {
+      foreach b [trd_builds $platform] {
+        foreach e [trd_extras $platform $b] {
+          lappend testset [list $b make $e]
+        }
+      }
+    }
+
+  } elseif {$first=="all"} {
+
+    set clist [trd_all_configs]
     set patternlist [lrange $patternlist 1 end]
+    foreach c $clist {
+      testset_append testset Default $c $patternlist
+    }
 
   } elseif {[info exists ::testspec($first)]} {
     set clist $first
     set patternlist [lrange $patternlist 1 end]
+
+    testset_append testset Default $first [lrange $patternlist 1 end]
   } elseif { [llength $patternlist]==0 } {
-    set clist veryquick
+    testset_append testset Default veryquick $patternlist
   } else {
-    set clist full
+    testset_append testset Default full $patternlist
   }
 
-  set testset [list]
+  set testset
+}
 
-  foreach config $clist {
-    catch { array unset O }
-    array set O $::testspec($config)
-    foreach f $O(-files) {
-      if {[file pathtype $f]!="absolute"} {
-        set f [file join $::testdir $f]
-      }
-      lappend testset [list $config [file normalize $f]]
-    }
-  }
+proc testset_append {listvar build config patternlist} {
+  upvar $listvar lvar
 
-  if {[llength $patternlist]>0} {
-    foreach t $testset {
-      set tail [file tail [lindex $t 1]]
+  catch { array unset O }
+  array set O $::testspec($config)
+
+  foreach f $O(-files) {
+    if {[llength $patternlist]>0} {
+      set bMatch 0
       foreach p $patternlist {
-        if {[string match $p $tail]} {
-          lappend ret $t
-          break;
+        if {[string match $p [file tail $f]]} {
+          set bMatch
+          break
         }
       }
+      if {$bMatch==0} continue
     }
-  } else {
-    set ret $testset
-  }
 
-  set ret
+    if {[file pathtype $f]!="absolute"} {
+      set f [file join $::testdir $f]
+    }
+    lappend lvar [list $build $config $f]
+  }
 }
+
 #--------------------------------------------------------------------------
 
 
@@ -264,6 +425,57 @@ proc r_write_db {tcl} {
   db close
 }
 
+# Obtain a new job to be run by worker $iJob (an integer). A job is
+# returned as a three element list:
+#
+#    {$build $config $file}
+#
+proc r_get_next_job {iJob} {
+
+  switch -- [expr $iJob%3] {
+    0 {
+      set orderby "ORDER BY ismake ASC"
+    }
+    1 {
+      set orderby "ORDER BY slow DESC, ismake ASC"
+    }
+    2 {
+      set orderby "ORDER BY ismake DESC"
+    }
+  }
+
+  r_write_db {
+    set f ""
+    set c ""
+    db eval "
+      SELECT build, config, filename 
+        FROM script 
+        WHERE state='ready' 
+        $orderby LIMIT 1
+    " {
+      set b $build
+      set c $config
+      set f $filename
+    }
+    if {$f!=""} {
+      db eval { 
+        UPDATE script SET state='running'
+        WHERE (build, config, filename) = ($b, $c, $f)
+      }
+    }
+  }
+
+  if {$f==""} { return "" }
+  list $b $c $f
+}
+
+#rename r_get_next_job r_get_next_job_r
+#proc r_get_next_job {iJob} {
+  #puts [time { set res [r_get_next_job_r $iJob] }]
+  #set res
+#}
+
+
 proc make_new_testset {} {
   global R
 
@@ -271,435 +483,205 @@ proc make_new_testset {} {
   r_write_db {
     db eval $R(schema)
     foreach t $tests {
-      foreach {c s} $t {}
+      foreach {b c s} $t {}
       set slow 0
 
-      set fd [open $s]
-      for {set ii 0} {$ii<100 && ![eof $fd]} {incr ii} {
-        set line [gets $fd]
-        if {[string match -nocase *testrunner:* $line]} {
-          regexp -nocase {.*testrunner:(.*)} $line -> properties
-          foreach p $properties {
-            if {$p=="slow"} { set slow 1 }
+      if {$c!="make"} {
+        set fd [open $s]
+        for {set ii 0} {$ii<100 && ![eof $fd]} {incr ii} {
+          set line [gets $fd]
+          if {[string match -nocase *testrunner:* $line]} {
+            regexp -nocase {.*testrunner:(.*)} $line -> properties
+            foreach p $properties {
+              if {$p=="slow"} { set slow 1 }
+              if {$p=="superslow"} { set slow 2 }
+            }
           }
         }
+        close $fd
+      }
+
+      if {$c=="veryquick"} {
+        set c "default"
       }
-      close $fd
 
       db eval { 
-        INSERT INTO script(config, filename, slow, state) 
-            VALUES ($c, $s, $slow, 'ready') 
+        INSERT INTO script(build, config, filename, slow, state) 
+            VALUES ($b, $c, $s, $slow, 'ready') 
       }
     }
   }
 }
 
-# Find the next job in the database and mark it as 'running'. Then return
-# a list consisting of the 
-#
-#   CONFIG FILENAME
-#
-# pair for the test.
-#
-proc get_next_test {} {
+proc script_input_ready {fd iJob b c f} {
   global R
-  set myid $R(helper_id)
+  global O
+  global T
+
+  if {[eof $fd]} {
+    set ::done 1
+    fconfigure $fd -blocking 1
+    set state "done"
+    set rc [catch { close $fd } msg]
+    if {$rc} { 
+      puts "FAILED: $b $c $f"
+      set state "failed" 
+    }
+
+    set tm [expr [clock milliseconds] - $T($iJob)]
+
+    puts $R(log) "### $b ### $c ### $f ${tm}ms ($state)"
+    puts $R(log) [string trim $O($iJob)]
+
+    incr R(nHelperRunning) -1
+    r_write_db {
+      set output $O($iJob)
+      db eval {
+        UPDATE script SET output = $output, state=$state, time=$tm
+        WHERE (build, config, filename) = ($b, $c, $f)
+      }
+    }
+
+    launch_another_job $iJob
+    incr ::wakeup
+  } else {
+    set rc [catch { gets $fd line } res]
+    if {$rc} {
+      puts "ERROR $res"
+    } 
+    if {$res>=0} {
+      append O($iJob) "$line\n"
+    }
+  }
+
+}
+
+proc dirname {ii} {
+  return "testdir$ii"
+}
+
+proc launch_another_job {iJob} {
+  global R
+  global O
+  global T
+
+  set testfixture [info nameofexec]
+  set script $R(info_script)
+
+  set dir [dirname $iJob]
+  create_or_clear_dir $dir
+
+  set O($iJob) ""
+  set T($iJob) [clock milliseconds]
+  
+  set job [r_get_next_job $iJob]
+  if {$job==""} return
+
+  foreach {b c f} $job {}
+
+  if {$c=="make"} {
+    set builddir [build_to_dirname $b]
+    copy_dir $builddir $dir
+    set cmd "make $f"
+  } else {
+    if {$b=="Default"} {
+      set testfixture [info nameofexec]
+    } else {
+      set testfixture [
+        file normalize [file join [build_to_dirname $b] testfixture]
+      ]
+    }
+
+    if {$c=="valgrind"} {
+      set testfixture "valgrind -v --error-exitcode=1 $testfixture"
+      set ::env(OMIT_MISUSE) 1
+    }
+    set cmd [concat $testfixture [list $script helper $c $f]]
+  }
+
+  set pwd [pwd]
+  cd $dir
+  set fd [open "|$cmd 2>@1" r]
+  cd $pwd
+  set pid [pid $fd]
+
+  fconfigure $fd -blocking false
+  fileevent $fd readable [list script_input_ready $fd $iJob $b $c $f]
+  incr R(nHelperRunning) +1
+  unset -nocomplain ::env(OMIT_MISUSE)
+}
+
+proc one_line_report {} {
+  global R
+
+  set tm [expr [clock milliseconds] - $R(starttime)]
+  set tm [format "%.2f" [expr $tm/1000.0]]
+
+  foreach s {ready running done failed} {
+    set v($s,0) 0
+    set v($s,1) 0
+  }
+  set t(0) 0
+  set t(1) 0
 
   r_write_db {
-    set f ""
-    set c ""
     db eval {
-      SELECT config, filename FROM script WHERE state='ready' 
-      ORDER BY 
-        (slow * (($myid+1) % 2)) DESC, 
-        config!='full', 
-        config,
-        filename
-      LIMIT 1
+      SELECT state, ismake, count(*) AS cnt 
+      FROM script 
+      GROUP BY state, ismake
     } {
-      set c $config
-      set f $filename
+      set v($state,$ismake) $cnt
+      incr t($ismake) $cnt
     }
-    if {$f!=""} {
-      db eval { 
-        UPDATE script SET state='running', testfixtureid=$myid 
-        WHERE (config, filename) = ($c, $f)
+  }
+
+  set d0 [expr $v(done,0)+$v(failed,0)]
+  set d1 [expr $v(done,1)+$v(failed,1)]
+
+  puts "${tm}s: scripts: ($d0/$t(0)) $v(failed,0) failed, $v(running,0) running, makes: ($d1/$t(1)) $v(failed,1) failed, $v(running,1) running"
+
+  after 1000 one_line_report
+}
+
+proc run_testset {} {
+  global R
+  set ii 0
+
+  set R(starttime) [clock milliseconds]
+  set R(log) [open $R(logname) w]
+
+  for {set ii 0} {$ii<$R(nJob)} {incr ii} {
+    launch_another_job $ii
+  }
+
+  one_line_report
+  while {$R(nHelperRunning)>0} {
+    after 500 {incr ::wakeup}
+    vwait ::wakeup
+  }
+  close $R(log)
+  one_line_report
+
+  r_write_db {
+    set nErr [db one {SELECT count(*) FROM script WHERE state='failed'}]
+    if {$nErr>0} {
+      puts "$nErr failures:"
+      db eval {
+        SELECT build, config, filename FROM script WHERE state='failed'
+      } {
+        puts "FAILED: $build $config $filename"
       }
     }
   }
 
-  if {$f==""} { return "" }
-  list $c $f
-}
-
-proc r_testname {config filename} {
-  set name [file tail $filename]
-  if {$config!="" && $config!="full" && $config!="veryquick"} {
-    set name "$config-$name"
-  }
-  return $name
-}
-
-proc r_set_test_result {config filename ms nerr ntest output} {
-  global R
-
-  set f [r_testname $config $filename]
-  if {$nerr==0} {
-    set msg "$f... Ok"
-  } else {
-    set msg "$f... FAILED - $nerr errors of $ntest tests"
-  }
-  append msg " (${ms}ms)"
-  if {$R(helper)} {
-    append msg " (helper $R(helper_id))"
-  }
-
-  sqlite3_shutdown
-  set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]
-  set nByte   [sqlite3_memory_used]
-  if {($nByte>0 || $nMalloc>0) && $R(leaker)==""} {
-    set R(leaker) $f
-  }
-
-  r_write_db {
-    db eval {
-      UPDATE script 
-        SET state='done', output=$output, nerr=$nerr, ntest=$ntest, time=$ms
-      WHERE (config, filename)=($config, $filename);
-
-      INSERT INTO msg(msg) VALUES ($msg);
-    }
-  }
-}
-
-set R(iNextMsg) 1
-proc r_get_messages {{db ""}} {
-  global R
-
-  sqlite3_test_control_pending_byte 0x010000
-
-  if {$db==""} {
-    sqlite3 rgmhandle $R(dbname)
-    set dbhandle rgmhandle
-    $dbhandle timeout $R(timeout)
-  } else {
-    set dbhandle $db
-  }
-
-  $dbhandle transaction {
-    set next $R(iNextMsg)
-    set ret [$dbhandle eval {SELECT msg FROM msg WHERE id>=$next}]
-    set R(iNextMsg) [$dbhandle one {SELECT COALESCE(max(id), 0)+1 FROM msg}]
-  }
-
-  if {$db==""} {
-    rgmhandle close
-  }
-
-  set ret
-}
-
-# This is called after all tests have been run to write the leaked memory
-# report into the malloc table of testrunner.db.
-#
-proc r_memory_report {} {
-  global R
-
-  sqlite3_shutdown
-
-  set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]
-  set nByte   [sqlite3_memory_used]
-  set id $R(helper_id)
-  set leaker $R(leaker)
-
-  r_write_db {
-    db eval {
-      INSERT INTO malloc(id, nMalloc, nByte, leaker) 
-        VALUES($id, $nMalloc, $nByte, $leaker)
-    }
-  }
-}
-
-
-#--------------------------------------------------------------------------
-#
-set ::R_INSTALL_PUTS_WRAPPER {
-  proc puts_sts_wrapper {args} {
-    set n [llength $args]
-    if {$n==1 || ($n==2 && [string first [lindex $args 0] -nonewline]==0)} {
-      uplevel puts_into_caller $args
-    } else {
-      # A channel was explicitly specified.
-      uplevel puts_sts_original $args
-    }
-  }
-  rename puts puts_sts_original
-  proc puts {args} { uplevel puts_sts_wrapper $args }
-}
-
-proc r_install_puts_wrapper {} $::R_INSTALL_PUTS_WRAPPER
-proc r_uninstall_puts_wrapper {} {
-  rename puts ""
-  rename puts_sts_original puts
-}
-
-proc slave_test_script {script} {
-
-  # Create the interpreter used to run the test script.
-  interp create tinterp
-
-  # Populate some global variables that tester.tcl expects to see.
-  foreach {var value} [list              \
-    ::argv0 $::argv0                     \
-    ::argv  {}                           \
-    ::SLAVE 1                            \
-  ] {
-    interp eval tinterp [list set $var $value]
-  }
-
-  # The alias used to access the global test counters.
-  tinterp alias set_test_counter set_test_counter
-
-  # Set up an empty ::cmdlinearg array in the slave.
-  interp eval tinterp [list array set ::cmdlinearg [array get ::cmdlinearg]]
-
-  # Set up the ::G array in the slave.
-  interp eval tinterp [list array set ::G [array get ::G]]
-  interp eval tinterp [list set ::G(runner.tcl) 1]
-
-  interp eval tinterp $::R_INSTALL_PUTS_WRAPPER
-  tinterp alias puts_into_caller puts_into_caller
-
-  # Load the various test interfaces implemented in C.
-  load_testfixture_extensions tinterp
-
-  # Run the test script.
-  set rc [catch { interp eval tinterp $script } msg opt]
-  if {$rc} {
-    puts_into_caller $msg
-    puts_into_caller [dict get $opt -errorinfo]
-    incr ::TC(errors)
-  }
-
-  # Check if the interpreter call [run_thread_tests]
-  if { [interp eval tinterp {info exists ::run_thread_tests_called}] } {
-    set ::run_thread_tests_called 1
-  }
-
-  # Delete the interpreter used to run the test script.
-  interp delete tinterp
-}
-
-proc slave_test_file {zFile} {
-  set tail [file tail $zFile]
-
-  # Remember the value of the shared-cache setting. So that it is possible
-  # to check afterwards that it was not modified by the test script.
-  #
-  ifcapable shared_cache { set scs [sqlite3_enable_shared_cache] }
-
-  # Run the test script in a slave interpreter.
-  #
-  unset -nocomplain ::run_thread_tests_called
-  reset_prng_state
-  set ::sqlite_open_file_count 0
-  set time [time { slave_test_script [list source $zFile] }]
-  set ms [expr [lindex $time 0] / 1000]
-
-  r_install_puts_wrapper
-
-  # Test that all files opened by the test script were closed. Omit this
-  # if the test script has "thread" in its name. The open file counter
-  # is not thread-safe.
-  #
-  if {[info exists ::run_thread_tests_called]==0} {
-    do_test ${tail}-closeallfiles { expr {$::sqlite_open_file_count>0} } {0}
-  }
-  set ::sqlite_open_file_count 0
-
-  # Test that the global "shared-cache" setting was not altered by
-  # the test script.
-  #
-  ifcapable shared_cache {
-    set res [expr {[sqlite3_enable_shared_cache] == $scs}]
-    do_test ${tail}-sharedcachesetting [list set {} $res] 1
-  }
-
-  # Add some info to the output.
-  #
-  output2 "Time: $tail $ms ms"
-  show_memstats
-
-  r_uninstall_puts_wrapper
-  return $ms
-}
-
-proc puts_into_caller {args} {
-  global R
-  if {[llength $args]==1} {
-    append R(output) [lindex $args 0]
-    append R(output) "\n"
-  } else {
-    append R(output) [lindex $args 1]
-  }
-}
-
-#-------------------------------------------------------------------------
-#
-proc r_final_report {} {
-  global R
-
-  sqlite3_test_control_pending_byte 0x010000
-  sqlite3 db $R(dbname)
-
-  db timeout $R(timeout)
-
-  set errcode 0
-
-  # Create the text log file. This is just the concatenation of the 
-  # 'output' column of the database for every script that was run.
-  set fd [open $R(logname) w]
-  db eval {SELECT output FROM script ORDER BY config!='full',config,filename} {
-    puts $fd $output
-  }
-  close $fd
-
-  # Check if any scripts reported errors. If so, print one line noting
-  # how many errors, and another identifying the scripts in which they
-  # occured. Or, if no errors occurred, print out "no errors at all!".
-  sqlite3 db $R(dbname)
-  db timeout $R(timeout)
-  db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { }
-  puts "$nerr errors from $ntest tests."
-  if {$nerr>0} {
-    db eval { SELECT config, filename FROM script WHERE nerr>0 } {
-      lappend errlist [r_testname $config $filename]
-    }
-    puts "Errors in: $errlist"
-    set errcode 1
-  }
-
-  # Check if any scripts were not run or did not finish. Print out a
-  # line identifying them if there are any. 
-  set errlist [list]
-  db eval { SELECT config, filename FROM script WHERE state!='done' } {
-    lappend errlist [r_testname $config $filename]
-  }
-  if {$errlist!=[list]} {
-    puts "Tests DID NOT FINISH (crashed?): $errlist"
-    set errcode 1
-  }
-
-  set bLeak 0
-  db eval {
-    SELECT id, nmalloc, nbyte, leaker FROM malloc 
-      WHERE nmalloc>0 OR nbyte>0
-  } {
-    if {$id==0} { 
-      set line "This process " 
-    } else {
-      set line "Helper $id "
-    }
-    append line "leaked $nbyte byte in $nmalloc allocations"
-    if {$leaker!=""} { append line " (perhaps in [file tail $leaker])" }
-    puts $line
-    set bLeak 1
-  }
-  if {$bLeak==0} {
-    puts "No leaks - all allocations freed."
-  }
-
-  db close
-
   puts "Test database is $R(dbname)"
-  puts "Test log file is $R(logname)"
-  if {$errcode} {
-    puts "This test has FAILED."
-  }
-  return $errcode
-}
-
-
-if {$R(helper)==0} {
-  make_new_testset
+  puts "Test log is $R(logname)"
 }
 
 set R(nHelperRunning) 0
-if {$R(helper)==0 && $R(nJob)>1} {
-  cd $cmdlinearg(TESTFIXTURE_HOME)
-  for {set ii 1} {$ii <= $R(nJob)} {incr ii} {
-    set cmd "[info nameofexec] $R(info_script) helper $ii 2>@1"
-    puts "Launching helper $ii ($cmd)"
-    set chan [open "|$cmd" r]
-    fconfigure $chan -blocking false
-    fileevent $chan readable [list r_helper_readable $ii $chan]
-    incr R(nHelperRunning) 
-  }
-  cd $cmdlinearg(testdir)
-}
+set tm [lindex [time { make_new_testset }] 0]
+puts "built testset in [expr $tm/1000]ms.."
 
-proc r_helper_readable {id chan} {
-  set data [gets $chan]
-  if {$data!=""} { puts "helper $id:$data" }
-  if {[eof $chan]} {
-    puts "helper $id is finished"
-    incr ::R(nHelperRunning) -1
-    close $chan
-  }
-}
-
-if {$R(nHelperRunning)==0} {
-  while { ""!=[set t [get_next_test]] } {
-    set R(output) ""
-    set TC(count) 0
-    set TC(errors) 0
-
-    foreach {config filename} $t {}
-
-    array set O $::testspec($config)
-    set ::G(perm:name)         $config
-    set ::G(perm:prefix)       $O(-prefix)
-    set ::G(isquick)           1
-    set ::G(perm:dbconfig)     $O(-dbconfig)
-    set ::G(perm:presql)       $O(-presql)
-
-    eval $O(-initialize)
-    set ms [slave_test_file $filename]
-    eval $O(-shutdown)
-
-    unset -nocomplain ::G(perm:sqlite3_args)
-    unset ::G(perm:name)
-    unset ::G(perm:prefix)
-    unset ::G(perm:dbconfig)
-    unset ::G(perm:presql)
-
-    r_set_test_result $config $filename $ms $TC(errors) $TC(count) $R(output)
-  
-    if {$R(helper)==0} {
-      foreach msg [r_get_messages] { puts $msg }
-    }
-  }
-
-  # Tests are finished - write a record into testrunner.db describing 
-  # any memory leaks. 
-  r_memory_report
-
-} else {
-  set TTT 0
-  sqlite3 db $R(dbname)
-  db timeout $R(timeout)
-  while {$R(nHelperRunning)>0} {
-    after 250 { incr TTT }
-    vwait TTT
-    foreach msg [r_get_messages db] { puts $msg }
-  }
-  db close
-}
-
-set errcode 0
-if {$R(helper)==0} {
-  set errcode [r_final_report]
-}
-
-exit $errcode
+run_testset
+#puts [pwd]
 
diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl
new file mode 100644
index 0000000000..f843680cdc
--- /dev/null
+++ b/test/testrunner_data.tcl
@@ -0,0 +1,123 @@
+
+
+
+namespace eval trd {
+  variable tcltest
+  variable extra
+  variable all_configs
+
+
+  # Tcl tests to run for various builds.
+  #
+  set tcltest(linux.Fast-One)             veryquick
+  set tcltest(linux.Debug-One)            veryquick
+  set tcltest(linux.Debug-Two)            veryquick
+  set tcltest(linux.Have-Not)             veryquick
+  set tcltest(linux.Secure-Delete)        veryquick
+  set tcltest(linux.Unlock-Notify)        veryquick
+  set tcltest(linux.User-Auth)            veryquick
+  set tcltest(linux.Update-Delete-Limit)  veryquick
+  set tcltest(linux.Extra-Robustness)     veryquick
+  set tcltest(linux.Device-Two)           veryquick
+  set tcltest(linux.No-lookaside)         veryquick
+  set tcltest(linux.Devkit)               veryquick
+  set tcltest(linux.Apple)                veryquick
+  set tcltest(linux.Sanitize)             veryquick
+  set tcltest(linux.Device-One)           all
+  set tcltest(linux.Default)              all_plus_autovacuum_crash
+  set tcltest(linux.Valgrind)             valgrind
+
+  # Extra [make xyz] tests that should be run for various builds.
+  #
+  set extra(linux.Check-Symbols)          checksymbols
+  set extra(linux.Fast-One)               {fuzztest sourcetest}
+  set extra(linux.Debug-One)              {fuzztest sourcetest mptest}
+  set extra(linux.Debug-Two)              {fuzztest sourcetest}
+  set extra(linux.Have-Not)               {fuzztest sourcetest}
+  set extra(linux.Secure-Delete)          {fuzztest sourcetest}
+  set extra(linux.Unlock-Notify)          {fuzztest sourcetest}
+  set extra(linux.Update-Delete-Limit)    {fuzztest sourcetest}
+  set extra(linux.Extra-Robustness)       {fuzztest sourcetest}
+  set extra(linux.Device-Two)             {fuzztest sourcetest threadtest}
+  set extra(linux.No-lookaside)           {fuzztest sourcetest}
+  set extra(linux.Devkit)                 {fuzztest sourcetest}
+  set extra(linux.Apple)                  {fuzztest sourcetest}
+  set extra(linux.Sanitize)               {fuzztest sourcetest}
+  set extra(linux.Default)                {fuzztest sourcetest threadtest}
+
+  # The following mirrors the set of test suites invoked by "all.test".
+  #
+  set all_configs {
+    full no_optimization memsubsys1 memsubsys2 singlethread 
+    multithread onefile utf16 exclusive persistent_journal 
+    persistent_journal_error no_journal no_journal_error
+    autovacuum_ioerr no_mutex_try fullmutex journaltest 
+    inmemory_journal pcache0 pcache10 pcache50 pcache90 
+    pcache100 prepare mmap
+  }
+}
+
+
+#-------------------------------------------------------------------------
+proc trd_import {} {
+  uplevel {
+    variable ::trd::tcltest
+    variable ::trd::extra
+    variable ::trd::all_configs
+  }
+}
+
+proc trd_builds {platform} {
+  trd_import
+
+  set klist [lsort -uniq [concat \
+      [array names tcltest ${platform}.*] \
+      [array names extra ${platform}.*]   \
+  ]]
+  if {[llength $klist]==0} {
+    error "no such platform: $platform"
+  }
+
+  set ret ""
+  foreach k $klist {
+    foreach {p c} [split $k "."] {}
+    lappend ret $c
+  }
+  set ret
+}
+
+proc trd_configs {platform build} {
+  trd_import
+
+  set clist [list]
+
+  if {[info exists tcltest($platform.$build)]} {
+    set clist $tcltest($platform.$build)
+    if {$clist=="all"} {
+      set clist $all_configs
+    } elseif {$clist=="all_plus_autovacuum_crash"} {
+      set clist [concat $all_configs autovacuum_crash]
+    }
+  }
+
+  set clist
+}
+
+proc trd_extras {platform build} {
+  trd_import
+
+  set elist [list]
+  if {[info exists extra($platform.$build)]} {
+    set elist $extra($platform.$build)
+  }
+
+  set elist
+}
+
+proc trd_all_configs {} {
+  trd_import
+  set all_configs
+}
+
+
+
diff --git a/test/vt02.c b/test/vt02.c
index 40bd6fcb3e..f83fc9af95 100644
--- a/test/vt02.c
+++ b/test/vt02.c
@@ -177,6 +177,8 @@
   #include <assert.h>
 #endif
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
 /* Forward declarations */
 typedef struct vt02_vtab vt02_vtab;
 typedef struct vt02_cur vt02_cur;
@@ -999,6 +1001,10 @@ static void vt02CoreInit(sqlite3 *db){
   sqlite3_create_module(db, "vt02pkabcd", &vt02Module, (void*)zPkABCDSchema);
 }
 
+#else
+# define vt02CoreInit(db)
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
 #ifdef TH3_VERSION
 static void vt02_init(th3state *p, int iDb, char *zArg){
   vt02CoreInit(th3dbPointer(p, iDb));
diff --git a/test/windowB.test b/test/windowB.test
index 2548165956..1c7234ce42 100644
--- a/test/windowB.test
+++ b/test/windowB.test
@@ -411,6 +411,7 @@ do_execsql_test 10.3 {
 }
 
 ifcapable json1 {
+if {[permutation]!="no_optimization"} {
 
   do_execsql_test 11.0 {
     SELECT value FROM json_each('[1,2,3,4,5]');
@@ -479,7 +480,7 @@ ifcapable json1 {
       |  `--USE TEMP B-TREE FOR ORDER BY
       `--SCAN (subquery-xxxxxx)
   }
-}
+}}
 
 finish_test