Changes to the way tcl test scripts work. No changes to production code.

FossilOrigin-Name: 2c5e48a4855e2ec4572697c8fcfe4433e302ef33

Makefile.vxworks

@@ -625,7 +625,7 @@ fulltest:	testfixture$(EXE) sqlite3$(EXE)
 	./testfixture$(EXE) $(TOP)/test/all.test
 
 soaktest:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/all.test -soak 1
+	./testfixture$(EXE) $(TOP)/test/all.test -soak=1
 
 test:	testfixture$(EXE) sqlite3$(EXE)
 	./testfixture$(EXE) $(TOP)/test/veryquick.test

main.mk

@@ -494,7 +494,7 @@ fulltest:	testfixture$(EXE) sqlite3$(EXE)
 	./testfixture$(EXE) $(TOP)/test/all.test
 
 soaktest:	testfixture$(EXE) sqlite3$(EXE)
-	./testfixture$(EXE) $(TOP)/test/all.test -soak 1
+	./testfixture$(EXE) $(TOP)/test/all.test -soak=1
 
 test:	testfixture$(EXE) sqlite3$(EXE)
 	./testfixture$(EXE) $(TOP)/test/veryquick.test

manifest

@@ -1,9 +1,9 @@
-C Add\sthe\snew\sxShmMap\s(formerly\sxShmPage)\sto\sos_win.c.
+C Changes\sto\sthe\sway\stcl\stest\sscripts\swork.\sNo\schanges\sto\sproduction\scode.
-D 2010-06-14T16:16:34
+D 2010-06-15T06:56:37
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
-F Makefile.vxworks accd27bfc55cfe3aa48a58c7b9cc9c5fb225a6de
+F Makefile.vxworks 4314cde20a1d9460ec5083526ea975442306ae7e
 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6
 F VERSION 4dce4379514b12d6bc5c30f7d1f64582ccb4f412
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -93,7 +93,7 @@ F ext/rtree/tkt3363.test 2bf324f7908084a5f463de3109db9c6e607feb1b
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
-F main.mk b39182f15fd980b86aceb7e4b9a1ba88abd75e2d
+F main.mk 1e576354113193ab68af6b06dad9e8a450f522ea
 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -232,7 +232,7 @@ F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
-F test/all.test 1f527d5ecd6614eeba91a02a24ab2baf5b5a1218
+F test/all.test 6745008c144bd2956d58864d21f7b304689c1cce
 F test/alter.test 15f9224868b290d6bf7a63f31437f31aee070636
 F test/alter2.test 0266160d61b0f7156b7e7a9905dbf85ebe067c63
 F test/alter3.test 25b95a136708f22b87184fa6a4309eea03d65153
@@ -537,13 +537,13 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
 F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58
 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
-F test/permutations.test 64fbafa685149be54a1ceb545942911f998c604d
+F test/permutations.test 28f46dfcc880e15ec7ab1ff36d12af765a0e3bc4
 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a
 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
-F test/quick.test f84a9aa4dbb1d208e8299430fae6dccb335cb41d
+F test/quick.test d2a697cf6d6db2725649670859365ef601289f96
 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6
 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459
 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a
@@ -608,7 +608,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3
 F test/tempdb.test 800c36623d67a2ad1f58784b9c5644e0405af6e6
 F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a
 F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05
-F test/tester.tcl 2e19f173752b743526c61e0ee9b84c0db05d46b0
+F test/tester.tcl e474c9aed346a905191afd7cbf4484948ab13a2a
 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f
 F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db
 F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca
@@ -745,7 +745,7 @@ F test/vacuum2.test ec57f21d394b7b72249b11f8e4b5d487bab56539
 F test/vacuum3.test f39ad1428347c5808cd2da7578c470f186a4d0ce
 F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9
 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102
-F test/veryquick.test e265401afefa994cdf2fe4b6f286b1e87c2f9b9d
+F test/veryquick.test 7701bb609fe8bf6535514e8b849a309e8f00573b
 F test/view.test 45f518205ecdb6dd23a86dd4a99bb4ae945e625d
 F test/vtab1.test 9bc4a349a1989bcd064eb3b8fac2f06aca64297a
 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d
@@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P fc0cabc15c97dde6a852b4f07df6d30f1d2c04bc
+P 13e7a8242206bca4b5bf356ef074e66474d39609
-R 25e35232d2e784a0633c25f463199a04
+R 61ffe7c12e775783a8deaa48b0886849
 U dan
-Z 06f3bcfd0aba7d97df7e9b4406c194ab
+Z 3e8efe7ef023bbc1e7120ae3f52832da

manifest.uuid

@@ -1 +1 @@
-13e7a8242206bca4b5bf356ef074e66474d39609
+2c5e48a4855e2ec4572697c8fcfe4433e302ef33

test/all.test

@@ -10,92 +10,40 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: all.test,v 1.62 2009/01/06 18:43:51 drh Exp $
 
 set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-db close
-
-if {[file exists ./sqlite_test_count]} {
-  set COUNT [exec cat ./sqlite_test_count]
-} else {
-  set COUNT 1
-}
-
-if {[llength $argv]>0} {
-  foreach {name value} $argv {
-    switch -- $name {
-      -count {
-         set COUNT $value
-      }
-      -quick {
-         set G(isquick) $value
-      }
-      -soak {
-         set G(issoak) $value
-      }
-      default {
-         puts stderr "Unknown option: $name"
-         exit
-      }
-    }
-  }
-}
-set argv {}
-
-set EXCLUDE {}
-lappend EXCLUDE all.test               ;# This file
-lappend EXCLUDE async.test
-lappend EXCLUDE crash.test             ;# Run separately later.
-lappend EXCLUDE crash2.test            ;# Run separately later.
-lappend EXCLUDE quick.test             ;# Alternate test driver script
-lappend EXCLUDE veryquick.test         ;# Alternate test driver script
-lappend EXCLUDE misuse.test            ;# Run separately later.
-lappend EXCLUDE memleak.test           ;# Alternate test driver script
-lappend EXCLUDE permutations.test      ;# Run separately later.
-lappend EXCLUDE soak.test              ;# Takes a very long time (default 1 hr)
-lappend EXCLUDE fts3.test              ;# Wrapper for muliple fts3*.tests
-lappend EXCLUDE mallocAll.test         ;# Wrapper for running all malloc tests
-
-# Files to include in the test.  If this list is empty then everything
-# that is not in the EXCLUDE list is run.
-#
-set INCLUDE {
-}
-
-for {set Counter 0} {$Counter<$COUNT} {incr Counter} {
-  foreach testfile [lsort -dictionary [glob $testdir/*.test]] {
-    set tail [file tail $testfile]
-    if {[lsearch -exact $EXCLUDE $tail]>=0} continue
-    if {[llength $INCLUDE]>0 && [lsearch -exact $INCLUDE $tail]<0} continue
-    reset_prng_state
-    slave_test_file $testfile
-  }
-
-  if {[set_test_counter errors]} break
-}
-set argv all
 source $testdir/permutations.test
-set argv ""
 
-# Run the crashtest only on unix and only once. If the library does not
+run_test_suite full
-# always create auto-vacuum databases, also run autovacuum_crash.test.
+
-#
+run_test_suite memsubsys1 
+run_test_suite memsubsys2 
+run_test_suite singlethread 
+run_test_suite multithread 
+run_test_suite onefile 
+run_test_suite utf16 
+run_test_suite exclusive
+run_test_suite persistent_journal 
+run_test_suite persistent_journal_error 
+run_test_suite no_journal 
+run_test_suite no_journal_error
+run_test_suite autovacuum_ioerr 
+run_test_suite no_mutex_try 
+run_test_suite fullmutex 
+run_test_suite journaltest 
+run_test_suite inmemory_journal
+run_test_suite pcache0 
+run_test_suite pcache10 
+run_test_suite pcache50 
+run_test_suite pcache90 
+run_test_suite pcache100
+
 if {$::tcl_platform(platform)=="unix"} {
-  slave_test_file [file join $testdir crash.test]
-  slave_test_file [file join $testdir crash2.test]
-
   ifcapable !default_autovacuum {
-    set argv autovacuum_crash
+    run_test_suite autovacuum_crash
-    source $testdir/permutations.test
-    set argv ""
   }
 }
 
-# Run the misuse test after memory leak detection. It may leak memory. 
-# Currently, misuse.test also leaks a handful of file descriptors. This is 
-# not considered a problem.
-#
-slave_test_file [file join $testdir misuse.test]
 finish_test
 
+

test/permutations.test

@@ -15,167 +15,168 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 db close
 
-# Argument processing.
+#-------------------------------------------------------------------------
-#
+# test_suite NAME OPTIONS
-#puts "PERM-DEBUG: argv=$argv"
-namespace eval ::perm {
-  variable testmode [lindex $::argv 0]
-  variable testfile [lindex $::argv 1]
-}
-set argv [lrange $argv 2 end]
-#puts "PERM-DEBUG: testmode=$::perm::testmode tstfile=$::perm::testfile"
-
-if {$::perm::testmode eq "veryquick"} {
-  set ::perm::testmode [list persistent_journal no_journal]
-  set G(isquick) 1
-}
-if {$::perm::testmode eq "quick"} {
-  set ::perm::testmode [list persistent_journal no_journal autovacuum_ioerr]
-  set G(isquick) 1
-}
-if {$::perm::testmode eq "all" || $::perm::testmode eq ""} {
-  set ::perm::testmode {
-    memsubsys1 memsubsys2 singlethread multithread onefile utf16 exclusive
-    persistent_journal persistent_journal_error no_journal no_journal_error
-    autovacuum_ioerr no_mutex_try fullmutex journaltest inmemory_journal
-    pcache0 pcache10 pcache50 pcache90 pcache100
-  }
-}
-if {$::perm::testmode eq "targets"} { 
-  puts ""
-  puts -nonewline "veryquick            "
-  puts "Same as persistent_journal and no_journal"
-  puts -nonewline "quick                "
-  puts "Same as persistent_journal, no_journal and autovacuum_ioerr"
-  puts -nonewline "all                  "
-  puts "Everything except autovacuum_crash"
-}
-#puts "PERM-DEBUG: testmode=$::perm::testmode"
-
-set EXCLUDE {
-  all.test                  in2.test                  onefile.test
-  async2.test               incrvacuum_ioerr.test     permutations.test
-  async.test                jrnlmode2.test            quick.test
-  autovacuum_crash.test     jrnlmode3.test            shared_err.test
-  autovacuum_ioerr.test     jrnlmode4.test            soak.test
-  btree8.test               loadext.test              speed1p.test
-  corrupt.test              malloc2.test              speed1.test
-  crash2.test               malloc3.test              speed2.test
-  crash3.test               malloc4.test              speed3.test
-  crash4.test               mallocAll.test            speed4p.test
-  crash6.test               malloc.test               speed4.test
-  crash7.test               memleak.test              sqllimits1.test
-  crash.test                memsubsys1.test           thread001.test
-  exclusive3.test           memsubsys2.test           thread002.test
-  fts3.test                 misc7.test                utf16.test
-  fuzz_malloc.test          misuse.test               veryquick.test
-  fuzz.test                 mutex2.test               vtab_err.test
-  lookaside.test            fuzz3.test                savepoint4.test
-}
-set ALLTESTS [list]
-foreach filename [glob $testdir/*.test] {
-  set filename [file tail $filename]
-  if {[lsearch $EXCLUDE $filename] < 0} { lappend ALLTESTS $filename }
-}
-set ALLTESTS [lsort $ALLTESTS]
-set WALTESTS [list]
-foreach filename $ALLTESTS {
-  if {[regexp {^wal} $filename]} {lappend WALTESTS $filename}
-}
-
-# run_tests NAME OPTIONS
 #
 # where available options are:  
 #
 #       -description TITLE                  (default "")
 #       -initialize  SCRIPT                 (default "")
 #       -shutdown    SCRIPT                 (default "")
-#       -include     LIST-OF-FILES          (default $::ALLTESTS)
-#       -exclude     LIST-OF-FILES          (default "")
 #       -presql      SQL                    (default "")
-#       -options     LIST                   (default "")
+#       -files       LIST-OF-FILES          (default $::ALLTESTS)
 #
-proc run_tests {name args} {
+#
+proc test_suite {name args} {
 
-  set default(-shutdown) ""
+  set default(-shutdown)    ""
-  set default(-initialize) ""
+  set default(-initialize)  ""
-  set default(-exclude) ""
+  set default(-presql)      ""
-  set default(-include) $::ALLTESTS
-  set default(-presql) ""
   set default(-description) "no description supplied (fixme)"
-  set default(-options) ""
+  set default(-files)     ""
 
   array set options [array get default]
-  array set options $args
+  if {[llength $args]%2} {
-  foreach k [array names options] {
+    error "uneven number of options/switches passed to test_suite"
-    if {[info exists default($k)]==0} {
-      error "Unknown option: $k"
-    }
   }
-  #puts "PERM-DEBUG: name=$name testfile=$::perm::testfile"
+  foreach {k v} $args {
-  #puts "PERM-DEBUG: [array get options]"
+    set o [array names options ${k}*]
-
+    if {[llength $o]>1}  { error "ambiguous option: $k" }
-  if {$::perm::testmode eq "targets"} {
+    if {[llength $o]==0} { error "unknown option: $k" }
-    puts [format "% -20s %s" $name [string trim $options(-description)]]
+    set options([lindex $o 0]) $v
-    return
-  }
-  if {$::perm::testmode ne "" && [lsearch $::perm::testmode $name]<0} {
-    puts "skipping permutation test $name..."
-    return
   }
 
+  set     ::testspec($name) [array get options]
+  lappend ::testsuitelist $name
+}
 
-  set ::G(perm:name)         $name
+#-------------------------------------------------------------------------
-  set ::G(perm:presql)       $options(-presql)
+# test_set ARGS...
-  set ::G(perm:sqlite3_args) $options(-options)
+#
-
+proc test_set {args} {
-  uplevel $options(-initialize)
+  set isExclude 0
-
+  foreach a $args {
-  foreach file [lsort $options(-include)] {
+    if {[string match -* $a]} {
-    if {[lsearch $options(-exclude) $file] < 0 &&
+      switch -- $a {
-       ( $::perm::testfile eq "" ||
+        -include { set isExclude 0 }
-         $::perm::testfile eq $file ||
+        -exclude { set isExclude 1 }
-        "$::perm::testfile.test" eq $file )
+        default {
-    } {
+          error "Unknown switch: $a"
-      set ::perm::shared_cache_setting [shared_cache_setting]
+        }
-
-      slave_test_file $::testdir/$file
-
-      if {$::perm::shared_cache_setting ne [shared_cache_setting]} {
-        error "File $::testdir/$file changed the shared cache setting from $::perm::shared_cache_setting to [shared_cache_setting]"
       }
+    } elseif {$isExclude == 0} {
+      foreach f $a { set t($f) 1 }
     } else {
-      # puts "skipping file $file"
+      foreach f $a { array unset t $f }
     }
   }
 
-
+  return [array names t]
-  uplevel $options(-shutdown)
-
-  unset ::G(perm:name)
-  unset ::G(perm:presql)
-  unset ::G(perm:sqlite3_args)
 }
 
-proc shared_cache_setting {} {
+#-------------------------------------------------------------------------
-  set ret 0
+# Set up the following global list variables containing the names of 
-  catch {
+# various test scripts:
-    set ret [sqlite3_enable_shared_cache]
+#
-  }
+#   $alltests
-  return $ret
+#   $slowtests
+#
+
+set alltests [list]
+foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
+if {$::tcl_platform(platform)!="unix"} {
+  set alltests [test_set $alltests -exclude crash.test crash2.test]
 }
+set alltests [test_set $alltests -exclude {
+  all.test        async.test         quick.test  veryquick.test
+  memleak.test    permutations.test  soak.test   fts3.test
+  mallocAll.test
+}]
+
+set allquicktests [test_set $alltests -exclude {
+  async2.test async3.test backup_ioerr.test corrupt.test
+  corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test
+  crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test
+  fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test
+  misc7.test mutex2.test notify2.test onefile.test 
+  savepoint4.test savepoint6.test select9.test 
+  speed1.test speed1p.test speed2.test speed3.test speed4.test 
+  speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
+  thread003.test thread004.test thread005.test trans2.test vacuum3.test 
+  incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
+  vtab_err.test walslow.test walcrash.test 
+  walthread.test
+}]
 
 #############################################################################
 # Start of tests
+#
+
+#-------------------------------------------------------------------------
+# Define the generic test suites:
+#
+#   veryquick
+#   quick
+#   full
+#   veryquick_plus_notify2
+#
+
+test_suite "veryquick" -description {
+  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
+  This test suite is the same as the "quick" tests, except that some files
+  that test malloc and IO errors are omitted.
+} -files [
+  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
+]
+
+test_suite "quick" -description {
+  Quick test suite. Runs in around 10 minutes on a workstation.
+} -files [
+  test_set $allquicktests
+]
+
+test_suite "veryquick_plus_notify2" -description {
+  Very quick test suite + file notify2.test. This is used by one of the 
+  sqlite.org release test configurations.
+} -files [ 
+  test_set $allquicktests notify2.test -exclude *malloc* *ioerr* *fault*
+]
+
+test_suite "full" -description {
+  Full test suite. Takes a long time.
+} -files [ 
+  test_set $alltests 
+] -initialize {
+  unset -nocomplain ::G(isquick)
+}
+
+#-------------------------------------------------------------------------
+# Define the coverage related test suites:
+#
+#   coverage-wal
+#   coverage-fkey
+#
+test_suite "coverage-wal" -description {
+  Coverage tests for file wal.c.
+} -files {
+  wal.test       wal2.test      wal3.test      walmode.test    
+  walbak.test    walhook.test  walcrash2.test  walcksum.test
+  walfault.test
+} 
+
+
+
+#-------------------------------------------------------------------------
+# Define the permutation test suites:
+#
 
 # Run some tests using pre-allocated page and scratch blocks.
 #
-run_tests "memsubsys1" -description {
+test_suite "memsubsys1" -description {
   Tests using pre-allocated page and scratch blocks
-} -exclude {
+} -files [
-  ioerr5.test
+  test_set $::allquicktests -exclude ioerr5.test malloc5.test
-  malloc5.test
+] -initialize {
-} -initialize {
   catch {db close}
   sqlite3_shutdown
   sqlite3_config_pagecache 4096 24
@@ -198,12 +199,11 @@ run_tests "memsubsys1" -description {
 # sqlite3_soft_heap_limit() and sqlite3_release_memory() functionality.
 # This functionality is disabled if a pre-allocated page block is provided.
 #
-run_tests "memsubsys2" -description {
+test_suite "memsubsys2" -description {
   Tests using small pre-allocated page and scratch blocks
-} -exclude {
+} -files [
-  ioerr5.test
+  test_set $::allquicktests -exclude ioerr5.test malloc5.test
-  malloc5.test
+] -initialize {
-} -initialize {
   catch {db close}
   sqlite3_shutdown
   sqlite3_config_pagecache 512 5
@@ -221,7 +221,7 @@ run_tests "memsubsys2" -description {
 
 # Run all tests with the lookaside allocator disabled.
 #
-run_tests "nolookaside" -description {
+test_suite "nolookaside" -description {
   OOM tests with lookaside disabled
 } -initialize {
   catch {db close}
@@ -235,11 +235,11 @@ run_tests "nolookaside" -description {
   sqlite3_config_lookaside 100 500
   sqlite3_initialize
   autoinstall_test_functions
-}
+} -files $::allquicktests
 
 # Run some tests in SQLITE_CONFIG_SINGLETHREAD mode.
 #
-run_tests "singlethread" -description {
+test_suite "singlethread" -description {
   Tests run in SQLITE_CONFIG_SINGLETHREAD mode
 } -initialize {
   catch {db close}
@@ -247,7 +247,7 @@ run_tests "singlethread" -description {
   catch {sqlite3_config singlethread}
   sqlite3_initialize
   autoinstall_test_functions
-} -include {
+} -files {
   delete.test   delete2.test  insert.test  rollback.test  select1.test
   select2.test  trans.test    update.test  vacuum.test    types.test
   types2.test   types3.test
@@ -259,7 +259,7 @@ run_tests "singlethread" -description {
   autoinstall_test_functions
 }
 
-run_tests "nomutex" -description {
+test_suite "nomutex" -description {
   Tests run with the SQLITE_OPEN_MULTITHREADED flag passed to sqlite3_open().
 } -initialize {
   rename sqlite3 sqlite3_nomutex
@@ -269,7 +269,7 @@ run_tests "nomutex" -description {
     }
     uplevel [concat sqlite3_nomutex $args]
   }
-} -include {
+} -files {
   delete.test   delete2.test  insert.test  rollback.test  select1.test
   select2.test  trans.test    update.test  vacuum.test    types.test
   types2.test   types3.test
@@ -280,7 +280,7 @@ run_tests "nomutex" -description {
 
 # Run some tests in SQLITE_CONFIG_MULTITHREAD mode.
 #
-run_tests "multithread" -description {
+test_suite "multithread" -description {
   Tests run in SQLITE_CONFIG_MULTITHREAD mode
 } -initialize {
   catch {db close}
@@ -288,7 +288,7 @@ run_tests "multithread" -description {
   catch {sqlite3_config multithread}
   sqlite3_initialize
   autoinstall_test_functions
-} -include {
+} -files {
   delete.test   delete2.test  insert.test  rollback.test  select1.test
   select2.test  trans.test    update.test  vacuum.test    types.test
   types2.test   types3.test
@@ -302,7 +302,7 @@ run_tests "multithread" -description {
 
 # Run some tests in SQLITE_OPEN_FULLMUTEX mode.
 #
-run_tests "fullmutex" -description {
+test_suite "fullmutex" -description {
   Tests run in SQLITE_OPEN_FULLMUTEX mode
 } -initialize {
   rename sqlite3 sqlite3_fullmutex
@@ -312,7 +312,7 @@ run_tests "fullmutex" -description {
     }
     uplevel [concat sqlite3_fullmutex $args]
   }
-} -include {
+} -files {
   delete.test   delete2.test  insert.test  rollback.test  select1.test
   select2.test  trans.test    update.test  vacuum.test    types.test
   types2.test   types3.test
@@ -323,7 +323,7 @@ run_tests "fullmutex" -description {
 
 # Run some tests using the "onefile" demo.
 #
-run_tests "onefile" -description {
+test_suite "onefile" -description {
   Run some tests using the "test_onefile.c" demo
 } -initialize {
   rename sqlite3 sqlite3_onefile
@@ -333,7 +333,7 @@ run_tests "onefile" -description {
     }
     uplevel [concat sqlite3_onefile $args]
   }
-} -include {
+} -files {
   conflict.test  insert.test   insert2.test  insert3.test
   rollback.test  select1.test  select2.test  select3.test
 } -shutdown {
@@ -343,11 +343,11 @@ run_tests "onefile" -description {
 
 # Run some tests using UTF-16 databases.
 #
-run_tests "utf16" -description {
+test_suite "utf16" -description {
   Run tests using UTF-16 databases
 } -presql {
   pragma encoding = 'UTF-16'
-} -include {
+} -files {
     alter.test alter3.test
     auth.test bind.test blob.test capi2.test capi3.test collate1.test
     collate2.test collate3.test collate4.test collate5.test collate6.test
@@ -366,45 +366,45 @@ run_tests "utf16" -description {
 
 # Run some tests in exclusive locking mode.
 #
-run_tests "exclusive" -description {
+test_suite "exclusive" -description {
   Run tests in exclusive locking mode.
 } -presql {
   pragma locking_mode = 'exclusive'
-} -include {
+} -files {
   rollback.test select1.test select2.test 
   malloc.test ioerr.test
 } 
 
 # Run some tests in exclusive locking mode with truncated journals.
 #
-run_tests "exclusive-truncate" -description {
+test_suite "exclusive-truncate" -description {
   Run tests in exclusive locking mode and truncate journal mode.
 } -presql {
   pragma locking_mode = 'exclusive';
   pragma journal_mode = TRUNCATE;
-} -include {
+} -files {
   delete.test delete2.test insert.test rollback.test select1.test
   select2.test update.test malloc.test ioerr.test
 } 
 
 # Run some tests in persistent journal mode.
 #
-run_tests "persistent_journal" -description {
+test_suite "persistent_journal" -description {
   Run tests in persistent-journal mode.
 } -presql {
   pragma journal_mode = persist
-} -include {
+} -files {
   delete.test delete2.test insert.test rollback.test select1.test
   select2.test trans.test update.test vacuum.test 
 }
 
 # Run some tests in truncating journal mode.
 #
-run_tests "truncate_journal" -description {
+test_suite "truncate_journal" -description {
   Run tests in persistent-journal mode.
 } -presql {
   pragma journal_mode = truncate
-} -include {
+} -files {
   delete.test delete2.test insert.test rollback.test select1.test
   select2.test trans.test update.test vacuum.test 
   malloc.test ioerr.test
@@ -412,58 +412,58 @@ run_tests "truncate_journal" -description {
 
 # Run some error tests in persistent journal mode.
 #
-run_tests "persistent_journal_error" -description {
+test_suite "persistent_journal_error" -description {
   Run malloc.test and ioerr.test in persistent-journal mode.
 } -presql {
   pragma journal_mode = persist
-} -include {
+} -files {
   malloc.test ioerr.test
 }
 
 # Run some tests in no journal mode.
 #
-run_tests "no_journal" -description {
+test_suite "no_journal" -description {
   Run tests in no-journal mode.
 } -presql {
   pragma journal_mode = persist
-} -include {
+} -files {
   delete.test delete2.test insert.test rollback.test select1.test
   select2.test trans.test update.test vacuum.test 
 }
 
 # Run some error tests in no journal mode.
 #
-run_tests "no_journal_error" -description {
+test_suite "no_journal_error" -description {
   Run malloc.test and ioerr.test in no-journal mode.
 } -presql {
   pragma journal_mode = persist
-} -include {
+} -files {
   malloc.test ioerr.test
 }
 
 # Run some crash-tests in autovacuum mode.
 #
-run_tests "autovacuum_crash" -description {
+test_suite "autovacuum_crash" -description {
   Run crash.test in autovacuum mode.
 } -presql {
   pragma auto_vacuum = 1
-} -include crash.test
+} -files crash.test
 
 # Run some ioerr-tests in autovacuum mode.
 #
-run_tests "autovacuum_ioerr" -description {
+test_suite "autovacuum_ioerr" -description {
   Run ioerr.test in autovacuum mode.
 } -presql {
   pragma auto_vacuum = 1
-} -include ioerr.test
+} -files ioerr.test
 
 # Run tests with an in-memory journal.
 #
-run_tests "inmemory_journal" -description {
+test_suite "inmemory_journal" -description {
   Run tests with an in-memory journal file.
 } -presql {
   pragma journal_mode = 'memory'
-} -exclude {
+} -files [test_set $::allquicktests -exclude {
   # Exclude all tests that simulate IO errors.
   autovacuum_ioerr2.test incrvacuum_ioerr.test ioerr.test
   ioerr.test ioerr2.test ioerr3.test ioerr4.test ioerr5.test
@@ -479,13 +479,13 @@ run_tests "inmemory_journal" -description {
   stmt.test
 
   # WAL mode is different.
-  walmode.test walfault.test wal.test wal2.test wal3.test
+  wal*
-}
+}]
 
 ifcapable mem3 {
-  run_tests "memsys3" -description {
+  test_suite "memsys3" -description {
     Run tests using the allocator in mem3.c.
-  } -exclude {
+  } [test_set $::allquicktests -exclude {
     autovacuum.test           delete3.test              manydb.test
     bigrow.test               incrblob2.test            memdb.test
     bitvec.test               index2.test               memsubsys1.test
@@ -493,7 +493,7 @@ ifcapable mem3 {
     capi3.test                join3.test                pagesize.test
     collate5.test             limit.test                backup_ioerr.test
     backup_malloc.test
-  } -initialize {
+  }] -initialize {
     catch {db close}
     sqlite3_reset_auto_extension
     sqlite3_shutdown
@@ -521,16 +521,16 @@ ifcapable mem3 {
 }
 
 ifcapable mem5 {
-  run_tests "memsys5" -description {
+  test_suite "memsys5" -description {
     Run tests using the allocator in mem5.c.
-  } -exclude {
+  } [test_set $::allquicktests -exclude {
     autovacuum.test           delete3.test              manydb.test
     bigrow.test               incrblob2.test            memdb.test
     bitvec.test               index2.test               memsubsys1.test
     capi3c.test               ioerr.test                memsubsys2.test
     capi3.test                join3.test                pagesize.test
     collate5.test             limit.test                zeroblob.test
-  } -initialize {
+  }] -initialize {
     catch {db close}
     sqlite3_shutdown
     sqlite3_config_heap 25000000 64
@@ -548,9 +548,9 @@ ifcapable mem5 {
     autoinstall_test_functions
   }
 
-  run_tests "memsys5-2" -description {
+  test_suite "memsys5-2" -description {
     Run tests using the allocator in mem5.c in a different configuration.
-  } -include {
+  } -files {
     select1.test 
   } -initialize {
     catch {db close}
@@ -572,10 +572,11 @@ ifcapable mem5 {
 }
 
 ifcapable threadsafe {
-  run_tests "no_mutex_try" -description {
+  test_suite "no_mutex_try" -description {
      The sqlite3_mutex_try() interface always fails
-  } -exclude [concat $EXCLUDE mutex1.test mutex2.test] \
+  } -files [
-  -initialize {
+    test_set $::allquicktests -exclude mutex1.test mutex2.test
+  ] -initialize {
     catch {db close}
     sqlite3_shutdown
     install_mutex_counters 1
@@ -611,9 +612,9 @@ ifcapable threadsafe {
 # } -shutdown {
 #   rename crashsql {}
 #   rename sa_crashsql crashsql
-# } -include crash.test
+# } -files crash.test
 
-run_tests "safe_append" -description {
+test_suite "safe_append" -description {
   Run some tests on a SAFE_APPEND file-system.
 } -initialize {
   rename sqlite3 sqlite3_safeappend
@@ -627,8 +628,9 @@ run_tests "safe_append" -description {
 } -shutdown {
   rename sqlite3 {}
   rename sqlite3_shutdown sqlite3
-} -include [lsort [concat shared_err.test $ALLTESTS]] \
+} -files [
-  -exclude async3.test
+  test_set $::allquicktests shared_err.test -exclude async3.test
+]
 
 # The set of tests to run on the alternative-pcache
 set perm-alt-pcache-testset {
@@ -644,7 +646,7 @@ set perm-alt-pcache-testset {
   update.test
 }
 
-run_tests "pcache0" -description {
+test_suite "pcache0" -description {
   Alternative pcache implementation without random discard
 } -initialize {
   catch {db close}
@@ -660,9 +662,9 @@ run_tests "pcache0" -description {
   install_malloc_faultsim 1 
   sqlite3_initialize
   autoinstall_test_functions
-} -include ${perm-alt-pcache-testset}
+} -files ${perm-alt-pcache-testset}
 
-run_tests "pcache10" -description {
+test_suite "pcache10" -description {
   Alternative pcache implementation without 10% random discard
 } -initialize {
   catch {db close}
@@ -676,9 +678,9 @@ run_tests "pcache10" -description {
   sqlite3_config_alt_pcache 0 0 0
   sqlite3_initialize
   autoinstall_test_functions
-} -include ${perm-alt-pcache-testset}
+} -files ${perm-alt-pcache-testset}
 
-run_tests "pcache50" -description {
+test_suite "pcache50" -description {
   Alternative pcache implementation without 50% random discard
 } -initialize {
   catch {db close}
@@ -692,9 +694,9 @@ run_tests "pcache50" -description {
   sqlite3_config_alt_pcache 0 0 0
   sqlite3_initialize
   autoinstall_test_functions
-} -include ${perm-alt-pcache-testset}
+} -files ${perm-alt-pcache-testset}
 
-run_tests "pcache90" -description {
+test_suite "pcache90" -description {
   Alternative pcache implementation without 90% random discard
 } -initialize {
   catch {db close}
@@ -708,9 +710,9 @@ run_tests "pcache90" -description {
   sqlite3_config_alt_pcache 0 0 0
   sqlite3_initialize
   autoinstall_test_functions
-} -include ${perm-alt-pcache-testset}
+} -files ${perm-alt-pcache-testset}
 
-run_tests "pcache100" -description {
+test_suite "pcache100" -description {
   Alternative pcache implementation that always discards when unpinning
 } -initialize {
   catch {db close}
@@ -724,55 +726,107 @@ run_tests "pcache100" -description {
   sqlite3_config_alt_pcache 0 0 0
   sqlite3_initialize
   autoinstall_test_functions
-} -include ${perm-alt-pcache-testset}
+} -files ${perm-alt-pcache-testset}
 
-run_tests "journaltest" -description {
+test_suite "journaltest" -description {
   Check that pages are synced before being written (test_journal.c).
 } -initialize {
-  set G(isquick) 1
   catch {db close}
   register_jt_vfs -default ""
-  #sqlite3_instvfs binarylog -default binarylog ostrace.bin
 } -shutdown {
-  #sqlite3_instvfs destroy binarylog
   unregister_jt_vfs
-} -include $::ALLTESTS -exclude [concat $::WALTESTS {
+} -files [test_set $::allquicktests -exclude {
-  incrvacuum.test
+  wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test 
-  ioerr.test
+  async4.test bigfile.test
-  corrupt4.test 
-  io.test 
-  crash8.test 
-  async4.test 
-  bigfile.test
 }]
 
 if {[info commands register_demovfs] != ""} {
-  run_tests "demovfs" -description {
+  test_suite "demovfs" -description {
     Check that the demovfs (code in test_demovfs.c) more or less works.
   } -initialize {
     register_demovfs
   } -shutdown {
     unregister_demovfs
-  } -include {
+  } -files {
     insert.test   insert2.test  insert3.test rollback.test 
     select1.test  select2.test  select3.test
   }
 }
 
-run_tests "wal" -description {
+test_suite "wal" -description {
   Run tests with journal_mode=WAL
 } -initialize {
   set ::G(savepoint6_iterations) 100
 } -shutdown {
   unset -nocomplain ::G(savepoint6_iterations)
-} -include {
+} -files {
   savepoint.test     savepoint2.test     savepoint6.test
   trans.test         avtrans.test
 }
-
 # End of tests
 #############################################################################
 
-if {$::perm::testmode eq "targets"} { puts "" ; exit }
+# run_tests NAME OPTIONS
+#
+# where available options are:  
+#
+#       -description TITLE                  (default "")
+#       -initialize  SCRIPT                 (default "")
+#       -shutdown    SCRIPT                 (default "")
+#       -presql      SQL                    (default "")
+#       -files       LIST-OF-FILES          (default "")
+#
+proc run_tests {name args} {
+  array set options $args
+
+  set ::G(perm:name)         $name
+  set ::G(perm:presql)       $options(-presql)
+  set ::G(isquick)           1
+
+  uplevel $options(-initialize)
+
+  foreach file [lsort $options(-files)] {
+    slave_test_file $::testdir/$file
+  }
+
+  uplevel $options(-shutdown)
+
+  unset ::G(perm:name)
+  unset ::G(perm:presql)
+}
+
+proc run_test_suite {name} {
+  if {[info exists ::testspec($name)]==0} {
+    error "No such test suite: $name"
+  }
+  uplevel run_tests $name $::testspec($name)
+}
+
+proc help {} {
+  foreach k $::testsuitelist {
+    array set o $::testspec($k)
+    puts "Test suite: \"$k\""
+    set d [string trim $o(-description)]
+    set d [regsub {\n *} $d "\n  "]
+    puts "  $d"
+    puts ""
+  }
+  exit -1
+}
+
+if {[info script] == $argv0} {
+  proc main {argv} {
+    if {[llength $argv]==0} {
+      help
+    } else {
+      set suite [lindex $argv 0]
+      if {[info exists ::testspec($suite)]==0} help
+      set extra ""
+      if {[llength $argv]>1} { set extra [list -files [lrange $argv 1 end]] }
+      eval run_tests $suite $::testspec($suite) $extra
+    }
+  }
+  main $argv
+  finish_test
+}
 
-finish_test

test/quick.test

@@ -8,149 +8,13 @@
 #
 
 set testdir [file dirname $argv0]
-source $testdir/tester.tcl
+source $testdir/permutations.test
-db close
 
-proc lshift {lvar} {
+if {[info exists env(QUICKTEST_INCLUDE]} {
-  upvar $lvar l
+  # A hack so that releasetest.tcl keeps working.
-  set ret [lindex $l 0]
+  run_test_suite quick_plus_notify2
-  set l [lrange $l 1 end]
+} else {
-  return $ret
+  run_test_suite quick
 }
-while {[set arg [lshift argv]] != ""} {
-  switch -- $arg {
-    -sharedpagercache {
-      sqlite3_enable_shared_cache 1
-    }
-    -soak {
-       set G(issoak) 1
-    }
-    -start {
-       set STARTAT "[lshift argv]*"
-    }
-    default {
-      set argv [linsert $argv 0 $arg]
-      break
-    }
-  }
-}
-
-set G(isquick) 1
-
-set EXCLUDE {
-  all.test
-  async.test
-  async2.test
-  async3.test
-  backup_ioerr.test
-  corrupt.test
-  corruptC.test
-  crash.test
-  crash2.test
-  crash3.test
-  crash4.test
-  crash5.test
-  crash6.test
-  crash7.test
-  delete3.test
-  e_fts3.test
-  fts3.test
-  fts3rnd.test
-  fkey_malloc.test
-  fuzz.test
-  fuzz3.test
-  fuzz_malloc.test
-  in2.test
-  loadext.test
-  memleak.test
-  misc7.test
-  misuse.test
-  mutex2.test
-  notify2.test
-  onefile.test
-  permutations.test
-  quick.test
-  savepoint4.test
-  savepoint6.test
-  select9.test
-  soak.test
-  speed1.test
-  speed1p.test
-  speed2.test
-  speed3.test
-  speed4.test
-  speed4p.test
-  sqllimits1.test
-  tkt2686.test
-  thread001.test
-  thread002.test
-  thread003.test
-  thread004.test
-  thread005.test
-  trans2.test
-  vacuum3.test
-
-  incrvacuum_ioerr.test
-  autovacuum_crash.test
-  btree8.test
-  shared_err.test
-  vtab_err.test
-  veryquick.test
-  mallocAll.test
-
-  walslow.test
-  walcrash.test
-  walthread.test
-}
-
-if {[sqlite3 -has-codec]} {
-  # lappend EXCLUDE \
-  #  conflict.test
-}
-
-
-# Files to include in the test.  If this list is empty then everything
-# that is not in the EXCLUDE list is run.
-#
-set INCLUDE {
-}
-
-# If the QUICKTEST_INCLUDE environment variable is set, then interpret
-# it as a list of test files. Always run these files, even if they
-# begin with "malloc*" or "ioerr*" or are part of the EXCLUDE list
-# defined above.
-#
-set QUICKTEST_INCLUDE {}
-catch { set QUICKTEST_INCLUDE $env(QUICKTEST_INCLUDE) }
-
-# Run all test files in directory $testdir, subject to the following:
-#
-#   1. If a test file is specified as part of the $INCLUDE or 
-#      $QUICKTEST_INCLUDE list variables, run it.
-#
-#   2. If $INCLUDE is non-empty, and rule 1 does not apply to it, do not run it.
-#
-#   3. If a test file is specified as part of $EXCLUDE, and rule 1 does not
-#      apply, do not run it.
-#
-foreach testfile [lsort -dictionary [glob $testdir/*.test]] {
-  set tail [file tail $testfile]
-
-  if {[info exists STARTAT] && [string match $STARTAT $tail]} {unset STARTAT}
-  if {[info exists STARTAT]} continue
-
-  set run [expr {[llength $INCLUDE]==0}]
-  if {[info exists ISVERYQUICK] && [string match *malloc* $tail]} { set run 0 }
-  if {[info exists ISVERYQUICK] && [string match *ioerr* $tail]}  { set run 0 }
-  if {[lsearch -exact $EXCLUDE $tail]>=0}                         { set run 0 }
-  if {[lsearch -exact $INCLUDE $tail]>=0}                         { set run 1 }
-  if {[lsearch -exact $QUICKTEST_INCLUDE $tail]>=0}               { set run 1 }
-
-  if {$run} {
-    slave_test_file $testfile
-  }
-}
-slave_test_file $testdir/misuse.test
 
 finish_test
-

test/tester.tcl

@@ -133,12 +133,14 @@ if {[info exists cmdlinearg]==0} {
   #   --malloctrace=N
   #   --backtrace=N
   #   --binarylog=N
+  #   --soak=N
   #
   set cmdlinearg(soft-heap-limit)    0
   set cmdlinearg(maxerror)        1000
   set cmdlinearg(malloctrace)        0
   set cmdlinearg(backtrace)         10
   set cmdlinearg(binarylog)          0
+  set cmdlinearg(soak)               0
 
   set leftover [list]
   foreach a $argv {
@@ -164,11 +166,15 @@ if {[info exists cmdlinearg]==0} {
       }
       {^-+backtrace=.+$} {
         foreach {dummy cmdlinearg(backtrace)} [split $a =] break
+        sqlite3_memdebug_backtrace $value
       }
-      sqlite3_memdebug_backtrace $value
       {^-+binarylog=.+$} {
         foreach {dummy cmdlinearg(binarylog)} [split $a =] break
       }
+      {^-+soak=.+$} {
+        foreach {dummy cmdlinearg(soak)} [split $a =] break
+        set ::G(issoak) $cmdlinearg(soak)
+      }
       default {
         lappend leftover $a
       }
@@ -1140,21 +1146,38 @@ proc slave_test_script {script} {
 }
 
 proc slave_test_file {zFile} {
+  set tail [file tail $zFile]
 
+  ifcapable shared_cache { 
+    set scs [sqlite3_enable_shared_cache] 
+  }
+
+  reset_prng_state
+  set ::sqlite_open_file_count 0
+  set time [time { slave_test_script [list source $zFile] }]
+  set ms [expr [lindex $time 0] / 1000]
+
+  # Test that all files opened by the test script were closed.
+  #
+  do_test ${tail}-closeallfiles {
+    expr {$::sqlite_open_file_count>0}
+  } {0}
   set ::sqlite_open_file_count 0
 
-  set time [time {
+  # Test that the global "shared-cache" setting was not altered by 
-    slave_test_script [list source $zFile]
+  # the test script.
-  }]
+  #
+  ifcapable shared_cache { 
+    set res [expr {[sqlite3_enable_shared_cache] == $scs}]
+    do_test ${tail}-sharedcachesetting [list set {} $res] 1
+  }
 
-  set tail [file tail $zFile]
   if {$::sqlite_open_file_count>0} {
     puts "$tail did not close all files: $::sqlite_open_file_count"
-    fail_test $tail
+    fail_test "$tail-closeallfiles"
     set ::sqlite_open_file_count 0
-    exit
   }
-  puts "time $tail [lrange $time 0 1]"
+  puts "Time: $tail $ms ms"
 
   show_memstats
 }

test/veryquick.test

@@ -11,5 +11,9 @@
 # $Id: veryquick.test,v 1.9 2008/07/12 14:52:21 drh Exp $
 
 set testdir [file dirname $argv0]
-set ISVERYQUICK 1
+source $testdir/permutations.test
-source $testdir/quick.test
+
+run_test_suite veryquick
+
+finish_test
+