Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5.

FossilOrigin-Name: 3a66c4e1bf311d38668dfcdcd77867feff6db7bd

manifest

@@ -1,5 +1,5 @@
-C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation.
+C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5.
-D 2014-05-03T13:53:37.085
+D 2014-05-03T14:28:14.676
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
 F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd
-F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2
+F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9
 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35
 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c
 F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df
@@ -241,7 +241,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8
 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e
 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f
-F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0
+F src/test_config.c bf2e0bf49ebd8fe3dccabb4542157f9571fe48fa
 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f
@@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4
 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b
 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac
 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447
-F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3
+F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0
 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
@@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
-F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d
+F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb
 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f
 F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24
-F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a
+F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73
-F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063
+F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010
-F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af
+F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2
-F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577
+F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512
 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
 F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8
@@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 13686035dd1cf67ad9c6d282ab13c3259e7273d1
+P f49ba1c926c63ee1c4609930138389fca182c845
-R 7c657c53542f045bb02fc7656a8fcff4
+R ecf8f8bb6befc88e6cfdbe14a65100aa
-U drh
+U dan
-Z af16da93d438f6495f6a372d4f94c5d4
+Z 4d5ad6cf671998327bccef4dcd098843

manifest.uuid

@@ -1 +1 @@
-f49ba1c926c63ee1c4609930138389fca182c845
+3a66c4e1bf311d38668dfcdcd77867feff6db7bd

src/test1.c

@@ -6336,6 +6336,40 @@ static int tclLoadStaticExtensionCmd(
   return TCL_OK;
 }
 
+/*
+**     sorter_test_fakeheap BOOL
+**
+*/
+static int sorter_test_fakeheap(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int bArg;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "BOOL");
+    return TCL_ERROR;
+  }
+
+  if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){
+    return TCL_ERROR;
+  }
+
+  if( bArg ){
+    if( sqlite3GlobalConfig.pHeap==0 ){
+      sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1);
+    }
+  }else{
+    if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){
+      sqlite3GlobalConfig.pHeap = 0;
+    }
+  }
+
+  Tcl_ResetResult(interp);
+  return TCL_OK;
+}
+
 
 /*
 ** Register commands with the TCL interpreter.
@@ -6569,6 +6603,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "getrusage", test_getrusage },
 #endif
      { "load_static_extension", tclLoadStaticExtensionCmd },
+     { "sorter_test_fakeheap", sorter_test_fakeheap },
   };
   static int bitmask_size = sizeof(Bitmask)*8;
   int i;

src/test_config.c

@@ -99,11 +99,9 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY);
 #endif
 
-#if SQLITE_MAX_WORKER_THREADS>0
+  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", 
-  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY);
+      STRINGVALUE(SQLITE_MAX_WORKER_THREADS), TCL_GLOBAL_ONLY
-#else
+  );
-  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY);
-#endif
 
 #if 1 /* def SQLITE_MEMDEBUG */
   Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY);

src/vdbesort.c

@@ -1556,7 +1556,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){
       if( aMem ){
         pSorter->list.aMemory = aMem;
         pSorter->nMemory = sqlite3MallocSize(aMem);
-      }else{
+      }else if( pSorter->list.aMemory ){
         pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory);
         if( !pSorter->list.aMemory ) return SQLITE_NOMEM;
       }

test/permutations.test

@@ -332,6 +332,12 @@ test_suite "coverage-analyze" -description {
   analyze.test analyzeB.test mallocA.test
 } 
 
+test_suite "coverage-sorter" -description {
+  Coverage tests for file vdbesort.c.
+} -files {
+  sort.test sortfault.test
+} 
+
 
 lappend ::testsuitelist xxx
 #-------------------------------------------------------------------------

test/sort.test

@@ -8,10 +8,10 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this file is testing the CREATE TABLE statement.
 #
-# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the sorter (code in vdbesort.c).
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -485,5 +485,125 @@ do_execsql_test sort-13.3 {
   SELECT a, b FROM t10 ORDER BY a;
 } [db eval {SELECT a, b FROM t10 ORDER BY a, b}]
 
+#-------------------------------------------------------------------------
+# Sort some large ( > 4KiB) records.
+#
+proc cksum {x} {
+  set i1 1
+  set i2 2
+  binary scan $x c* L
+  foreach {a b} $L {
+    set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF]
+    set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF]
+  }
+  list $i1 $i2
+}
+db func cksum cksum
+
+do_execsql_test sort-14.0 {
+  PRAGMA cache_size = 5;
+  CREATE TABLE t11(a, b);
+  INSERT INTO t11 VALUES(randomblob(5000), NULL);
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8
+  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9
+  UPDATE t11 SET b = cksum(a);
+}
+
+foreach {tn mmap_limit} {
+  1 0
+  2 1000000
+} {
+  do_test sort-14.$tn {
+    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
+    set prev ""
+    db eval { SELECT * FROM t11 ORDER BY b } {
+      if {$b != [cksum $a]} {error "checksum failed"}
+      if {[string compare $b $prev] < 0} {error "sort failed"}
+      set prev $b
+    }
+    set {} {}
+  } {}
+}
+
+#-------------------------------------------------------------------------
+#
+foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} {
+          1          0       3     file      true    false
+          2          0       3     file      true     true
+          3          0       0     file      true    false
+          4    1000000       3     file      true    false
+          5          0       0   memory     false     true
+} {
+  db close
+
+  sqlite3_shutdown
+  sqlite3_config_worker_threads $nWorker
+  if {$coremutex} {
+    sqlite3_config multithread
+  } else {
+    sqlite3_config singlethread
+  }
+  sqlite3_initialize
+
+  sorter_test_fakeheap $fakeheap
+
+  reset_db
+  sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
+  execsql "PRAGMA temp_store = $tmpstore"
+  
+  set ten [string repeat X 10300]
+  set one [string repeat y   200]
+
+  do_execsql_test 15.$tn.1 {
+    PRAGMA cache_size = 5;
+    WITH rr AS (
+      SELECT 4, $ten UNION ALL
+      SELECT 2, $one UNION ALL
+      SELECT 1, $ten UNION ALL
+      SELECT 3, $one
+    )
+    SELECT * FROM rr ORDER BY 1;
+  } [list 1 $ten 2 $one 3 $one 4 $ten]
+
+  do_execsql_test 15.$tn.2 {
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES(4);
+    INSERT INTO t1 VALUES(5);
+    INSERT INTO t1 VALUES(3);
+    INSERT INTO t1 VALUES(2);
+    INSERT INTO t1 VALUES(6);
+    INSERT INTO t1 VALUES(1);
+    CREATE INDEX i1 ON t1(a);
+    SELECT * FROM t1 ORDER BY a;
+  } {1 2 3 4 5 6}
+
+  do_execsql_test 15.$tn.3 {
+    PRAGMA cache_size = 5;
+    WITH rr AS (
+      SELECT 4, $ten UNION ALL
+      SELECT 2, $one
+    )
+    SELECT * FROM rr ORDER BY 1;
+  } [list 2 $one 4 $ten]
+
+  sorter_test_fakeheap 0
+}
+
+db close
+sqlite3_shutdown
+#sqlite3_config_worker_threads $sqlite_options(worker_threads)
+sqlite3_config_worker_threads 0
+set t(0) singlethread
+set t(1) multithread
+set t(2) serialized
+sqlite3_config $t($sqlite_options(threadsafe))
+sqlite3_initialize
 
 finish_test
+

test/sort2.test

@@ -10,6 +10,9 @@
 #***********************************************************************
 # This file implements regression tests for SQLite library. 
 #
+# Specifically, the tests in this file attempt to verify that 
+# multi-threaded sorting works.
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl

test/sort3.test

@@ -10,6 +10,10 @@
 #***********************************************************************
 # This file implements regression tests for SQLite library. 
 #
+# The tests in this file verify that sorting works when the library is
+# configured to use mmap(), but the temporary files generated by the
+# sorter are too large to be completely mapped.
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl

test/sortfault.test

@@ -10,6 +10,9 @@
 #***********************************************************************
 # This file implements regression tests for SQLite library. 
 #
+# Specifically, it tests the effects of fault injection on the sorter
+# module (code in vdbesort.c).
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -20,20 +23,26 @@ do_execsql_test 1.0 {
   PRAGMA cache_size = 5;
 }
 
-do_faultsim_test 1 -prep {
+foreach {tn mmap_limit} {
-  sqlite3 db test.db
+  1 0
-} -body {
+  2 100000
-  execsql { 
+} {
-    WITH r(x,y) AS (
+  do_faultsim_test 1.$tn -prep {
-      SELECT 1, randomblob(1000)
+    sqlite3 db test.db
-      UNION ALL
+    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit
-      SELECT x+1, randomblob(1000) FROM r
+  } -body {
-      LIMIT 500
+    execsql { 
-    )
+      WITH r(x,y) AS (
-    SELECT count(x), length(y) FROM r GROUP BY (x%5)
+          SELECT 1, randomblob(1000)
-  } 
+          UNION ALL
-} -test {
+          SELECT x+1, randomblob(1000) FROM r
-  faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}}
+          LIMIT 500
+          )
+        SELECT count(x), length(y) FROM r GROUP BY (x%5)
+    } 
+  } -test {
+    faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}}
+  }
 }
 
 finish_test