Even if compile time option SQLITE_MAX_WORKER_THREADS is set to one or greater, set the default number of worker threads to zero. Distribute data more evenly between threads in sqlite3VdbeSorterWrite() to improve performance when sorting large amounts of data. Add new test file sort2.test.

FossilOrigin-Name: 643c86a056168e39fcb7f39b8a72731f1eb246db

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode.
+C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test.
-D 2014-04-01T10:19:02.635
+D 2014-04-01T15:38:44.990
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf
 F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811
-F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202
+F src/global.c deadd872189b92aca4ee2566332a86315839f811
 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd
 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
@@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa
-F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df
+F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e
 F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
@@ -240,7 +240,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8
 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e
 F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16
-F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585
+F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0
 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f
@@ -251,7 +251,7 @@ F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61
 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d
 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64
 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4
-F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3
+F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307
 F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8
 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1
 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f
@@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4
 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea
 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa
 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447
-F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e
+F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1
 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
@@ -818,6 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24
 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a
+F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574
 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
 F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8
@@ -1160,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a
+P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425
-R a6fe6d7d652069432584e1a691602cb5
+R 574aa31e9a2482d16cbf0b6e42831c35
 U dan
-Z ac080311c2ad2c3d7bf733448ffc0f03
+Z 53e6ec40073ce4b7a6ae1cb9efd4c01b

manifest.uuid

@@ -1 +1 @@
-59cd5229e2b5be5272cf57c7e7d09e97d16a5425
+643c86a056168e39fcb7f39b8a72731f1eb246db

src/global.c

@@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* nPage */
    0,                         /* mxParserStack */
    0,                         /* sharedCacheEnabled */
-   SQLITE_MAX_WORKER_THREADS, /* nWorker */
+   0,                         /* nWorker */
    /* All the rest should always be initialized to zero */
    0,                         /* isInit */
    0,                         /* inProgress */

src/shell.c

@@ -3532,6 +3532,8 @@ static void main_init(struct callback_data *data) {
   sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
   sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
   sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
+  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
+  sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3);
 }
 
 /*

src/test_config.c

@@ -99,6 +99,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#if SQLITE_MAX_WORKER_THREADS>0
+  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #if 1 /* def SQLITE_MEMDEBUG */
   Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY);
 #else

src/test_malloc.c

@@ -1253,6 +1253,32 @@ static int test_config_cis(
   return TCL_OK;
 }
 
+/*
+** Usage:    sqlite3_config_worker_threads N
+*/
+static int test_config_worker_threads(
+  void * clientData, 
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  int nThread;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "N");
+    return TCL_ERROR;
+  }
+  if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){
+    return TCL_ERROR;
+  }
+
+  rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread);
+  Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
+
+  return TCL_OK;
+}
+
 /*
 ** Usage:    sqlite3_dump_memsys3  FILENAME
 **           sqlite3_dump_memsys5  FILENAME
@@ -1506,6 +1532,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){
      { "sqlite3_config_error",       test_config_error             ,0 },
      { "sqlite3_config_uri",         test_config_uri               ,0 },
      { "sqlite3_config_cis",         test_config_cis               ,0 },
+     { "sqlite3_config_worker_threads", test_config_worker_threads ,0 },
      { "sqlite3_db_config_lookaside",test_db_config_lookaside      ,0 },
      { "sqlite3_dump_memsys3",       test_dump_memsys3             ,3 },
      { "sqlite3_dump_memsys5",       test_dump_memsys3             ,5 },

src/vdbesort.c

@@ -180,6 +180,7 @@ struct VdbeSorter {
   u8 *aMemory;                    /* Block of memory to alloc records from */
   int iMemory;                    /* Offset of first free byte in aMemory */
   int nMemory;                    /* Size of aMemory allocation in bytes */
+  int iPrev;                      /* Previous thread used to flush PMA */
   int nThread;                    /* Size of aThread[] array */
   SorterThread aThread[1];
 };
@@ -1251,11 +1252,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
   VdbeSorter *pSorter = pCsr->pSorter;
   int rc = SQLITE_OK;
   int i;
-  SorterThread *pThread;        /* Thread context used to create new PMA */
+  SorterThread *pThread = 0;    /* Thread context used to create new PMA */
+  int nWorker = (pSorter->nThread-1);
 
   pSorter->bUsePMA = 1;
-  for(i=0; ALWAYS( i<pSorter->nThread ); i++){
+  for(i=0; i<nWorker; i++){
-    pThread = &pSorter->aThread[i];
+    int iTest = (pSorter->iPrev + i + 1) % nWorker;
+    pThread = &pSorter->aThread[iTest];
 #if SQLITE_MAX_WORKER_THREADS>0
     if( pThread->bDone ){
       void *pRet;
@@ -1269,7 +1272,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
     }
 #endif
     if( pThread->pThread==0 ) break;
+    pThread = 0;
   }
+  if( pThread==0 ){
+    pThread = &pSorter->aThread[nWorker];
+  }
+  pSorter->iPrev = (pThread - pSorter->aThread);
 
   if( rc==SQLITE_OK ){
     assert( pThread->pThread==0 && pThread->bDone==0 );
@@ -1286,7 +1294,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
     }
 
 #if SQLITE_MAX_WORKER_THREADS>0
-    if( bFg || i==(pSorter->nThread-1) ){
+    if( !bFg && pThread!=&pSorter->aThread[nWorker] ){
       /* Launch a background thread for this operation */
       void *pCtx = (void*)pThread;
       assert( pSorter->aMemory==0 || pThread->aListMemory!=0 );
@@ -1464,7 +1472,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){
     for(i=0; rc==SQLITE_OK && i<pSorter->nThread; i++){
       SorterThread *pThread = &pSorter->aThread[i];
       if( pThread->pTemp1 ){
-        pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread;
+        pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread;
         pThread->eWork = SORTER_THREAD_CONS;
 
 #if SQLITE_MAX_WORKER_THREADS>0

test/sort2.test

@@ -0,0 +1,41 @@
+# 2014 March 25.
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix sort2
+
+db close
+sqlite3_shutdown
+sqlite3_config_worker_threads 7
+reset_db
+
+do_execsql_test 1 {
+  PRAGMA cache_size = 5;
+  WITH r(x,y) AS (
+    SELECT 1, randomblob(100)
+    UNION ALL
+    SELECT x+1, randomblob(100) FROM r
+    LIMIT 100000
+  )
+  SELECT count(x), length(y) FROM r GROUP BY (x%5)
+} {
+  20000 100 20000 100 20000 100 20000 100 20000 100
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_worker_threads 0
+sqlite3_initialize
+finish_test
+