diff --git a/manifest b/manifest index b819b3428f..73c52a7c23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode. -D 2014-04-01T10:19:02.635 +C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. +D 2014-04-01T15:38:44.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 +F src/global.c deadd872189b92aca4ee2566332a86315839f811 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa -F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df +F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -240,7 +240,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 +F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -251,7 +251,7 @@ F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 +F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307 F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e +F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,6 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a +F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1160,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a -R a6fe6d7d652069432584e1a691602cb5 +P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 +R 574aa31e9a2482d16cbf0b6e42831c35 U dan -Z ac080311c2ad2c3d7bf733448ffc0f03 +Z 53e6ec40073ce4b7a6ae1cb9efd4c01b diff --git a/manifest.uuid b/manifest.uuid index 7a6826cad1..957ecbe9a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59cd5229e2b5be5272cf57c7e7d09e97d16a5425 \ No newline at end of file +643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file diff --git a/src/global.c b/src/global.c index 4ed597d292..091b750803 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - SQLITE_MAX_WORKER_THREADS, /* nWorker */ + 0, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/shell.c b/src/shell.c index c9abd9ec7d..6fc85ae394 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3532,6 +3532,8 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3); } /* diff --git a/src/test_config.c b/src/test_config.c index 1db8198641..2201fba007 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,6 +99,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif +#if SQLITE_MAX_WORKER_THREADS>0 + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/test_malloc.c b/src/test_malloc.c index e3cfcaa9f0..6ac030f23c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1253,6 +1253,32 @@ static int test_config_cis( return TCL_OK; } +/* +** Usage: sqlite3_config_worker_threads N +*/ +static int test_config_worker_threads( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + int nThread; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "N"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){ + return TCL_ERROR; + } + + rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + + return TCL_OK; +} + /* ** Usage: sqlite3_dump_memsys3 FILENAME ** sqlite3_dump_memsys5 FILENAME @@ -1506,6 +1532,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_config_cis", test_config_cis ,0 }, + { "sqlite3_config_worker_threads", test_config_worker_threads ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/vdbesort.c b/src/vdbesort.c index 60a023c8f5..746618cc25 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -180,6 +180,7 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ + int iPrev; /* Previous thread used to flush PMA */ int nThread; /* Size of aThread[] array */ SorterThread aThread[1]; }; @@ -1251,11 +1252,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SorterThread *pThread; /* Thread context used to create new PMA */ + SorterThread *pThread = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nThread-1); pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ - pThread = &pSorter->aThread[i]; + for(i=0; iiPrev + i + 1) % nWorker; + pThread = &pSorter->aThread[iTest]; #if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; @@ -1269,7 +1272,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #endif if( pThread->pThread==0 ) break; + pThread = 0; } + if( pThread==0 ){ + pThread = &pSorter->aThread[nWorker]; + } + pSorter->iPrev = (pThread - pSorter->aThread); if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); @@ -1286,7 +1294,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #if SQLITE_MAX_WORKER_THREADS>0 - if( bFg || i==(pSorter->nThread-1) ){ + if( !bFg && pThread!=&pSorter->aThread[nWorker] ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); @@ -1464,7 +1472,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; #if SQLITE_MAX_WORKER_THREADS>0 diff --git a/test/sort2.test b/test/sort2.test new file mode 100644 index 0000000000..2a77b61ed1 --- /dev/null +++ b/test/sort2.test @@ -0,0 +1,41 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort2 + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 7 +reset_db + +do_execsql_test 1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) +} { + 20000 100 20000 100 20000 100 20000 100 20000 100 +} + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 0 +sqlite3_initialize +finish_test +