From 70a1b71fb64c51b33abf7f8e751c6d633ff824a0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 28 Sep 2012 18:13:35 +0000 Subject: [PATCH 01/40] Modify the clearCell function to use SQLITE_CORRUPT_BKPT in the one place it was not. FossilOrigin-Name: 472beb306a4fa7103837d4417aef7d66eef49993 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6358e6d360..1dc244597a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Query\splanner\senhancements\sto\sbe\smore\sagressive\sabout\soptimizing\sout\sORDER\sBY\nclauses\s-\sin\sparticular\sthe\squery\splanner\snow\shas\sthe\sability\sto\somit\sORDER\sBY\nclauses\sthat\sspan\smultiple\stables\sin\sa\sjoin. -D 2012-09-28T00:44:28.903 +C Modify\sthe\sclearCell\sfunction\sto\suse\sSQLITE_CORRUPT_BKPT\sin\sthe\sone\splace\sit\swas\snot. +D 2012-09-28T18:13:35.369 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 5b31b24d6814b11de763debf342c8cd0a15a4910 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 9cf6de113d23d47967df24b8d8ce6501c879d7e6 +F src/btree.c 6e1b481435d04055eda89d24ae93ecd7a99cdf56 F src/btree.h 4aee02e879211bfcfd3f551769578d2e940ab6c2 F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 20caf80cb3b65e69a99dbc142db8ee435fb2dbdc 98b633717a1c9a08f6a1d00bc6bc891564ae7e9b -R 4c8024fc13aa6a5fc65357b963a76180 -U drh -Z 885caf237f331087362c9bb98b2de660 +P 1e874629d7cf568368b912b295bd3001147d0b52 +R f05735be438496d52bd0a75864235756 +U mistachkin +Z ea13a29c0e73e27415324aa966ef2598 diff --git a/manifest.uuid b/manifest.uuid index bdeb814a07..9dc405edef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e874629d7cf568368b912b295bd3001147d0b52 \ No newline at end of file +472beb306a4fa7103837d4417aef7d66eef49993 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 47dd7db99d..7eab9fe158 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5256,7 +5256,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ - return SQLITE_CORRUPT; /* Cell extends past end of page */ + return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); From 0094f37e1f2483d9b65d3058ccdcf3642e672c85 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Sep 2012 20:23:42 +0000 Subject: [PATCH 02/40] Avoid calling sqlite3BtreeEnter() in a corner case where the corresponding database handle mutex (sqlite3.mutex) may not be held. This prevents a potential deadlock or crash that can occur if the backup API, shared-cache mode and SQLITE_HAVE_CODEC are all in use. FossilOrigin-Name: 89b8c377a6f03d9fa885f3f94c1f0b1eec263dea --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/backup.c | 7 +++++-- src/btree.c | 16 ++++++++++++++++ src/btree.h | 1 + 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 1dc244597a..8cde120245 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sclearCell\sfunction\sto\suse\sSQLITE_CORRUPT_BKPT\sin\sthe\sone\splace\sit\swas\snot. -D 2012-09-28T18:13:35.369 +C Avoid\scalling\ssqlite3BtreeEnter()\sin\sa\scorner\scase\swhere\sthe\scorresponding\sdatabase\shandle\smutex\s(sqlite3.mutex)\smay\snot\sbe\sheld.\sThis\sprevents\sa\spotential\sdeadlock\sor\scrash\sthat\scan\soccur\sif\sthe\sbackup\sAPI,\sshared-cache\smode\sand\sSQLITE_HAVE_CODEC\sare\sall\sin\suse. +D 2012-09-28T20:23:42.567 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,11 +118,11 @@ F src/alter.c 0c1716aa8d248bd6bc750e23be4c68ad05f8668c F src/analyze.c 7553068d21e32a57fc33ab6b2393fc8c1ba41410 F src/attach.c 577bf5675b0c50495fc28549f2fcbdb1bac71143 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 5b31b24d6814b11de763debf342c8cd0a15a4910 +F src/backup.c afc067b9a9050ff48b9d46285c53d096c556a73d F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 6e1b481435d04055eda89d24ae93ecd7a99cdf56 -F src/btree.h 4aee02e879211bfcfd3f551769578d2e940ab6c2 +F src/btree.c 821615a1e1002346d84d2e341ecd5e947196454b +F src/btree.h 5e6482bcabf39455385e39c9739a5954e4775bba F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 1e874629d7cf568368b912b295bd3001147d0b52 -R f05735be438496d52bd0a75864235756 -U mistachkin -Z ea13a29c0e73e27415324aa966ef2598 +P 472beb306a4fa7103837d4417aef7d66eef49993 +R ea9a9130bee4f18f6369bb310e898b80 +U dan +Z 63a6a40692fb8e85a92f7431b790c782 diff --git a/manifest.uuid b/manifest.uuid index 9dc405edef..7b4ca77e44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -472beb306a4fa7103837d4417aef7d66eef49993 \ No newline at end of file +89b8c377a6f03d9fa885f3f94c1f0b1eec263dea \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 4881215e96..6abc11c20c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -219,13 +219,16 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ const int nCopy = MIN(nSrcPgsz, nDestPgsz); const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; #ifdef SQLITE_HAS_CODEC - int nSrcReserve = sqlite3BtreeGetReserve(p->pSrc); + /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is + ** guaranteed that the shared-mutex is held by this thread, handle + ** p->pSrc may not actually be the owner. */ + int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); int nDestReserve = sqlite3BtreeGetReserve(p->pDest); #endif - int rc = SQLITE_OK; i64 iOff; + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); assert( p->bDestLocked ); assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); diff --git a/src/btree.c b/src/btree.c index 7eab9fe158..eff70f3b36 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2200,6 +2200,22 @@ int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behaviour. +*/ +int sqlite3BtreeGetReserveNoMutex(Btree *p){ + assert( sqlite3_mutex_held(p->pBt->mutex) ); + return p->pBt->pageSize - p->pBt->usableSize; +} + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* ** Return the number of bytes of space at the end of every page that diff --git a/src/btree.h b/src/btree.h index 95897d5662..5a894f5f22 100644 --- a/src/btree.h +++ b/src/btree.h @@ -71,6 +71,7 @@ int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); int sqlite3BtreeGetReserve(Btree*); +int sqlite3BtreeGetReserveNoMutex(Btree *p); int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); int sqlite3BtreeBeginTrans(Btree*,int); From b3f787f4c70543a9bd8354c2e39020332af5b292 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Sep 2012 14:45:54 +0000 Subject: [PATCH 03/40] Fix compiler warnings found on Lion. FossilOrigin-Name: fd74d3d91721ca404537f195fed04c9edef20bf2 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/tclsqlite.c | 2 +- src/test1.c | 6 +++--- src/test_intarray.c | 5 +++-- src/test_quota.c | 9 +++++---- 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 8cde120245..e5dcbb464b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3BtreeEnter()\sin\sa\scorner\scase\swhere\sthe\scorresponding\sdatabase\shandle\smutex\s(sqlite3.mutex)\smay\snot\sbe\sheld.\sThis\sprevents\sa\spotential\sdeadlock\sor\scrash\sthat\scan\soccur\sif\sthe\sbackup\sAPI,\sshared-cache\smode\sand\sSQLITE_HAVE_CODEC\sare\sall\sin\suse. -D 2012-09-28T20:23:42.567 +C Fix\scompiler\swarnings\sfound\son\sLion. +D 2012-09-29T14:45:54.660 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,8 +183,8 @@ F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c e4de2458b3ef38fdd0498bc4e5ea5367a241b0f3 -F src/test1.c 0354b555639c92d2a63c0ad4e74ed152ba47e604 +F src/tclsqlite.c 22de426c579d01c7bd2c24fb9229fbd774613b42 +F src/test1.c 936afc02766403e5debca49a1817a780e116df7e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -204,7 +204,7 @@ F src/test_func.c 3a8dd37c08ab43b76d38eea2836e34a3897bf170 F src/test_fuzzer.c 1d26aa965120420bc14807da29d4d4541bfa6148 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a -F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 +F src/test_intarray.c 07ddcebe4097d400ffca362770f1d883c112387a F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e @@ -215,7 +215,7 @@ F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 8ab295092c70903ca6f3209fa4c75f5cb6c1bf8e +F src/test_quota.c e5fdb7d28e5afae1b619922804e544db0041ec81 F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 472beb306a4fa7103837d4417aef7d66eef49993 -R ea9a9130bee4f18f6369bb310e898b80 -U dan -Z 63a6a40692fb8e85a92f7431b790c782 +P 89b8c377a6f03d9fa885f3f94c1f0b1eec263dea +R 2026aca949a95171eccb43b1c93409df +U drh +Z af8b2e345718b556d866a3b69f467a92 diff --git a/manifest.uuid b/manifest.uuid index 7b4ca77e44..0172c2d13e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89b8c377a6f03d9fa885f3f94c1f0b1eec263dea \ No newline at end of file +fd74d3d91721ca404537f195fed04c9edef20bf2 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 0a586e489f..47110cb8d2 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2367,7 +2367,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ const char *zDb = "main"; const char *zTable; const char *zColumn; - sqlite_int64 iRow; + Tcl_WideInt iRow; /* Check for the -readonly option */ if( objc>3 && strcmp(Tcl_GetString(objv[2]), "-readonly")==0 ){ diff --git a/src/test1.c b/src/test1.c index 3d2fb02034..e7a2242210 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3067,7 +3067,7 @@ static int test_bind_int64( ){ sqlite3_stmt *pStmt; int idx; - i64 value; + Tcl_WideInt value; int rc; if( objc!=4 ){ @@ -4703,7 +4703,7 @@ static int test_soft_heap_limit( Tcl_Obj *CONST objv[] ){ sqlite3_int64 amt; - sqlite3_int64 N = -1; + Tcl_WideInt N = -1; if( objc!=1 && objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "?N?"); return TCL_ERROR; @@ -5096,7 +5096,7 @@ static int file_control_sizehint_test( int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ - sqlite3_int64 nSize; /* Hinted size */ + Tcl_WideInt nSize; /* Hinted size */ char *zDb; /* Db name ("main", "temp" etc.) */ sqlite3 *db; /* Database handle */ int rc; /* file_control() return code */ diff --git a/src/test_intarray.c b/src/test_intarray.c index 8651d01cfa..6ea7756176 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -346,8 +346,9 @@ static int test_intarray_bind( return TCL_ERROR; } for(i=0; i=0 ? rc : 0; + size_t nWritten = rc; sqlite3_int64 iNewEnd = iOfst + size*nWritten; if( iNewEndinterp, pEval, TCL_EVAL_GLOBAL); if( rc==TCL_OK ){ + Tcl_WideInt x; Tcl_Obj *pLimit = Tcl_ObjGetVar2(p->interp, pVarname, 0, 0); - rc = Tcl_GetWideIntFromObj(p->interp, pLimit, piLimit); + rc = Tcl_GetWideIntFromObj(p->interp, pLimit, &x); + *piLimit = x; Tcl_UnsetVar(p->interp, Tcl_GetString(pVarname), 0); } @@ -1437,7 +1439,7 @@ static int test_quota_set( Tcl_Obj *CONST objv[] ){ const char *zPattern; /* File pattern to configure */ - sqlite3_int64 iLimit; /* Initial quota in bytes */ + Tcl_WideInt iLimit; /* Initial quota in bytes */ Tcl_Obj *pScript; /* Tcl script to invoke to increase quota */ int rc; /* Value returned by quota_set() */ TclQuotaCallback *p; /* Callback object */ @@ -1613,7 +1615,6 @@ static int test_quota_fread( return TCL_ERROR; } got = sqlite3_quota_fread(zBuf, sz, nElem, p); - if( got<0 ) got = 0; zBuf[got*sz] = 0; Tcl_SetResult(interp, zBuf, TCL_VOLATILE); sqlite3_free(zBuf); From befda465984fb0df88d43fba04f98ea8400cfe1b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Sep 2012 15:45:12 +0000 Subject: [PATCH 04/40] Disable the bigfile tests on Macs. FossilOrigin-Name: d869eddaf208c4bf03f6bd1848f510392f9dba49 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/bigfile.test | 1 + test/bigfile2.test | 1 + 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e5dcbb464b..9706885348 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sfound\son\sLion. -D 2012-09-29T14:45:54.660 +C Disable\sthe\sbigfile\stests\son\sMacs. +D 2012-09-29T15:45:12.074 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,8 +295,8 @@ F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 -F test/bigfile.test 8f88b5ef065e31c615c49d725ede94155fbe9609 -F test/bigfile2.test 8a3c242c3c3481e7cde5a6ef2a66fdc367a095f7 +F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 +F test/bigfile2.test 7c79f1ef0c6c2c2bc1e7bd895596fab32bfb4796 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 F test/bind.test 3c7b320969000c441a70952b0b15938fbb66237c F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 89b8c377a6f03d9fa885f3f94c1f0b1eec263dea -R 2026aca949a95171eccb43b1c93409df +P fd74d3d91721ca404537f195fed04c9edef20bf2 +R acef59cbf66888f83c956993bdbbc818 U drh -Z af8b2e345718b556d866a3b69f467a92 +Z 60d0d33f11d050cbca81ed7def74feb5 diff --git a/manifest.uuid b/manifest.uuid index 0172c2d13e..3a58b39b0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd74d3d91721ca404537f195fed04c9edef20bf2 \ No newline at end of file +d869eddaf208c4bf03f6bd1848f510392f9dba49 \ No newline at end of file diff --git a/test/bigfile.test b/test/bigfile.test index 59e9f18c95..31c632322c 100644 --- a/test/bigfile.test +++ b/test/bigfile.test @@ -16,6 +16,7 @@ # if {[file exists skip-big-file]} return +if {$tcl_platform(os)=="Darwin"} return set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/bigfile2.test b/test/bigfile2.test index 1f0ea85e51..b67cb34b43 100644 --- a/test/bigfile2.test +++ b/test/bigfile2.test @@ -14,6 +14,7 @@ # if {[file exists skip-big-file]} return +if {$tcl_platform(os)=="Darwin"} return set testdir [file dirname $argv0] source $testdir/tester.tcl From 60441af4f258a4f4d6a2bf6965c145de01b75e22 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Sep 2012 19:10:29 +0000 Subject: [PATCH 05/40] Improved ORDER BY optimization when outer loops of a join return a single row. FossilOrigin-Name: 62225b4a4c4bfe1820ef54cb202edf2cd866429f --- manifest | 13 +- manifest.uuid | 2 +- src/where.c | 340 +++++++++++++++++++++++---------------------- test/orderby2.test | 71 ++++++++++ 4 files changed, 252 insertions(+), 174 deletions(-) create mode 100644 test/orderby2.test diff --git a/manifest b/manifest index 9706885348..9f30517df5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sbigfile\stests\son\sMacs. -D 2012-09-29T15:45:12.074 +C Improved\sORDER\sBY\soptimization\swhen\souter\sloops\sof\sa\sjoin\sreturn\sa\ssingle\srow. +D 2012-09-29T19:10:29.355 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c d836df3a2096c41c39e48ab5636f09f94ba02676 +F src/where.c acc2ec5f6879721f332223da393777438ea5a606 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -635,6 +635,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/orderby1.test 31c9865626046666e81cd22ecf8e1c24a4ea41b6 +F test/orderby2.test 295d6639e1ce522195354b88ab298d7ede169736 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1017,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P fd74d3d91721ca404537f195fed04c9edef20bf2 -R acef59cbf66888f83c956993bdbbc818 +P d869eddaf208c4bf03f6bd1848f510392f9dba49 +R 13af348b1bf68a7c36428c6258d6aba5 U drh -Z 60d0d33f11d050cbca81ed7def74feb5 +Z 7bbfe622f493f625120e8c012f0943a5 diff --git a/manifest.uuid b/manifest.uuid index 3a58b39b0f..2aa22ae871 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d869eddaf208c4bf03f6bd1848f510392f9dba49 \ No newline at end of file +62225b4a4c4bfe1820ef54cb202edf2cd866429f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7f386289c6..708fe7a1d8 100644 --- a/src/where.c +++ b/src/where.c @@ -257,10 +257,11 @@ struct WhereCost { #define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xpParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - int nPriorSat; /* ORDER BY terms satisfied by outer loops */ - int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ - int nEqOneRow; /* Idx columns that ref unique values */ - - if( p->i==0 ){ - nPriorSat = 0; - nEqOneRow = nEqCol; - }else{ - if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; - nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - sortOrder = bOuterRev; - nEqOneRow = 0; - } - if( p->i>0 && nEqCol==0 /*&& !allOuterLoopsUnique(p)*/ ) return nPriorSat; - pOrderBy = p->pOrderBy; - if( !pOrderBy ) return nPriorSat; - if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; - if( pIdx->bUnordered ) return nPriorSat; - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ - - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return nPriorSat; - } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqOneRow ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - break; - } - }else{ - sortOrder = termSortOrder; - } - j++; - pTerm++; - if( iColumn<0 ){ - seenRowid = 1; - break; - } - } - *pbRev = sortOrder; - - /* If there was an "ORDER BY rowid" term that matched, or it is only - ** possible for a single row from this table to match, then skip over - ** any additional ORDER BY terms dealing with this table. - */ - if( seenRowid || - ( (wsFlags & WHERE_COLUMN_NULL)==0 - && i>=pIdx->nColumn - && indexIsUniqueNotNull(pIdx, nEqCol) - ) - ){ - /* Advance j over additional ORDER BY terms associated with base */ - WhereMaskSet *pMS = p->pWC->pMaskSet; - Bitmask m = ~getMask(pMS, base); - while( ja[j].pExpr)&m)==0 ){ - j++; - } - } - return j; -} - /* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating @@ -2884,6 +2725,9 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ u8 sortOrder; for(i=p->i-1; i>=0; i--, pLevel--){ if( pLevel->iTabCur!=iTab ) continue; + if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + return 1; + } if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ pIdx = pLevel->plan.u.pIdx; if( iCol<0 ){ @@ -2926,9 +2770,6 @@ static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ assert( pExpr->op==TK_EQ ); assert( pExpr->pLeft!=0 && pExpr->pLeft->op==TK_COLUMN ); assert( pExpr->pRight!=0 ); - if( p->i==0 ){ - return 1; /* All == are ordered in the outer loop */ - } if( pTerm->prereqRight==0 ){ return 1; /* RHS of the == is a constant */ } @@ -2942,6 +2783,168 @@ static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ return 0; } +/* +** This routine decides if pIdx can be used to satisfy the ORDER BY +** clause, either in whole or in part. The return value is the +** cumulative number of terms in the ORDER BY clause that are satisfied +** by the index pIdx and other indices in outer loops. +** +** The table being queried has a cursor number of "base". pIdx is the +** index that is postulated for use to access the table. +** +** nEqCol is the number of columns of pIdx that are used as equality +** constraints and where the other side of the == is an ordered column +** or constant. An "order column" in the previous sentence means a column +** in table from an outer loop whose values will always appear in the +** correct order due to othre index, or because the outer loop generates +** a unique result. Any of the first nEqCol columns of pIdx may be missing +** from the ORDER BY clause and the match can still be a success. +** +** The *pbRev value is set to 0 order 1 depending on whether or not +** pIdx should be run in the forward order or in reverse order. +*/ +static int isSortingIndex( + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + int nEqCol, /* Number of index columns with ordered == constraints */ + int wsFlags, /* Index usages flags */ + int bOuterRev, /* True if outer loops scan in reverse order */ + int *pbRev /* Set to 1 for reverse-order scan of pIdx */ +){ + int i; /* Number of pIdx terms used */ + int j; /* Number of ORDER BY terms satisfied */ + int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Parse *pParse = p->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + int nEqOneRow; /* Idx columns that ref unique values */ + + if( p->i==0 ){ + nPriorSat = 0; + }else{ + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return nPriorSat; + } + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + nEqOneRow = nEqCol; + }else{ + if( nEqCol==0 ) return nPriorSat; + sortOrder = bOuterRev; + nEqOneRow = 0; + } + pOrderBy = p->pOrderBy; + assert( pOrderBy!=0 ); + if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; + if( pIdx->bUnordered ) return nPriorSat; + nTerm = pOrderBy->nExpr; + assert( nTerm>0 ); + + /* Argument pIdx must either point to a 'real' named index structure, + ** or an index structure allocated on the stack by bestBtreeIndex() to + ** represent the rowid index that is part of every table. */ + assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + + /* Match terms of the ORDER BY clause against columns of + ** the index. + ** + ** Note that indices have pIdx->nColumn regular columns plus + ** one additional column containing the rowid. The rowid column + ** of the index is also allowed to match against the ORDER BY + ** clause. + */ + for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ + Expr *pExpr; /* The expression of the ORDER BY pTerm */ + CollSeq *pColl; /* The collating sequence of pExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + const char *zColl; /* Name of the collating sequence for i-th index term */ + + pExpr = pTerm->pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ + /* Can not use an index sort on anything that is not a column in the + ** left-most table of the FROM clause */ + break; + } + pColl = sqlite3ExprCollSeq(pParse, pExpr); + if( !pColl ){ + pColl = db->pDfltColl; + } + if( pIdx->zName && inColumn ){ + iColumn = pIdx->aiColumn[i]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = -1; + } + iSortOrder = pIdx->aSortOrder[i]; + zColl = pIdx->azColl[i]; + }else{ + iColumn = -1; + iSortOrder = 0; + zColl = pColl->zName; + } + if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ + /* Term j of the ORDER BY clause does not match column i of the index */ + if( inColumn ){ + /* Index column i is the rowid. All other terms match. */ + break; + }else{ + /* If an index column fails to match and is not constrained by == + ** then the index cannot satisfy the ORDER BY constraint. + */ + return nPriorSat; + } + } + assert( pIdx->aSortOrder!=0 || iColumn==-1 ); + assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = iSortOrder ^ pTerm->sortOrder; + if( i>nEqOneRow ){ + if( termSortOrder!=sortOrder ){ + /* Indices can only be used if all ORDER BY terms past the + ** equality constraints are all either DESC or ASC. */ + break; + } + }else{ + sortOrder = termSortOrder; + } + j++; + pTerm++; + if( iColumn<0 ){ + seenRowid = 1; + break; + } + } + *pbRev = sortOrder; + + /* If there was an "ORDER BY rowid" term that matched, or it is only + ** possible for a single row from this table to match, then skip over + ** any additional ORDER BY terms dealing with this table. + */ + if( seenRowid || + ( (wsFlags & WHERE_COLUMN_NULL)==0 + && i>=pIdx->nColumn + && indexIsUniqueNotNull(pIdx, nEqCol) + ) + ){ + /* Advance j over additional ORDER BY terms associated with base */ + WhereMaskSet *pMS = p->pWC->pMaskSet; + Bitmask m = ~getMask(pMS, base); + while( ja[j].pExpr)&m)==0 ){ + j++; + } + } + return j; +} /* ** Find the best query plan for accessing a particular table. Write the @@ -3177,6 +3180,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ testcase( wsFlags & WHERE_COLUMN_NULL ); if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ wsFlags |= WHERE_UNIQUE; + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + wsFlags |= WHERE_ALL_UNIQUE; + } } }else if( pProbe->bUnordered==0 ){ int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); diff --git a/test/orderby2.test b/test/orderby2.test new file mode 100644 index 0000000000..f07cc57b00 --- /dev/null +++ b/test/orderby2.test @@ -0,0 +1,71 @@ +# 2012 Sept 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the optimizations that disable +# ORDER BY clauses when the natural order of a query is correct. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby2 + +# Generate test data for a join. Verify that the join gets the +# correct answer. +# +do_test 1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1,11), (2,22); + CREATE TABLE t2(d, e, UNIQUE(d,e)); + INSERT INTO t2 VALUES(10, 'ten'), (11,'eleven'), (12,'twelve'), + (11, 'oneteen'); + } +} {} + +do_test 1.1a { + db eval { + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY d, e; + } +} {eleven oneteen} +do_test 1.1b { + db eval { + EXPLAIN QUERY PLAN + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY d, e; + } +} {~/ORDER BY/} + +do_test 1.2a { + db eval { + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY e; + } +} {eleven oneteen} +do_test 1.2b { + db eval { + EXPLAIN QUERY PLAN + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY e; + } +} {~/ORDER BY/} + +do_test 1.3a { + db eval { + SELECT e, b FROM t1, t2 WHERE a=1 ORDER BY d, e; + } +} {ten 11 eleven 11 oneteen 11 twelve 11} +do_test 1.3b { + db eval { + EXPLAIN QUERY PLAN + SELECT e, b FROM t1, t2 WHERE a=1 ORDER BY d, e; + } +} {~/ORDER BY/} + + +finish_test From c9a532697400a62240e4ccfbf7f8f7cd404b576b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Oct 2012 06:50:55 +0000 Subject: [PATCH 06/40] Ensure that the value returned by xSectorSize() is reasonable (currently defined as between 2^5 and 2^16 bytes) before using it to calculate the amount of padding to add to a wal file. FossilOrigin-Name: 6b4ff83bff07d427af585c9fd03be90abf2fc82f --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/pager.c | 24 ++++++++++++++++-------- src/pager.h | 1 + src/wal.c | 2 +- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 9f30517df5..07b54cc849 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sORDER\sBY\soptimization\swhen\souter\sloops\sof\sa\sjoin\sreturn\sa\ssingle\srow. -D 2012-09-29T19:10:29.355 +C Ensure\sthat\sthe\svalue\sreturned\sby\sxSectorSize()\sis\sreasonable\s(currently\sdefined\sas\sbetween\s2^5\sand\s2^16\sbytes)\sbefore\susing\sit\sto\scalculate\sthe\samount\sof\spadding\sto\sadd\sto\sa\swal\sfile. +D 2012-10-01T06:50:55.576 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,8 +162,8 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 69b2fe66316524eebf5f1ce85c1fdfe2952307e9 F src/os_win.c 90c7a1fe2698867555ba4266f5bd436c85d0d1dc -F src/pager.c 9c59818c480261c1c5a4772532e0df92a27745a1 -F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 +F src/pager.c 9f5f2823594cc2848e151510f726af02896485b5 +F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c @@ -246,7 +246,7 @@ F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 -F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d +F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c acc2ec5f6879721f332223da393777438ea5a606 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P d869eddaf208c4bf03f6bd1848f510392f9dba49 -R 13af348b1bf68a7c36428c6258d6aba5 -U drh -Z 7bbfe622f493f625120e8c012f0943a5 +P 62225b4a4c4bfe1820ef54cb202edf2cd866429f +R 09d3e5b05c9dfd1b20715f593e6818b6 +U dan +Z 07828d460d32254f1342208677faf52f diff --git a/manifest.uuid b/manifest.uuid index 2aa22ae871..19bd1d6a97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62225b4a4c4bfe1820ef54cb202edf2cd866429f \ No newline at end of file +6b4ff83bff07d427af585c9fd03be90abf2fc82f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index a094e0da20..28f6546301 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2509,6 +2509,21 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ return rc; } +/* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method @@ -2544,14 +2559,7 @@ static void setSectorSize(Pager *pPager){ ** call will segfault. */ pPager->sectorSize = 512; }else{ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - if( pPager->sectorSize<32 ){ - pPager->sectorSize = 512; - } - if( pPager->sectorSize>MAX_SECTOR_SIZE ){ - assert( MAX_SECTOR_SIZE>=512 ); - pPager->sectorSize = MAX_SECTOR_SIZE; - } + pPager->sectorSize = sqlite3SectorSize(pPager->fd); } } diff --git a/src/pager.h b/src/pager.h index 2b60e058da..5fb0f0133f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -160,6 +160,7 @@ void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); void sqlite3PagerClearCache(Pager *); +int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/wal.c b/src/wal.c index cc166ba430..8394bfa29f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2828,7 +2828,7 @@ int sqlite3WalFrames( */ if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ if( pWal->padToSectorBoundary ){ - int sectorSize = sqlite3OsSectorSize(pWal->pWalFd); + int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; while( iOffset Date: Mon, 1 Oct 2012 12:16:26 +0000 Subject: [PATCH 07/40] Make sure the size parameter to read and write VFS methods in the unix VFS do not become too big or go negative. This was not actually possible in the current code. The checks are added to make sure some future bug does not make it possible. FossilOrigin-Name: daebe3bd2d9bd7b6f876a8110cf5045eb3fee078 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 07b54cc849..0bf6940ec3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\svalue\sreturned\sby\sxSectorSize()\sis\sreasonable\s(currently\sdefined\sas\sbetween\s2^5\sand\s2^16\sbytes)\sbefore\susing\sit\sto\scalculate\sthe\samount\sof\spadding\sto\sadd\sto\sa\swal\sfile. -D 2012-10-01T06:50:55.576 +C Make\ssure\sthe\ssize\sparameter\sto\sread\sand\swrite\sVFS\smethods\sin\sthe\sunix\sVFS\ndo\snot\sbecome\stoo\sbig\sor\sgo\snegative.\s\sThis\swas\snot\sactually\spossible\sin\sthe\ncurrent\scode.\s\sThe\schecks\sare\sadded\sto\smake\ssure\ssome\sfuture\sbug\sdoes\snot\nmake\sit\spossible. +D 2012-10-01T12:16:26.771 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 69b2fe66316524eebf5f1ce85c1fdfe2952307e9 +F src/os_unix.c a5a45a2857c43b37bac145b521064a85a544cd7a F src/os_win.c 90c7a1fe2698867555ba4266f5bd436c85d0d1dc F src/pager.c 9f5f2823594cc2848e151510f726af02896485b5 F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 62225b4a4c4bfe1820ef54cb202edf2cd866429f -R 09d3e5b05c9dfd1b20715f593e6818b6 -U dan -Z 07828d460d32254f1342208677faf52f +P 6b4ff83bff07d427af585c9fd03be90abf2fc82f +R 43398c1dd9f4f691234256568baab7a2 +U drh +Z 321bb2af8cac9de7aef8d0d7efaf3889 diff --git a/manifest.uuid b/manifest.uuid index 19bd1d6a97..1db2bb2963 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b4ff83bff07d427af585c9fd03be90abf2fc82f \ No newline at end of file +daebe3bd2d9bd7b6f876a8110cf5045eb3fee078 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c0df66e8e0..0852eb1a89 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3010,6 +3010,8 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ i64 newOffset; #endif TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -3099,6 +3101,8 @@ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ #if (!defined(USE_PREAD) && !defined(USE_PREAD64)) i64 newOffset; #endif + assert( cnt==(cnt&0x1ffff) ); + cnt &= 0x1ffff; TIMER_START; #if defined(USE_PREAD) do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); From a1f385379391549f44e87687956da0136f39d142 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Oct 2012 12:44:26 +0000 Subject: [PATCH 08/40] Omit the sqlite3GetReservedNoMutex() routine in build configurations where it is not used. FossilOrigin-Name: f193dbb6b96b2f49dd0d6345802c9c006cd99e3c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 ++ src/btree.h | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0bf6940ec3..8f4fbf72ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\ssize\sparameter\sto\sread\sand\swrite\sVFS\smethods\sin\sthe\sunix\sVFS\ndo\snot\sbecome\stoo\sbig\sor\sgo\snegative.\s\sThis\swas\snot\sactually\spossible\sin\sthe\ncurrent\scode.\s\sThe\schecks\sare\sadded\sto\smake\ssure\ssome\sfuture\sbug\sdoes\snot\nmake\sit\spossible. -D 2012-10-01T12:16:26.771 +C Omit\sthe\ssqlite3GetReservedNoMutex()\sroutine\sin\sbuild\sconfigurations\swhere\nit\sis\snot\sused. +D 2012-10-01T12:44:26.787 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c afc067b9a9050ff48b9d46285c53d096c556a73d F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 821615a1e1002346d84d2e341ecd5e947196454b -F src/btree.h 5e6482bcabf39455385e39c9739a5954e4775bba +F src/btree.c 95ee3c1e788ac1f68d3ffc37b599c811a34c080b +F src/btree.h 078f76b28c338ab6eb6dd7324d63ee54463aeb6c F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 6b4ff83bff07d427af585c9fd03be90abf2fc82f -R 43398c1dd9f4f691234256568baab7a2 +P daebe3bd2d9bd7b6f876a8110cf5045eb3fee078 +R d91a487579faabb2d53225715f6d270e U drh -Z 321bb2af8cac9de7aef8d0d7efaf3889 +Z ba2684210aeb6310a87308d43a7fd6e8 diff --git a/manifest.uuid b/manifest.uuid index 1db2bb2963..7fb135f8f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -daebe3bd2d9bd7b6f876a8110cf5045eb3fee078 \ No newline at end of file +f193dbb6b96b2f49dd0d6345802c9c006cd99e3c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index eff70f3b36..296469378a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2200,6 +2200,7 @@ int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) /* ** This function is similar to sqlite3BtreeGetReserve(), except that it ** may only be called if it is guaranteed that the b-tree mutex is already @@ -2215,6 +2216,7 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ assert( sqlite3_mutex_held(p->pBt->mutex) ); return p->pBt->pageSize - p->pBt->usableSize; } +#endif /* SQLITE_HAS_CODEC || SQLITE_DEBUG */ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) /* diff --git a/src/btree.h b/src/btree.h index 5a894f5f22..0efa0cdcdc 100644 --- a/src/btree.h +++ b/src/btree.h @@ -71,7 +71,9 @@ int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); int sqlite3BtreeBeginTrans(Btree*,int); From 04b85bc609a6b9fb45c9e15b2762f0527c71d604 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Oct 2012 17:44:05 +0000 Subject: [PATCH 09/40] Minor changes to the query planner for improved test coverage. FossilOrigin-Name: c12044df322283c9deb0d7dacd8a699ac53f27b6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8f4fbf72ad..069ed605a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\ssqlite3GetReservedNoMutex()\sroutine\sin\sbuild\sconfigurations\swhere\nit\sis\snot\sused. -D 2012-10-01T12:44:26.787 +C Minor\schanges\sto\sthe\squery\splanner\sfor\simproved\stest\scoverage. +D 2012-10-01T17:44:05.444 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c acc2ec5f6879721f332223da393777438ea5a606 +F src/where.c e676efbfa8d5fe04bf2e3955d83f88d699e6eecd F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P daebe3bd2d9bd7b6f876a8110cf5045eb3fee078 -R d91a487579faabb2d53225715f6d270e +P f193dbb6b96b2f49dd0d6345802c9c006cd99e3c +R cc743e0e8cc0407b063f795edee17263 U drh -Z ba2684210aeb6310a87308d43a7fd6e8 +Z 2fb8150c8641aac32ecd5d693f13df87 diff --git a/manifest.uuid b/manifest.uuid index 7fb135f8f2..f781c3b765 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f193dbb6b96b2f49dd0d6345802c9c006cd99e3c \ No newline at end of file +c12044df322283c9deb0d7dacd8a699ac53f27b6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 708fe7a1d8..b95e6bf6f6 100644 --- a/src/where.c +++ b/src/where.c @@ -1441,7 +1441,8 @@ static int indexIsUniqueNotNull(Index *pIdx, int nSkip){ if( pIdx->onError==OE_None ) return 0; for(i=nSkip; inColumn; i++){ int j = pIdx->aiColumn[i]; - if( j>=0 && pTab->aCol[j].notNull==0 ) return 0; + assert( j>=0 && jnCol ); + if( pTab->aCol[j].notNull==0 ) return 0; } return 1; } @@ -1505,7 +1506,8 @@ static int isDistinctIndex( Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ int i; /* Iterator variable */ - if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + assert( pDistinct!=0 ); + if( pIdx->zName==0 || pDistinct->nExpr>=BMS ) return 0; testcase( pDistinct->nExpr==BMS-1 ); /* Loop through all the expressions in the distinct list. If any of them @@ -3097,7 +3099,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** external sort (i.e. scanning the index being evaluated will not ** correctly order records). ** - ** bDistinct: + ** bDist: ** Boolean. True if there is a DISTINCT clause that will require an ** external btree. ** From 80bb6f821747967aa6b553ddfc44ac4dc6462062 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Oct 2012 18:44:33 +0000 Subject: [PATCH 10/40] Add SQLITE_FCNTL_BUSYHANDLER. Invoked to provide access to the busy-handler callback to custom VFS implementations. FossilOrigin-Name: 1262d2f471214a3417dd02159a7aa2355b18f789 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/pager.c | 9 ++++++++- src/pragma.c | 1 + src/sqlite.h.in | 12 ++++++++++++ 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 069ed605a7..2c021a017c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\sthe\squery\splanner\sfor\simproved\stest\scoverage. -D 2012-10-01T17:44:05.444 +C Add\sSQLITE_FCNTL_BUSYHANDLER.\sInvoked\sto\sprovide\saccess\sto\sthe\sbusy-handler\scallback\sto\scustom\sVFS\simplementations. +D 2012-10-01T18:44:33.687 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,13 +162,13 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a5a45a2857c43b37bac145b521064a85a544cd7a F src/os_win.c 90c7a1fe2698867555ba4266f5bd436c85d0d1dc -F src/pager.c 9f5f2823594cc2848e151510f726af02896485b5 +F src/pager.c 52bfc72f7f8c6e49668803657e8f239e38c529dd F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 7f5a0fa6dead752d9af3a1e1d16bace81d22ef95 +F src/pragma.c 4f31fe4ae4b885a8be8b1db9d94f9f5b6c8b5624 F src/prepare.c 931ad0d852a0df48f79adcba6ce79ca5f475625c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,7 +176,7 @@ F src/resolve.c 9e28280ec98035f31900fdd1db01f86f68ca6c32 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 75c5e37cc882c468383c9d9e07496b9a16cfae3e F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 -F src/sqlite.h.in cbe846facaba903654b4136c97e7f57b3ac0bac7 +F src/sqlite.h.in c7be05ad191d2634292fcc77bdb2bcfa4526eb98 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P f193dbb6b96b2f49dd0d6345802c9c006cd99e3c -R cc743e0e8cc0407b063f795edee17263 -U drh -Z 2fb8150c8641aac32ecd5d693f13df87 +P c12044df322283c9deb0d7dacd8a699ac53f27b6 +R d094915a20a7714c82c93282dde9246b +U dan +Z a39135b23d456d91a1a24c9caf254ec8 diff --git a/manifest.uuid b/manifest.uuid index f781c3b765..1a1f385f50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c12044df322283c9deb0d7dacd8a699ac53f27b6 \ No newline at end of file +1262d2f471214a3417dd02159a7aa2355b18f789 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 28f6546301..21279824ad 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3476,9 +3476,16 @@ void sqlite3PagerSetBusyhandler( Pager *pPager, /* Pager object */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ -){ +){ pPager->xBusyHandler = xBusyHandler; pPager->pBusyHandlerArg = pBusyHandlerArg; + + if( isOpen(pPager->fd) ){ + void **ap = &pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); + } } /* diff --git a/src/pragma.c b/src/pragma.c index f41f2db06f..8eaec51ec3 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -357,6 +357,7 @@ void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; + db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bf97679ab8..0d619e96c0 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -852,6 +852,17 @@ struct sqlite3_io_methods { ** file control occurs at the beginning of pragma statement analysis and so ** it is able to override built-in [PRAGMA] statements. ** +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^This file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -867,6 +878,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 /* ** CAPI3REF: Mutex Handle From c197eedbb4cb93138ef51bd0c59e880cb02b4eb0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 1 Oct 2012 20:12:30 +0000 Subject: [PATCH 11/40] Fix several compiler warnings seen with MSVC. FossilOrigin-Name: 8b5f617c2a5f0bf7bdbd9144aee58d69564db1e2 --- ext/rtree/rtree.c | 4 ++-- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/main.c | 2 +- src/pager.c | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 66da481e0f..731cb32be0 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -2660,7 +2660,7 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ */ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ int rc; /* Return code */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ int iCell; /* Index of iDelete cell in pLeaf */ RtreeNode *pRoot; /* Root node of rtree structure */ @@ -2863,7 +2863,7 @@ static int rtreeUpdate( */ if( rc==SQLITE_OK && nData>1 ){ /* Insert the new record into the r-tree */ - RtreeNode *pLeaf; + RtreeNode *pLeaf = 0; /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ diff --git a/manifest b/manifest index 2c021a017c..fd3d07a71d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_FCNTL_BUSYHANDLER.\sInvoked\sto\sprovide\saccess\sto\sthe\sbusy-handler\scallback\sto\scustom\sVFS\simplementations. -D 2012-10-01T18:44:33.687 +C Fix\sseveral\scompiler\swarnings\sseen\swith\sMSVC. +D 2012-10-01T20:12:30.885 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -83,7 +83,7 @@ F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d17aecb7a92762efa7b1f5d5fd7c88fd77d70827 +F ext/rtree/rtree.c 47064ee2995a396bfb626337d2b43f12cc0af687 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test e474a2b5eff231496dbd073fe67e5fbaf7f444c9 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -143,7 +143,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c 5249486037f88e765cce7c4870846dc5a2cd2ed1 +F src/main.c ca6784455f3207524b82d24141676d641d44f137 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a5a45a2857c43b37bac145b521064a85a544cd7a F src/os_win.c 90c7a1fe2698867555ba4266f5bd436c85d0d1dc -F src/pager.c 52bfc72f7f8c6e49668803657e8f239e38c529dd +F src/pager.c a7ad8c38809edf0be545e8f52da5bcbb88885b38 F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1018,7 +1018,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P c12044df322283c9deb0d7dacd8a699ac53f27b6 -R d094915a20a7714c82c93282dde9246b -U dan -Z a39135b23d456d91a1a24c9caf254ec8 +P 1262d2f471214a3417dd02159a7aa2355b18f789 +R 0f744c0892eb0257285d51198fe0d093 +T *branch * msvcWarnings +T *sym-msvcWarnings * +T -sym-trunk * +U mistachkin +Z b42d95f34ab630c00d546c5b9df935b7 diff --git a/manifest.uuid b/manifest.uuid index 1a1f385f50..a5fb3fa918 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1262d2f471214a3417dd02159a7aa2355b18f789 \ No newline at end of file +8b5f617c2a5f0bf7bdbd9144aee58d69564db1e2 \ No newline at end of file diff --git a/src/main.c b/src/main.c index b2826c0c7b..8f3d25dd22 100644 --- a/src/main.c +++ b/src/main.c @@ -3019,7 +3019,7 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + db->dbOptFlags = (u8)(va_arg(ap, int) & 0xff); break; } diff --git a/src/pager.c b/src/pager.c index 21279824ad..a767d747fe 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3481,7 +3481,7 @@ void sqlite3PagerSetBusyhandler( pPager->pBusyHandlerArg = pBusyHandlerArg; if( isOpen(pPager->fd) ){ - void **ap = &pPager->xBusyHandler; + void **ap = (void **)&pPager->xBusyHandler; assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); assert( ap[1]==pBusyHandlerArg ); sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); From 1afcaaea7aae81e9b34efac3cf2cae6481323149 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Oct 2012 01:10:00 +0000 Subject: [PATCH 12/40] Improvements to the wheretrace output. FossilOrigin-Name: 76b277655646a9fe0f336ff106157f1efcd32019 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 05bf29b768..2efd869be0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2012-10-01T20:32:38.373 +C Improvements\sto\sthe\swheretrace\soutput. +D 2012-10-02T01:10:00.432 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c e676efbfa8d5fe04bf2e3955d83f88d699e6eecd +F src/where.c 9f706f54008083dac6079a75463d6921eab2fecd F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 1262d2f471214a3417dd02159a7aa2355b18f789 8b5f617c2a5f0bf7bdbd9144aee58d69564db1e2 -R 0f744c0892eb0257285d51198fe0d093 +P 59c2be75ff8b8ccd77320b45e4088e5bf456a1d9 +R c1d8d9fb6c53d3d30af8c26e9e5fd207 U drh -Z 8e271de1fa8eae3da9424d1b1b2f682e +Z 969a36303c6eae35969bc68961950155 diff --git a/manifest.uuid b/manifest.uuid index e83a48038b..2594ba29b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59c2be75ff8b8ccd77320b45e4088e5bf456a1d9 \ No newline at end of file +76b277655646a9fe0f336ff106157f1efcd32019 \ No newline at end of file diff --git a/src/where.c b/src/where.c index b95e6bf6f6..7f08832437 100644 --- a/src/where.c +++ b/src/where.c @@ -5034,8 +5034,8 @@ WhereInfo *sqlite3WhereBegin( sWBI.notReady = (isOptimal ? m : sWBI.notValid); if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); + WHERETRACE(("=== trying table %d (%s) with isOptimal=%d ===\n", + j, sWBI.pSrc->pTab->zName, isOptimal)); assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(sWBI.pSrc->pTab) ){ @@ -5089,9 +5089,10 @@ WhereInfo *sqlite3WhereBegin( || (sWBI.cost.rCost<=bestPlan.rCost && sWBI.cost.plan.nRowpTab->zName, + sWBI.cost.rCost, sWBI.cost.plan.nRow, sWBI.cost.plan.nOBSat)); bestPlan = sWBI.cost; bestJ = j; @@ -5101,9 +5102,10 @@ WhereInfo *sqlite3WhereBegin( } assert( bestJ>=0 ); assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d with:\n" + WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" " cost=%.1f, nRow=%.1f, nOBSat=%d wsFlags=0x%08x\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, + bestJ, pTabList->a[bestJ].pTab->zName, + pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ pWInfo->nOBSat = pOrderBy->nExpr; From a578d048c64859965f7f48397eb68e4fc6b4bb35 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Oct 2012 01:25:16 +0000 Subject: [PATCH 13/40] Make sure the outer loop cursor numbers are recorded in time for them to be used by the ORDER BY optimizer. FossilOrigin-Name: 351dc8d9cab8a222f28fb865d3a7d1c622d60d6f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2efd869be0..f2a8ebb39a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\swheretrace\soutput. -D 2012-10-02T01:10:00.432 +C Make\ssure\sthe\souter\sloop\scursor\snumbers\sare\srecorded\sin\stime\sfor\sthem\sto\sbe\nused\sby\sthe\sORDER\sBY\soptimizer. +D 2012-10-02T01:25:16.426 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 9f706f54008083dac6079a75463d6921eab2fecd +F src/where.c 0a309f52e9ca5e77250a242c2f0527754e3e30c4 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 59c2be75ff8b8ccd77320b45e4088e5bf456a1d9 -R c1d8d9fb6c53d3d30af8c26e9e5fd207 +P 76b277655646a9fe0f336ff106157f1efcd32019 +R 8e3740840a007142381cf2f0396df8e5 U drh -Z 969a36303c6eae35969bc68961950155 +Z 2be46057d17778455d3bb7f9393d3220 diff --git a/manifest.uuid b/manifest.uuid index 2594ba29b7..53d466cfe9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76b277655646a9fe0f336ff106157f1efcd32019 \ No newline at end of file +351dc8d9cab8a222f28fb865d3a7d1c622d60d6f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7f08832437..739222338e 100644 --- a/src/where.c +++ b/src/where.c @@ -5116,6 +5116,7 @@ WhereInfo *sqlite3WhereBegin( } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; + pLevel->iTabCur = pTabList->a[bestJ].iCursor; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ @@ -5189,7 +5190,6 @@ WhereInfo *sqlite3WhereBegin( pTabItem = &pTabList->a[pLevel->iFrom]; pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ From 3be57d2abadd1507a9114eb920916a319c069715 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Oct 2012 01:46:48 +0000 Subject: [PATCH 14/40] Factor an invariant out the loop termination condition for the ORDER BY satisfied-by-index analyzer routine. FossilOrigin-Name: 545bb33688663066cf3f09e4cdc4d5cfe59fb8db --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f2a8ebb39a..0d1e3cf201 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\souter\sloop\scursor\snumbers\sare\srecorded\sin\stime\sfor\sthem\sto\sbe\nused\sby\sthe\sORDER\sBY\soptimizer. -D 2012-10-02T01:25:16.426 +C Factor\san\sinvariant\sout\sthe\sloop\stermination\scondition\sfor\sthe\sORDER\sBY\nsatisfied-by-index\sanalyzer\sroutine. +D 2012-10-02T01:46:48.203 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 0a309f52e9ca5e77250a242c2f0527754e3e30c4 +F src/where.c 58a99bf57e1e216210fa143f3ae3cd1a5928a36f F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 76b277655646a9fe0f336ff106157f1efcd32019 -R 8e3740840a007142381cf2f0396df8e5 +P 351dc8d9cab8a222f28fb865d3a7d1c622d60d6f +R e2343532e1485c60fb209fa3ba6a8cc6 U drh -Z 2be46057d17778455d3bb7f9393d3220 +Z 8f9278868782471bac046c34faf9fbb1 diff --git a/manifest.uuid b/manifest.uuid index 53d466cfe9..bd7521b1fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -351dc8d9cab8a222f28fb865d3a7d1c622d60d6f \ No newline at end of file +545bb33688663066cf3f09e4cdc4d5cfe59fb8db \ No newline at end of file diff --git a/src/where.c b/src/where.c index 739222338e..77f9fb2b7c 100644 --- a/src/where.c +++ b/src/where.c @@ -2859,7 +2859,7 @@ static int isSortingIndex( ** of the index is also allowed to match against the ORDER BY ** clause. */ - for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ + for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn ); pExpr = pTerm->pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ /* Can not use an index sort on anything that is not a column in the From 178eb61c6b18d16be898a73578937c8cd5b9eefa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Oct 2012 14:11:29 +0000 Subject: [PATCH 15/40] Only continue an ORDER BY optimization into inner loops if the equality constraints on the inner loop match terms of an outer ordered index that are actually used by the ORDER BY clause. FossilOrigin-Name: b0e7b4df6c2a8c479f8d210bde50c737eaa248f0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 7 +++++-- test/orderby1.test | 28 ++++++++++++++++++++-------- test/orderby2.test | 26 ++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 0d1e3cf201..a0b9b77d8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\san\sinvariant\sout\sthe\sloop\stermination\scondition\sfor\sthe\sORDER\sBY\nsatisfied-by-index\sanalyzer\sroutine. -D 2012-10-02T01:46:48.203 +C Only\scontinue\san\sORDER\sBY\soptimization\sinto\sinner\sloops\sif\sthe\sequality\nconstraints\son\sthe\sinner\sloop\smatch\sterms\sof\san\souter\sordered\sindex\sthat\nare\sactually\sused\sby\sthe\sORDER\sBY\sclause. +D 2012-10-02T14:11:29.949 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 58a99bf57e1e216210fa143f3ae3cd1a5928a36f +F src/where.c d40f2fa13bbda673cc681a2baeb490fe7a22f17e F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -634,8 +634,8 @@ F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 31c9865626046666e81cd22ecf8e1c24a4ea41b6 -F test/orderby2.test 295d6639e1ce522195354b88ab298d7ede169736 +F test/orderby1.test 1f7e942aec6074725d823edfff258e9187dd255f +F test/orderby2.test d8fa5991d8948ae1f335c2f91d751e955bfee815 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 351dc8d9cab8a222f28fb865d3a7d1c622d60d6f -R e2343532e1485c60fb209fa3ba6a8cc6 +P 545bb33688663066cf3f09e4cdc4d5cfe59fb8db +R 0244ec6263deff2d19e204ecc996e53d U drh -Z 8f9278868782471bac046c34faf9fbb1 +Z 17cee407676b72dedd8bc51219c654fe diff --git a/manifest.uuid b/manifest.uuid index bd7521b1fc..73689c0af9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -545bb33688663066cf3f09e4cdc4d5cfe59fb8db \ No newline at end of file +b0e7b4df6c2a8c479f8d210bde50c737eaa248f0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 77f9fb2b7c..ca998269b9 100644 --- a/src/where.c +++ b/src/where.c @@ -2736,10 +2736,13 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ sortOrder = 0; testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); }else{ - for(j=0; jnColumn; j++){ + int n = pLevel->plan.nOBSat; + if( p->i>=2 ) n -= pLevel[-1].plan.nOBSat; + assert( n<=pIdx->nColumn ); + for(j=0; jaiColumn[j] ) break; } - if( j>=pIdx->nColumn ) return 0; + if( j>=n ) return 0; sortOrder = pIdx->aSortOrder[j]; testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); } diff --git a/test/orderby1.test b/test/orderby1.test index 400659b475..77ceadae09 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -170,14 +170,14 @@ do_test 2.0 { name TEXT, UNIQUE(aid, tn) ); - INSERT INTO album VALUES(1, '1-one'), (2, '2-two'), (3, '3-three'); + INSERT INTO album VALUES(1, '1-one'), (20, '2-two'), (3, '3-three'); INSERT INTO track VALUES - (1, 1, 'one-a'), - (2, 2, 'two-b'), - (3, 3, 'three-c'), - (1, 3, 'one-c'), - (2, 1, 'two-a'), - (3, 1, 'three-a'); + (1, 1, 'one-a'), + (20, 2, 'two-b'), + (3, 3, 'three-c'), + (1, 3, 'one-c'), + (20, 1, 'two-a'), + (3, 1, 'three-a'); COMMIT; } } {} @@ -194,6 +194,18 @@ do_test 2.1b { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } +} {/ORDER BY/} ;# ORDER BY required because of missing aid term in ORDER BY + +do_test 2.1c { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, aid, tn + } +} {one-a one-c two-a two-b three-a three-c} +do_test 2.1d { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, aid, tn + } } {~/ORDER BY/} ;# ORDER BY optimized out # The same query with ORDER BY clause optimization disabled via + operators @@ -284,7 +296,7 @@ do_test 2.6c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } -} {~/ORDER BY/} ;# ORDER BY optimized-out +} {/ORDER BY/} ;# ORDER BY required # Generate another test dataset, but this time using mixed ASC/DESC indices. diff --git a/test/orderby2.test b/test/orderby2.test index f07cc57b00..7d543229c0 100644 --- a/test/orderby2.test +++ b/test/orderby2.test @@ -67,5 +67,31 @@ do_test 1.3b { } } {~/ORDER BY/} +# After where34.314 in TH3: +do_test 2.0 { + db eval { + CREATE TABLE t31(a,b); CREATE INDEX t31ab ON t31(a,b); + CREATE TABLE t32(c,d); CREATE INDEX t32cd ON t32(c,d); + CREATE TABLE t33(e,f); CREATE INDEX t33ef ON t33(e,f); + CREATE TABLE t34(g,h); CREATE INDEX t34gh ON t34(g,h); + + INSERT INTO t31 VALUES(1,4), (2,3), (1,3); + INSERT INTO t32 VALUES(4,5), (3,6), (3,7), (4,8); + INSERT INTO t33 VALUES(5,9), (7,10), (6,11), (8,12), (8,13), (7,14); + INSERT INTO t34 VALUES(11,20), (10,21), (12,22), (9,23), (13,24), + (14,25), (12,26); + SELECT a||','||c||','||e||','||g FROM t31, t32, t33, t34 + WHERE c=b AND e=d AND g=f + ORDER BY a ASC, c ASC, e DESC, g ASC; + } +} {1,3,7,10 1,3,7,14 1,3,6,11 1,4,8,12 1,4,8,12 1,4,8,13 1,4,5,9 2,3,7,10 2,3,7,14 2,3,6,11} +do_test 2.1 { + db eval { + SELECT a||','||c||','||e||','||g FROM t31, t32, t33, t34 + WHERE c=b AND e=d AND g=f + ORDER BY +a ASC, +c ASC, +e DESC, +g ASC; + } +} {1,3,7,10 1,3,7,14 1,3,6,11 1,4,8,12 1,4,8,12 1,4,8,13 1,4,5,9 2,3,7,10 2,3,7,14 2,3,6,11} + finish_test From afcd522af800c9e46d72edf3a6f8583f417d6805 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Oct 2012 15:19:19 +0000 Subject: [PATCH 16/40] More lenient handling of ORDER BY optimization in joins with mixed ASC/DESC. This is a better and less restrictive fix for the problem addressed by the previous check-in. FossilOrigin-Name: abcf6a5d054559ee5a093ba39180c47b4958d9cd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 8 +++----- test/orderby1.test | 4 ++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index a0b9b77d8a..58498bcbfe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\scontinue\san\sORDER\sBY\soptimization\sinto\sinner\sloops\sif\sthe\sequality\nconstraints\son\sthe\sinner\sloop\smatch\sterms\sof\san\souter\sordered\sindex\sthat\nare\sactually\sused\sby\sthe\sORDER\sBY\sclause. -D 2012-10-02T14:11:29.949 +C More\slenient\shandling\sof\sORDER\sBY\soptimization\sin\sjoins\swith\smixed\sASC/DESC.\nThis\sis\sa\sbetter\sand\sless\srestrictive\sfix\sfor\sthe\sproblem\saddressed\sby\nthe\sprevious\scheck-in. +D 2012-10-02T15:19:19.625 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c d40f2fa13bbda673cc681a2baeb490fe7a22f17e +F src/where.c 69398e95e9c1012ae07ce9ea00f21b0e7bab8df1 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -634,7 +634,7 @@ F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 1f7e942aec6074725d823edfff258e9187dd255f +F test/orderby1.test 4875a2a0a87d81920f3600a3405dc42f233b8c82 F test/orderby2.test d8fa5991d8948ae1f335c2f91d751e955bfee815 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 545bb33688663066cf3f09e4cdc4d5cfe59fb8db -R 0244ec6263deff2d19e204ecc996e53d +P b0e7b4df6c2a8c479f8d210bde50c737eaa248f0 +R a1842b2ba16ad28b21201e48479c524d U drh -Z 17cee407676b72dedd8bc51219c654fe +Z 4c91eefd164f9f4e974324de15060f38 diff --git a/manifest.uuid b/manifest.uuid index 73689c0af9..22cffbf9b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0e7b4df6c2a8c479f8d210bde50c737eaa248f0 \ No newline at end of file +abcf6a5d054559ee5a093ba39180c47b4958d9cd \ No newline at end of file diff --git a/src/where.c b/src/where.c index ca998269b9..16b0c1c710 100644 --- a/src/where.c +++ b/src/where.c @@ -2736,9 +2736,7 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ sortOrder = 0; testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); }else{ - int n = pLevel->plan.nOBSat; - if( p->i>=2 ) n -= pLevel[-1].plan.nOBSat; - assert( n<=pIdx->nColumn ); + int n = pIdx->nColumn; for(j=0; jaiColumn[j] ) break; } @@ -2840,7 +2838,7 @@ static int isSortingIndex( }else{ if( nEqCol==0 ) return nPriorSat; sortOrder = bOuterRev; - nEqOneRow = 0; + nEqOneRow = -1; } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); @@ -2917,7 +2915,7 @@ static int isSortingIndex( if( i>nEqOneRow ){ if( termSortOrder!=sortOrder ){ /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ + ** equality constraints have the correct DESC or ASC. */ break; } }else{ diff --git a/test/orderby1.test b/test/orderby1.test index 77ceadae09..8087857931 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -194,7 +194,7 @@ do_test 2.1b { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } -} {/ORDER BY/} ;# ORDER BY required because of missing aid term in ORDER BY +} {~/ORDER BY/} ;# ORDER BY optimized out do_test 2.1c { db eval { @@ -296,7 +296,7 @@ do_test 2.6c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } -} {/ORDER BY/} ;# ORDER BY required +} {~/ORDER BY/} ;# ORDER BY optimized out # Generate another test dataset, but this time using mixed ASC/DESC indices. From e7c54168faa48c0e589b5fbbe9fd96409801b177 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 2 Oct 2012 22:54:27 +0000 Subject: [PATCH 17/40] Work around an optimization issue with the MSVC compiler for ARM. FossilOrigin-Name: 7d301fdfeec540e4a58f43bff04d219e9f769dc4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58498bcbfe..178efcf35c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\slenient\shandling\sof\sORDER\sBY\soptimization\sin\sjoins\swith\smixed\sASC/DESC.\nThis\sis\sa\sbetter\sand\sless\srestrictive\sfix\sfor\sthe\sproblem\saddressed\sby\nthe\sprevious\scheck-in. -D 2012-10-02T15:19:19.625 +C Work\saround\san\soptimization\sissue\swith\sthe\sMSVC\scompiler\sfor\sARM. +D 2012-10-02T22:54:27.390 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c afc067b9a9050ff48b9d46285c53d096c556a73d F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 95ee3c1e788ac1f68d3ffc37b599c811a34c080b +F src/btree.c 14de53ebb334633ee632ab3c5b9262cfe7cbe455 F src/btree.h 078f76b28c338ab6eb6dd7324d63ee54463aeb6c F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P b0e7b4df6c2a8c479f8d210bde50c737eaa248f0 -R a1842b2ba16ad28b21201e48479c524d -U drh -Z 4c91eefd164f9f4e974324de15060f38 +P abcf6a5d054559ee5a093ba39180c47b4958d9cd +R e2af9a84c8984f66075b1b5542126d99 +U mistachkin +Z 2e5e5eca8b24072e2977c29980c9c33a diff --git a/manifest.uuid b/manifest.uuid index 22cffbf9b5..6ff004a2db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abcf6a5d054559ee5a093ba39180c47b4958d9cd \ No newline at end of file +7d301fdfeec540e4a58f43bff04d219e9f769dc4 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 296469378a..4ee0c860a7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5940,6 +5940,9 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) +#pragma optimize("", off) +#endif static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ @@ -6570,6 +6573,9 @@ balance_cleanup: return rc; } +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) +#pragma optimize("", on) +#endif /* From d663b5bdefd6f5fc6752f66df21d9cdfc601a779 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Oct 2012 00:25:54 +0000 Subject: [PATCH 18/40] Further attempts to optimize out unnecessary ORDER BY clauses. FossilOrigin-Name: 6744d9a37faffed59b4d5cb96c8671ec46a87ea7 --- manifest | 21 ++-- manifest.uuid | 2 +- src/where.c | 254 +++++++++++++++++++++++++-------------------- test/fuzzer1.test | 2 - test/orderby2.test | 2 +- 5 files changed, 156 insertions(+), 125 deletions(-) diff --git a/manifest b/manifest index 178efcf35c..019544c892 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\san\soptimization\sissue\swith\sthe\sMSVC\scompiler\sfor\sARM. -D 2012-10-02T22:54:27.390 +C Further\sattempts\sto\soptimize\sout\sunnecessary\sORDER\sBY\sclauses. +D 2012-10-03T00:25:54.662 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 69398e95e9c1012ae07ce9ea00f21b0e7bab8df1 +F src/where.c 76de1934899015c71d044093de51b308d608e1e5 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -512,7 +512,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 69cf1036b92fd3b8e1fd65bef4d7ee3f085c28fb +F test/fuzzer1.test a2e93bb1e19513dd6bf9c63d3d7c4673c983ca19 F test/fuzzerfault.test ff2282c81797b6a355f0748d8b54c7287c5d2b25 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 @@ -635,7 +635,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/orderby1.test 4875a2a0a87d81920f3600a3405dc42f233b8c82 -F test/orderby2.test d8fa5991d8948ae1f335c2f91d751e955bfee815 +F test/orderby2.test b799f7c96b5b00daa0aa914c22309423a4b56bc8 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1018,7 +1018,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P abcf6a5d054559ee5a093ba39180c47b4958d9cd -R e2af9a84c8984f66075b1b5542126d99 -U mistachkin -Z 2e5e5eca8b24072e2977c29980c9c33a +P 7d301fdfeec540e4a58f43bff04d219e9f769dc4 +R 062d037eebc135368ed6a1f30b2d19fc +T *branch * qp-enhancements +T *sym-qp-enhancements * +T -sym-trunk * +U drh +Z f0dc62fc9e20f7c7c36c5a741064bb38 diff --git a/manifest.uuid b/manifest.uuid index 6ff004a2db..33ca9fe4da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d301fdfeec540e4a58f43bff04d219e9f769dc4 \ No newline at end of file +6744d9a37faffed59b4d5cb96c8671ec46a87ea7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 16b0c1c710..613215a3f7 100644 --- a/src/where.c +++ b/src/where.c @@ -258,7 +258,7 @@ struct WhereCost { #define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xrCostrCost ) return 1; + if( pProbe->rCost>pBaseline->rCost ) return 0; + if( pProbe->plan.nOBSat>pBaseline->plan.nOBSat ) return 1; + if( pProbe->plan.nRowplan.nRow ) return 1; + return 0; +} + /* ** Initialize a preallocated WhereClause structure. */ @@ -1762,6 +1773,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ p->cost.rCost = rTotal; p->cost.used = used; p->cost.plan.nRow = nRow; + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; p->cost.plan.wsFlags = flags; p->cost.plan.u.pTerm = pTerm; } @@ -2304,7 +2316,10 @@ static void bestVirtualIndex(WhereBestIdx *p){ } p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - p->cost.plan.wsFlags |= WHERE_ORDERBY; + p->cost.plan.wsFlags |= WHERE_ORDERED; + p->cost.plan.nOBSat = nOrderBy; + }else{ + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; } p->cost.plan.nEq = 0; pIdxInfo->nOrderBy = nOrderBy; @@ -2730,8 +2745,10 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ return 1; } - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - pIdx = pLevel->plan.u.pIdx; + if( (pLevel->plan.wsFlags & WHERE_ORDERED)==0 ){ + return 0; + } + if( (pIdx = pLevel->plan.u.pIdx)!=0 ){ if( iCol<0 ){ sortOrder = 0; testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); @@ -2833,10 +2850,14 @@ static int isSortingIndex( nPriorSat = p->aLevel[p->i-1].plan.nOBSat; if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return nPriorSat; } - if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + if( nEqCol==0 ){ + if( p->i && (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ + return nPriorSat; + } + nEqOneRow = 0; + }else if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ nEqOneRow = nEqCol; }else{ - if( nEqCol==0 ) return nPriorSat; sortOrder = bOuterRev; nEqOneRow = -1; } @@ -3043,18 +3064,16 @@ static void bestBtreeIndex(WhereBestIdx *p){ */ for(; pProbe; pIdx=pProbe=pProbe->pNext){ const tRowcnt * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ + WhereCost pc; /* Cost of using pProbe */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ int bRev = 2; /* 0=forward scan. 1=reverse. 2=undecided */ - int wsFlags = 0; - Bitmask used = 0; + memset(&pc, 0, sizeof(pc)); /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected ** cost and number of rows returned. ** - ** nEq: + ** pc.plan.nEq: ** Number of equality terms that can be implemented using the index. ** In other words, the number of initial fields in the index that ** are used in == or IN or NOT NULL constraints of the WHERE clause. @@ -3120,7 +3139,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** SELECT a, b FROM tbl WHERE a = 1; ** SELECT a, b, c FROM tbl WHERE a = 1; */ - int nEq; /* Number of == or IN terms matching index */ int nOrdered; /* Number of ordered terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ @@ -3129,7 +3147,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ int bSort; /* True if external sort required */ int bDist; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ - int nOBSat = 0; /* Number of ORDER BY terms satisfied */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int nOrderBy; /* Number of ORDER BY terms */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 @@ -3137,19 +3155,26 @@ static void bestBtreeIndex(WhereBestIdx *p){ #endif nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; - bSort = nOrderBy>0 && (p->i==0 || p->aLevel[p->i-1].plan.nOBSati==0 && p->pDistinct!=0; + if( p->i ){ + nPriorSat = pc.plan.nOBSat = p->aLevel[p->i-1].plan.nOBSat; + bSort = nPriorSat0; + bDist = p->pDistinct!=0; + } - /* Determine the values of nEq and nInMul */ - for(nEq=nOrdered=0; nEqnColumn; nEq++){ - int j = pProbe->aiColumn[nEq]; + /* Determine the values of pc.plan.nEq and nInMul */ + for(pc.plan.nEq=nOrdered=0; pc.plan.nEqnColumn; pc.plan.nEq++){ + int j = pProbe->aiColumn[pc.plan.nEq]; pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; - wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); + pc.plan.wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); testcase( pTerm->pWC!=pWC ); if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; + pc.plan.wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ nInMul *= 25; @@ -3159,15 +3184,15 @@ static void bestBtreeIndex(WhereBestIdx *p){ nInMul *= pExpr->x.pList->nExpr; } }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; - if( nEq==nOrdered ) nOrdered++; - }else if( bSort && nEq==nOrdered && isOrderedTerm(p, pTerm, &bRev) ){ + pc.plan.wsFlags |= WHERE_COLUMN_NULL; + if( pc.plan.nEq==nOrdered ) nOrdered++; + }else if( bSort && pc.plan.nEq==nOrdered && isOrderedTerm(p, pTerm, &bRev) ){ nOrdered++; } #ifdef SQLITE_ENABLE_STAT3 - if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; + if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; #endif - used |= pTerm->prereqRight; + pc.used |= pTerm->prereqRight; } /* If the index being considered is UNIQUE, and there is an equality @@ -3176,75 +3201,80 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** indicate this to the caller. ** ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (nEq+1) that can be + ** there is a range constraint on indexed column (pc.plan.nEq+1) that can be ** optimized using the index. */ - if( nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; + if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ + testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); + testcase( pc.plan.wsFlags & WHERE_COLUMN_NULL ); + if( (pc.plan.wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ + pc.plan.wsFlags |= WHERE_UNIQUE; if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ - wsFlags |= WHERE_ALL_UNIQUE; + pc.plan.wsFlags |= WHERE_ALL_UNIQUE; } } }else if( pProbe->bUnordered==0 ){ - int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); + int j; + j = (pc.plan.nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[pc.plan.nEq]); if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ WhereTerm *pTop, *pBtm; pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx); pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); + whereRangeScanEst(pParse, pProbe, pc.plan.nEq, pBtm, pTop, &rangeDiv); if( pTop ){ nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; + pc.plan.wsFlags |= WHERE_TOP_LIMIT; + pc.used |= pTop->prereqRight; testcase( pTop->pWC!=pWC ); } if( pBtm ){ nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; + pc.plan.wsFlags |= WHERE_BTM_LIMIT; + pc.used |= pBtm->prereqRight; testcase( pBtm->pWC!=pWC ); } - wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); + pc.plan.wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); } } /* If there is an ORDER BY clause and the index being considered will ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ + ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but + ** the index will scan rows in a different order, set the bSort + ** variable. */ assert( bRev>=0 && bRev<=2 ); if( bSort ){ testcase( bRev==0 ); testcase( bRev==1 ); testcase( bRev==2 ); - nOBSat = isSortingIndex(p, pProbe, iCur, nOrdered, - wsFlags, bRev&1, &bRev); - if( nOrderBy==nOBSat ){ - bSort = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, nOrdered, + pc.plan.wsFlags, bRev&1, &bRev); + if( nPriorSatpDistinct, nEq) - && (wsFlags & WHERE_COLUMN_IN)==0 + && isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, pc.plan.nEq) + && (pc.plan.wsFlags & WHERE_COLUMN_IN)==0 ){ bDist = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; + pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK ** index), determine if all required column data may be obtained without ** using the main table (i.e. if the index is a covering ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. */ + ** pc.plan.wsFlags. Otherwise, set the bLookup variable to true. */ if( pIdx ){ Bitmask m = pSrc->colUsed; int j; @@ -3255,7 +3285,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ } } if( m==0 ){ - wsFlags |= WHERE_IDX_ONLY; + pc.plan.wsFlags |= WHERE_IDX_ONLY; }else{ bLookup = 1; } @@ -3265,10 +3295,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** Estimate the number of rows of output. For an "x IN (SELECT...)" ** constraint, do not let the estimate exceed half the rows in the table. */ - nRow = (double)(aiRowEst[nEq] * nInMul); - if( bInEst && nRow*2>aiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); + pc.plan.nRow = (double)(aiRowEst[pc.plan.nEq] * nInMul); + if( bInEst && pc.plan.nRow*2>aiRowEst[0] ){ + pc.plan.nRow = aiRowEst[0]/2; + nInMul = (int)(pc.plan.nRow / aiRowEst[pc.plan.nEq]); } #ifdef SQLITE_ENABLE_STAT3 @@ -3278,15 +3308,18 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ + if( pc.plan.nRow>(double)1 && pc.plan.nEq==1 + && pFirstTerm!=0 && aiRowEst[1]>1 ){ assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ testcase( pFirstTerm->eOperator==WO_EQ ); testcase( pFirstTerm->eOperator==WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); + whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, + &pc.plan.nRow); }else if( bInEst==0 ){ assert( pFirstTerm->eOperator==WO_IN ); - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); + whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, + &pc.plan.nRow); } } #endif /* SQLITE_ENABLE_STAT3 */ @@ -3294,8 +3327,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ /* Adjust the number of output rows and downward to reflect rows ** that are excluded by range constraints. */ - nRow = nRow/rangeDiv; - if( nRow<1 ) nRow = 1; + pc.plan.nRow = pc.plan.nRow/rangeDiv; + if( pc.plan.nRow<1 ) pc.plan.nRow = 1; /* Experiments run on real SQLite databases show that the time needed ** to do a binary search to locate a row in a table or index is roughly @@ -3310,7 +3343,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** So this computation assumes table records are about twice as big ** as index records */ - if( (wsFlags&~WHERE_REVERSE)==WHERE_IDX_ONLY + if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED))==WHERE_IDX_ONLY && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) @@ -3319,9 +3352,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** A full-scan of the index might be a little faster than a full-scan ** of the table, so give this case a cost slightly less than a table ** scan. */ - cost = aiRowEst[0]*3 + pProbe->nColumn; - wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE; - }else if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + pc.rCost = aiRowEst[0]*3 + pProbe->nColumn; + pc.plan.wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE; + }else if( (pc.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ /* The cost of a full table scan is a number of move operations equal ** to the number of rows in the table. ** @@ -3331,11 +3364,11 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** decision and one which we expect to revisit in the future. But ** it seems to be working well enough at the moment. */ - cost = aiRowEst[0]*4; - wsFlags &= ~WHERE_IDX_ONLY; + pc.rCost = aiRowEst[0]*4; + pc.plan.wsFlags &= ~WHERE_IDX_ONLY; }else{ log10N = estLog(aiRowEst[0]); - cost = nRow; + pc.rCost = pc.plan.nRow; if( pIdx ){ if( bLookup ){ /* For an index lookup followed by a table lookup: @@ -3343,20 +3376,20 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** + nRow steps through the index ** + nRow table searches to lookup the table entry using the rowid */ - cost += (nInMul + nRow)*log10N; + pc.rCost += (nInMul + pc.plan.nRow)*log10N; }else{ /* For a covering index: ** nInMul index searches to find the initial entry ** + nRow steps through the index */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } }else{ /* For a rowid primary key lookup: ** nInMult table searches to find the initial entry for each range ** + nRow steps through the table */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } } @@ -3367,10 +3400,12 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** difference and select C of 3.0. */ if( bSort ){ - cost += nRow*estLog(nRow*(nOrderBy - nOBSat)/nOrderBy)*3; + double m = estLog(pc.plan.nRow*(nOrderBy - pc.plan.nOBSat)/nOrderBy); + m *= (double)(pc.plan.nOBSat ? 2 : 3); + pc.rCost += pc.plan.nRow*m; } if( bDist ){ - cost += nRow*estLog(nRow)*3; + pc.rCost += pc.plan.nRow*estLog(pc.plan.nRow)*3; } /**** Cost of using this index has now been computed ****/ @@ -3391,25 +3426,25 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** might be selected even when there exists an optimal index that has ** no such dependency. */ - if( nRow>2 && cost<=p->cost.rCost ){ + if( pc.plan.nRow>2 && pc.rCost<=p->cost.rCost ){ int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ + int nSkipEq = pc.plan.nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ Bitmask thisTab; /* Bitmap for pSrc */ thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ + for(pTerm=pWC->a, k=pWC->nTerm; pc.plan.nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index + /* Ignore the first pc.plan.nEq equality matches since the index ** has already accounted for these */ nSkipEq--; }else{ /* Assume each additional equality match reduces the result ** set size by a factor of 10 */ - nRow /= 10; + pc.plan.nRow /= 10; } }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ if( nSkipRange ){ @@ -3423,14 +3458,14 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** more selective intentionally because of the subjective ** observation that indexed range constraints really are more ** selective in practice, on average. */ - nRow /= 3; + pc.plan.nRow /= 3; } }else if( pTerm->eOperator!=WO_NOOP ){ /* Any other expression lowers the output row count by half */ - nRow /= 2; + pc.plan.nRow /= 2; } } - if( nRow<2 ) nRow = 2; + if( pc.plan.nRow<2 ) pc.plan.nRow = 2; } @@ -3440,22 +3475,17 @@ static void bestBtreeIndex(WhereBestIdx *p){ " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" " used=0x%llx nOrdered=%d nOBSat=%d\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - p->notReady, log10N, nRow, cost, used, nOrdered, nOBSat + pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, + p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, nOrdered, + pc.plan.nOBSat )); /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. + ** index and its cost in the p->cost structure. */ - if( (!pIdx || wsFlags) - && (costcost.rCost || (cost<=p->cost.rCost && nRowcost.plan.nRow)) - ){ - p->cost.rCost = cost; - p->cost.used = used; - p->cost.plan.nRow = nRow; - p->cost.plan.wsFlags = (wsFlags&wsFlagMask); - p->cost.plan.nEq = nEq; - p->cost.plan.nOBSat = nOBSat; + if( (!pIdx || pc.plan.wsFlags) && compareCost(&pc, &p->cost) ){ + p->cost = pc; + p->cost.plan.wsFlags &= wsFlagMask; p->cost.plan.u.pIdx = pIdx; } @@ -3477,17 +3507,15 @@ static void bestBtreeIndex(WhereBestIdx *p){ p->cost.plan.wsFlags |= WHERE_REVERSE; } - assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERBY)==0 ); + assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERED)==0 ); assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 ); assert( pSrc->pIndex==0 || p->cost.plan.u.pIdx==0 || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE(("best index is: %s\n", - ((p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk") - )); + WHERETRACE(("best index is: %s\n", + p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk")); bestOrClauseIndex(p); bestAutomaticIndex(p); @@ -4215,7 +4243,7 @@ static Bitmask codeOneLoopStart( ** this requires some special handling. */ if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) + && (pLevel->plan.wsFlags&WHERE_ORDERED) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ @@ -5078,8 +5106,8 @@ WhereInfo *sqlite3WhereBegin( ** The NEVER() comes about because rule (2) above prevents ** An indexable full-table-scan from reaching rule (3). ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. + ** (4) The plan cost must be lower than prior plans, where "cost" + ** is defined by the compareCost() function above. */ if( (sWBI.cost.used&sWBI.notValid)==0 /* (1) */ && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ @@ -5087,15 +5115,13 @@ WhereInfo *sqlite3WhereBegin( || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) && (nUnconstrained==0 || sWBI.pSrc->pIndex==0 /* (3) */ || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sWBI.cost.rCostpTab->zName, sWBI.cost.rCost, sWBI.cost.plan.nRow, - sWBI.cost.plan.nOBSat)); + sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags)); bestPlan = sWBI.cost; bestJ = j; } @@ -5105,13 +5131,10 @@ WhereInfo *sqlite3WhereBegin( assert( bestJ>=0 ); assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" - " cost=%.1f, nRow=%.1f, nOBSat=%d wsFlags=0x%08x\n", + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", bestJ, pTabList->a[bestJ].pTab->zName, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ - pWInfo->nOBSat = pOrderBy->nExpr; - } if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ assert( pWInfo->eDistinct==0 ); pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; @@ -5160,11 +5183,18 @@ WhereInfo *sqlite3WhereBegin( if( pParse->nErr || db->mallocFailed ){ goto whereBeginError; } + if( nTabList ){ + pLevel--; + pWInfo->nOBSat = pLevel->plan.nOBSat; + }else{ + pWInfo->nOBSat = 0; + } /* If the total query only selects a single row, then the ORDER BY ** clause is irrelevant. */ if( (andFlags & WHERE_UNIQUE)!=0 && pOrderBy ){ + assert( nTabList==0 || (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ); pWInfo->nOBSat = pOrderBy->nExpr; } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index dc8b445ee9..827d6a05f9 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1864,5 +1864,3 @@ do_execsql_test 10.3 { } {1 21 41 61 81} finish_test - - diff --git a/test/orderby2.test b/test/orderby2.test index 7d543229c0..a2bb4e13a9 100644 --- a/test/orderby2.test +++ b/test/orderby2.test @@ -92,6 +92,6 @@ do_test 2.1 { ORDER BY +a ASC, +c ASC, +e DESC, +g ASC; } } {1,3,7,10 1,3,7,14 1,3,6,11 1,4,8,12 1,4,8,12 1,4,8,13 1,4,5,9 2,3,7,10 2,3,7,14 2,3,6,11} - + finish_test From c45e67162de9ad08991b095db0f5ae6d089f3535 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Oct 2012 11:02:33 +0000 Subject: [PATCH 19/40] Fix the TCL interface so that SQL functions implemented in TCL honor the "nullvalue" setting. Also remove from the TCL interface some unused legacy UTF8 translation code left over from SQLite2. FossilOrigin-Name: c1f10a2643179ec58f3879764e9e89676d4b5f91 --- manifest | 16 +++++++------- manifest.uuid | 2 +- src/tclsqlite.c | 52 ++++++++++++--------------------------------- test/tclsqlite.test | 11 +++++++++- 4 files changed, 33 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 178efcf35c..05b97bd3c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\san\soptimization\sissue\swith\sthe\sMSVC\scompiler\sfor\sARM. -D 2012-10-02T22:54:27.390 +C Fix\sthe\sTCL\sinterface\sso\sthat\sSQL\sfunctions\simplemented\sin\sTCL\shonor\sthe\n"nullvalue"\ssetting.\s\sAlso\sremove\sfrom\sthe\sTCL\sinterface\ssome\sunused\s\nlegacy\sUTF8\stranslation\scode\sleft\sover\sfrom\sSQLite2. +D 2012-10-03T11:02:33.741 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 22de426c579d01c7bd2c24fb9229fbd774613b42 +F src/tclsqlite.c fc9027118d5fa1b3d16a7b22c63684a54adeeb42 F src/test1.c 936afc02766403e5debca49a1817a780e116df7e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d @@ -737,7 +737,7 @@ F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 -F test/tclsqlite.test a3d2df21ee98957f5de4f9dc1db0eab68047ab5d +F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P abcf6a5d054559ee5a093ba39180c47b4958d9cd -R e2af9a84c8984f66075b1b5542126d99 -U mistachkin -Z 2e5e5eca8b24072e2977c29980c9c33a +P 7d301fdfeec540e4a58f43bff04d219e9f769dc4 +R 6327212080c41cdbc10c1e2eb094be0d +U drh +Z 25d327c4e46a0608f83a0eb5d2f156c2 diff --git a/manifest.uuid b/manifest.uuid index 6ff004a2db..c4e20d4544 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d301fdfeec540e4a58f43bff04d219e9f769dc4 \ No newline at end of file +c1f10a2643179ec58f3879764e9e89676d4b5f91 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 47110cb8d2..5ea67782a0 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -53,15 +53,8 @@ #define NUM_PREPARED_STMTS 10 #define MAX_PREPARED_STMTS 100 -/* -** If TCL uses UTF-8 and SQLite is configured to use iso8859, then we -** have to do a translation when going between the two. Set the -** UTF_TRANSLATION_NEEDED macro to indicate that we need to do -** this translation. -*/ -#if defined(TCL_UTF_MAX) && !defined(SQLITE_UTF8) -# define UTF_TRANSLATION_NEEDED 1 -#endif +/* Forward declaration */ +typedef struct SqliteDb SqliteDb; /* ** New SQL functions can be created as TCL scripts. Each such function @@ -71,6 +64,7 @@ typedef struct SqlFunc SqlFunc; struct SqlFunc { Tcl_Interp *interp; /* The TCL interpret to execute the function */ Tcl_Obj *pScript; /* The Tcl_Obj representation of the script */ + SqliteDb *pDb; /* Database connection that owns this function */ int useEvalObjv; /* True if it is safe to use Tcl_EvalObjv */ char *zName; /* Name of this function */ SqlFunc *pNext; /* Next function on the list of them all */ @@ -113,7 +107,6 @@ typedef struct IncrblobChannel IncrblobChannel; ** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements. ** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used. */ -typedef struct SqliteDb SqliteDb; struct SqliteDb { sqlite3 *db; /* The "real" database structure. MUST BE FIRST */ Tcl_Interp *interp; /* The interpreter used for this database */ @@ -431,6 +424,7 @@ static SqlFunc *findSqlFunc(SqliteDb *pDb, const char *zName){ } } pNew->interp = pDb->interp; + pNew->pDb = pDb; pNew->pScript = 0; pNew->pNext = pDb->pFunc; pDb->pFunc = pNew; @@ -478,6 +472,7 @@ static void DbDeleteCmd(void *db){ while( pDb->pFunc ){ SqlFunc *pFunc = pDb->pFunc; pDb->pFunc = pFunc->pNext; + assert( pFunc->pDb==pDb ); Tcl_DecrRefCount(pFunc->pScript); Tcl_Free((char*)pFunc); } @@ -794,7 +789,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ break; } case SQLITE_NULL: { - pVal = Tcl_NewStringObj("", 0); + pVal = Tcl_NewStringObj(p->pDb->zNull, -1); break; } default: { @@ -933,26 +928,6 @@ static int auth_callback( } #endif /* SQLITE_OMIT_AUTHORIZATION */ -/* -** zText is a pointer to text obtained via an sqlite3_result_text() -** or similar interface. This routine returns a Tcl string object, -** reference count set to 0, containing the text. If a translation -** between iso8859 and UTF-8 is required, it is preformed. -*/ -static Tcl_Obj *dbTextToObj(char const *zText){ - Tcl_Obj *pVal; -#ifdef UTF_TRANSLATION_NEEDED - Tcl_DString dCol; - Tcl_DStringInit(&dCol); - Tcl_ExternalToUtfDString(NULL, zText, -1, &dCol); - pVal = Tcl_NewStringObj(Tcl_DStringValue(&dCol), -1); - Tcl_DStringFree(&dCol); -#else - pVal = Tcl_NewStringObj(zText, -1); -#endif - return pVal; -} - /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer @@ -1140,13 +1115,13 @@ static int dbPrepareAndBind( int nByte; if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){ - Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db))); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(pDb->db), -1)); return TCL_ERROR; } if( pStmt==0 ){ if( SQLITE_OK!=sqlite3_errcode(pDb->db) ){ /* A compile-time error in the statement. */ - Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db))); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(pDb->db), -1)); return TCL_ERROR; }else{ /* The statement was a no-op. Continue to the next statement @@ -1365,7 +1340,7 @@ static void dbEvalRowInfo( if( nCol>0 && (papColName || p->pArray) ){ apColName = (Tcl_Obj**)Tcl_Alloc( sizeof(Tcl_Obj*)*nCol ); for(i=0; iapColName = apColName; @@ -1452,7 +1427,8 @@ static int dbEvalStep(DbEvalContext *p){ continue; } #endif - Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); + Tcl_SetObjResult(pDb->interp, + Tcl_NewStringObj(sqlite3_errmsg(pDb->db), -1)); return TCL_ERROR; }else{ dbReleaseStmt(pDb, pPreStmt, 0); @@ -1509,11 +1485,11 @@ static Tcl_Obj *dbEvalColumnValue(DbEvalContext *p, int iCol){ return Tcl_NewDoubleObj(sqlite3_column_double(pStmt, iCol)); } case SQLITE_NULL: { - return dbTextToObj(p->pDb->zNull); + return Tcl_NewStringObj(p->pDb->zNull, -1); } } - return dbTextToObj((char *)sqlite3_column_text(pStmt, iCol)); + return Tcl_NewStringObj(sqlite3_column_text(pStmt, iCol), -1); } /* @@ -2433,7 +2409,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ pDb->zNull = 0; } } - Tcl_SetObjResult(interp, dbTextToObj(pDb->zNull)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(pDb->zNull, -1)); break; } diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 5761e1604c..3d9cd46ac6 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -319,14 +319,23 @@ do_test tcl-8.1 { execsql {INSERT INTO t1 VALUES(30,NULL)} db eval {SELECT * FROM t1 WHERE b IS NULL} } {30 NaN} +proc concatFunc args {return [join $args {}]} do_test tcl-8.2 { + db function concat concatFunc + db eval {SELECT concat('a', b, 'z') FROM t1 WHERE b is NULL} +} {aNaNz} +do_test tcl-8.3 { db nullvalue NULL db nullvalue } {NULL} -do_test tcl-8.3 { +do_test tcl-8.4 { db nullvalue {} db eval {SELECT * FROM t1 WHERE b IS NULL} } {30 {}} +do_test tcl-8.5 { + db function concat concatFunc + db eval {SELECT concat('a', b, 'z') FROM t1 WHERE b is NULL} +} {az} # Test the return type of user-defined functions # From 6f0e400cba68d8cf91a6c3d0a0525b14cd5052e2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Oct 2012 12:38:19 +0000 Subject: [PATCH 20/40] Fix a query planner problem that only occurs when covering-index-scan is disabled. Fix to tests whose output changed due to the new and more aggressive ORDER BY optimization. FossilOrigin-Name: 0f9bb90100aa304a7f28023ca4173e68b445e8bd --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 1 + test/where.test | 4 ++-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 019544c892..47363407dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sattempts\sto\soptimize\sout\sunnecessary\sORDER\sBY\sclauses. -D 2012-10-03T00:25:54.662 +C Fix\sa\squery\splanner\sproblem\sthat\sonly\soccurs\swhen\scovering-index-scan\sis\ndisabled.\s\sFix\sto\stests\swhose\soutput\schanged\sdue\sto\sthe\snew\sand\smore\naggressive\sORDER\sBY\soptimization. +D 2012-10-03T12:38:19.551 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 76de1934899015c71d044093de51b308d608e1e5 +F src/where.c cd99218c11dc8df4836f18df0d8a8cc5c10aeb30 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -956,7 +956,7 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test a6bfb5a29286811d798d326a8f1153a58c0fb2bd +F test/where.test 41b65069a227a61238c1387b050f029480ca5677 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 @@ -1018,10 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 7d301fdfeec540e4a58f43bff04d219e9f769dc4 -R 062d037eebc135368ed6a1f30b2d19fc -T *branch * qp-enhancements -T *sym-qp-enhancements * -T -sym-trunk * +P 6744d9a37faffed59b4d5cb96c8671ec46a87ea7 +R 603241d0c48a2df1479c4d8cfb353c66 U drh -Z f0dc62fc9e20f7c7c36c5a741064bb38 +Z e697da84737ed0e1f148e3cf30d77940 diff --git a/manifest.uuid b/manifest.uuid index 33ca9fe4da..a9af475df2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6744d9a37faffed59b4d5cb96c8671ec46a87ea7 \ No newline at end of file +0f9bb90100aa304a7f28023ca4173e68b445e8bd \ No newline at end of file diff --git a/src/where.c b/src/where.c index 613215a3f7..c6af796d86 100644 --- a/src/where.c +++ b/src/where.c @@ -3366,6 +3366,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ */ pc.rCost = aiRowEst[0]*4; pc.plan.wsFlags &= ~WHERE_IDX_ONLY; + if( pIdx ) pc.plan.wsFlags &= ~WHERE_ORDERED; }else{ log10N = estLog(aiRowEst[0]); pc.rCost = pc.plan.nRow; diff --git a/test/where.test b/test/where.test index 83fec2cf60..461ab87277 100644 --- a/test/where.test +++ b/test/where.test @@ -1088,12 +1088,12 @@ do_test where-14.1 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, y.b } -} {1/4 1/1 4/4 4/1 sort} +} {1/4 1/1 4/4 4/1 nosort} do_test where-14.2 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, y.b DESC } -} {1/1 1/4 4/1 4/4 sort} +} {1/1 1/4 4/1 4/4 nosort} do_test where-14.3 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, x.b From 613a53a029b83773facb5f923c8d662f08b0e102 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Oct 2012 18:09:32 +0000 Subject: [PATCH 21/40] Fix an out-of-order memset() that occurs before all variable declarations are finished. Also fix a line that exceeds the 80-character line length limit. FossilOrigin-Name: ba2f492f957ab5556cd540e21a76ebb75efea725 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 373e4e653b..f15d875432 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\squery\splanner\sto\srecognize\smore\scases\swhere\sORDER\sBY\sclauses\ncan\sbe\soptimized\sout.\s\sAdd\stest\scases\sto\sverify\scorrect\sbehavior\sof\sthe\nORDER\sBY\soptimization\swhen\sthe\scovering-index-scan\soptimization\sis\sdisabled.\nFix\sa\sharmless\scompiler\swarning\sin\sthe\sTCL\sinterface. -D 2012-10-03T12:56:18.398 +C Fix\san\sout-of-order\smemset()\sthat\soccurs\sbefore\sall\svariable\sdeclarations\nare\sfinished.\s\sAlso\sfix\sa\sline\sthat\sexceeds\sthe\s80-character\sline\slength\nlimit. +D 2012-10-03T18:09:32.822 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c cd99218c11dc8df4836f18df0d8a8cc5c10aeb30 +F src/where.c fabdb473752fb465b0cae66d446b7af678033d0c F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P c1f10a2643179ec58f3879764e9e89676d4b5f91 0f9bb90100aa304a7f28023ca4173e68b445e8bd -R 8774a06a6467942bbeb55b1c69f79903 +P 956e4d7f8958e7065ff2d61cd71519d6f4113d4a +R 9f4b1a764df6137e6f14304dffc8ad1d U drh -Z dc6130441c34298b8082b4166a85e60e +Z a3c3e4f99944b9339f658e73f3668f5f diff --git a/manifest.uuid b/manifest.uuid index 7aace3cdf7..d51ad9182c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -956e4d7f8958e7065ff2d61cd71519d6f4113d4a \ No newline at end of file +ba2f492f957ab5556cd540e21a76ebb75efea725 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c6af796d86..0df9bb08d5 100644 --- a/src/where.c +++ b/src/where.c @@ -3067,7 +3067,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ WhereCost pc; /* Cost of using pProbe */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ int bRev = 2; /* 0=forward scan. 1=reverse. 2=undecided */ - memset(&pc, 0, sizeof(pc)); /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected @@ -3154,6 +3153,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif + memset(&pc, 0, sizeof(pc)); nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; if( p->i ){ nPriorSat = pc.plan.nOBSat = p->aLevel[p->i-1].plan.nOBSat; @@ -3186,7 +3186,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ }else if( pTerm->eOperator & WO_ISNULL ){ pc.plan.wsFlags |= WHERE_COLUMN_NULL; if( pc.plan.nEq==nOrdered ) nOrdered++; - }else if( bSort && pc.plan.nEq==nOrdered && isOrderedTerm(p, pTerm, &bRev) ){ + }else if( bSort && pc.plan.nEq==nOrdered + && isOrderedTerm(p,pTerm,&bRev) ){ nOrdered++; } #ifdef SQLITE_ENABLE_STAT3 From 4f68d6c8c0784cc1969b395e471ab7b2f0447c49 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Oct 2012 12:10:25 +0000 Subject: [PATCH 22/40] Yet another refactoring of ORDER BY logic in the query planner. This particular check-in works mostly, but still has a few minor issues. FossilOrigin-Name: 8f4487450be1a2b0371f8251a967cbe341b2dea1 --- manifest | 15 +++-- manifest.uuid | 2 +- src/where.c | 154 ++++++++++++++++++++++---------------------------- 3 files changed, 78 insertions(+), 93 deletions(-) diff --git a/manifest b/manifest index f15d875432..311e37d60d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sout-of-order\smemset()\sthat\soccurs\sbefore\sall\svariable\sdeclarations\nare\sfinished.\s\sAlso\sfix\sa\sline\sthat\sexceeds\sthe\s80-character\sline\slength\nlimit. -D 2012-10-03T18:09:32.822 +C Yet\sanother\srefactoring\sof\sORDER\sBY\slogic\sin\sthe\squery\splanner.\s\sThis\nparticular\scheck-in\sworks\smostly,\sbut\sstill\shas\sa\sfew\sminor\sissues. +D 2012-10-04T12:10:25.997 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c fabdb473752fb465b0cae66d446b7af678033d0c +F src/where.c f2468071088a73703cd0fa9c9a2a3014b6a5501b F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 956e4d7f8958e7065ff2d61cd71519d6f4113d4a -R 9f4b1a764df6137e6f14304dffc8ad1d +P ba2f492f957ab5556cd540e21a76ebb75efea725 +R 9e8b25b7e134451a912caf8748b81c9a +T *branch * qp-enhancements +T *sym-qp-enhancements * +T -sym-trunk * U drh -Z a3c3e4f99944b9339f658e73f3668f5f +Z 0d6c8df6b2d44ceaf575eaf63391c9ce diff --git a/manifest.uuid b/manifest.uuid index d51ad9182c..c7968b7212 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba2f492f957ab5556cd540e21a76ebb75efea725 \ No newline at end of file +8f4487450be1a2b0371f8251a967cbe341b2dea1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0df9bb08d5..bce9b4041a 100644 --- a/src/where.c +++ b/src/where.c @@ -1442,22 +1442,6 @@ static void exprAnalyze( pTerm->prereqRight |= extraRight; } -/* -** Return TRUE if the given index is UNIQUE and all columns past the -** first nSkip columns are NOT NULL. -*/ -static int indexIsUniqueNotNull(Index *pIdx, int nSkip){ - Table *pTab = pIdx->pTable; - int i; - if( pIdx->onError==OE_None ) return 0; - for(i=nSkip; inColumn; i++){ - int j = pIdx->aiColumn[i]; - assert( j>=0 && jnCol ); - if( pTab->aCol[j].notNull==0 ) return 0; - } - return 1; -} - /* ** This function searches the expression list passed as the second argument ** for an expression of type TK_COLUMN that refers to the same column and @@ -2781,27 +2765,49 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ } /* -** pTerm is an == constraint. Check to see if the other side of -** the == is a constant or a value that is guaranteed to be ordered -** by outer loops. Return 1 if pTerm is ordered, and 0 if not. +** Check to see if there is an == or IS NULL constraint in the WHERE clause +** that restricts base.iColumn to be well-ordered. If base.iColumn must +** be a constant or must be NULL, that qualifies as well-ordered. If +** base.iColumn must equal the value of a column in an outer loop that is +** ordered, that also qualifies as being well ordered. +** +** In the second case (when base.iColumn == an ordered value in an outer +** loop) set or verify the sort order. If *pbRev is initially 2, then set +** it approprately. If *pbRev is 0 or 1, make sure it matches the sort +** order of the outer loop constraint. */ -static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ - Expr *pExpr = pTerm->pExpr; - assert( pExpr->op==TK_EQ ); - assert( pExpr->pLeft!=0 && pExpr->pLeft->op==TK_COLUMN ); - assert( pExpr->pRight!=0 ); - if( pTerm->prereqRight==0 ){ - return 1; /* RHS of the == is a constant */ - } - if( pExpr->pRight->op==TK_COLUMN - && isOrderedColumn(p, pExpr->pRight->iTable, pExpr->pRight->iColumn, pbRev) - ){ - return 1; - } +static int existsEqualityColumnConstraint( + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* Constraint must be compatible with this index */ + int base, /* Cursor number for the table to be sorted */ + int iColumn, /* Index of a column on the "base" table */ + int *pbRev, /* Set to 1 for reverse-order constraint */ + int *notNull /* Set to 0 if an IS NULL constraint is seen */ +){ + WhereTerm *pTerm; + int rc; - /* If we cannot prove that the constraint is ordered, assume it is not */ - return 0; +WHERETRACE(("EQ Constraint on %d.%d: pbRev-in=%d", base, iColumn, *pbRev)); + pTerm = findTerm(p->pWC, base, iColumn, p->notReady, WO_EQ|WO_ISNULL, pIdx); + if( pTerm==0 ){ + rc = 0; + }else if( pTerm->prereqRight==0 ){ + rc = 1; + }else if( pTerm->eOperator & WO_ISNULL ){ + *notNull = 0; + rc = 1; + }else{ + Expr *pRight = pTerm->pExpr->pRight; + if( pRight->op==TK_COLUMN ){ + rc = isOrderedColumn(p, pRight->iTable, pRight->iColumn, pbRev); + }else{ + rc = 0; + } + } +WHERETRACE((" rc=%d pbRev-out=%d\n", rc, *pbRev)); + return rc; } + /* ** This routine decides if pIdx can be used to satisfy the ORDER BY @@ -2827,22 +2833,20 @@ static int isSortingIndex( WhereBestIdx *p, /* Best index search context */ Index *pIdx, /* The index we are testing */ int base, /* Cursor number for the table to be sorted */ - int nEqCol, /* Number of index columns with ordered == constraints */ - int wsFlags, /* Index usages flags */ - int bOuterRev, /* True if outer loops scan in reverse order */ int *pbRev /* Set to 1 for reverse-order scan of pIdx */ ){ int i; /* Number of pIdx terms used */ int j; /* Number of ORDER BY terms satisfied */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ + int sortOrder = 2; /* 0: forward. 1: backward. 2: unknown */ int nTerm; /* Number of ORDER BY terms */ struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + Table *pTab = pIdx->pTable; /* Table that owns index pIdx */ ExprList *pOrderBy; /* The ORDER BY clause */ Parse *pParse = p->pParse; /* Parser context */ sqlite3 *db = pParse->db; /* Database connection */ int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ - int nEqOneRow; /* Idx columns that ref unique values */ + int uniqueNotNull = 1; /* pIdx is UNIQUE with all terms are NOT NULL */ if( p->i==0 ){ nPriorSat = 0; @@ -2850,22 +2854,11 @@ static int isSortingIndex( nPriorSat = p->aLevel[p->i-1].plan.nOBSat; if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return nPriorSat; } - if( nEqCol==0 ){ - if( p->i && (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ - return nPriorSat; - } - nEqOneRow = 0; - }else if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ - nEqOneRow = nEqCol; - }else{ - sortOrder = bOuterRev; - nEqOneRow = -1; - } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); - if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; if( pIdx->bUnordered ) return nPriorSat; nTerm = pOrderBy->nExpr; + uniqueNotNull = pIdx->onError==OE_None; assert( nTerm>0 ); /* Argument pIdx must either point to a 'real' named index structure, @@ -2881,15 +2874,15 @@ static int isSortingIndex( ** of the index is also allowed to match against the ORDER BY ** clause. */ - for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; ja[j]; jnColumn; i++){ Expr *pExpr; /* The expression of the ORDER BY pTerm */ CollSeq *pColl; /* The collating sequence of pExpr */ int termSortOrder; /* Sort order for this term */ int iColumn; /* The i-th column of the index. -1 for rowid */ int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + int isEq; /* Subject to an == or IS NULL constraint */ const char *zColl; /* Name of the collating sequence for i-th index term */ - assert( i<=pIdx->nColumn ); pExpr = pTerm->pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ /* Can not use an index sort on anything that is not a column in the @@ -2912,16 +2905,20 @@ static int isSortingIndex( iSortOrder = 0; zColl = pColl->zName; } + assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = pTerm->sortOrder; + isEq = existsEqualityColumnConstraint(p, pIdx, base, iColumn, + &termSortOrder, &uniqueNotNull); + termSortOrder = iSortOrder ^ pTerm->sortOrder; if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; }else{ /* If an index column fails to match and is not constrained by == ** then the index cannot satisfy the ORDER BY constraint. @@ -2929,12 +2926,8 @@ static int isSortingIndex( return nPriorSat; } } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqOneRow ){ - if( termSortOrder!=sortOrder ){ + if( sortOrder<2 ){ + if( sortOrder!=termSortOrder ){ /* Indices can only be used if all ORDER BY terms past the ** equality constraints have the correct DESC or ASC. */ break; @@ -2947,20 +2940,17 @@ static int isSortingIndex( if( iColumn<0 ){ seenRowid = 1; break; + }else if( pTab->aCol[iColumn].notNull==0 ){ + uniqueNotNull = 0; } } - *pbRev = sortOrder; + *pbRev = sortOrder & 1; /* If there was an "ORDER BY rowid" term that matched, or it is only ** possible for a single row from this table to match, then skip over ** any additional ORDER BY terms dealing with this table. */ - if( seenRowid || - ( (wsFlags & WHERE_COLUMN_NULL)==0 - && i>=pIdx->nColumn - && indexIsUniqueNotNull(pIdx, nEqCol) - ) - ){ + if( seenRowid || (uniqueNotNull && i>=pIdx->nColumn) ){ /* Advance j over additional ORDER BY terms associated with base */ WhereMaskSet *pMS = p->pWC->pMaskSet; Bitmask m = ~getMask(pMS, base); @@ -3097,10 +3087,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** the sub-select is assumed to return 25 rows for the purposes of ** determining nInMul. ** - ** nOrdered: - ** The number of equality terms that are constrainted by outer loop - ** variables that are well-ordered. - ** ** bInEst: ** Set to true if there was at least one "x IN (SELECT ...)" term used ** in determining the value of nInMul. Note that the RHS of the @@ -3138,7 +3124,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** SELECT a, b FROM tbl WHERE a = 1; ** SELECT a, b, c FROM tbl WHERE a = 1; */ - int nOrdered; /* Number of ordered terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ @@ -3166,7 +3151,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ } /* Determine the values of pc.plan.nEq and nInMul */ - for(pc.plan.nEq=nOrdered=0; pc.plan.nEqnColumn; pc.plan.nEq++){ + for(pc.plan.nEq=0; pc.plan.nEqnColumn; pc.plan.nEq++){ int j = pProbe->aiColumn[pc.plan.nEq]; pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; @@ -3185,10 +3170,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ } }else if( pTerm->eOperator & WO_ISNULL ){ pc.plan.wsFlags |= WHERE_COLUMN_NULL; - if( pc.plan.nEq==nOrdered ) nOrdered++; - }else if( bSort && pc.plan.nEq==nOrdered - && isOrderedTerm(p,pTerm,&bRev) ){ - nOrdered++; } #ifdef SQLITE_ENABLE_STAT3 if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; @@ -3248,8 +3229,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ testcase( bRev==0 ); testcase( bRev==1 ); testcase( bRev==2 ); - pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, nOrdered, - pc.plan.wsFlags, bRev&1, &bRev); +WHERETRACE(("--> before isSortingIndex: bRev=%d nPriorSat=%d\n", bRev, nPriorSat)); + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev); +WHERETRACE(("--> after isSortingIndex: bRev=%d nOBSat=%d\n", bRev, pc.plan.nOBSat)); if( nPriorSatpTab->zName, (pIdx ? pIdx->zName : "ipk"), pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, - p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, nOrdered, + p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, pc.plan.nOBSat )); From 41fb5cd103b8faf4b16bda1230717821679acf4b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Oct 2012 19:33:00 +0000 Subject: [PATCH 23/40] Shared-cache mode fixes for views and virtual tables. FossilOrigin-Name: 2b370dea704b59262c604af0efcef5660b194454 --- manifest | 36 +++++++++++---------- manifest.uuid | 2 +- src/alter.c | 4 +-- src/attach.c | 8 +++-- src/build.c | 31 +++++++++++++++--- src/delete.c | 2 +- src/expr.c | 1 + src/insert.c | 2 +- src/select.c | 3 +- src/sqliteInt.h | 3 ++ src/vtab.c | 9 +++++- test/shared9.test | 78 ++++++++++++++++++++++++++++++++++++++++++++++ test/trigger1.test | 6 ++-- 13 files changed, 151 insertions(+), 34 deletions(-) create mode 100644 test/shared9.test diff --git a/manifest b/manifest index f15d875432..a089e5ebea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sout-of-order\smemset()\sthat\soccurs\sbefore\sall\svariable\sdeclarations\nare\sfinished.\s\sAlso\sfix\sa\sline\sthat\sexceeds\sthe\s80-character\sline\slength\nlimit. -D 2012-10-03T18:09:32.822 +C Shared-cache\smode\sfixes\sfor\sviews\sand\svirtual\stables. +D 2012-10-04T19:33:00.992 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,9 +114,9 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c 0c1716aa8d248bd6bc750e23be4c68ad05f8668c +F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c 7553068d21e32a57fc33ab6b2393fc8c1ba41410 -F src/attach.c 577bf5675b0c50495fc28549f2fcbdb1bac71143 +F src/attach.c 34c15ecd686e58f08e5bb1389e28a0b65c2c83db F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c afc067b9a9050ff48b9d46285c53d096c556a73d F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 @@ -124,13 +124,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 14de53ebb334633ee632ab3c5b9262cfe7cbe455 F src/btree.h 078f76b28c338ab6eb6dd7324d63ee54463aeb6c F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 -F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 +F src/build.c 74baa2d6d63535c9fc82e0d51d3e5d1145614473 F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 -F src/delete.c 4f7d773ec44b7db22b30ec9144f58a69154e09b7 -F src/expr.c 4de967b85f577ba00a7cdcb53d22070def6198db +F src/delete.c e35684ad93c741266b086610d2efd709b7946853 +F src/expr.c 57fb8e7a05d4147e40b9f4c439e37ed2abab9332 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af F src/func.c cbb90dc84b22eea25caf39528d342279e61b8898 @@ -138,7 +138,7 @@ F src/global.c fb44b11e02e06c995e6ed6642509edd23599d584 F src/hash.c a4031441741932da9e7a65bee2b36b5d0e81c073 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 6e2aa7fbb5d4c5f34d412772751ed0aff0b9e87b +F src/insert.c aefc97951fbac7f022d423170b9f74e82410bc13 F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 @@ -174,12 +174,12 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9e28280ec98035f31900fdd1db01f86f68ca6c32 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 75c5e37cc882c468383c9d9e07496b9a16cfae3e +F src/select.c 2a82736faeca1fe93315eda20c691d68ec13bb6c F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 F src/sqlite.h.in c7be05ad191d2634292fcc77bdb2bcfa4526eb98 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a +F src/sqliteInt.h 27105bbf399f65842fb8eb9e343123a628cc6e5a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -245,7 +245,7 @@ F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 -F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 +F src/vtab.c c7671587d07b76582c495d9305b35c0172cad5d3 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b @@ -702,6 +702,7 @@ F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 F test/shared7.test 960760bc8d03e1419e70dea69cf41db62853616e F test/shared8.test b27befbefbe7f4517f1d6b7ff8f64a41ec74165d +F test/shared9.test 61cf645c716451642ae9be2342fada8b200649c3 F test/shared_err.test 91e26ec4f3fbe07951967955585137e2f18993de F test/sharedlock.test ffa0a3c4ac192145b310f1254f8afca4d553eabf F test/shell1.test 272384163432c0efd2c6817396beb0d119565d53 @@ -884,7 +885,7 @@ F test/trace2.test c1dc104a8d11a347c870cfea6235e3fc6f6cb06d F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 -F test/trigger1.test de42feb7cd442787d38185ae74f5a1d7afa400cb +F test/trigger1.test 70acedb76532117bbcc873f8601cdc6905f1f455 F test/trigger2.test 834187beafd1db383af0c659cfa49b0576832816 F test/trigger3.test d2c60d8be271c355d61727411e753181e877230a F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -1018,7 +1019,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 956e4d7f8958e7065ff2d61cd71519d6f4113d4a -R 9f4b1a764df6137e6f14304dffc8ad1d -U drh -Z a3c3e4f99944b9339f658e73f3668f5f +P ba2f492f957ab5556cd540e21a76ebb75efea725 +R a18cfd3cd9e6232f2cdfda7383e14e1c +T *branch * shared-cache-fix +T *sym-shared-cache-fix * +T -sym-trunk * +U dan +Z f1b4bf4b6486e150199936af81c3ec9e diff --git a/manifest.uuid b/manifest.uuid index d51ad9182c..8362af0600 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba2f492f957ab5556cd540e21a76ebb75efea725 \ No newline at end of file +2b370dea704b59262c604af0efcef5660b194454 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 8a96ab704d..a49d3349d7 100644 --- a/src/alter.c +++ b/src/alter.c @@ -414,7 +414,7 @@ void sqlite3AlterRenameTable( assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; @@ -757,7 +757,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/src/attach.c b/src/attach.c index e295c30111..3e045aa366 100644 --- a/src/attach.c +++ b/src/attach.c @@ -434,6 +434,7 @@ int sqlite3FixInit( assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; return 1; @@ -464,14 +465,15 @@ int sqlite3FixSrcList( if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } + sqlite3_free(pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; diff --git a/src/build.c b/src/build.c index 8910bbd981..2a54721e37 100644 --- a/src/build.c +++ b/src/build.c @@ -319,6 +319,31 @@ Table *sqlite3LocateTable( return p; } +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + /* ** Locate the in-memory structure that describes ** a particular index given the name of that index @@ -2114,8 +2139,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ @@ -2555,8 +2579,7 @@ Index *sqlite3CreateIndex( ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); if( !pTab || db->mallocFailed ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ diff --git a/src/delete.c b/src/delete.c index f7f52865e1..c83ea98970 100644 --- a/src/delete.c +++ b/src/delete.c @@ -32,7 +32,7 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ struct SrcList_item *pItem = pSrc->a; Table *pTab; assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ diff --git a/src/expr.c b/src/expr.c index 2d1fb38ede..04216b1711 100644 --- a/src/expr.c +++ b/src/expr.c @@ -930,6 +930,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ struct SrcList_item *pNewItem = &pNew->a[i]; struct SrcList_item *pOldItem = &p->a[i]; Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); diff --git a/src/insert.c b/src/insert.c index 969f9fcf64..4b81d5ddaa 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1743,7 +1743,7 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } diff --git a/src/select.c b/src/select.c index 2da14d93ed..51847ffbdb 100644 --- a/src/select.c +++ b/src/select.c @@ -3332,8 +3332,7 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a03b7ac1e7..86b6597392 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1869,6 +1869,7 @@ struct SrcList { i16 nSrc; /* Number of tables or subqueries in the FROM clause */ i16 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { + Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ @@ -2437,6 +2438,7 @@ struct TriggerStep { typedef struct DbFixer DbFixer; struct DbFixer { Parse *pParse; /* The parsing context. Error messages written here */ + Schema *pSchema; /* Fix items to this schema */ const char *zDb; /* Make sure all objects are contained in this database */ const char *zType; /* Type of the container - used for error messages */ const Token *pName; /* Name of the container - used for error messages */ @@ -2847,6 +2849,7 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int); void sqlite3ExprIfFalse(Parse*, Expr*, int, int); Table *sqlite3FindTable(sqlite3*,const char*, const char*); Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); +Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); Index *sqlite3FindIndex(sqlite3*,const char*, const char*); void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); diff --git a/src/vtab.c b/src/vtab.c index 29f097da44..d3f8126adb 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -263,6 +263,7 @@ void sqlite3VtabClear(sqlite3 *db, Table *p){ if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->azModuleArg ){ int i; + assert( p->nModuleArg<2 || p->azModuleArg[1]==0 ); for(i=0; inModuleArg; i++){ sqlite3DbFree(db, p->azModuleArg[i]); } @@ -324,7 +325,7 @@ void sqlite3VtabBeginParse( pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); + addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); @@ -481,6 +482,7 @@ static int vtabCallConstructor( int nArg = pTab->nModuleArg; char *zErr = 0; char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + int iDb; if( !zModuleName ){ return SQLITE_NOMEM; @@ -494,6 +496,10 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; + assert( pTab->azModuleArg[1]==0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->azModuleArg[1] = db->aDb[iDb].zName; + /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); @@ -504,6 +510,7 @@ static int vtabCallConstructor( rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); db->pVtabCtx = pPriorCtx; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + pTab->azModuleArg[1] = 0; if( SQLITE_OK!=rc ){ if( zErr==0 ){ diff --git a/test/shared9.test b/test/shared9.test new file mode 100644 index 0000000000..99c2b29f8f --- /dev/null +++ b/test/shared9.test @@ -0,0 +1,78 @@ +# 2012 October 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file are intended to show if two connections attach +# to the same shared cache using different database names, views and +# virtual tables may still be accessed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix shared9 +db close +set enable_shared_cache [sqlite3_enable_shared_cache 1] + +# Test organization: +# +# 1.* - Views. +# 2.* - Virtual tables. +# + +sqlite3 db1 test.db +sqlite3 db2 test.db +forcedelete test.db2 + +do_test 1.1 { + db1 eval { + ATTACH 'test.db2' AS 'fred'; + CREATE TABLE fred.t1(a, b, c); + CREATE VIEW fred.v1 AS SELECT * FROM t1; + + CREATE TABLE fred.t2(a, b); + CREATE TABLE fred.t3(a, b); + CREATE TRIGGER fred.trig AFTER INSERT ON t2 BEGIN + DELETE FROM t3; + INSERT INTO t3 SELECT * FROM t2; + END; + INSERT INTO t2 VALUES(1, 2); + SELECT * FROM t3; + } +} {1 2} + +do_test 1.2 { db2 eval "ATTACH 'test.db2' AS 'jones'" } {} +do_test 1.2 { db2 eval "SELECT * FROM v1" } {} +do_test 1.3 { db2 eval "INSERT INTO t2 VALUES(3, 4)" } {} + +do_test 2.1 { + db1 eval { + CREATE VIRTUAL TABLE fred.t4 USING fts4; + INSERT INTO t4 VALUES('hello world'); + } +} {} + +do_test 2.2 { + db2 eval { + INSERT INTO t4 VALUES('shared cache'); + SELECT * FROM t4 WHERE t4 MATCH 'hello'; + } +} {{hello world}} + +do_test 2.3 { + db1 eval { + SELECT * FROM t4 WHERE t4 MATCH 'c*'; + } +} {{shared cache}} + +db1 close +db2 close +sqlite3_enable_shared_cache $::enable_shared_cache +finish_test + diff --git a/test/trigger1.test b/test/trigger1.test index 9d917bdb38..38806db3b5 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -210,7 +210,7 @@ do_test trigger1-1.12 { delete from t1 WHERE a=old.a+2; end; } -} {1 {cannot create INSTEAD OF trigger on table: main.t1}} +} {1 {cannot create INSTEAD OF trigger on table: t1}} ifcapable view { # Ensure that we cannot create BEFORE triggers on views @@ -221,7 +221,7 @@ do_test trigger1-1.13 { delete from t1 WHERE a=old.a+2; end; } -} {1 {cannot create BEFORE trigger on view: main.v1}} +} {1 {cannot create BEFORE trigger on view: v1}} # Ensure that we cannot create AFTER triggers on views do_test trigger1-1.14 { catchsql { @@ -231,7 +231,7 @@ do_test trigger1-1.14 { delete from t1 WHERE a=old.a+2; end; } -} {1 {cannot create AFTER trigger on view: main.v1}} +} {1 {cannot create AFTER trigger on view: v1}} } ;# ifcapable view # Check for memory leaks in the trigger parser From d744fccdbde021bb622d5d6499a89baf8c72fc8a Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 5 Oct 2012 07:36:34 +0000 Subject: [PATCH 24/40] Rename a tool batch file subroutine to avoid confusion about its purpose. FossilOrigin-Name: 43155b1543bddbb84a8bc13a5b7344b228ddacb9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/build-all-msvc.bat | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f15d875432..1dd4fd127e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sout-of-order\smemset()\sthat\soccurs\sbefore\sall\svariable\sdeclarations\nare\sfinished.\s\sAlso\sfix\sa\sline\sthat\sexceeds\sthe\s80-character\sline\slength\nlimit. -D 2012-10-03T18:09:32.822 +C Rename\sa\stool\sbatch\sfile\ssubroutine\sto\savoid\sconfusion\sabout\sits\spurpose. +D 2012-10-05T07:36:34.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -974,7 +974,7 @@ F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/win32lock.test b2a539e85ae6b2d78475e016a9636b4451dc7fb9 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c -F tool/build-all-msvc.bat 1a18aa39983ae7354d834bc55a850a54fc007576 x +F tool/build-all-msvc.bat cd505f24717ca4ecd94c651f763d7bb4e7fe68af x F tool/build-shell.sh b64a481901fc9ffe5ca8812a2a9255b6cfb77381 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 956e4d7f8958e7065ff2d61cd71519d6f4113d4a -R 9f4b1a764df6137e6f14304dffc8ad1d -U drh -Z a3c3e4f99944b9339f658e73f3668f5f +P ba2f492f957ab5556cd540e21a76ebb75efea725 +R 5327149995bfae724325c397eda64b6a +U mistachkin +Z 0e16eea60d3d1df7334d6e6fee7ae0d5 diff --git a/manifest.uuid b/manifest.uuid index d51ad9182c..ff06a4f574 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba2f492f957ab5556cd540e21a76ebb75efea725 \ No newline at end of file +43155b1543bddbb84a8bc13a5b7344b228ddacb9 \ No newline at end of file diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index a2d7dae3a3..58e2dca3e3 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -142,7 +142,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM be used for the name of the platform-specific binary directory via REM the environment variables setup earlier. REM - CALL :fn_SetVariable %%P_NAME PLATFORMNAME + CALL :fn_CopyVariable %%P_NAME PLATFORMNAME REM REM NOTE: This is the inner loop. There should be exactly one iteration. @@ -230,7 +230,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM file used to setup the MSVC environment. REM IF DEFINED SET_NSDKLIBPATH ( - CALL :fn_SetVariable WindowsSdkDir NSDKLIBPATH + CALL :fn_CopyVariable WindowsSdkDir NSDKLIBPATH CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86 ) @@ -339,7 +339,7 @@ GOTO no_errors VERIFY MAYBE 2> NUL GOTO :EOF -:fn_SetVariable +:fn_CopyVariable SETLOCAL IF NOT DEFINED %1 GOTO :EOF IF "%2" == "" GOTO :EOF From 79e72a50aac070d6d42d38738d198bc0f4dddb33 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Oct 2012 14:43:40 +0000 Subject: [PATCH 25/40] Consolidate all occurrences of the "no such collation sequence" error message into a single spot. FossilOrigin-Name: 8ea67522fb03134f355ecb776b70c6917241cab2 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 5 +---- src/callback.c | 12 +++++++----- src/sqliteInt.h | 2 +- src/where.c | 4 +--- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 1dd4fd127e..e198ba9848 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sa\stool\sbatch\sfile\ssubroutine\sto\savoid\sconfusion\sabout\sits\spurpose. -D 2012-10-05T07:36:34.233 +C Consolidate\sall\soccurrences\sof\sthe\s"no\ssuch\scollation\ssequence"\serror\smessage\ninto\sa\ssingle\sspot. +D 2012-10-05T14:43:40.494 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,8 +124,8 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 14de53ebb334633ee632ab3c5b9262cfe7cbe455 F src/btree.h 078f76b28c338ab6eb6dd7324d63ee54463aeb6c F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 -F src/build.c c4555e16f8ccdadb2616014c617ed8166c5a93f7 -F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd +F src/build.c b87aff4aa847680b90556d639dedac829ad9363a +F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 @@ -179,7 +179,7 @@ F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 F src/sqlite.h.in c7be05ad191d2634292fcc77bdb2bcfa4526eb98 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a +F src/sqliteInt.h 6066ed7dac3824c0b362e789863622f1589a94d2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c fabdb473752fb465b0cae66d446b7af678033d0c +F src/where.c bf45ec770aa6e5da55d6e9e9405ce990ff6ca201 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P ba2f492f957ab5556cd540e21a76ebb75efea725 -R 5327149995bfae724325c397eda64b6a -U mistachkin -Z 0e16eea60d3d1df7334d6e6fee7ae0d5 +P 43155b1543bddbb84a8bc13a5b7344b228ddacb9 +R 1ba72200797152f8f0b84e511e307440 +U drh +Z 6fe8cb59728e23f06e655a7d282a5a09 diff --git a/manifest.uuid b/manifest.uuid index ff06a4f574..ce97c943d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43155b1543bddbb84a8bc13a5b7344b228ddacb9 \ No newline at end of file +8ea67522fb03134f355ecb776b70c6917241cab2 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 8910bbd981..221e476f14 100644 --- a/src/build.c +++ b/src/build.c @@ -1295,10 +1295,7 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } return pColl; diff --git a/src/callback.c b/src/callback.c index a515e05e2a..d40c65cb92 100644 --- a/src/callback.c +++ b/src/callback.c @@ -75,17 +75,18 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ ** ** The return value is either the collation sequence to be used in database ** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** sequence can be found. If no collation is found, leave an error message. ** ** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ + Parse *pParse, /* Parsing context */ u8 enc, /* The desired encoding for the collating sequence */ CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ const char *zName /* Collating sequence name */ ){ CollSeq *p; + sqlite3 *db = pParse->db; p = pColl; if( !p ){ @@ -102,6 +103,9 @@ CollSeq *sqlite3GetCollSeq( p = 0; } assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } return p; } @@ -120,10 +124,8 @@ int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ if( pColl ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; return SQLITE_ERROR; } assert( p==pColl ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a03b7ac1e7..fe0ab3e1fc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3077,7 +3077,7 @@ int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); -CollSeq *sqlite3GetCollSeq(sqlite3*, u8, CollSeq *, const char*); +CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*); diff --git a/src/where.c b/src/where.c index 0df9bb08d5..725e45cb7a 100644 --- a/src/where.c +++ b/src/where.c @@ -2415,10 +2415,8 @@ static int whereKeyStats( pColl = db->pDfltColl; assert( pColl->enc==SQLITE_UTF8 ); }else{ - pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); + pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl); if( pColl==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", - *pIdx->azColl); return SQLITE_ERROR; } z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); From 1e92fb59b26f423d01b803724c29c092b1bbb392 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Oct 2012 16:30:10 +0000 Subject: [PATCH 26/40] Fix a problem with shared-cache mode and CHECK constraints causing one db handle to invoke a collation sequence function registered with another. FossilOrigin-Name: c2c776ab73828374836af008aa320cc670c081b5 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/insert.c | 5 ++++- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a089e5ebea..f6c4043cc1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Shared-cache\smode\sfixes\sfor\sviews\sand\svirtual\stables. -D 2012-10-04T19:33:00.992 +C Fix\sa\sproblem\swith\sshared-cache\smode\sand\sCHECK\sconstraints\scausing\sone\sdb\shandle\sto\sinvoke\sa\scollation\ssequence\sfunction\sregistered\swith\sanother. +D 2012-10-05T16:30:10.692 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -138,7 +138,7 @@ F src/global.c fb44b11e02e06c995e6ed6642509edd23599d584 F src/hash.c a4031441741932da9e7a65bee2b36b5d0e81c073 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c aefc97951fbac7f022d423170b9f74e82410bc13 +F src/insert.c b1d413e792e8f5e6747647625cb7c0e9f2e89e50 F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 @@ -1019,10 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P ba2f492f957ab5556cd540e21a76ebb75efea725 -R a18cfd3cd9e6232f2cdfda7383e14e1c -T *branch * shared-cache-fix -T *sym-shared-cache-fix * -T -sym-trunk * +P 2b370dea704b59262c604af0efcef5660b194454 +R b8d4e5a86ac362df32fe82058e93d169 U dan -Z f1b4bf4b6486e150199936af81c3ec9e +Z c2b386d7cf024f01e0a45b9a845b3c47 diff --git a/manifest.uuid b/manifest.uuid index 8362af0600..a72d51b876 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b370dea704b59262c604af0efcef5660b194454 \ No newline at end of file +c2c776ab73828374836af008aa320cc670c081b5 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 4b81d5ddaa..6301dab54d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1274,7 +1274,9 @@ void sqlite3GenerateConstraintChecks( onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ int allOk = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + Expr *pDup = sqlite3ExprDup(db, pCheck->a[i].pExpr, 0); + if( pDup==0 ) break; + sqlite3ExprIfTrue(pParse, pDup, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ @@ -1288,6 +1290,7 @@ void sqlite3GenerateConstraintChecks( sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); } sqlite3VdbeResolveLabel(v, allOk); + sqlite3ExprDelete(db, pDup); } } #endif /* !defined(SQLITE_OMIT_CHECK) */ From a2ebe4b418c20099d5582dcc8c31a180bb853b11 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Oct 2012 17:18:16 +0000 Subject: [PATCH 27/40] Add a test for the collation-sequence/CHECK constraint problem fixed by the previous commit. FossilOrigin-Name: 82b6aa77c8d8de4c6fad1960f5958457a929a821 --- manifest | 12 ++--- manifest.uuid | 2 +- test/shared9.test | 112 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 94 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index f6c4043cc1..00aaa5060f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sshared-cache\smode\sand\sCHECK\sconstraints\scausing\sone\sdb\shandle\sto\sinvoke\sa\scollation\ssequence\sfunction\sregistered\swith\sanother. -D 2012-10-05T16:30:10.692 +C Add\sa\stest\sfor\sthe\scollation-sequence/CHECK\sconstraint\sproblem\sfixed\sby\sthe\sprevious\scommit. +D 2012-10-05T17:18:16.288 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -702,7 +702,7 @@ F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 F test/shared7.test 960760bc8d03e1419e70dea69cf41db62853616e F test/shared8.test b27befbefbe7f4517f1d6b7ff8f64a41ec74165d -F test/shared9.test 61cf645c716451642ae9be2342fada8b200649c3 +F test/shared9.test 3a5b09583e3ba3139a4bd66958061306b4331c7e F test/shared_err.test 91e26ec4f3fbe07951967955585137e2f18993de F test/sharedlock.test ffa0a3c4ac192145b310f1254f8afca4d553eabf F test/shell1.test 272384163432c0efd2c6817396beb0d119565d53 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 2b370dea704b59262c604af0efcef5660b194454 -R b8d4e5a86ac362df32fe82058e93d169 +P c2c776ab73828374836af008aa320cc670c081b5 +R 944a2743bc72c249d323dde91a15aa61 U dan -Z c2b386d7cf024f01e0a45b9a845b3c47 +Z fb89a86dbc14aadae3bf562a37bf56e5 diff --git a/manifest.uuid b/manifest.uuid index a72d51b876..65775bb214 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2c776ab73828374836af008aa320cc670c081b5 \ No newline at end of file +82b6aa77c8d8de4c6fad1960f5958457a929a821 \ No newline at end of file diff --git a/test/shared9.test b/test/shared9.test index 99c2b29f8f..bb0dfee186 100644 --- a/test/shared9.test +++ b/test/shared9.test @@ -17,15 +17,15 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix shared9 + +ifcapable !view||!trigger { + finish_test + return +} + db close set enable_shared_cache [sqlite3_enable_shared_cache 1] -# Test organization: -# -# 1.* - Views. -# 2.* - Virtual tables. -# - sqlite3 db1 test.db sqlite3 db2 test.db forcedelete test.db2 @@ -48,31 +48,93 @@ do_test 1.1 { } {1 2} do_test 1.2 { db2 eval "ATTACH 'test.db2' AS 'jones'" } {} -do_test 1.2 { db2 eval "SELECT * FROM v1" } {} -do_test 1.3 { db2 eval "INSERT INTO t2 VALUES(3, 4)" } {} +do_test 1.3 { db2 eval "SELECT * FROM v1" } {} +do_test 1.4 { db2 eval "INSERT INTO t2 VALUES(3, 4)" } {} -do_test 2.1 { - db1 eval { - CREATE VIRTUAL TABLE fred.t4 USING fts4; - INSERT INTO t4 VALUES('hello world'); - } -} {} +ifcapable fts3 { + do_test 1.5 { + db1 eval { + CREATE VIRTUAL TABLE fred.t4 USING fts4; + INSERT INTO t4 VALUES('hello world'); + } + } {} -do_test 2.2 { - db2 eval { - INSERT INTO t4 VALUES('shared cache'); - SELECT * FROM t4 WHERE t4 MATCH 'hello'; - } -} {{hello world}} + do_test 1.6 { + db2 eval { + INSERT INTO t4 VALUES('shared cache'); + SELECT * FROM t4 WHERE t4 MATCH 'hello'; + } + } {{hello world}} -do_test 2.3 { - db1 eval { - SELECT * FROM t4 WHERE t4 MATCH 'c*'; - } -} {{shared cache}} + do_test 1.7 { + db1 eval { + SELECT * FROM t4 WHERE t4 MATCH 'c*'; + } + } {{shared cache}} +} db1 close db2 close + +#------------------------------------------------------------------------- +# The following tests attempt to find a similar problem with collation +# sequence names - pointers to database handle specific allocations leaking +# into schema objects and being used after the original handle has been +# closed. +# +forcedelete test.db test.db2 +sqlite3 db1 test.db +sqlite3 db2 test.db +foreach x {collate1 collate2 collate3} { + proc $x {a b} { string compare $a $b } + db1 collate $x $x + db2 collate $x $x +} +do_test 2.1 { + db1 eval { + CREATE TABLE t1(a, b, c COLLATE collate1); + CREATE INDEX i1 ON t1(a COLLATE collate2, c, b); + } +} {} +do_test 2.2 { + db1 close + db2 eval "INSERT INTO t1 VALUES('abc', 'def', 'ghi')" +} {} +db2 close + +#------------------------------------------------------------------------- +# At one point, the following would cause a collation sequence belonging +# to connection [db1] to be invoked by a call to [db2 eval]. Which is a +# problem if [db1] has already been closed. +# +forcedelete test.db test.db2 +sqlite3 db1 test.db +sqlite3 db2 test.db + +proc mycollate_db1 {a b} {set ::invoked_mycollate_db1 1 ; string compare $a $b} +proc mycollate_db2 {a b} {string compare $a $b} + +db1 collate mycollate mycollate_db1 +db2 collate mycollate mycollate_db2 + +do_test 2.3 { + set ::invoked_mycollate_db1 0 + db1 eval { + CREATE TABLE t1(a COLLATE mycollate, CHECK (a IN ('one', 'two', 'three'))); + INSERT INTO t1 VALUES('one'); + } + db1 close + set ::invoked_mycollate_db1 +} {1} +do_test 2.4 { + set ::invoked_mycollate_db1 0 + db2 eval { + INSERT INTO t1 VALUES('two'); + } + db2 close + set ::invoked_mycollate_db1 +} {0} + sqlite3_enable_shared_cache $::enable_shared_cache finish_test From 6b9bb59f82e40a81818005e9511c598b02d86a69 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Oct 2012 19:43:02 +0000 Subject: [PATCH 28/40] Fix a problem in shared-cache mode where a COMMIT statement might cause a busy-handler belonging to a shared-cache connection other than the current writer to be invoked. FossilOrigin-Name: e0c889d66ccf4af12cc77ac38c1e6477da63ac72 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/vdbeaux.c | 2 ++ test/shared9.test | 67 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a68030c33f..8483654893 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sshared-cache\srelated\sfixes\sfrom\sthe\sshared-cache-fix\sbranch\sto\sthe\strunk. -D 2012-10-05T17:44:51.191 +C Fix\sa\sproblem\sin\sshared-cache\smode\swhere\sa\sCOMMIT\sstatement\smight\scause\sa\sbusy-handler\sbelonging\sto\sa\sshared-cache\sconnection\sother\sthan\sthe\scurrent\swriter\sto\sbe\sinvoked. +D 2012-10-05T19:43:02.225 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/vdbe.c 31523df2b986fc6c959dd54ca640ba865884641b F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 573a43ab5697b648a1e8f3dfc7d8667d5ca55729 F src/vdbeapi.c 4c2418161cf45392ba76a7ca92f9a5f06b96f89c -F src/vdbeaux.c fac025c798ad19070451b41eddc5dcd4696fdd1e +F src/vdbeaux.c 674e969e026d1af1938942eba17071127839fc15 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b @@ -702,7 +702,7 @@ F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 F test/shared7.test 960760bc8d03e1419e70dea69cf41db62853616e F test/shared8.test b27befbefbe7f4517f1d6b7ff8f64a41ec74165d -F test/shared9.test 3a5b09583e3ba3139a4bd66958061306b4331c7e +F test/shared9.test 43cbbe6f59bd49abd7e40ac653a4aa8596feafcd F test/shared_err.test 91e26ec4f3fbe07951967955585137e2f18993de F test/sharedlock.test ffa0a3c4ac192145b310f1254f8afca4d553eabf F test/shell1.test 272384163432c0efd2c6817396beb0d119565d53 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 8ea67522fb03134f355ecb776b70c6917241cab2 82b6aa77c8d8de4c6fad1960f5958457a929a821 -R 25900bb7c1310d28535793e9cff3e6d9 +P 698ec7769d4feea1782401479c61ce67d2113fd4 +R f57e31f66af3653aafd36cef29db2f9f U dan -Z 6fd03f9d2f9680aaf35bc79001b16899 +Z 7483039414b69be1ce1710f91dbcd117 diff --git a/manifest.uuid b/manifest.uuid index 3e7db5616a..88cc72dd6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -698ec7769d4feea1782401479c61ce67d2113fd4 \ No newline at end of file +e0c889d66ccf4af12cc77ac38c1e6477da63ac72 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 9db3bf435c..405ea48b89 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1770,7 +1770,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + sqlite3BtreeEnter(pBt); rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); } } if( rc!=SQLITE_OK ){ diff --git a/test/shared9.test b/test/shared9.test index bb0dfee186..98485be68a 100644 --- a/test/shared9.test +++ b/test/shared9.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/lock_common.tcl set testprefix shared9 ifcapable !view||!trigger { @@ -135,6 +136,72 @@ do_test 2.4 { set ::invoked_mycollate_db1 } {0} +#------------------------------------------------------------------------- +# This test verifies that a bug causing a busy-handler belonging to one +# shared-cache connection to be executed as a result of an sqlite3_step() +# on another has been fixed. +# +forcedelete test.db test.db2 +sqlite3 db1 test.db +sqlite3 db2 test.db + +proc busyhandler {handle args} { + set ::busyhandler_invoked_for $handle + return 1 +} +db1 busy [list busyhandler db1] +db2 busy [list busyhandler db2] + +do_test 3.1 { + db1 eval { + BEGIN; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(1, 2); + } + # Keep this next COMMIT as a separate statement. This ensures that COMMIT + # has already been compiled and loaded into the tcl interface statement + # cache when it is attempted below. + db1 eval COMMIT + db1 eval { + BEGIN; + INSERT INTO t1 VALUES(3, 4); + } +} {} + +do_test 3.3 { + set ::tf [launch_testfixture] + testfixture $::tf { + sqlite3 db test.db + db eval { + BEGIN; + SELECT * FROM t1; + } + } +} {1 2} + +do_test 3.2 { + db2 eval { SELECT * FROM t2 } +} {1 2} + +do_test 3.3 { + list [catch { db1 eval COMMIT } msg] $msg +} {1 {database is locked}} + +# At one point the following would fail, showing that the busy-handler +# belonging to [db2] was invoked instead. +do_test 3.4 { + set ::busyhandler_invoked_for +} {db1} +do_test 3.5 { + close $::tf + db1 eval COMMIT +} {} + +db1 close +db2 close + sqlite3_enable_shared_cache $::enable_shared_cache finish_test From 08c1c3150b2b3c2b40a7a98e5f2603d1e2318b0d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 6 Oct 2012 03:48:25 +0000 Subject: [PATCH 29/40] Changes for WinRT compatibility. Also, allow version resource compilation and embedding to be disabled at compile-time. FossilOrigin-Name: 4b0facc13b1026419f9b89dab3453ba43071455c --- Makefile.msc | 46 ++++++++++++++++++++++++++++++++-------------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 15 ++++++++++++--- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 1f8b5bfe82..4b83eee739 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -30,6 +30,10 @@ XCOMPILE = 0 # USE_NATIVE_LIBPATHS = 0 +# Set this 0 to skip the compiling and embedding of version resources. +# +USE_RC = 1 + # Set this non-0 to compile binaries suitable for the WinRT environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -168,8 +172,8 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src !IF $(FOR_WINRT)!=0 TCC = $(TCC) -DSQLITE_OS_WINRT=1 RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_PARTITION_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_PARTITION_APP +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF # Also, we need to dynamically link to the correct MSVC runtime @@ -180,14 +184,18 @@ RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_PARTITION_APP !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>0 TCC = $(TCC) -MDd +BCC = $(BCC) -MDd !ELSE TCC = $(TCC) -MD +BCC = $(BCC) -MD !ENDIF !ELSE !IF $(DEBUG)>0 TCC = $(TCC) -MTd +BCC = $(BCC) -MTd !ELSE TCC = $(TCC) -MT +BCC = $(BCC) -MT !ENDIF !ENDIF @@ -472,6 +480,14 @@ LIBOBJ = $(LIBOBJS0) LIBOBJ = $(LIBOBJS1) !ENDIF +# Determine if embedded resource compilation and usage are enabled. +# +!IF $(USE_RC)!=0 +LIBRESOBJS = sqlite3res.lo +!ELSE +LIBRESOBJS = +!ENDIF + # All of the source code files. # SRC = \ @@ -766,10 +782,10 @@ libsqlite3.lib: $(LIBOBJ) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) -sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3res.lo sqlite3.h +sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h $(LTLINK) $(READLINE_FLAGS) \ $(TOP)\src\shell.c \ - /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib sqlite3res.lo $(LIBREADLINE) $(LTLIBS) $(TLIBS) + /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to @@ -819,14 +835,16 @@ opcodes.lo: opcodes.c # Rule to build the Win32 resources object file. # -sqlite3res.lo: $(TOP)\src\sqlite3.rc $(HDR) +!IF $(USE_RC)!=0 +$(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(HDR) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ | $(NAWK) "/.*/ { gsub(/[.]/,\",\");print }" >> sqlite3rc.h \ ) echo #endif >> sqlite3rc.h - $(LTRCOMPILE) -fo sqlite3res.lo $(TOP)\src\sqlite3.rc + $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc +!ENDIF # Rules to build individual *.lo files from files in the src directory. # @@ -1043,8 +1061,8 @@ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib sqlite3res.lo - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ libsqlite3.lib tclsqlite-shell.lo sqlite3res.lo $(LTLIBS) $(TLIBS) +tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib $(LIBRESOBJS) + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ libsqlite3.lib tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # Rules to build opcodes.c and opcodes.h # @@ -1157,11 +1175,11 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF -testfixture.exe: $(TESTFIXTURE_SRC) sqlite3res.lo $(HDR) +testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ - /link $(LTLINKOPTS) $(LTLIBPATHS) sqlite3res.lo $(LTLIBS) $(TLIBS) + /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) fulltest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\all.test @@ -1182,9 +1200,9 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)\src\test_stat.c $(TOP)\src\tclsqlite.c $(TO $(NAWK) -f $(TOP)\tool\tostr.awk $(TOP)\tool\spaceanal.tcl >> $@ echo ; return zMainloop; } >> $@ -sqlite3_analyzer.exe: sqlite3_analyzer.c sqlite3res.lo +sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) -DBUILD_sqlite -DTCLSH=2 -I$(TCLINCDIR) sqlite3_analyzer.c \ - /link $(LTLINKOPTS) $(LTLIBPATHS) sqlite3res.lo $(LTLIBS) $(TLIBS) + /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) clean: del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib @@ -1217,5 +1235,5 @@ sqlite3.def: libsqlite3.lib | $(NAWK) "/ 1 _?sqlite3_/ { sub(/^.* _?/,\"\");print }" \ | sort >> sqlite3.def -sqlite3.dll: $(LIBOBJ) sqlite3res.lo sqlite3.def - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) sqlite3res.lo $(LTLIBS) $(TLIBS) +sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) sqlite3.def + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index 8483654893..24d29501c4 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\sproblem\sin\sshared-cache\smode\swhere\sa\sCOMMIT\sstatement\smight\scause\sa\sbusy-handler\sbelonging\sto\sa\sshared-cache\sconnection\sother\sthan\sthe\scurrent\swriter\sto\sbe\sinvoked. -D 2012-10-05T19:43:02.225 +C Changes\sfor\sWinRT\scompatibility.\s\sAlso,\sallow\sversion\sresource\scompilation\sand\sembedding\sto\sbe\sdisabled\sat\scompile-time. +D 2012-10-06T03:48:25.094 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 1102ce2c75965b9b5534efce380007ffcb08c52e +F Makefile.msc a8495ef567c1872cd9e0679c96a02aafbf9c43b5 F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION edab4af5a4623f8198833ea481ce98ab53750a8d @@ -161,7 +161,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a5a45a2857c43b37bac145b521064a85a544cd7a -F src/os_win.c 90c7a1fe2698867555ba4266f5bd436c85d0d1dc +F src/os_win.c 44f39059279dfd40ead4f259001420105c20e710 F src/pager.c a7ad8c38809edf0be545e8f52da5bcbb88885b38 F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 698ec7769d4feea1782401479c61ce67d2113fd4 -R f57e31f66af3653aafd36cef29db2f9f -U dan -Z 7483039414b69be1ce1710f91dbcd117 +P e0c889d66ccf4af12cc77ac38c1e6477da63ac72 +R 7aa3a9ae530d62ad432ec99f0941dd9b +U mistachkin +Z dfd5a7d9cdc76e6d57eba7e1ef9d075b diff --git a/manifest.uuid b/manifest.uuid index 88cc72dd6b..aebbf63d72 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0c889d66ccf4af12cc77ac38c1e6477da63ac72 \ No newline at end of file +4b0facc13b1026419f9b89dab3453ba43071455c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index c4d87b92c5..352fe06b48 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -402,7 +402,11 @@ static struct win_syscall { #define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ DWORD,va_list*))aSyscall[15].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif #define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) @@ -483,6 +487,7 @@ static struct win_syscall { #define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) #if SQLITE_OS_WINCE /* The GetProcAddressA() routine is only available on Windows CE. */ { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, @@ -491,6 +496,9 @@ static struct win_syscall { ** an ANSI string regardless of the _UNICODE setting */ { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, #endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ LPCSTR))aSyscall[27].pCurrent) @@ -594,7 +602,7 @@ static struct win_syscall { #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[41].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, #else { "LoadLibraryA", (SYSCALL)0, 0 }, @@ -602,7 +610,8 @@ static struct win_syscall { #define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else { "LoadLibraryW", (SYSCALL)0, 0 }, @@ -791,7 +800,7 @@ static struct win_syscall { #define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, #else { "LoadPackagedLibrary", (SYSCALL)0, 0 }, From f1dacbfc5b889cb8cf17bbdfcc6114da2fafd97c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 7 Oct 2012 00:52:22 +0000 Subject: [PATCH 30/40] Manually define the Win32 file-mapping APIs for WAL if SQLITE_WIN32_FILEMAPPING_API is defined. FossilOrigin-Name: 585e2070eef3bc273f23d8e384a1261ee5fff5bd --- manifest | 15 +++++++------ manifest.uuid | 2 +- src/os_win.c | 59 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 24d29501c4..0f8673d462 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sfor\sWinRT\scompatibility.\s\sAlso,\sallow\sversion\sresource\scompilation\sand\sembedding\sto\sbe\sdisabled\sat\scompile-time. -D 2012-10-06T03:48:25.094 +C Manually\sdefine\sthe\sWin32\sfile-mapping\sAPIs\sfor\sWAL\sif\sSQLITE_WIN32_FILEMAPPING_API\sis\sdefined. +D 2012-10-07T00:52:22.053 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a5a45a2857c43b37bac145b521064a85a544cd7a -F src/os_win.c 44f39059279dfd40ead4f259001420105c20e710 +F src/os_win.c 8a549824ca5cf85a06b550fa95090fa911c4e5ee F src/pager.c a7ad8c38809edf0be545e8f52da5bcbb88885b38 F src/pager.h bdbc379557eb2e233dfec10986b3086877e72db7 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1019,7 +1019,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P e0c889d66ccf4af12cc77ac38c1e6477da63ac72 -R 7aa3a9ae530d62ad432ec99f0941dd9b +P 4b0facc13b1026419f9b89dab3453ba43071455c +R 3a4914baa2fdfc1f1361069820af579b +T *branch * winFileMapping +T *sym-winFileMapping * +T -sym-trunk * U mistachkin -Z dfd5a7d9cdc76e6d57eba7e1ef9d075b +Z 4c6d5f7ae10ca61313253da78e41bee6 diff --git a/manifest.uuid b/manifest.uuid index aebbf63d72..6dd9adc3a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b0facc13b1026419f9b89dab3453ba43071455c \ No newline at end of file +585e2070eef3bc273f23d8e384a1261ee5fff5bd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 352fe06b48..0463b3eb23 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -33,6 +33,57 @@ with SQLITE_OMIT_WAL." #endif +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode (e.g. these APIs are available in the Windows CE SDK; however, they +** are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +HANDLE WINAPI CreateFileMappingFromApp(HANDLE, LPSECURITY_ATTRIBUTES, \ + ULONG, ULONG64, LPCWSTR); + +LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, DWORD, \ + DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, DWORD, \ + DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** This file mapping API is common to both Win32 and WinRT. +*/ +BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ + /* ** Macro to find the minimum of two numeric values. */ @@ -238,14 +289,6 @@ int sqlite3_os_type = 0; static int sqlite3_os_type = 0; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_ANSI -#endif - -#if SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT -# define SQLITE_WIN32_HAS_WIDE -#endif - #ifndef SYSCALL # define SYSCALL sqlite3_syscall_ptr #endif From 3038cfe90a86b62c765d730ec43851e8ed9dc109 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 7 Oct 2012 05:34:39 +0000 Subject: [PATCH 31/40] Corrections to test names to eliminate duplicates and follow naming conventions. FossilOrigin-Name: 50679889c75cd3205253b1682abfbddc5ff09175 --- manifest | 14 ++++++------- manifest.uuid | 2 +- test/shared9.test | 10 +++++----- test/trigger1.test | 50 +++++++++++++++++++++++----------------------- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 24d29501c4..6999191df1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sfor\sWinRT\scompatibility.\s\sAlso,\sallow\sversion\sresource\scompilation\sand\sembedding\sto\sbe\sdisabled\sat\scompile-time. -D 2012-10-06T03:48:25.094 +C Corrections\sto\stest\snames\sto\seliminate\sduplicates\sand\sfollow\snaming\sconventions. +D 2012-10-07T05:34:39.472 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -702,7 +702,7 @@ F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 F test/shared7.test 960760bc8d03e1419e70dea69cf41db62853616e F test/shared8.test b27befbefbe7f4517f1d6b7ff8f64a41ec74165d -F test/shared9.test 43cbbe6f59bd49abd7e40ac653a4aa8596feafcd +F test/shared9.test 614a3ca431adc73c857632deb4eff75bcaee40ec F test/shared_err.test 91e26ec4f3fbe07951967955585137e2f18993de F test/sharedlock.test ffa0a3c4ac192145b310f1254f8afca4d553eabf F test/shell1.test 272384163432c0efd2c6817396beb0d119565d53 @@ -885,7 +885,7 @@ F test/trace2.test c1dc104a8d11a347c870cfea6235e3fc6f6cb06d F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 -F test/trigger1.test 70acedb76532117bbcc873f8601cdc6905f1f455 +F test/trigger1.test 30f343f91586765874a28ad539c06f5a5f049931 F test/trigger2.test 834187beafd1db383af0c659cfa49b0576832816 F test/trigger3.test d2c60d8be271c355d61727411e753181e877230a F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P e0c889d66ccf4af12cc77ac38c1e6477da63ac72 -R 7aa3a9ae530d62ad432ec99f0941dd9b +P 4b0facc13b1026419f9b89dab3453ba43071455c +R 963a11ab0ec405d25d1ad37b4f26863a U mistachkin -Z dfd5a7d9cdc76e6d57eba7e1ef9d075b +Z 2db2f94384e1edd2a3e02fd4f28955ba diff --git a/manifest.uuid b/manifest.uuid index aebbf63d72..0d8ab7fd78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b0facc13b1026419f9b89dab3453ba43071455c \ No newline at end of file +50679889c75cd3205253b1682abfbddc5ff09175 \ No newline at end of file diff --git a/test/shared9.test b/test/shared9.test index 98485be68a..4ebdc1743d 100644 --- a/test/shared9.test +++ b/test/shared9.test @@ -170,7 +170,7 @@ do_test 3.1 { } } {} -do_test 3.3 { +do_test 3.2 { set ::tf [launch_testfixture] testfixture $::tf { sqlite3 db test.db @@ -181,20 +181,20 @@ do_test 3.3 { } } {1 2} -do_test 3.2 { +do_test 3.3 { db2 eval { SELECT * FROM t2 } } {1 2} -do_test 3.3 { +do_test 3.4 { list [catch { db1 eval COMMIT } msg] $msg } {1 {database is locked}} # At one point the following would fail, showing that the busy-handler # belonging to [db2] was invoked instead. -do_test 3.4 { +do_test 3.5 { set ::busyhandler_invoked_for } {db1} -do_test 3.5 { +do_test 3.6 { close $::tf db1 eval COMMIT } {} diff --git a/test/trigger1.test b/test/trigger1.test index 38806db3b5..c49bc659ad 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -11,20 +11,20 @@ # with the database COMMIT/ROLLBACK logic. # # 1. CREATE and DROP TRIGGER tests -# trig-1.1: Error if table does not exist -# trig-1.2: Error if trigger already exists -# trig-1.3: Created triggers are deleted if the transaction is rolled back -# trig-1.4: DROP TRIGGER removes trigger -# trig-1.5: Dropped triggers are restored if the transaction is rolled back -# trig-1.6: Error if dropped trigger doesn't exist -# trig-1.7: Dropping the table automatically drops all triggers -# trig-1.8: A trigger created on a TEMP table is not inserted into sqlite_master -# trig-1.9: Ensure that we cannot create a trigger on sqlite_master -# trig-1.10: -# trig-1.11: -# trig-1.12: Ensure that INSTEAD OF triggers cannot be created on tables -# trig-1.13: Ensure that AFTER triggers cannot be created on views -# trig-1.14: Ensure that BEFORE triggers cannot be created on views +# trigger1-1.1: Error if table does not exist +# trigger1-1.2: Error if trigger already exists +# trigger1-1.3: Created triggers are deleted if the transaction is rolled back +# trigger1-1.4: DROP TRIGGER removes trigger +# trigger1-1.5: Dropped triggers are restored if the transaction is rolled back +# trigger1-1.6: Error if dropped trigger doesn't exist +# trigger1-1.7: Dropping the table automatically drops all triggers +# trigger1-1.8: A trigger created on a TEMP table is not inserted into sqlite_master +# trigger1-1.9: Ensure that we cannot create a trigger on sqlite_master +# trigger1-1.10: +# trigger1-1.11: +# trigger1-1.12: Ensure that INSTEAD OF triggers cannot be created on tables +# trigger1-1.13: Ensure that AFTER triggers cannot be created on views +# trigger1-1.14: Ensure that BEFORE triggers cannot be created on views # set testdir [file dirname $argv0] @@ -265,32 +265,32 @@ ifcapable tempdb { END; } } {0 {}} - do_test trigger-3.2 { + do_test trigger1-3.2 { catchsql { INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; } } {1 {no such table: main.t2}} - do_test trigger-3.3 { + do_test trigger1-3.3 { db close set rc [catch {sqlite3 db test.db} err] if {$rc} {lappend rc $err} set rc } {0} - do_test trigger-3.4 { + do_test trigger1-3.4 { catchsql { INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; } } {1 {no such table: main.t2}} - do_test trigger-3.5 { + do_test trigger1-3.5 { catchsql { CREATE TEMP TABLE t2(x,y); INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; } } {1 {no such table: main.t2}} - do_test trigger-3.6.1 { + do_test trigger1-3.6.1 { catchsql { DROP TRIGGER r1; CREATE TEMP TRIGGER r1 AFTER INSERT ON t1 BEGIN @@ -300,7 +300,7 @@ ifcapable tempdb { SELECT * FROM t2; } } {0 {1 2 200 100}} - do_test trigger-3.6.2 { + do_test trigger1-3.6.2 { catchsql { DROP TRIGGER r1; DELETE FROM t1; @@ -312,7 +312,7 @@ ifcapable tempdb { SELECT * FROM t2; } } {0 {1 2}} - do_test trigger-3.7 { + do_test trigger1-3.7 { execsql { DROP TABLE t2; CREATE TABLE t2(x,y); @@ -320,7 +320,7 @@ ifcapable tempdb { } } {} - # There are two versions of trigger-3.8 and trigger-3.9. One that uses + # There are two versions of trigger1-3.8 and trigger1-3.9. One that uses # compound SELECT statements, and another that does not. ifcapable compound { do_test trigger1-3.8 { @@ -446,7 +446,7 @@ do_test trigger1-6.8 { execsql {SELECT * FROM t2} } {3 4 7 8} -integrity_check trigger-7.1 +integrity_check trigger1-7.1 # Check to make sure the name of a trigger can be quoted so that keywords # can be used as trigger names. Ticket #468 @@ -491,7 +491,7 @@ do_test trigger1-8.6 { ifcapable conflict { # Make sure REPLACE works inside of triggers. # - # There are two versions of trigger-9.1 and trigger-9.2. One that uses + # There are two versions of trigger1-9.1 and trigger1-9.2. One that uses # compound SELECT statements, and another that does not. ifcapable compound { do_test trigger1-9.1 { @@ -612,7 +612,7 @@ ifcapable tempdb&&attach { SELECT * FROM insert_log; } } {main 11 12 13 temp 14 15 16 aux 17 18 19} - do_test trigger1-10.8 { + do_test trigger1-10.9 { # Drop and re-create the insert_log table in a different database. Note # that we can change the column names because the trigger programs don't # use them explicitly. From 8e4af1b997200992222705ffb182245b7cf2a1e7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Oct 2012 18:23:51 +0000 Subject: [PATCH 32/40] Continued refactoring of the ORDER BY optimization logic. This check-in is close to working, but it still has issues. A few test cases fail. FossilOrigin-Name: adbdc663f3d22ff03f21040a811d585cf2218626 --- manifest | 15 ++-- manifest.uuid | 2 +- src/where.c | 231 +++++++++++++++++++++++++------------------------- 3 files changed, 121 insertions(+), 127 deletions(-) diff --git a/manifest b/manifest index 311e37d60d..ff4e46d0cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Yet\sanother\srefactoring\sof\sORDER\sBY\slogic\sin\sthe\squery\splanner.\s\sThis\nparticular\scheck-in\sworks\smostly,\sbut\sstill\shas\sa\sfew\sminor\sissues. -D 2012-10-04T12:10:25.997 +C Continued\srefactoring\sof\sthe\sORDER\sBY\soptimization\slogic.\s\sThis\scheck-in\nis\sclose\sto\sworking,\sbut\sit\sstill\shas\sissues.\s\sA\sfew\stest\scases\sfail. +D 2012-10-08T18:23:51.612 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c f2468071088a73703cd0fa9c9a2a3014b6a5501b +F src/where.c 968bea256016bdc32de4051fc2e921771ed945e6 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,10 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P ba2f492f957ab5556cd540e21a76ebb75efea725 -R 9e8b25b7e134451a912caf8748b81c9a -T *branch * qp-enhancements -T *sym-qp-enhancements * -T -sym-trunk * +P 8f4487450be1a2b0371f8251a967cbe341b2dea1 +R b898c44fee2dd1d4ef226b8a2707fa76 U drh -Z 0d6c8df6b2d44ceaf575eaf63391c9ce +Z cd5853ba0dbcfea98f0be88905f219b5 diff --git a/manifest.uuid b/manifest.uuid index c7968b7212..9a42175bcf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f4487450be1a2b0371f8251a967cbe341b2dea1 \ No newline at end of file +adbdc663f3d22ff03f21040a811d585cf2218626 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bce9b4041a..c309b456f0 100644 --- a/src/where.c +++ b/src/where.c @@ -2712,14 +2712,20 @@ static int whereInScanEst( /* ** Check to see if column iCol of the table with cursor iTab will appear -** in sorted order according to the current query plan. Return true if -** it will and false if not. +** in sorted order according to the current query plan. ** -** If *pbRev is initially 2 (meaning "unknown") then set *pbRev to the -** sort order of iTab.iCol. If *pbRev is 0 or 1 but does not match -** the sort order of iTab.iCol, then consider the column to be unordered. +** Return values: +** +** 0 iCol is not ordered +** 1 iCol has only a single value +** 2 iCol is in ASC order +** 3 iCol is in DESC order */ -static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ +static int isOrderedColumn( + WhereBestIdx *p, + int iTab, + int iCol +){ int i, j; WhereLevel *pLevel = &p->aLevel[p->i-1]; Index *pIdx; @@ -2755,60 +2761,11 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ testcase( sortOrder==1 ); sortOrder = 1 - sortOrder; } - if( *pbRev==2 ){ - *pbRev = sortOrder; - return 1; - } - return (*pbRev==sortOrder); + return sortOrder+2; } return 0; } -/* -** Check to see if there is an == or IS NULL constraint in the WHERE clause -** that restricts base.iColumn to be well-ordered. If base.iColumn must -** be a constant or must be NULL, that qualifies as well-ordered. If -** base.iColumn must equal the value of a column in an outer loop that is -** ordered, that also qualifies as being well ordered. -** -** In the second case (when base.iColumn == an ordered value in an outer -** loop) set or verify the sort order. If *pbRev is initially 2, then set -** it approprately. If *pbRev is 0 or 1, make sure it matches the sort -** order of the outer loop constraint. -*/ -static int existsEqualityColumnConstraint( - WhereBestIdx *p, /* Best index search context */ - Index *pIdx, /* Constraint must be compatible with this index */ - int base, /* Cursor number for the table to be sorted */ - int iColumn, /* Index of a column on the "base" table */ - int *pbRev, /* Set to 1 for reverse-order constraint */ - int *notNull /* Set to 0 if an IS NULL constraint is seen */ -){ - WhereTerm *pTerm; - int rc; - -WHERETRACE(("EQ Constraint on %d.%d: pbRev-in=%d", base, iColumn, *pbRev)); - pTerm = findTerm(p->pWC, base, iColumn, p->notReady, WO_EQ|WO_ISNULL, pIdx); - if( pTerm==0 ){ - rc = 0; - }else if( pTerm->prereqRight==0 ){ - rc = 1; - }else if( pTerm->eOperator & WO_ISNULL ){ - *notNull = 0; - rc = 1; - }else{ - Expr *pRight = pTerm->pExpr->pRight; - if( pRight->op==TK_COLUMN ){ - rc = isOrderedColumn(p, pRight->iTable, pRight->iColumn, pbRev); - }else{ - rc = 0; - } - } -WHERETRACE((" rc=%d pbRev-out=%d\n", rc, *pbRev)); - return rc; -} - - /* ** This routine decides if pIdx can be used to satisfy the ORDER BY ** clause, either in whole or in part. The return value is the @@ -2839,7 +2796,7 @@ static int isSortingIndex( int j; /* Number of ORDER BY terms satisfied */ int sortOrder = 2; /* 0: forward. 1: backward. 2: unknown */ int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + struct ExprList_item *pOBItem;/* A term of the ORDER BY clause */ Table *pTab = pIdx->pTable; /* Table that owns index pIdx */ ExprList *pOrderBy; /* The ORDER BY clause */ Parse *pParse = p->pParse; /* Parser context */ @@ -2858,7 +2815,7 @@ static int isSortingIndex( assert( pOrderBy!=0 ); if( pIdx->bUnordered ) return nPriorSat; nTerm = pOrderBy->nExpr; - uniqueNotNull = pIdx->onError==OE_None; + uniqueNotNull = pIdx->onError!=OE_None; assert( nTerm>0 ); /* Argument pIdx must either point to a 'real' named index structure, @@ -2874,25 +2831,28 @@ static int isSortingIndex( ** of the index is also allowed to match against the ORDER BY ** clause. */ - for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - int isEq; /* Subject to an == or IS NULL constraint */ - const char *zColl; /* Name of the collating sequence for i-th index term */ + j = nPriorSat; + for(i=0,pOBItem=&pOrderBy->a[j]; jnColumn; i++){ + Expr *pOBExpr; /* The expression of the ORDER BY pOBItem */ + CollSeq *pColl; /* The collating sequence of pOBExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + int isEq; /* Subject to an == or IS NULL constraint */ + int isMatch; /* ORDER BY term matches the index term */ + const char *zColl; /* Name of collating sequence for i-th index term */ + WhereTerm *pConstraint; /* A constraint in the WHERE clause */ - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ + /* If the next term of the ORDER BY clause refers to anything other than + ** a column in the "base" table, then this index will not be of any + ** further use in handling the ORDER BY. */ + pOBExpr = pOBItem->pExpr; + if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){ break; } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } + + /* Find column number and collating sequence for the next entry + ** in the index */ if( pIdx->zName && inColumn ){ iColumn = pIdx->aiColumn[i]; if( iColumn==pIdx->pTable->iPKey ){ @@ -2900,43 +2860,75 @@ static int isSortingIndex( } iSortOrder = pIdx->aSortOrder[i]; zColl = pIdx->azColl[i]; + assert( zColl!=0 ); }else{ iColumn = -1; iSortOrder = 0; - zColl = pColl->zName; + zColl = 0; } - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = pTerm->sortOrder; - isEq = existsEqualityColumnConstraint(p, pIdx, base, iColumn, - &termSortOrder, &uniqueNotNull); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( isEq ){ - /* If an index column that is constrained by == or IS NULL fails to - ** match an ORDER BY term, that is OK. Just ignore that column of - ** the index - */ - continue; + + /* Check to see if the column number and collating sequence of the + ** index match the column number and collating sequence of the ORDER BY + ** clause entry. Set isMatch to 1 if they both match. */ + if( pOBExpr->iColumn==iColumn ){ + if( zColl ){ + pColl = sqlite3ExprCollSeq(pParse, pOBExpr); + if( !pColl ) pColl = db->pDfltColl; + isMatch = sqlite3StrICmp(pColl->zName, zColl)==0; }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return nPriorSat; - } - } - if( sortOrder<2 ){ - if( sortOrder!=termSortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints have the correct DESC or ASC. */ - break; + isMatch = 1; } }else{ + isMatch = 0; + } + + /* termSortOrder is 0 or 1 for whether or not the access loop should + ** run forward or backwards (respectively) in order to satisfy this + ** term of the ORDER BY clause. */ + termSortOrder = iSortOrder ^ pOBItem->sortOrder; + + /* If X is the column in the index and ORDER BY clause, check to see + ** if there are any X= or X IS NULL constraints in the WHERE clause. */ + pConstraint = findTerm(p->pWC, base, iColumn, p->notReady, + WO_EQ|WO_ISNULL|WO_IN, pIdx); + if( pConstraint==0 ){ + isEq = 0; + }else if( pConstraint->eOperator==WO_IN ){ + break; + }else if( pConstraint->prereqRight==0 ){ + isEq = 1; + }else if( pConstraint->eOperator==WO_ISNULL ){ + uniqueNotNull = 0; + isEq = 1; + }else{ + Expr *pRight = pConstraint->pExpr->pRight; + if( pRight->op==TK_COLUMN ){ + WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", + pRight->iTable, pRight->iColumn)); + isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); + WHERETRACE((" -> isEq=%d\n", isEq)); + if( isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ + break; + } + }else{ + isEq = 0; + } + } + assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + if( !isMatch ){ + if( isEq==0 ){ + break; + }else{ + continue; + } + }else if( sortOrder==2 ){ sortOrder = termSortOrder; + }else if( termSortOrder!=sortOrder ){ + break; } j++; - pTerm++; + pOBItem++; if( iColumn<0 ){ seenRowid = 1; break; @@ -3058,6 +3050,11 @@ static void bestBtreeIndex(WhereBestIdx *p){ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ int bRev = 2; /* 0=forward scan. 1=reverse. 2=undecided */ + WHERETRACE(( + " %s(%s):\n", + pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") + )); + /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected ** cost and number of rows returned. @@ -3225,13 +3222,12 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** the index will scan rows in a different order, set the bSort ** variable. */ assert( bRev>=0 && bRev<=2 ); - if( bSort ){ - testcase( bRev==0 ); - testcase( bRev==1 ); - testcase( bRev==2 ); -WHERETRACE(("--> before isSortingIndex: bRev=%d nPriorSat=%d\n", bRev, nPriorSat)); + if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ + int bRev = 2; + WHERETRACE((" --> before isSortingIndex: nPriorSat=%d\n",nPriorSat)); pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev); -WHERETRACE(("--> after isSortingIndex: bRev=%d nOBSat=%d\n", bRev, pc.plan.nOBSat)); + WHERETRACE((" --> after isSortingIndex: bRev=%d nOBSat=%d\n", + bRev, pc.plan.nOBSat)); if( nPriorSat after isSortingIndex: bRev=%d nOBSat=%d\n", bRev, pc.plan.nOBS */ pc.rCost = aiRowEst[0]*4; pc.plan.wsFlags &= ~WHERE_IDX_ONLY; - if( pIdx ) pc.plan.wsFlags &= ~WHERE_ORDERED; + if( pIdx ){ + pc.plan.wsFlags &= ~WHERE_ORDERED; + pc.plan.nOBSat = nPriorSat; + } }else{ log10N = estLog(aiRowEst[0]); pc.rCost = pc.plan.nRow; @@ -3454,11 +3453,9 @@ WHERETRACE(("--> after isSortingIndex: bRev=%d nOBSat=%d\n", bRev, pc.plan.nOBS WHERETRACE(( - "%s(%s):\n" - " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" - " used=0x%llx nOBSat=%d\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), + " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" + " used=0x%llx nOBSat=%d\n", pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, pc.plan.nOBSat @@ -3498,7 +3495,7 @@ WHERETRACE(("--> after isSortingIndex: bRev=%d nOBSat=%d\n", bRev, pc.plan.nOBS || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE(("best index is: %s\n", + WHERETRACE((" best index is: %s\n", p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk")); bestOrClauseIndex(p); @@ -5048,7 +5045,7 @@ WhereInfo *sqlite3WhereBegin( sWBI.notReady = (isOptimal ? m : sWBI.notValid); if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; - WHERETRACE(("=== trying table %d (%s) with isOptimal=%d ===\n", + WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", j, sWBI.pSrc->pTab->zName, isOptimal)); assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -5101,8 +5098,8 @@ WhereInfo *sqlite3WhereBegin( || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) && (bestJ<0 || compareCost(&sWBI.cost, &bestPlan)) /* (4) */ ){ - WHERETRACE(("=== table %d (%s) is best so far\n" - " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", + WHERETRACE((" === table %d (%s) is best so far\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", j, sWBI.pSrc->pTab->zName, sWBI.cost.rCost, sWBI.cost.plan.nRow, sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags)); From 0a4c741cab66da8d599e68e660718adeb50f7d24 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Oct 2012 19:41:38 +0000 Subject: [PATCH 33/40] Bug fixes in the ORDER BY optimizer. FossilOrigin-Name: 301bbee4045aa169e29fb4fb75743b71eb4760a1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 25 ++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ff4e46d0cb..a505a09dff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continued\srefactoring\sof\sthe\sORDER\sBY\soptimization\slogic.\s\sThis\scheck-in\nis\sclose\sto\sworking,\sbut\sit\sstill\shas\sissues.\s\sA\sfew\stest\scases\sfail. -D 2012-10-08T18:23:51.612 +C Bug\sfixes\sin\sthe\sORDER\sBY\soptimizer. +D 2012-10-08T19:41:38.447 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 968bea256016bdc32de4051fc2e921771ed945e6 +F src/where.c d4a39bc7ea96ac2580b662df73c16d5ddacd9723 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 8f4487450be1a2b0371f8251a967cbe341b2dea1 -R b898c44fee2dd1d4ef226b8a2707fa76 +P adbdc663f3d22ff03f21040a811d585cf2218626 +R 3f2fb8221173011002c406e4b1569bb1 U drh -Z cd5853ba0dbcfea98f0be88905f219b5 +Z 5a939ce30180c1cb84b0892ff09f4ab1 diff --git a/manifest.uuid b/manifest.uuid index 9a42175bcf..1996a19ef2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adbdc663f3d22ff03f21040a811d585cf2218626 \ No newline at end of file +301bbee4045aa169e29fb4fb75743b71eb4760a1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c309b456f0..2701bbda08 100644 --- a/src/where.c +++ b/src/where.c @@ -2803,17 +2803,30 @@ static int isSortingIndex( sqlite3 *db = pParse->db; /* Database connection */ int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ - int uniqueNotNull = 1; /* pIdx is UNIQUE with all terms are NOT NULL */ + int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ if( p->i==0 ){ nPriorSat = 0; }else{ nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return nPriorSat; + if( (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ + /* This loop cannot be ordered unless the next outer loop is + ** also ordered */ + return nPriorSat; + } + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ){ + /* Only look at the outer-most loop if the OrderByIdxJoin + ** optimization is disabled */ + return nPriorSat; + } } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); - if( pIdx->bUnordered ) return nPriorSat; + if( pIdx->bUnordered ){ + /* Hash indices (indicated by the "unordered" tag on sqlite_stat1) cannot + ** be used for sorting */ + return nPriorSat; + } nTerm = pOrderBy->nExpr; uniqueNotNull = pIdx->onError!=OE_None; assert( nTerm>0 ); @@ -2936,6 +2949,12 @@ static int isSortingIndex( uniqueNotNull = 0; } } + + /* If we have not found at least one ORDER BY term that matches the + ** index, then show no progress. */ + if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; + + /* Return the necessary scan order back to the caller */ *pbRev = sortOrder & 1; /* If there was an "ORDER BY rowid" term that matched, or it is only From 6b9e565e54fb7277234fd50939091ab0daf8c079 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Oct 2012 20:27:35 +0000 Subject: [PATCH 34/40] Further tweaks to the ORDER BY optimizer, to fix a bug and to get the optimizer to recognize some additional cases. FossilOrigin-Name: bcb4f262476193cfb17818d8c62bab528dddeef9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 16 +++++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a505a09dff..60325045c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfixes\sin\sthe\sORDER\sBY\soptimizer. -D 2012-10-08T19:41:38.447 +C Further\stweaks\sto\sthe\sORDER\sBY\soptimizer,\sto\sfix\sa\sbug\sand\sto\sget\sthe\s\noptimizer\sto\srecognize\ssome\sadditional\scases. +D 2012-10-08T20:27:35.617 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c d4a39bc7ea96ac2580b662df73c16d5ddacd9723 +F src/where.c 6bc65388a642277ede4449bf852ef09f630997e8 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P adbdc663f3d22ff03f21040a811d585cf2218626 -R 3f2fb8221173011002c406e4b1569bb1 +P 301bbee4045aa169e29fb4fb75743b71eb4760a1 +R efd94f360278497c63522273f6fc0188 U drh -Z 5a939ce30180c1cb84b0892ff09f4ab1 +Z e6fe7146356897869a54564ecea90f21 diff --git a/manifest.uuid b/manifest.uuid index 1996a19ef2..2a7390063c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -301bbee4045aa169e29fb4fb75743b71eb4760a1 \ No newline at end of file +bcb4f262476193cfb17818d8c62bab528dddeef9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2701bbda08..21b34a6175 100644 --- a/src/where.c +++ b/src/where.c @@ -2908,11 +2908,11 @@ static int isSortingIndex( isEq = 0; }else if( pConstraint->eOperator==WO_IN ){ break; - }else if( pConstraint->prereqRight==0 ){ - isEq = 1; }else if( pConstraint->eOperator==WO_ISNULL ){ uniqueNotNull = 0; isEq = 1; + }else if( pConstraint->prereqRight==0 ){ + isEq = 1; }else{ Expr *pRight = pConstraint->pExpr->pRight; if( pRight->op==TK_COLUMN ){ @@ -2935,17 +2935,19 @@ static int isSortingIndex( }else{ continue; } - }else if( sortOrder==2 ){ - sortOrder = termSortOrder; - }else if( termSortOrder!=sortOrder ){ - break; + }else if( isEq!=1 ){ + if( sortOrder==2 ){ + sortOrder = termSortOrder; + }else if( termSortOrder!=sortOrder ){ + break; + } } j++; pOBItem++; if( iColumn<0 ){ seenRowid = 1; break; - }else if( pTab->aCol[iColumn].notNull==0 ){ + }else if( pTab->aCol[iColumn].notNull==0 && isEq==0 ){ uniqueNotNull = 0; } } From 2fad815c52f9ba596b6cfc1cbbf980bd9d8a2566 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Oct 2012 21:01:15 +0000 Subject: [PATCH 35/40] All test cases (veryquick.tcl and min.rc) pass. A few branch operations in ORDER BY optimization logic are untested by min.rc. FossilOrigin-Name: 8314fd6078a3a71aea9d6386ff277cfeb592b4ba --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/orderby1.test | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 60325045c0..763c801a91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stweaks\sto\sthe\sORDER\sBY\soptimizer,\sto\sfix\sa\sbug\sand\sto\sget\sthe\s\noptimizer\sto\srecognize\ssome\sadditional\scases. -D 2012-10-08T20:27:35.617 +C All\stest\scases\s(veryquick.tcl\sand\smin.rc)\spass.\s\sA\sfew\sbranch\soperations\sin\nORDER\sBY\soptimization\slogic\sare\suntested\sby\smin.rc. +D 2012-10-08T21:01:15.797 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 6bc65388a642277ede4449bf852ef09f630997e8 +F src/where.c b74866c4c15f0971a997e194699837815e5e628d F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -634,7 +634,7 @@ F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 4875a2a0a87d81920f3600a3405dc42f233b8c82 +F test/orderby1.test ef4f7c40df81b9a4303a718433d34052f07db47d F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab @@ -1018,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 301bbee4045aa169e29fb4fb75743b71eb4760a1 -R efd94f360278497c63522273f6fc0188 +P bcb4f262476193cfb17818d8c62bab528dddeef9 +R 530e49166e8d0f53a60bc95ac28c268f U drh -Z e6fe7146356897869a54564ecea90f21 +Z efffd1a95fdeb1f2abbf95a65e64a523 diff --git a/manifest.uuid b/manifest.uuid index 2a7390063c..b7ef933b27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcb4f262476193cfb17818d8c62bab528dddeef9 \ No newline at end of file +8314fd6078a3a71aea9d6386ff277cfeb592b4ba \ No newline at end of file diff --git a/src/where.c b/src/where.c index 21b34a6175..4016e93d07 100644 --- a/src/where.c +++ b/src/where.c @@ -2920,7 +2920,7 @@ static int isSortingIndex( pRight->iTable, pRight->iColumn)); isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); WHERETRACE((" -> isEq=%d\n", isEq)); - if( isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ + if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ break; } }else{ diff --git a/test/orderby1.test b/test/orderby1.test index 8087857931..2001e34009 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -114,7 +114,7 @@ do_test 1.4c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } -} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC +} {~/ORDER BY/} ;# optimized out do_test 1.5a { @@ -132,7 +132,7 @@ do_test 1.5c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } -} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC +} {~/ORDER BY/} ;# optimized out do_test 1.6a { db eval { @@ -261,7 +261,7 @@ do_test 2.4c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } -} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC +} {~/ORDER BY/} ;# optimized out do_test 2.5a { @@ -279,7 +279,7 @@ do_test 2.5c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } -} {/ORDER BY/} ;# separate sorting pass due to mixed ASC/DESC +} {~/ORDER BY/} ;# optimized out do_test 2.6a { db eval { @@ -396,7 +396,7 @@ do_test 3.4c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } -} {/ORDER BY/} ;# separate sorting pass due to mismatched DESC/ASC +} {~/ORDER BY/} ;# optimized out do_test 3.5a { @@ -414,7 +414,7 @@ do_test 3.5c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } -} {/ORDER BY/} ;# separate sorting pass due to mismatched ASC/DESC +} {~/ORDER BY/} ;# optimzed out do_test 3.6a { From c31c7c1c6ec079f2f7abe99422704b8bda3f2c52 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Oct 2012 23:25:07 +0000 Subject: [PATCH 36/40] Changes to facilitate full test coverage. FossilOrigin-Name: 28d1eb40bfc4c993010ce670ee10780ca2ae5c9c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 3 ++- src/insert.c | 26 ++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 5329b43150..94539ac691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sORDER\sBY\soptimization\srefactoring\sand\srepair\sinto\strunk. -D 2012-10-08T21:51:58.040 +C Changes\sto\sfacilitate\sfull\stest\scoverage. +D 2012-10-08T23:25:07.490 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 14de53ebb334633ee632ab3c5b9262cfe7cbe455 F src/btree.h 078f76b28c338ab6eb6dd7324d63ee54463aeb6c F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 -F src/build.c aaff8dbeb50c4a7fc48c4f9b65e633b991755ccf +F src/build.c b21c130017820dfe8721e380325e128fd0c6e019 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e @@ -138,7 +138,7 @@ F src/global.c fb44b11e02e06c995e6ed6642509edd23599d584 F src/hash.c a4031441741932da9e7a65bee2b36b5d0e81c073 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c b1d413e792e8f5e6747647625cb7c0e9f2e89e50 +F src/insert.c ad0d8943d2a4f0b52ebf75b9e459f2da00117dbd F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 1c2c0a2880dd0ee8862352e307142b8de7e80d13 8314fd6078a3a71aea9d6386ff277cfeb592b4ba -R b10014ed9089c50b134287e60971b9d3 +P c027a9af9137f3346dbb5c5e100a087c2e89797b +R 7421bd1f29e4f1c03e6a7c48cc004fd6 U drh -Z c680ce318c399b8bae240279b61ec223 +Z d39c58ca060ec602a0c6c79ebdb80011 diff --git a/manifest.uuid b/manifest.uuid index cd70e28e94..aa78661df2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c027a9af9137f3346dbb5c5e100a087c2e89797b \ No newline at end of file +28d1eb40bfc4c993010ce670ee10780ca2ae5c9c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 456adf34c8..00ff705f77 100644 --- a/src/build.c +++ b/src/build.c @@ -2577,7 +2577,8 @@ Index *sqlite3CreateIndex( assert(0); } pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); - if( !pTab || db->mallocFailed ) goto exit_create_index; + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); diff --git a/src/insert.c b/src/insert.c index 6301dab54d..47a2b104ed 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1275,21 +1275,23 @@ void sqlite3GenerateConstraintChecks( for(i=0; inExpr; i++){ int allOk = sqlite3VdbeMakeLabel(v); Expr *pDup = sqlite3ExprDup(db, pCheck->a[i].pExpr, 0); - if( pDup==0 ) break; - sqlite3ExprIfTrue(pParse, pDup, allOk, SQLITE_JUMPIFNULL); - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - char *zConsName = pCheck->a[i].zName; - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - if( zConsName ){ - zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + if( !db->mallocFailed ){ + assert( pDup!=0 ); + sqlite3ExprIfTrue(pParse, pDup, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ - zConsName = 0; + char *zConsName = pCheck->a[i].zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( zConsName ){ + zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + }else{ + zConsName = 0; + } + sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); } - sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); sqlite3ExprDelete(db, pDup); } } From 69a76bad73257518a051b628515c5ae3405c9948 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Oct 2012 01:23:25 +0000 Subject: [PATCH 37/40] Replace an always-false conditional with an assert(). FossilOrigin-Name: 981b9943e4b2b660690ec1ac145abdd3c74f8dd2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 94539ac691..2d49dfe330 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sfacilitate\sfull\stest\scoverage. -D 2012-10-08T23:25:07.490 +C Replace\san\salways-false\sconditional\swith\san\sassert(). +D 2012-10-09T01:23:25.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c c7671587d07b76582c495d9305b35c0172cad5d3 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 56916abbd9572487713df7dd41a7cace9e5fae2a +F src/where.c 50d1d0d3e62a561e8e3057d90f0ff5506a115690 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P c027a9af9137f3346dbb5c5e100a087c2e89797b -R 7421bd1f29e4f1c03e6a7c48cc004fd6 +P 28d1eb40bfc4c993010ce670ee10780ca2ae5c9c +R c49291c07fb5c66de29c1d2358325d86 U drh -Z d39c58ca060ec602a0c6c79ebdb80011 +Z d8af7ea73f1923e9b2fa65f8c9082c37 diff --git a/manifest.uuid b/manifest.uuid index aa78661df2..11b99fd3b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28d1eb40bfc4c993010ce670ee10780ca2ae5c9c \ No newline at end of file +981b9943e4b2b660690ec1ac145abdd3c74f8dd2 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1502bd50f6..1a15897f47 100644 --- a/src/where.c +++ b/src/where.c @@ -2733,9 +2733,7 @@ static int isOrderedColumn( if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ return 1; } - if( (pLevel->plan.wsFlags & WHERE_ORDERED)==0 ){ - return 0; - } + assert( (pLevel->plan.wsFlags & WHERE_ORDERED)!=0 ); if( (pIdx = pLevel->plan.u.pIdx)!=0 ){ if( iCol<0 ){ sortOrder = 0; From 86257fff8733e6c52bb355cf5bfee31dcc896aa3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Oct 2012 01:39:25 +0000 Subject: [PATCH 38/40] Remove an unused variable. Fix code that occurs before a variable declaration. FossilOrigin-Name: 01dc032b5bbd9c9ebb1965f176ca5d732cda85ea --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 11 ++++------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2d49dfe330..87603bebec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\san\salways-false\sconditional\swith\san\sassert(). -D 2012-10-09T01:23:25.030 +C Remove\san\sunused\svariable.\s\sFix\scode\sthat\soccurs\sbefore\sa\svariable\ndeclaration. +D 2012-10-09T01:39:25.125 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c c7671587d07b76582c495d9305b35c0172cad5d3 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 50d1d0d3e62a561e8e3057d90f0ff5506a115690 +F src/where.c 7d24dda2b2e8cf62c74e4f5f38bab160596ad658 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 28d1eb40bfc4c993010ce670ee10780ca2ae5c9c -R c49291c07fb5c66de29c1d2358325d86 +P 981b9943e4b2b660690ec1ac145abdd3c74f8dd2 +R 14bbf14e4e201d652ae4e981e1902683 U drh -Z d8af7ea73f1923e9b2fa65f8c9082c37 +Z 22e61519cc26696da476a58e182c0b3a diff --git a/manifest.uuid b/manifest.uuid index 11b99fd3b4..7ec1865ca4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -981b9943e4b2b660690ec1ac145abdd3c74f8dd2 \ No newline at end of file +01dc032b5bbd9c9ebb1965f176ca5d732cda85ea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1a15897f47..f9deb3d418 100644 --- a/src/where.c +++ b/src/where.c @@ -3065,12 +3065,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ const tRowcnt * const aiRowEst = pProbe->aiRowEst; WhereCost pc; /* Cost of using pProbe */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - int bRev = 2; /* 0=forward scan. 1=reverse. 2=undecided */ - - WHERETRACE(( - " %s(%s):\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") - )); /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected @@ -3152,6 +3146,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif + WHERETRACE(( + " %s(%s):\n", + pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") + )); memset(&pc, 0, sizeof(pc)); nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; if( p->i ){ @@ -3238,7 +3236,6 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but ** the index will scan rows in a different order, set the bSort ** variable. */ - assert( bRev>=0 && bRev<=2 ); if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ int bRev = 2; WHERETRACE((" --> before isSortingIndex: nPriorSat=%d\n",nPriorSat)); From 2f546ee914eadadc20f37e0e03e74fd0e4878533 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Oct 2012 09:26:28 +0000 Subject: [PATCH 39/40] Bring some comments up to date in the query planner, and especially the ORDER BY optimizer. Also add testcase() macros. FossilOrigin-Name: bb5c0f1c92b48631d4a12cd4f321b53cd07885c8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 32 ++++++++++++++++++-------------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 87603bebec..b5e839254a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\svariable.\s\sFix\scode\sthat\soccurs\sbefore\sa\svariable\ndeclaration. -D 2012-10-09T01:39:25.125 +C Bring\ssome\scomments\sup\sto\sdate\sin\sthe\squery\splanner,\sand\sespecially\sthe\nORDER\sBY\soptimizer.\s\sAlso\sadd\stestcase()\smacros. +D 2012-10-09T09:26:28.770 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c c7671587d07b76582c495d9305b35c0172cad5d3 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 7d24dda2b2e8cf62c74e4f5f38bab160596ad658 +F src/where.c 410017c604713ab6c010a9e3731a7688a0c7a059 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 981b9943e4b2b660690ec1ac145abdd3c74f8dd2 -R 14bbf14e4e201d652ae4e981e1902683 +P 01dc032b5bbd9c9ebb1965f176ca5d732cda85ea +R 321fa7d1d84b8152f712a4c09dc171f1 U drh -Z 22e61519cc26696da476a58e182c0b3a +Z b6de6b5036402a601a629813c2656e44 diff --git a/manifest.uuid b/manifest.uuid index 7ec1865ca4..181f486db3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01dc032b5bbd9c9ebb1965f176ca5d732cda85ea \ No newline at end of file +bb5c0f1c92b48631d4a12cd4f321b53cd07885c8 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f9deb3d418..7a994943cd 100644 --- a/src/where.c +++ b/src/where.c @@ -2771,14 +2771,6 @@ static int isOrderedColumn( ** The table being queried has a cursor number of "base". pIdx is the ** index that is postulated for use to access the table. ** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints and where the other side of the == is an ordered column -** or constant. An "order column" in the previous sentence means a column -** in table from an outer loop whose values will always appear in the -** correct order due to othre index, or because the outer loop generates -** a unique result. Any of the first nEqCol columns of pIdx may be missing -** from the ORDER BY clause and the match can still be a success. -** ** The *pbRev value is set to 0 order 1 depending on whether or not ** pIdx should be run in the forward order or in reverse order. */ @@ -2894,6 +2886,8 @@ static int isSortingIndex( /* termSortOrder is 0 or 1 for whether or not the access loop should ** run forward or backwards (respectively) in order to satisfy this ** term of the ORDER BY clause. */ + assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); termSortOrder = iSortOrder ^ pOBItem->sortOrder; /* If X is the column in the index and ORDER BY clause, check to see @@ -2903,12 +2897,15 @@ static int isSortingIndex( if( pConstraint==0 ){ isEq = 0; }else if( pConstraint->eOperator==WO_IN ){ + /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY + ** because we do not know in what order the values on the RHS of the IN + ** operator will occur. */ break; }else if( pConstraint->eOperator==WO_ISNULL ){ uniqueNotNull = 0; - isEq = 1; + isEq = 1; /* "X IS NULL" means X has only a single value */ }else if( pConstraint->prereqRight==0 ){ - isEq = 1; + isEq = 1; /* Constraint "X=constant" means X has only a single value */ }else{ Expr *pRight = pConstraint->pExpr->pRight; if( pRight->op==TK_COLUMN ){ @@ -2916,15 +2913,19 @@ static int isSortingIndex( pRight->iTable, pRight->iColumn)); isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); WHERETRACE((" -> isEq=%d\n", isEq)); + + /* If the constraint is of the form X=Y where Y is an ordered value + ** in an outer loop, then make sure the sort order of Y matches the + ** sort order required for X. */ if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ + testcase( isEq==2 ); + testcase( isEq==3 ); break; } }else{ - isEq = 0; + isEq = 0; /* "X=expr" places no ordering constraints on X */ } } - assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); if( !isMatch ){ if( isEq==0 ){ break; @@ -2943,7 +2944,10 @@ static int isSortingIndex( if( iColumn<0 ){ seenRowid = 1; break; - }else if( pTab->aCol[iColumn].notNull==0 && isEq==0 ){ + }else if( pTab->aCol[iColumn].notNull==0 && isEq!=1 ){ + testcase( isEq==0 ); + testcase( isEq==2 ); + testcase( isEq==3 ); uniqueNotNull = 0; } } From 5a55826bcda624f8fa4a3b7cc0c8845a689d8957 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Oct 2012 14:36:47 +0000 Subject: [PATCH 40/40] Avoid an assertion fault and/or freeing memory while it is still in use when an error occurs during virtual table construction. FossilOrigin-Name: a02599ad85d02470c9effa51c02dbda7796bfa16 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vtab.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b5e839254a..820f87d17f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bring\ssome\scomments\sup\sto\sdate\sin\sthe\squery\splanner,\sand\sespecially\sthe\nORDER\sBY\soptimizer.\s\sAlso\sadd\stestcase()\smacros. -D 2012-10-09T09:26:28.770 +C Avoid\san\sassertion\sfault\sand/or\sfreeing\smemory\swhile\sit\sis\sstill\sin\suse\swhen\nan\serror\soccurs\sduring\svirtual\stable\sconstruction. +D 2012-10-09T14:36:47.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 -F src/vtab.c c7671587d07b76582c495d9305b35c0172cad5d3 +F src/vtab.c 9c64ae18af78c740610df841c6f49fc2d240a279 F src/wal.c e1fe8f92a0ea0fef8faa87ec43a127a478589d22 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b @@ -1019,7 +1019,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 01dc032b5bbd9c9ebb1965f176ca5d732cda85ea -R 321fa7d1d84b8152f712a4c09dc171f1 +P bb5c0f1c92b48631d4a12cd4f321b53cd07885c8 +R a8d4530b0140a5399b6136d03ddec9af U drh -Z b6de6b5036402a601a629813c2656e44 +Z a4f133ff7d13d53e026b03f59090cbe2 diff --git a/manifest.uuid b/manifest.uuid index 181f486db3..a002cb666a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb5c0f1c92b48631d4a12cd4f321b53cd07885c8 \ No newline at end of file +a02599ad85d02470c9effa51c02dbda7796bfa16 \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index d3f8126adb..39fbacacbc 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -263,9 +263,8 @@ void sqlite3VtabClear(sqlite3 *db, Table *p){ if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->azModuleArg ){ int i; - assert( p->nModuleArg<2 || p->azModuleArg[1]==0 ); for(i=0; inModuleArg; i++){ - sqlite3DbFree(db, p->azModuleArg[i]); + if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); } sqlite3DbFree(db, p->azModuleArg); }