From 7a4b1642a732f6ef183931bb51dc5b5a0dcdb502 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Mar 2014 21:16:07 +0000 Subject: [PATCH 1/5] Experiments in picking better query plans, especially when the usage of one index is a subset of another. FossilOrigin-Name: 8f869ca7a6eaa9ca7a08102290e6c606735f9090 --- manifest | 15 +++-- manifest.uuid | 2 +- src/where.c | 176 +++++++++++++++++++++++++++++++------------------- 3 files changed, 119 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index 52d212de1d..9ef65fa6b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. -D 2014-03-28T14:41:35.536 +C Experiments\sin\spicking\sbetter\squery\splans,\sespecially\swhen\sthe\susage\sof\sone\nindex\sis\sa\ssubset\sof\sanother. +D 2014-03-29T21:16:07.103 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb +F src/where.c 26c6f6260b0cc2b0038dcc68ace0d7db20c7dcee F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a4e47150f32b3a4120b1f89ccc66d633d829e3bb -R 578568415288cd31fe0adba6128329da +P 27deb6e49bcc76714dbdc61b34748603155ac770 +R 1df7985395668b26c614cc4e641f29c8 +T *branch * query-plan-experiments +T *sym-query-plan-experiments * +T -sym-trunk * U drh -Z 045e2748905f8bd05ecf1197b97d7f20 +Z 1c5ff3f986da6bbe350c60cf72c963dd diff --git a/manifest.uuid b/manifest.uuid index d9c988d4d5..05916c40aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file +8f869ca7a6eaa9ca7a08102290e6c606735f9090 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 93ee8c59c3..c2b2cf2111 100644 --- a/src/where.c +++ b/src/where.c @@ -3712,59 +3712,24 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Insert or replace a WhereLoop entry using the template supplied. +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. ** -** An existing WhereLoop entry might be overwritten if the new template -** is better and has fewer dependencies. Or the template will be ignored -** and no insert will occur if an existing WhereLoop is faster and has -** fewer dependencies than the template. Otherwise a new WhereLoop is -** added based on the template. +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. ** -** If pBuilder->pOrSet is not NULL then we only care about only the -** prerequisites and rRun and nOut costs of the N best loops. That -** information is gathered in the pBuilder->pOrSet object. This special -** processing mode is used only for OR clause processing. +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. ** -** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we -** still might overwrite similar loops with the new template if the -** template is better. Loops may be overwritten if the following -** conditions are met: -** -** (1) They have the same iTab. -** (2) They have the same iSortIdx. -** (3) The template has same or fewer dependencies than the current loop -** (4) The template has the same or lower cost than the current loop -** (5) The template uses more terms of the same index but has no additional -** dependencies +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. */ -static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p, *pNext = 0; - WhereInfo *pWInfo = pBuilder->pWInfo; - sqlite3 *db = pWInfo->pParse->db; - - /* If pBuilder->pOrSet is defined, then only keep track of the costs - ** and prereqs. - */ - if( pBuilder->pOrSet!=0 ){ -#if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = -#endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, - pTemplate->nOut); -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; - } - - /* Search for an existing WhereLoop to overwrite, or which takes - ** priority over pTemplate. - */ - for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ /* If either the iTab or iSortIdx values for two WhereLoop are different ** then those WhereLoops need to be considered separately. Neither is @@ -3799,12 +3764,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with an similar one that uses ** more terms of the index */ - pNext = p->pNextLoop; break; }else{ - /* pTemplate is not helpful. - ** Return without changing or adding anything */ - goto whereLoopInsert_noop; + /* There is an existing WhereLoop that is better than pTemplate */ + return 0; } } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq @@ -3816,10 +3779,79 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** or (4) number of output rows, and is no worse in any of those ** categories. */ assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - pNext = p->pNextLoop; break; } } + return ppPrev; +} + +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +** (5) The template uses more terms of the same index but has no additional +** dependencies +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; + + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-noop: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; + } /* If we reach this point it means that either p[] should be overwritten ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new @@ -3836,13 +3868,33 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } #endif if( p==0 ){ - p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( NEVER(ppTail==0) ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } } whereLoopXfer(db, p, pTemplate); - p->pNextLoop = pNext; - *ppPrev = p; if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ @@ -3850,16 +3902,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } } return SQLITE_OK; - - /* Jump here if the insert is a no-op */ -whereLoopInsert_noop: -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; } /* From 53cd10afcd77c4ff2296999a9fe5116c51333688 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 18:24:18 +0000 Subject: [PATCH 2/5] Make sure that an index that covers a proper superset of the WHERE clause terms of some other index has a lower cost than the other index. FossilOrigin-Name: ea8b0910040198751551b0b960e6b783913607df --- manifest | 15 ++++---- manifest.uuid | 2 +- src/where.c | 99 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 9ef65fa6b9..ac1faf182f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\sin\spicking\sbetter\squery\splans,\sespecially\swhen\sthe\susage\sof\sone\nindex\sis\sa\ssubset\sof\sanother. -D 2014-03-29T21:16:07.103 +C Make\ssure\sthat\san\sindex\sthat\scovers\sa\sproper\ssuperset\sof\sthe\sWHERE\sclause\nterms\sof\ssome\sother\sindex\shas\sa\slower\scost\sthan\sthe\sother\sindex. +D 2014-03-31T18:24:18.867 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 26c6f6260b0cc2b0038dcc68ace0d7db20c7dcee +F src/where.c 50ac3154473b5c8df15c7b8dbd19da385fa859e1 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,10 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 27deb6e49bcc76714dbdc61b34748603155ac770 -R 1df7985395668b26c614cc4e641f29c8 -T *branch * query-plan-experiments -T *sym-query-plan-experiments * -T -sym-trunk * +P 8f869ca7a6eaa9ca7a08102290e6c606735f9090 +R 76b8d8088ba83ec014dc8bf476fa6749 U drh -Z 1c5ff3f986da6bbe350c60cf72c963dd +Z ecebe13d1c9f7ea1da970c5f6118dc52 diff --git a/manifest.uuid b/manifest.uuid index 05916c40aa..649fc9530b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f869ca7a6eaa9ca7a08102290e6c606735f9090 \ No newline at end of file +ea8b0910040198751551b0b960e6b783913607df \ No newline at end of file diff --git a/src/where.c b/src/where.c index c2b2cf2111..38c3712977 100644 --- a/src/where.c +++ b/src/where.c @@ -3711,6 +3711,51 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } +/* +** Compare every WhereLoop X on the list p against pTemplate. If: +** +** (1) both X and pTemplate refer to the same table, and +** (2) both X and pTemplate use a single index, and +** (3) pTemplate uses all the same WHERE clause terms as X plus +** at least one more term, +** +** then make sure the cost pTemplate is less than the cost of X, adjusting +** the cost of pTemplate downward if necessary. +** +** Example: When computing the query plan for the SELECT below: +** +** CREATE TABLE t1(a,b,c,d); +** CREATE INDEX t1abc ON t1(a,b,c); +** CREATE INDEX t1bc ON t1(b,c); +** SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +** +** Make sure the cost of using three three columns of index t1abc is less +** than the cost of using both columns of t1bc. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( pTemplate->nLTerm==0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab==pTemplate->iTab + && (p->wsFlags & WHERE_INDEXED)!=0 + && p->nLTermnLTerm + && (p->rRunrRun || (p->rRun==pTemplate->rRun && + p->nOut<=pTemplate->nOut)) + ){ + int i, j; + for(j=0, i=p->nLTerm-1; i>=0 && j>=0; i--){ + for(j=pTemplate->nLTerm-1; j>=0; j--){ + if( pTemplate->aLTerm[j]==p->aLTerm[i] ) break; + } + } + if( j>=0 ){ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + } + } + } +} + /* ** Search the list of WhereLoops in *ppPrev looking for one that can be ** supplanted by pTemplate. @@ -3747,39 +3792,30 @@ static WhereLoop **whereLoopFindLesser( ** rSetup. Call this SETUP-INVARIANT */ assert( p->rSetup>=pTemplate->rSetup ); - if( (p->prereq & pTemplate->prereq)==p->prereq - && p->rSetup<=pTemplate->rSetup - && p->rRun<=pTemplate->rRun - && p->nOut<=pTemplate->nOut + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ ){ - /* This branch taken when p is equal or better than pTemplate in - ** all of (1) dependencies (2) setup-cost, (3) run-cost, and - ** (4) number of output rows. */ - assert( p->rSetup==pTemplate->rSetup ); - if( p->prereq==pTemplate->prereq - && p->nLTermnLTerm - && (p->wsFlags & pTemplate->wsFlags & WHERE_INDEXED)!=0 - && (p->u.btree.pIndex==pTemplate->u.btree.pIndex - || pTemplate->rRun+p->nLTerm<=p->rRun+pTemplate->nLTerm) - ){ - /* Overwrite an existing WhereLoop with an similar one that uses - ** more terms of the index */ - break; - }else{ - /* There is an existing WhereLoop that is better than pTemplate */ - return 0; - } + return 0; /* Discard pTemplate */ } - if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rRun>=pTemplate->rRun - && p->nOut>=pTemplate->nOut + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ ){ - /* Overwrite an existing WhereLoop with a better one: one that is - ** better at one of (1) dependencies, (2) setup-cost, (3) run-cost - ** or (4) number of output rows, and is no worse in any of those - ** categories. */ assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - break; + break; /* Cause p to be overwritten by pTemplate */ } } return ppPrev; @@ -3801,15 +3837,13 @@ static WhereLoop **whereLoopFindLesser( ** ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we ** still might overwrite similar loops with the new template if the -** template is better. Loops may be overwritten if the following +** new template is better. Loops may be overwritten if the following ** conditions are met: ** ** (1) They have the same iTab. ** (2) They have the same iSortIdx. ** (3) The template has same or fewer dependencies than the current loop ** (4) The template has the same or lower cost than the current loop -** (5) The template uses more terms of the same index but has no additional -** dependencies */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p; @@ -3837,6 +3871,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ /* Look for an existing WhereLoop to replace with pTemplate */ + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); if( ppPrev==0 ){ From 3fb183d2f7f83cdc9cb406dccbe97d731f1e0644 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 19:49:00 +0000 Subject: [PATCH 3/5] Also make sure an index that is a proper subset of some other index has a higher cost than that other index. Add test cases. FossilOrigin-Name: b7830d232b073a197aa1092e78cb24e88cb10fd3 --- manifest | 13 +++-- manifest.uuid | 2 +- src/where.c | 68 +++++++++++++---------- test/whereH.test | 139 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 37 deletions(-) create mode 100644 test/whereH.test diff --git a/manifest b/manifest index ac1faf182f..7824fe6caa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\san\sindex\sthat\scovers\sa\sproper\ssuperset\sof\sthe\sWHERE\sclause\nterms\sof\ssome\sother\sindex\shas\sa\slower\scost\sthan\sthe\sother\sindex. -D 2014-03-31T18:24:18.867 +C Also\smake\ssure\san\sindex\sthat\sis\sa\sproper\ssubset\sof\ssome\sother\sindex\shas\sa\nhigher\scost\sthan\sthat\sother\sindex.\s\sAdd\stest\scases. +D 2014-03-31T19:49:00.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 50ac3154473b5c8df15c7b8dbd19da385fa859e1 +F src/where.c 182f16d91060418dfcc7401d24e43d8ec24e026c F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1093,6 +1093,7 @@ F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test eb3a46b3eaf38e25e3013433b2db8a25a866c215 +F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1159,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8f869ca7a6eaa9ca7a08102290e6c606735f9090 -R 76b8d8088ba83ec014dc8bf476fa6749 +P ea8b0910040198751551b0b960e6b783913607df +R 176e1ff66c14893be5de17b3890f1972 U drh -Z ecebe13d1c9f7ea1da970c5f6118dc52 +Z 3bf4657410567331f2f5534ba4eae60e diff --git a/manifest.uuid b/manifest.uuid index 649fc9530b..a0ffd4b3db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea8b0910040198751551b0b960e6b783913607df \ No newline at end of file +b7830d232b073a197aa1092e78cb24e88cb10fd3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 38c3712977..9882b4873f 100644 --- a/src/where.c +++ b/src/where.c @@ -3712,46 +3712,54 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Compare every WhereLoop X on the list p against pTemplate. If: +** Return TRUE if the set of WHERE clause terms used by pA is a proper +** subset of the WHERE clause terms used by pB. +*/ +static int whereLoopProperSubset(const WhereLoop *pA, const WhereLoop *pB){ + int i, j; + if( pA->nLTerm>=pB->nLTerm ) return 0; + for(j=0, i=pA->nLTerm-1; i>=0 && j>=0; i--){ + for(j=pB->nLTerm-1; j>=0; j--){ + if( pB->aLTerm[j]==pA->aLTerm[i] ) break; + } + } + return j>=0; +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: ** -** (1) both X and pTemplate refer to the same table, and -** (2) both X and pTemplate use a single index, and -** (3) pTemplate uses all the same WHERE clause terms as X plus -** at least one more term, +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate ** -** then make sure the cost pTemplate is less than the cost of X, adjusting -** the cost of pTemplate downward if necessary. +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. ** -** Example: When computing the query plan for the SELECT below: -** -** CREATE TABLE t1(a,b,c,d); -** CREATE INDEX t1abc ON t1(a,b,c); -** CREATE INDEX t1bc ON t1(b,c); -** SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; -** -** Make sure the cost of using three three columns of index t1abc is less -** than the cost of using both columns of t1bc. +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( pTemplate->nLTerm==0 ) return; for(; p; p=p->pNextLoop){ - if( p->iTab==pTemplate->iTab - && (p->wsFlags & WHERE_INDEXED)!=0 - && p->nLTermnLTerm + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( p->nLTermnLTerm && (p->rRunrRun || (p->rRun==pTemplate->rRun && p->nOut<=pTemplate->nOut)) + && whereLoopProperSubset(p, pTemplate) ){ - int i, j; - for(j=0, i=p->nLTerm-1; i>=0 && j>=0; i--){ - for(j=pTemplate->nLTerm-1; j>=0; j--){ - if( pTemplate->aLTerm[j]==p->aLTerm[i] ) break; - } - } - if( j>=0 ){ - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut - 1; - } + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else + if( p->nLTerm>pTemplate->nLTerm + && (p->rRun>pTemplate->rRun || (p->rRun==pTemplate->rRun && + p->nOut>=pTemplate->nOut)) + && whereLoopProperSubset(pTemplate, p) + ){ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; } } } diff --git a/test/whereH.test b/test/whereH.test new file mode 100644 index 0000000000..c252fe1ef7 --- /dev/null +++ b/test/whereH.test @@ -0,0 +1,139 @@ +# 2014-03-31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for query planning decisions where one candidate index +# covers a proper superset of the WHERE clause terms of another +# candidate index. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test whereH-1.1 { + CREATE TABLE t1(a,b,c,d); + CREATE INDEX t1abc ON t1(a,b,c); + CREATE INDEX t1bc ON t1(b,c); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {/INDEX t1abc /} +do_execsql_test whereH-1.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-2.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d); + CREATE INDEX t1bc ON t1(b,c); + CREATE INDEX t1abc ON t1(a,b,c); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {/INDEX t1abc /} +do_execsql_test whereH-2.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-3.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-3.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-4.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-4.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-5.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-5.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-6.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1cd ON t1(c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-6.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-7.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1cd ON t1(c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-7.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-8.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-8.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + + + +finish_test From d3c156638a227b72ecee81b8d87e539ca044102d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 20:05:20 +0000 Subject: [PATCH 4/5] Remove an unnecessary conditional. FossilOrigin-Name: 7473c4dfc10a47594affa6f4e071a08dc8838c0c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7824fe6caa..35a72e84b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Also\smake\ssure\san\sindex\sthat\sis\sa\sproper\ssubset\sof\ssome\sother\sindex\shas\sa\nhigher\scost\sthan\sthat\sother\sindex.\s\sAdd\stest\scases. -D 2014-03-31T19:49:00.374 +C Remove\san\sunnecessary\sconditional. +D 2014-03-31T20:05:20.135 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 182f16d91060418dfcc7401d24e43d8ec24e026c +F src/where.c 7b5010f5c4d6cf81a8d6fcc9e09b05faad71b395 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ea8b0910040198751551b0b960e6b783913607df -R 176e1ff66c14893be5de17b3890f1972 +P b7830d232b073a197aa1092e78cb24e88cb10fd3 +R ddf1c9d2d849c5f13c78d81687d216d7 U drh -Z 3bf4657410567331f2f5534ba4eae60e +Z 7ba197d4f98bb2ab782217a2f7675fd9 diff --git a/manifest.uuid b/manifest.uuid index a0ffd4b3db..2b145830ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7830d232b073a197aa1092e78cb24e88cb10fd3 \ No newline at end of file +7473c4dfc10a47594affa6f4e071a08dc8838c0c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9882b4873f..299acbaf0f 100644 --- a/src/where.c +++ b/src/where.c @@ -3717,7 +3717,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ */ static int whereLoopProperSubset(const WhereLoop *pA, const WhereLoop *pB){ int i, j; - if( pA->nLTerm>=pB->nLTerm ) return 0; + assert( pA->nLTermnLTerm ); /* Checked by calling function */ for(j=0, i=pA->nLTerm-1; i>=0 && j>=0; i--){ for(j=pB->nLTerm-1; j>=0; j--){ if( pB->aLTerm[j]==pA->aLTerm[i] ) break; From 1fb6a11072900e7f347980ee07612d2e37ae2c54 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 14:12:52 +0000 Subject: [PATCH 5/5] Ensure the "PRAGMA journal_mode=WAL" works coming from any other journal_mode with ATTACH-ed databases. FossilOrigin-Name: e54330b43127e46fc6494748cbb353a6fc91cfd7 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/pager.c | 3 +-- test/wal.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4bcee0a832..26b58d03cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sOP_Copy\sinstead\sof\sOP_SCopy\swhen\smoving\sresults\sout\sof\sa\ssubquery,\nto\sprevent\sthe\ssubquery\sresults\sfrom\schanging\sout\sfrom\sunder\sthe\souter\nquery.\s\sFix\sfor\sticket\s[1e64dd782a126f48d78]. -D 2014-04-03T16:29:31.330 +C Ensure\sthe\s"PRAGMA\sjournal_mode=WAL"\sworks\scoming\sfrom\sany\sother\sjournal_mode\nwith\sATTACH-ed\sdatabases. +D 2014-04-04T14:12:52.073 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e +F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1050,7 +1050,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test 40073e54359d43354925b2b700b7eebd5e207285 +F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1159,8 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 48ecdd4aff03741f96c070dced69c3c273b652cb -Q +ec6a06246e04eee5f25f1c28507df73b697099c0 -R b6a2fefa5b61ea0e64da4db2aaa10286 +P d5513dfa23baa0b0a095aaf17d19aacd30dcef61 +R dda37fe73fefd9a3a9bbbb33bf2f4700 U drh -Z f1f1ceb5f31f4e26d736a13a2a5aca0f +Z ec7ec754ade71c9dc3d6dc5092f63d72 diff --git a/manifest.uuid b/manifest.uuid index 601b0b4f2e..f9214073e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5513dfa23baa0b0a095aaf17d19aacd30dcef61 \ No newline at end of file +e54330b43127e46fc6494748cbb353a6fc91cfd7 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c6485a4d45..b09d6cb930 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1624,12 +1624,11 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_OFF + || !isOpen(pPager->jfd) ){ return SQLITE_OK; } pPager->setMaster = 1; - assert( isOpen(pPager->jfd) ); assert( pPager->journalHdr <= pPager->journalOff ); /* Calculate the length in bytes and the checksum of zMaster */ diff --git a/test/wal.test b/test/wal.test index 3a69de54f5..675be73791 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1574,4 +1574,17 @@ sqlite3_shutdown test_sqlite3_log sqlite3_initialize +# Make sure PRAGMA journal_mode=WAL works with ATTACHED databases in +# all journal modes. +# +foreach mode {OFF MEMORY PERSIST DELETE TRUNCATE WAL} { + delete_file test.db test2.db + sqlite3 db test.db + do_test wal-25.$mode { + db eval "PRAGMA journal_mode=$mode" + db eval {ATTACH 'test2.db' AS t2; PRAGMA journal_mode=WAL;} + } {wal} + db close +} + finish_test