From f1b5f5b855d767170826f2cbb79b802df676f09c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 May 2013 00:15:01 +0000 Subject: [PATCH 001/109] Begin inserting some experimental code for the next generation query planner. FossilOrigin-Name: ccaf4c3f7e1ec45e058d594d9b5c26818a37722a --- manifest | 17 +-- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 273 +++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 249 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index a5bd0b436b..75d4e526f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. -D 2013-05-01T17:58:35.871 +C Begin\sinserting\ssome\sexperimental\scode\sfor\sthe\snext\sgeneration\squery\splanner. +D 2013-05-02T00:15:01.231 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e +F src/sqliteInt.h 0d76a0aa7c64536c6f55d11a8f9f40df0636af6a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 +F src/where.c fc62bea654f27d5787bc428cd8f3f176d764f785 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 329478cbed06f93652de50abdb31a6b41af02b9e -R cc457983507eee39e1b071292dee5b26 +P faedaeace9c7ed9a8aaf96700caee09db0c0c061 +R 84f8c9b09f28282800936ed72686b19f +T *branch * nextgen-query-plan-exp +T *sym-nextgen-query-plan-exp * +T -sym-trunk * U drh -Z 944723cfd5dfa8d16765eaab2fd82886 +Z 18faca4b1f68b4b3514e189f395d77ff diff --git a/manifest.uuid b/manifest.uuid index f88e8ae7a8..016ae611b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file +ccaf4c3f7e1ec45e058d594d9b5c26818a37722a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b041b3fb6e..7a405b2fb1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2065,6 +2065,7 @@ struct WhereInfo { int iBreak; /* Jump here to break out of the loop */ int nLevel; /* Number of nested loop */ struct WhereClause *pWC; /* Decomposition of the WHERE clause */ + struct WhereLoop *pLoops; /* List of all WhereLoop objects */ double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ double nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ diff --git a/src/where.c b/src/where.c index 2de894c3e7..f44ab496c0 100644 --- a/src/where.c +++ b/src/where.c @@ -39,6 +39,43 @@ typedef struct WhereMaskSet WhereMaskSet; typedef struct WhereOrInfo WhereOrInfo; typedef struct WhereAndInfo WhereAndInfo; typedef struct WhereCost WhereCost; +typedef struct WhereLoop WhereLoop; +typedef struct WherePath WherePath; +typedef struct WhereTerm WhereTerm; + +/* +** Each instance of this object represents a way of evaluating one +** term of a join. The WhereClause object holds a table of these +** objects using (iTab,prereq,iOb,nOb) as the primary key. Note that the +** same join term might have multiple associated WhereLoop objects. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + int iTab; /* Index of the table coded by this loop */ + u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ + double rSetup; /* One-time setup cost (ex: create transient index) */ + double rRun; /* Cost of running each loop */ + double nOut; /* Estimated number of output rows */ + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nEq; /* Number of equality constraints */ + u16 nTerm; /* Number of entries in aTerm[] */ + Index *pIndex; /* Index used */ + WhereTerm *aTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ +}; + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of the entire query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + double nRow; /* Estimated number of rows generated by this path */ + double rCost; /* Total cost of this path */ + WhereLoop *aLoop[1]; /* Array of WhereLoop objects implementing this path */ + WherePath *pNextPath; /* Next path in order of increasing cost */ + WherePath *pPrevPath; /* Previous path in cost order */ +}; /* ** The query generator uses an array of instances of this structure to @@ -91,7 +128,6 @@ typedef struct WhereCost WhereCost; ** in prereqRight and prereqAll. The default is 64 bits, hence SQLite ** is only able to process joins with 64 or fewer tables. */ -typedef struct WhereTerm WhereTerm; struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ @@ -1793,9 +1829,9 @@ static void bestOrClauseIndex(WhereBestIdx *p){ sBOI.pDistinct = 0; sBOI.ppIdxInfo = 0; for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) - )); + ));*/ if( (pOrTerm->eOperator& WO_AND)!=0 ){ sBOI.pWC = &pOrTerm->u.pAndInfo->wc; bestIndex(&sBOI); @@ -1822,15 +1858,15 @@ static void bestOrClauseIndex(WhereBestIdx *p){ /* If there is an ORDER BY clause, increase the scan cost to account ** for the cost of the sort. */ if( p->pOrderBy!=0 ){ - WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", - rTotal, rTotal+nRow*estLog(nRow))); + /*WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", + rTotal, rTotal+nRow*estLog(nRow)));*/ rTotal += nRow*estLog(nRow); } /* If the cost of scanning using this OR term for optimization is ** less than the current cost stored in pCost, replace the contents ** of pCost. */ - WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); + /*WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));*/ if( rTotalcost.rCost ){ p->cost.rCost = rTotal; p->cost.used = used; @@ -1927,8 +1963,8 @@ static void bestAutomaticIndex(WhereBestIdx *p){ pWCEnd = &pWC->a[pWC->nTerm]; for(pTerm=pWC->a; pTermnotReady) ){ - WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", - p->cost.rCost, costTempIdx)); + /*WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", + p->cost.rCost, costTempIdx));*/ p->cost.rCost = costTempIdx; p->cost.plan.nRow = logN + 1; p->cost.plan.wsFlags = WHERE_TEMP_INDEX; @@ -2113,7 +2149,7 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ int nOrderBy; sqlite3_index_info *pIdxInfo; - WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName)); + /*WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName));*/ /* Count the number of possible WHERE clause constraints referring ** to this virtual table */ @@ -2222,7 +2258,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ int i; int rc; - WHERETRACE(("xBestIndex for %s\n", pTab->zName)); + /*WHERETRACE(("xBestIndex for %s\n", pTab->zName));*/ TRACE_IDX_INPUTS(p); rc = pVtab->pModule->xBestIndex(pVtab, p); TRACE_IDX_OUTPUTS(p); @@ -2733,8 +2769,8 @@ static int whereRangeScanEst( }else{ *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower); } - WHERETRACE(("range scan regions: %u..%u div=%g\n", - (u32)iLower, (u32)iUpper, *pRangeDiv)); + /*WHERETRACE(("range scan regions: %u..%u div=%g\n", + (u32)iLower, (u32)iUpper, *pRangeDiv));*/ return SQLITE_OK; } } @@ -2791,7 +2827,7 @@ static int whereEqualScanEst( if( pRhs==0 ) return SQLITE_NOTFOUND; rc = whereKeyStats(pParse, p, pRhs, 0, a); if( rc==SQLITE_OK ){ - WHERETRACE(("equality scan regions: %d\n", (int)a[1])); + /*WHERETRACE(("equality scan regions: %d\n", (int)a[1]));*/ *pnRow = a[1]; } whereEqualScanEst_cancel: @@ -2837,7 +2873,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; *pnRow = nRowEst; - WHERETRACE(("IN row estimate: est=%g\n", nRowEst)); + /*WHERETRACE(("IN row estimate: est=%g\n", nRowEst));*/ } return rc; } @@ -3049,10 +3085,10 @@ static int isSortingIndex( }else{ Expr *pRight = pConstraint->pExpr->pRight; if( pRight->op==TK_COLUMN ){ - WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", - pRight->iTable, pRight->iColumn)); + /*WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", + pRight->iTable, pRight->iColumn));*/ isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); - WHERETRACE((" -> isEq=%d\n", isEq)); + /*WHERETRACE((" -> isEq=%d\n", isEq));*/ /* If the constraint is of the form X=Y where Y is an ordered value ** in an outer loop, then make sure the sort order of Y matches the @@ -3319,10 +3355,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif - WHERETRACE(( + /*WHERETRACE(( " %s(%s):\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") - )); + ));*/ memset(&pc, 0, sizeof(pc)); pc.plan.nOBSat = nPriorSat; @@ -3403,10 +3439,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ int bRev = 2; int bObUnique = 0; - WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat)); + /*WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat));*/ pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); - WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", - bRev, bObUnique, pc.plan.nOBSat)); + /*WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", + bRev, bObUnique, pc.plan.nOBSat));*/ if( nPriorSatnotReady, log10N, pc.plan.nRow, pc.rCost, pc.used, pc.plan.nOBSat - )); + ));*/ /* If this index is the best we have seen so far, then record this ** index and its cost in the p->cost structure. @@ -3677,9 +3713,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE((" best index is %s cost=%.1f\n", + /*WHERETRACE((" best index is %s cost=%.1f\n", p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk", - p->cost.rCost)); + p->cost.rCost));*/ bestOrClauseIndex(p); bestAutomaticIndex(p); @@ -4928,6 +4964,13 @@ static int nQPlan = 0; /* Next free slow in _query_plan[] */ #endif /* SQLITE_TEST */ +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + sqlite3DbFree(db, p->aTerm); + sqlite3DbFree(db, p); +} /* ** Free a WhereInfo structure @@ -4953,10 +4996,140 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } whereClauseClear(pWInfo->pWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } sqlite3DbFree(db, pWInfo); } } +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based no the template. +*/ +static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + sqlite3 *db = pWInfo->pParse->db; + + /* Search for an existing WhereLoop to overwrite, or which takes + ** priority over pTemplate. + */ + for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->prereq & pTemplate->prereq)==p->prereq + && p->nOb>=pTemplate->nOb + && p->iOb==pTemplate->iOb + && p->rSetup<=pTemplate->rSetup + && p->rRun<=pTemplate->rRun + ){ + /* Already holding an equal or better WhereLoop. + ** Return without changing or adding anything */ + return SQLITE_OK; + } + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq + && p->nOb<=pTemplate->nOb + && p->iOb==pTemplate->iOb + && p->rSetup>=pTemplate->rSetup + && p->rRun>=pTemplate->rRun + ){ + /* Overwrite an existing WhereLoop with a better one */ + sqlite3DbFree(db, p->aTerm); + *ppPrev = p->pNextLoop; + break; + } + } + + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ + if( p==0 ){ + p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM; + } + *p = *pTemplate; + p->pNextLoop = pWInfo->pLoops; + pWInfo->pLoops = p; + if( pTemplate->nTerm<=0 ) return SQLITE_OK; + p->aTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); + if( p->aTerm==0 ){ + p->nTerm = 0; + return SQLITE_NOMEM; + } + memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); + return SQLITE_OK; +} + +/* +** Add all WhereLoop objects for the iTab-th table of the join. That +** table is guaranteed to be a b-tree table, not a virtual table. +*/ +static void whereLoopAddBtree( + WhereInfo *pWInfo, /* WHERE clause information */ + int iTab, /* The table to process */ + Bitmask mExtra /* Extra prerequesites for using this table */ +){ + WhereLoop *pNew; /* Template WhereLoop object */ + sqlite3 *db = pWInfo->pParse->db; + + pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); + if( pNew==0 ) return; + + /* Insert a full table scan */ + pNew->iTab = iTab; + pNew->rSetup = (double)0; + pNew->rRun = (double)1000000; + pNew->nOut = (double)1000000; + whereLoopInsert(pWInfo, pNew); + + whereLoopDelete(db, pNew); +} + +/* +** Add all WhereLoop objects for the iTab-th table of the join. That +** table is guaranteed to be a virtual table. +*/ +static void whereLoopAddVirtual( + WhereInfo *pWInfo, /* WHERE clause information */ + int iTab, /* The table to process */ + Bitmask mExtra /* Extra prerequesites for using this table */ +){ +} + +/* +** Add all WhereLoop objects for all tables +*/ +static void whereLoopAddAll(WhereInfo *pWInfo){ + Bitmask mExtra = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + struct SrcList_item *pItem; + WhereClause *pWC = pWInfo->pWC; + sqlite3 *db = pWInfo->pParse->db; + + /* Loop over the tables in the join, from left to right */ + for(iTab=0, pItem=pTabList->a; iTabnSrc; iTab++, pItem++){ + if( IsVirtual(pItem->pTab) ){ + whereLoopAddVirtual(pWInfo, iTab, mExtra); + }else{ + whereLoopAddBtree(pWInfo, iTab, mExtra); + } + mPrior |= getMask(pWC->pMaskSet, pItem->iCursor); + if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ + mExtra = mPrior; + } + if( db->mallocFailed ) break; + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -5187,6 +5360,35 @@ WhereInfo *sqlite3WhereBegin( pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } + /* Construct the WhereLoop objects */ + WHERETRACE(("*** Optimizer Start ***\n")); + whereLoopAddAll(pWInfo); + + /* Display all of the WhereLoop objects if wheretrace is enabled */ +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) + if( sqlite3WhereTrace ){ + WhereLoop *p; + int nb = 2*((nTabList+15)/16); + for(p=pWInfo->pLoops; p; p=p->pNextLoop){ + struct SrcList_item *pItem = pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); + sqlite3DebugPrintf(" %s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( p->pIndex ){ + sqlite3DebugPrintf(" index %s nEq %d", p->pIndex->zName, p->nEq); + }else{ + sqlite3DebugPrintf("\n"); + } + sqlite3DebugPrintf(" wsFlags %08x OB %d,%d nTerm %d\n", + p->wsFlags, p->iOb, p->nOb, p->nTerm); + sqlite3DebugPrintf(" cost %.2e + %.2e nOut %.2e\n", + p->prereq, p->rSetup, p->rRun, p->nOut); + } + } +#endif + /* Chose the best index to use for each table in the FROM clause. ** ** This loop fills in the following fields: @@ -5207,7 +5409,6 @@ WhereInfo *sqlite3WhereBegin( sWBI.n = nTabList; sWBI.pDistinct = pDistinct; andFlags = ~0; - WHERETRACE(("*** Optimizer Start ***\n")); for(sWBI.i=iFrom=0, pLevel=pWInfo->a; sWBI.ipIndex==0 ) nUnconstrained++; - WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", - j, sWBI.pSrc->pTab->zName, isOptimal)); + /*WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", + j, sWBI.pSrc->pTab->zName, isOptimal));*/ assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(sWBI.pSrc->pTab) ){ @@ -5335,9 +5536,9 @@ WhereInfo *sqlite3WhereBegin( ** for the outer loop that table which benefits the least from ** being in the inner loop. The following code scales the ** outer loop cost estimate to accomplish that. */ - WHERETRACE((" scaling cost from %.1f to %.1f\n", + /*WHERETRACE((" scaling cost from %.1f to %.1f\n", sWBI.cost.rCost, - sWBI.cost.rCost/pWInfo->a[j].rOptCost)); + sWBI.cost.rCost/pWInfo->a[j].rOptCost));*/ sWBI.cost.rCost /= pWInfo->a[j].rOptCost; } @@ -5367,11 +5568,11 @@ WhereInfo *sqlite3WhereBegin( || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) && (bestJ<0 || compareCost(&sWBI.cost, &bestPlan)) /* (4) */ ){ - WHERETRACE((" === table %d (%s) is best so far\n" + /*WHERETRACE((" === table %d (%s) is best so far\n" " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", j, sWBI.pSrc->pTab->zName, sWBI.cost.rCost, sWBI.cost.plan.nRow, - sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags)); + sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags));*/ bestPlan = sWBI.cost; bestJ = j; } @@ -5388,11 +5589,11 @@ WhereInfo *sqlite3WhereBegin( testcase( bestJ>iFrom && (pTabList->a[iFrom].jointype & JT_CROSS)!=0 ); testcase( bestJ>iFrom && bestJa[bestJ+1].jointype & JT_LEFT)!=0 ); - WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" + /*WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", bestJ, pTabList->a[bestJ].pTab->zName, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, - bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); + bestPlan.plan.nOBSat, bestPlan.plan.wsFlags));*/ if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ assert( pWInfo->eDistinct==0 ); pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; From 1c8148fffb49cb540e0c97f0700a0f3c0c0db52d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 May 2013 20:25:23 +0000 Subject: [PATCH 002/109] In where.c, make findTerm() a wrapper around methods to a new WhereScan object which is capable of finding all suitable matching terms, not just the first. This check-in includes some prototype functions for building WhereLoop objects. FossilOrigin-Name: dd92b8fa929badaf2f79e8a00c83667a9d589096 --- manifest | 15 +- manifest.uuid | 2 +- src/where.c | 381 +++++++++++++++++++++++++++++++++++++------------- 3 files changed, 292 insertions(+), 106 deletions(-) diff --git a/manifest b/manifest index 75d4e526f5..a36fbf25aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sinserting\ssome\sexperimental\scode\sfor\sthe\snext\sgeneration\squery\splanner. -D 2013-05-02T00:15:01.231 +C In\swhere.c,\smake\sfindTerm()\sa\swrapper\saround\smethods\sto\sa\snew\sWhereScan\sobject\nwhich\sis\scapable\sof\sfinding\sall\ssuitable\smatching\sterms,\snot\sjust\sthe\sfirst.\nThis\scheck-in\sincludes\ssome\sprototype\sfunctions\sfor\sbuilding\sWhereLoop\sobjects. +D 2013-05-04T20:25:23.612 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c fc62bea654f27d5787bc428cd8f3f176d764f785 +F src/where.c 173347e4598207467401e1952bdee983cf0bf087 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,10 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P faedaeace9c7ed9a8aaf96700caee09db0c0c061 -R 84f8c9b09f28282800936ed72686b19f -T *branch * nextgen-query-plan-exp -T *sym-nextgen-query-plan-exp * -T -sym-trunk * +P ccaf4c3f7e1ec45e058d594d9b5c26818a37722a +R d516c3475d503fb543da6770e30ae39a U drh -Z 18faca4b1f68b4b3514e189f395d77ff +Z 4a8c637fc587086861ee6817f8b965a6 diff --git a/manifest.uuid b/manifest.uuid index 016ae611b1..36eaec293a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ccaf4c3f7e1ec45e058d594d9b5c26818a37722a \ No newline at end of file +dd92b8fa929badaf2f79e8a00c83667a9d589096 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f44ab496c0..14120339d2 100644 --- a/src/where.c +++ b/src/where.c @@ -42,6 +42,8 @@ typedef struct WhereCost WhereCost; typedef struct WhereLoop WhereLoop; typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; +typedef struct WhereLoopBuilder WhereLoopBuilder; +typedef struct WhereScan WhereScan; /* ** Each instance of this object represents a way of evaluating one @@ -60,7 +62,7 @@ struct WhereLoop { u16 nEq; /* Number of equality constraints */ u16 nTerm; /* Number of entries in aTerm[] */ Index *pIndex; /* Index used */ - WhereTerm *aTerm; /* WhereTerms used */ + WhereTerm **aTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ }; @@ -161,6 +163,23 @@ struct WhereTerm { # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereTerm *pCurrent; /* Most recent match */ + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + char *zCollName; /* Must have this collating sequence, if not NULL */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aEquiv[] */ + unsigned char iEquiv; /* Next unused slot in aEquiv[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ +}; + /* ** An instance of the following structure holds all information about a ** WHERE clause. Mostly this is a container for one or more WhereTerms. @@ -247,6 +266,19 @@ struct WhereCost { Bitmask used; /* Bitmask of cursors used by this plan */ }; +/* +** This object is a factory for WhereLoop objects for a particular query. +*/ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + sqlite3 *db; /* Database connection */ + Parse *pParse; /* Parsing context */ + WhereClause *pWC; /* WHERE clause terms */ + SrcList *pTabList; /* FROM clause */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ +}; + /* ** Bitmasks for the operators that indices are able to exploit. An ** OR-ed combination of these values can be used when searching for @@ -661,6 +693,114 @@ static u16 operatorMask(int op){ return c; } +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + int iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aEquiv[pScan->iEquiv-2]; + iColumn = pScan->aEquiv[pScan->iEquiv-1]; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+pScan->k; pScan->knTerm; pScan->k++, pTerm++){ + if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaEquiv) + ){ + int j; + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jnEquiv; j+=2){ + if( pScan->aEquiv[j]==pX->iTable + && pScan->aEquiv[j+1]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aEquiv[j] = pX->iTable; + pScan->aEquiv[j+1] = pX->iColumn; + pScan->nEquiv += 2; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pWC->pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pWC->pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + pScan->pCurrent = pTerm; + pScan->k++; + return pTerm; + } + } + } + pWC = pScan->pWC = pScan->pWC->pOuter; + pScan->k = 0; + } + pScan->pWC = pScan->pOrigWC; + pScan->k = 0; + pScan->iEquiv += 2; + } + pScan->pCurrent = 0; + return 0; +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + int j; + + memset(pScan, 0, sizeof(*pScan)); + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + pScan->zCollName = pIdx->azColl[j]; + } + pScan->opMask = opMask; + pScan->aEquiv[0] = iCur; + pScan->aEquiv[1] = iColumn; + pScan->nEquiv = 2; + pScan->iEquiv = 2; + return whereScanNext(pScan); +} + /* ** Search for a term in the WHERE clause that is of the form "X " ** where X is a reference to the iColumn of table iCur and is one of @@ -692,84 +832,20 @@ static WhereTerm *findTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - WhereTerm *pTerm; /* Term being examined as possible result */ - WhereTerm *pResult = 0; /* The answer to return */ - WhereClause *pWCOrig = pWC; /* Original pWC value */ - int j, k; /* Loop counters */ - Expr *pX; /* Pointer to an expression */ - Parse *pParse; /* Parsing context */ - int iOrigCol = iColumn; /* Original value of iColumn */ - int nEquiv = 2; /* Number of entires in aEquiv[] */ - int iEquiv = 2; /* Number of entries of aEquiv[] processed so far */ - int aEquiv[22]; /* iCur,iColumn and up to 10 other equivalents */ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; - assert( iCur>=0 ); - aEquiv[0] = iCur; - aEquiv[1] = iColumn; - for(;;){ - for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn - ){ - if( (pTerm->prereqRight & notReady)==0 - && (pTerm->eOperator & op & WO_ALL)!=0 - ){ - if( iOrigCol>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; - char idxaff; - - pX = pTerm->pExpr; - pParse = pWC->pParse; - idxaff = pIdx->pTable->aCol[iOrigCol].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ){ - continue; - } - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - - for(j=0; pIdx->aiColumn[j]!=iOrigCol; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; - } - if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ){ - continue; - } - } - if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ - pResult = pTerm; - goto findTerm_success; - }else if( pResult==0 ){ - pResult = pTerm; - } - } - if( (pTerm->eOperator & WO_EQUIV)!=0 - && nEquivpExpr->pRight); - assert( pX->op==TK_COLUMN ); - for(j=0; jiTable && aEquiv[j+1]==pX->iColumn ) break; - } - if( j==nEquiv ){ - aEquiv[j] = pX->iTable; - aEquiv[j+1] = pX->iColumn; - nEquiv += 2; - } - } - } + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ + return p; } + if( pResult==0 ) pResult = p; } - if( iEquiv>=nEquiv ) break; - iCur = aEquiv[iEquiv++]; - iColumn = aEquiv[iEquiv++]; + p = whereScanNext(&scan); } -findTerm_success: return pResult; } @@ -5058,38 +5134,139 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ p->pNextLoop = pWInfo->pLoops; pWInfo->pLoops = p; if( pTemplate->nTerm<=0 ) return SQLITE_OK; + if( p->pIndex && p->pIndex->tnum==0 ) p->pIndex = 0; p->aTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); if( p->aTerm==0 ){ p->nTerm = 0; + sqlite3DbFree(db, p); return SQLITE_NOMEM; } memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); return SQLITE_OK; } +/* +** We have so far matched pBuilder->pNew->nEq terms of the index pIndex. +** Try to match one more. +** +** If pProbe->tnum==0, that means pIndex is a fake index used for the +** INTEGER PRIMARY KEY. +*/ +static void whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + struct SrcList_item *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + int nInMul /* Number of iterations due to IN */ +){ + sqlite3 *db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int eqTermMask; /* Valid equality operators */ + WhereScan scan; /* Iterator for WHERE terms */ + + db = pBuilder->db; + pNew = pBuilder->pNew; + if( db->mallocFailed ) return; + + if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + eqTermMask = WO_EQ|WO_IN; + }else{ + eqTermMask = WO_EQ|WO_IN|WO_ISNULL; + } + + + if( pNew->nEqnColumn ){ + int iCol; /* Index of the column in the table */ + + + iCol = pProbe->aiColumn[pNew->nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + eqTermMask, iCol>=0 ? pProbe : 0); + pNew->nEq++; + for(; pTerm!=0; pTerm = whereScanNext(&scan)){ + pNew->aTerm[pNew->nEq-1] = pTerm; + + } + pNew->nEq--; + + } +} + /* ** Add all WhereLoop objects for the iTab-th table of the join. That ** table is guaranteed to be a b-tree table, not a virtual table. */ static void whereLoopAddBtree( - WhereInfo *pWInfo, /* WHERE clause information */ - int iTab, /* The table to process */ - Bitmask mExtra /* Extra prerequesites for using this table */ + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + int iTab, /* The table to process */ + Bitmask mExtra /* Extra prerequesites for using this table */ ){ - WhereLoop *pNew; /* Template WhereLoop object */ - sqlite3 *db = pWInfo->pParse->db; + Index *pProbe; /* An index we are evaluating */ + int eqTermMask; /* Current mask of valid equality operators */ + Index sPk; /* A fake index object for the primary key */ + tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + struct SrcList_item *pSrc; /* The FROM clause btree term to add */ + sqlite3 *db; /* The database connection */ + WhereLoop *pNew; /* Template WhereLoop object */ - pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); - if( pNew==0 ) return; - + pNew = pBuilder->pNew; + db = pBuilder->db; + pSrc = pBuilder->pTabList->a + iTab; + + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->pIndex; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pSrc->pTab; + aiRowEstPk[0] = pSrc->pTab->nRowEst; + aiRowEstPk[1] = 1; + pFirst = pSrc->pTab->pIndex; + if( pSrc->notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + } + + /* Loop over all indices + */ + for(; pProbe; pProbe=pProbe->pNext){ + pNew->prereq = mExtra; + pNew->iTab = iTab; + pNew->nEq = 0; + pNew->nTerm = 0; + pNew->aTerm = sqlite3DbRealloc(db, pNew->aTerm, + (pProbe->nColumn+1)*sizeof(pNew->aTerm[0])); + if( pNew->aTerm==0 ) break; + pNew->pIndex = pProbe; + + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); + + /* If there was an INDEXED BY clause, then only that one index is + ** considered. */ + if( pSrc->pIndex ) break; + } + +#if 0 /* Insert a full table scan */ pNew->iTab = iTab; pNew->rSetup = (double)0; pNew->rRun = (double)1000000; pNew->nOut = (double)1000000; - whereLoopInsert(pWInfo, pNew); - - whereLoopDelete(db, pNew); + whereLoopInsert(pBuilder->pWInfo, pNew); +#endif } /* @@ -5097,30 +5274,32 @@ static void whereLoopAddBtree( ** table is guaranteed to be a virtual table. */ static void whereLoopAddVirtual( - WhereInfo *pWInfo, /* WHERE clause information */ - int iTab, /* The table to process */ - Bitmask mExtra /* Extra prerequesites for using this table */ + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + int iTab, /* The table to process */ + Bitmask mExtra /* Extra prerequesites for using this table */ ){ } /* ** Add all WhereLoop objects for all tables */ -static void whereLoopAddAll(WhereInfo *pWInfo){ +static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ Bitmask mExtra = 0; Bitmask mPrior = 0; int iTab; - SrcList *pTabList = pWInfo->pTabList; + SrcList *pTabList = pBuilder->pTabList; struct SrcList_item *pItem; - WhereClause *pWC = pWInfo->pWC; - sqlite3 *db = pWInfo->pParse->db; + WhereClause *pWC = pBuilder->pWC; + sqlite3 *db = pBuilder->db; /* Loop over the tables in the join, from left to right */ + pBuilder->pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); + if( pBuilder->pNew==0 ) return; for(iTab=0, pItem=pTabList->a; iTabnSrc; iTab++, pItem++){ if( IsVirtual(pItem->pTab) ){ - whereLoopAddVirtual(pWInfo, iTab, mExtra); + whereLoopAddVirtual(pBuilder, iTab, mExtra); }else{ - whereLoopAddBtree(pWInfo, iTab, mExtra); + whereLoopAddBtree(pBuilder, iTab, mExtra); } mPrior |= getMask(pWC->pMaskSet, pItem->iCursor); if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ @@ -5128,6 +5307,8 @@ static void whereLoopAddAll(WhereInfo *pWInfo){ } if( db->mallocFailed ) break; } + whereLoopDelete(db, pBuilder->pNew); + pBuilder->pNew = 0; } @@ -5234,6 +5415,7 @@ WhereInfo *sqlite3WhereBegin( Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ WhereBestIdx sWBI; /* Best index search context */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ WhereLevel *pLevel; /* A single level in pWInfo->a[] */ int iFrom; /* First unused FROM clause element */ @@ -5245,6 +5427,11 @@ WhereInfo *sqlite3WhereBegin( /* Variable initialization */ memset(&sWBI, 0, sizeof(sWBI)); sWBI.pParse = pParse; + memset(&sWLB, 0, sizeof(sWLB)); + sWLB.pParse = pParse; + sWLB.db = pParse->db; + sWLB.pTabList = pTabList; + sWLB.pOrderBy = pOrderBy; /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask @@ -5290,6 +5477,8 @@ WhereInfo *sqlite3WhereBegin( pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; sWBI.aLevel = pWInfo->a; + sWLB.pWInfo = pWInfo; + sWLB.pWC = pWInfo->pWC; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -5362,7 +5551,7 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); - whereLoopAddAll(pWInfo); + /*whereLoopAddAll(&sWLB);*/ /* Display all of the WhereLoop objects if wheretrace is enabled */ #if defined(SQLITE_DEBUG) \ From f1645f084376428d018930ef72f1e304cfab29dc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 May 2013 19:44:38 +0000 Subject: [PATCH 003/109] Inserting a few WhereLoop objects without leaking memory. Costs are not correct. Inequality and IN constraints are not implemented. FossilOrigin-Name: e8881a8b2f25f38bc8ff77619f96f38fe530d13b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 41 ++++++++++++++++++++++++++++------------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index a36fbf25aa..29d28aecb0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swhere.c,\smake\sfindTerm()\sa\swrapper\saround\smethods\sto\sa\snew\sWhereScan\sobject\nwhich\sis\scapable\sof\sfinding\sall\ssuitable\smatching\sterms,\snot\sjust\sthe\sfirst.\nThis\scheck-in\sincludes\ssome\sprototype\sfunctions\sfor\sbuilding\sWhereLoop\sobjects. -D 2013-05-04T20:25:23.612 +C Inserting\sa\sfew\sWhereLoop\sobjects\swithout\sleaking\smemory.\s\sCosts\sare\nnot\scorrect.\s\sInequality\sand\sIN\sconstraints\sare\snot\simplemented. +D 2013-05-07T19:44:38.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 173347e4598207467401e1952bdee983cf0bf087 +F src/where.c e987a4ff2818b812f902361c1131fef4a8f95a41 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ccaf4c3f7e1ec45e058d594d9b5c26818a37722a -R d516c3475d503fb543da6770e30ae39a +P dd92b8fa929badaf2f79e8a00c83667a9d589096 +R 20c9cc42b7ec6059ab1cb985542962e1 U drh -Z 4a8c637fc587086861ee6817f8b965a6 +Z 6dee1758f1f268139b7d8139f9ade772 diff --git a/manifest.uuid b/manifest.uuid index 36eaec293a..62ed5b5e8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd92b8fa929badaf2f79e8a00c83667a9d589096 \ No newline at end of file +e8881a8b2f25f38bc8ff77619f96f38fe530d13b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 14120339d2..4776a2b231 100644 --- a/src/where.c +++ b/src/where.c @@ -5163,6 +5163,7 @@ static void whereLoopAddBtreeIndex( WhereTerm *pTerm; /* A WhereTerm under consideration */ int eqTermMask; /* Valid equality operators */ WhereScan scan; /* Iterator for WHERE terms */ + WhereLoop savedLoop; db = pBuilder->db; pNew = pBuilder->pNew; @@ -5182,13 +5183,26 @@ static void whereLoopAddBtreeIndex( iCol = pProbe->aiColumn[pNew->nEq]; pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, eqTermMask, iCol>=0 ? pProbe : 0); + savedLoop = *pNew; pNew->nEq++; + pNew->nTerm++; for(; pTerm!=0; pTerm = whereScanNext(&scan)){ pNew->aTerm[pNew->nEq-1] = pTerm; - + pNew->nOut = (double)(pProbe->aiRowEst[pNew->nEq] * nInMul); + pNew->rSetup = (double)0; + pNew->rRun = pNew->nOut; + pNew->prereq = savedLoop.prereq | pTerm->prereqRight; + if( pProbe->tnum<=0 ){ + pNew->wsFlags = savedLoop.wsFlags | WHERE_ROWID_EQ; + }else{ + pNew->wsFlags = savedLoop.wsFlags | WHERE_COLUMN_EQ; + } + whereLoopInsert(pBuilder->pWInfo, pNew); + if( pNew->nEqnColumn ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + } } - pNew->nEq--; - + *pNew = savedLoop; } } @@ -5202,7 +5216,6 @@ static void whereLoopAddBtree( Bitmask mExtra /* Extra prerequesites for using this table */ ){ Index *pProbe; /* An index we are evaluating */ - int eqTermMask; /* Current mask of valid equality operators */ Index sPk; /* A fake index object for the primary key */ tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ @@ -5243,13 +5256,15 @@ static void whereLoopAddBtree( /* Loop over all indices */ for(; pProbe; pProbe=pProbe->pNext){ + WhereTerm **paTerm; pNew->prereq = mExtra; pNew->iTab = iTab; pNew->nEq = 0; pNew->nTerm = 0; - pNew->aTerm = sqlite3DbRealloc(db, pNew->aTerm, - (pProbe->nColumn+1)*sizeof(pNew->aTerm[0])); - if( pNew->aTerm==0 ) break; + paTerm = sqlite3DbRealloc(db, pNew->aTerm, + (pProbe->nColumn+1)*sizeof(pNew->aTerm[0])); + if( paTerm==0 ) break; + pNew->aTerm = paTerm; pNew->pIndex = pProbe; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); @@ -5551,7 +5566,7 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); - /*whereLoopAddAll(&sWLB);*/ + whereLoopAddAll(&sWLB); /* Display all of the WhereLoop objects if wheretrace is enabled */ #if defined(SQLITE_DEBUG) \ @@ -5563,16 +5578,16 @@ WhereInfo *sqlite3WhereBegin( struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); - sqlite3DebugPrintf(" %s", + sqlite3DebugPrintf(" %5s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( p->pIndex ){ - sqlite3DebugPrintf(" index %s nEq %d", p->pIndex->zName, p->nEq); + sqlite3DebugPrintf(".%-5s %2d", p->pIndex->zName, p->nEq); }else{ - sqlite3DebugPrintf("\n"); + sqlite3DebugPrintf("%9s",""); } - sqlite3DebugPrintf(" wsFlags %08x OB %d,%d nTerm %d\n", + sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", p->wsFlags, p->iOb, p->nOb, p->nTerm); - sqlite3DebugPrintf(" cost %.2e + %.2e nOut %.2e\n", + sqlite3DebugPrintf(" cost %.2g+%.2g,%.2g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } } From 43fe25fcef6f0fa329c2a9748bc3794d783e12b9 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 May 2013 23:06:23 +0000 Subject: [PATCH 004/109] Continued progress on generating good WhereLoop objects for the new query planner. FossilOrigin-Name: 15cc8a16482777d8e138c4d0863faf8d54fef33a --- manifest | 12 ++--- manifest.uuid | 2 +- src/where.c | 134 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 101 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index 29d28aecb0..c79d6f38fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Inserting\sa\sfew\sWhereLoop\sobjects\swithout\sleaking\smemory.\s\sCosts\sare\nnot\scorrect.\s\sInequality\sand\sIN\sconstraints\sare\snot\simplemented. -D 2013-05-07T19:44:38.531 +C Continued\sprogress\son\sgenerating\sgood\sWhereLoop\sobjects\sfor\sthe\snew\squery\nplanner. +D 2013-05-07T23:06:23.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e987a4ff2818b812f902361c1131fef4a8f95a41 +F src/where.c e957c2c6fd7b421ea6dd1ff11d254c80d0b11638 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dd92b8fa929badaf2f79e8a00c83667a9d589096 -R 20c9cc42b7ec6059ab1cb985542962e1 +P e8881a8b2f25f38bc8ff77619f96f38fe530d13b +R 90f142cd2d5094cb6ddc049cebfe5b6d U drh -Z 6dee1758f1f268139b7d8139f9ade772 +Z d9519f5b83b11415ad77247122aa0d25 diff --git a/manifest.uuid b/manifest.uuid index 62ed5b5e8a..82bc167f3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8881a8b2f25f38bc8ff77619f96f38fe530d13b \ No newline at end of file +15cc8a16482777d8e138c4d0863faf8d54fef33a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4776a2b231..56568989aa 100644 --- a/src/where.c +++ b/src/where.c @@ -315,6 +315,7 @@ struct WhereLoopBuilder { */ #define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ #define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ +#define WHERE_IPK 0x00004000 /* x is the INTEGER PRIMARY KEY */ #define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ #define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ #define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ @@ -710,6 +711,15 @@ WhereTerm *whereScanNext(WhereScan *pScan){ iColumn = pScan->aEquiv[pScan->iEquiv-1]; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+pScan->k; pScan->knTerm; pScan->k++, pTerm++){ + if( pTerm->iParent>=0 ){ + WhereTerm *pParent = &pWC->a[pTerm->iParent]; + int j; + for(j=pScan->iEquiv-4; j>=0; j-=2 ){ + if( pParent->leftCursor==pScan->aEquiv[j] + && pParent->u.leftColumn==pScan->aEquiv[j+1] ) break; + } + if( j>=0 ) continue; + } if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaEquiv) @@ -5091,7 +5101,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** added based no the template. */ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p; + WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; sqlite3 *db = pWInfo->pParse->db; /* Search for an existing WhereLoop to overwrite, or which takes @@ -5117,7 +5127,7 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with a better one */ sqlite3DbFree(db, p->aTerm); - *ppPrev = p->pNextLoop; + pNext = p->pNextLoop; break; } } @@ -5127,18 +5137,19 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ ** WhereLoop and insert it. */ if( p==0 ){ - p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + p = pToFree = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; } *p = *pTemplate; - p->pNextLoop = pWInfo->pLoops; - pWInfo->pLoops = p; - if( pTemplate->nTerm<=0 ) return SQLITE_OK; + p->pNextLoop = pNext; + *ppPrev = p; + p->aTerm = 0; if( p->pIndex && p->pIndex->tnum==0 ) p->pIndex = 0; + if( pTemplate->nTerm<=0 ) return SQLITE_OK; p->aTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); if( p->aTerm==0 ){ p->nTerm = 0; - sqlite3DbFree(db, p); + sqlite3DbFree(db, pToFree); return SQLITE_NOMEM; } memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); @@ -5154,6 +5165,7 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ */ static void whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + Bitmask maskSelf, /* Bitmask for table being scanned */ struct SrcList_item *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ int nInMul /* Number of iterations due to IN */ @@ -5161,18 +5173,22 @@ static void whereLoopAddBtreeIndex( sqlite3 *db; /* Database connection malloc context */ WhereLoop *pNew; /* Template WhereLoop under construction */ WhereTerm *pTerm; /* A WhereTerm under consideration */ - int eqTermMask; /* Valid equality operators */ + int opMask; /* Valid operators for constraints */ WhereScan scan; /* Iterator for WHERE terms */ - WhereLoop savedLoop; + WhereLoop savedLoop; /* Saved original content of pNew[] */ db = pBuilder->db; pNew = pBuilder->pNew; if( db->mallocFailed ) return; - if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ - eqTermMask = WO_EQ|WO_IN; + assert( pNew->nEqnColumn ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ - eqTermMask = WO_EQ|WO_IN|WO_ISNULL; + opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } @@ -5182,24 +5198,42 @@ static void whereLoopAddBtreeIndex( iCol = pProbe->aiColumn[pNew->nEq]; pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - eqTermMask, iCol>=0 ? pProbe : 0); + opMask, iCol>=0 ? pProbe : 0); savedLoop = *pNew; - pNew->nEq++; - pNew->nTerm++; + pNew->rSetup = (double)0; for(; pTerm!=0; pTerm = whereScanNext(&scan)){ - pNew->aTerm[pNew->nEq-1] = pTerm; - pNew->nOut = (double)(pProbe->aiRowEst[pNew->nEq] * nInMul); - pNew->rSetup = (double)0; - pNew->rRun = pNew->nOut; - pNew->prereq = savedLoop.prereq | pTerm->prereqRight; - if( pProbe->tnum<=0 ){ - pNew->wsFlags = savedLoop.wsFlags | WHERE_ROWID_EQ; - }else{ - pNew->wsFlags = savedLoop.wsFlags | WHERE_COLUMN_EQ; + int nIn = 1; + pNew->nEq = savedLoop.nEq; + pNew->nTerm = savedLoop.nTerm; + pNew->aTerm[pNew->nTerm++] = pTerm; + pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~maskSelf; + if( pTerm->eOperator & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ + nIn = 25; + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = pExpr->x.pList->nExpr; + } + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; + }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; + }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->nOut = savedLoop.nOut/3; + }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->nOut = savedLoop.nOut/3; } + pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; whereLoopInsert(pBuilder->pWInfo, pNew); - if( pNew->nEqnColumn ){ - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ + whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, nInMul*nIn); } } *pNew = savedLoop; @@ -5222,10 +5256,12 @@ static void whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ sqlite3 *db; /* The database connection */ WhereLoop *pNew; /* Template WhereLoop object */ + Bitmask maskSelf; /* Mask for iTab */ pNew = pBuilder->pNew; db = pBuilder->db; pSrc = pBuilder->pTabList->a + iTab; + maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5253,6 +5289,19 @@ static void whereLoopAddBtree( pProbe = &sPk; } + /* Insert a full table scan */ + pNew->iTab = iTab; + pNew->nEq = 0; + pNew->nTerm = 0; + pNew->rSetup = (double)0; + pNew->prereq = 0; + pNew->pIndex = 0; + pNew->wsFlags = 0; + pNew->iOb = pNew->nOb = 0; + pNew->rRun = (double)pSrc->pTab->nRowEst; + pNew->nOut = (double)pSrc->pTab->nRowEst; + whereLoopInsert(pBuilder->pWInfo, pNew); + /* Loop over all indices */ for(; pProbe; pProbe=pProbe->pNext){ @@ -5261,27 +5310,32 @@ static void whereLoopAddBtree( pNew->iTab = iTab; pNew->nEq = 0; pNew->nTerm = 0; + if( pProbe->tnum<=0 ){ + /* Integer primary key index */ + pNew->wsFlags = WHERE_IPK; + }else{ + Bitmask m = pSrc->colUsed; + int j; + for(j=pProbe->nColumn-1; j>=0; j--){ + int x = pProbe->aiColumn[j]; + if( xwsFlags = m==0 ? WHERE_IDX_ONLY : 0; + } paTerm = sqlite3DbRealloc(db, pNew->aTerm, (pProbe->nColumn+1)*sizeof(pNew->aTerm[0])); if( paTerm==0 ) break; pNew->aTerm = paTerm; pNew->pIndex = pProbe; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); + whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ if( pSrc->pIndex ) break; } - -#if 0 - /* Insert a full table scan */ - pNew->iTab = iTab; - pNew->rSetup = (double)0; - pNew->rRun = (double)1000000; - pNew->nOut = (double)1000000; - whereLoopInsert(pBuilder->pWInfo, pNew); -#endif } /* @@ -5578,16 +5632,16 @@ WhereInfo *sqlite3WhereBegin( struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); - sqlite3DebugPrintf(" %5s", + sqlite3DebugPrintf(" %6s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( p->pIndex ){ - sqlite3DebugPrintf(".%-5s %2d", p->pIndex->zName, p->nEq); + sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); }else{ - sqlite3DebugPrintf("%9s",""); + sqlite3DebugPrintf("%12s",""); } sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", p->wsFlags, p->iOb, p->nOb, p->nTerm); - sqlite3DebugPrintf(" cost %.2g+%.2g,%.2g\n", + sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } } From a18f3d271e52dd964491417cce401dee96449148 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 03:05:41 +0000 Subject: [PATCH 005/109] Add the NGQP solver. FossilOrigin-Name: 5d37587c50d8932b6357bfd03152a851510a4317 --- manifest | 14 +-- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 273 +++++++++++++++++++++++++++++++++++------------- 4 files changed, 207 insertions(+), 83 deletions(-) diff --git a/manifest b/manifest index c79d6f38fd..53780d9f27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continued\sprogress\son\sgenerating\sgood\sWhereLoop\sobjects\sfor\sthe\snew\squery\nplanner. -D 2013-05-07T23:06:23.630 +C Add\sthe\sNGQP\ssolver. +D 2013-05-08T03:05:41.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 0d76a0aa7c64536c6f55d11a8f9f40df0636af6a +F src/sqliteInt.h aee09cf5ee7d5e899246affdd8c3cf58f086ade5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e957c2c6fd7b421ea6dd1ff11d254c80d0b11638 +F src/where.c cac7c10d422ece684740285366285ef06529796f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e8881a8b2f25f38bc8ff77619f96f38fe530d13b -R 90f142cd2d5094cb6ddc049cebfe5b6d +P 15cc8a16482777d8e138c4d0863faf8d54fef33a +R 6dc017dbdcd8ab4e7013a1e96308cbfe U drh -Z d9519f5b83b11415ad77247122aa0d25 +Z 6f2b95a5959d577b5b6c3e6985beb9a5 diff --git a/manifest.uuid b/manifest.uuid index 82bc167f3b..379ce7b07e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15cc8a16482777d8e138c4d0863faf8d54fef33a \ No newline at end of file +5d37587c50d8932b6357bfd03152a851510a4317 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7a405b2fb1..86b123749c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2021,6 +2021,7 @@ struct WhereLevel { Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ } u; double rOptCost; /* "Optimal" cost for this level */ + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ /* The following field is really not part of the current level. But ** we need a place to cache virtual table index information for each diff --git a/src/where.c b/src/where.c index 56568989aa..e87d2ae643 100644 --- a/src/where.c +++ b/src/where.c @@ -53,6 +53,7 @@ typedef struct WhereScan WhereScan; */ struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ int iTab; /* Index of the table coded by this loop */ u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ double rSetup; /* One-time setup cost (ex: create transient index) */ @@ -74,9 +75,7 @@ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ double nRow; /* Estimated number of rows generated by this path */ double rCost; /* Total cost of this path */ - WhereLoop *aLoop[1]; /* Array of WhereLoop objects implementing this path */ - WherePath *pNextPath; /* Next path in order of increasing cost */ - WherePath *pPrevPath; /* Previous path in cost order */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; /* @@ -5050,6 +5049,30 @@ static int nQPlan = 0; /* Next free slow in _query_plan[] */ #endif /* SQLITE_TEST */ +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ + int nb = 2*((pTabList->nSrc+15)/16); + struct SrcList_item *pItem = pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); + sqlite3DebugPrintf(" %6s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( p->pIndex ){ + sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); + }else{ + sqlite3DebugPrintf("%12s",""); + } + sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", + p->wsFlags, p->iOb, p->nOb, p->nTerm); + sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", + p->prereq, p->rSetup, p->rRun, p->nOut); +} +#endif + /* ** Delete a WhereLoop object */ @@ -5102,6 +5125,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ */ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; + WhereTerm **paTerm = 0; sqlite3 *db = pWInfo->pParse->db; /* Search for an existing WhereLoop to overwrite, or which takes @@ -5127,6 +5151,8 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with a better one */ sqlite3DbFree(db, p->aTerm); + p->aTerm = 0; + p->nTerm = 0; pNext = p->pNextLoop; break; } @@ -5140,19 +5166,21 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ p = pToFree = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; } + if( pTemplate->nTerm ){ + paTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); + if( paTerm==0 ){ + sqlite3DbFree(db, pToFree); + return SQLITE_NOMEM; + } + } *p = *pTemplate; p->pNextLoop = pNext; *ppPrev = p; - p->aTerm = 0; + p->aTerm = paTerm; if( p->pIndex && p->pIndex->tnum==0 ) p->pIndex = 0; - if( pTemplate->nTerm<=0 ) return SQLITE_OK; - p->aTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); - if( p->aTerm==0 ){ - p->nTerm = 0; - sqlite3DbFree(db, pToFree); - return SQLITE_NOMEM; + if( pTemplate->nTerm ){ + memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); } - memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); return SQLITE_OK; } @@ -5165,7 +5193,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ */ static void whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ - Bitmask maskSelf, /* Bitmask for table being scanned */ struct SrcList_item *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ int nInMul /* Number of iterations due to IN */ @@ -5176,6 +5203,7 @@ static void whereLoopAddBtreeIndex( int opMask; /* Valid operators for constraints */ WhereScan scan; /* Iterator for WHERE terms */ WhereLoop savedLoop; /* Saved original content of pNew[] */ + int iCol; /* Index of the column in the table */ db = pBuilder->db; pNew = pBuilder->pNew; @@ -5191,53 +5219,47 @@ static void whereLoopAddBtreeIndex( opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } - - if( pNew->nEqnColumn ){ - int iCol; /* Index of the column in the table */ - - - iCol = pProbe->aiColumn[pNew->nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, iCol>=0 ? pProbe : 0); - savedLoop = *pNew; - pNew->rSetup = (double)0; - for(; pTerm!=0; pTerm = whereScanNext(&scan)){ - int nIn = 1; - pNew->nEq = savedLoop.nEq; - pNew->nTerm = savedLoop.nTerm; - pNew->aTerm[pNew->nTerm++] = pTerm; - pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~maskSelf; - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 25; - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nIn = pExpr->x.pList->nExpr; - } - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; - }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_EQ; - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->nOut = savedLoop.nOut/3; - }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->nOut = savedLoop.nOut/3; - } - pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; - whereLoopInsert(pBuilder->pWInfo, pNew); - if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ - whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, nInMul*nIn); + iCol = pProbe->aiColumn[pNew->nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, iCol>=0 ? pProbe : 0); + savedLoop = *pNew; + pNew->rSetup = (double)0; + for(; pTerm!=0; pTerm = whereScanNext(&scan)){ + int nIn = 1; + pNew->nEq = savedLoop.nEq; + pNew->nTerm = savedLoop.nTerm; + pNew->aTerm[pNew->nTerm++] = pTerm; + pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; + if( pTerm->eOperator & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ + nIn = 25; + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = pExpr->x.pList->nExpr; } + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; + }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; + }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->nOut = savedLoop.nOut/3; + }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->nOut = savedLoop.nOut/3; + } + pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; + whereLoopInsert(pBuilder->pWInfo, pNew); + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); } - *pNew = savedLoop; } + *pNew = savedLoop; } /* @@ -5256,12 +5278,11 @@ static void whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ sqlite3 *db; /* The database connection */ WhereLoop *pNew; /* Template WhereLoop object */ - Bitmask maskSelf; /* Mask for iTab */ pNew = pBuilder->pNew; db = pBuilder->db; pSrc = pBuilder->pTabList->a + iTab; - maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); + pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5330,7 +5351,7 @@ static void whereLoopAddBtree( pNew->aTerm = paTerm; pNew->pIndex = pProbe; - whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, 1); + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ @@ -5360,11 +5381,12 @@ static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ struct SrcList_item *pItem; WhereClause *pWC = pBuilder->pWC; sqlite3 *db = pBuilder->db; + int nTabList = pBuilder->pWInfo->nLevel; /* Loop over the tables in the join, from left to right */ pBuilder->pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); if( pBuilder->pNew==0 ) return; - for(iTab=0, pItem=pTabList->a; iTabnSrc; iTab++, pItem++){ + for(iTab=0, pItem=pTabList->a; iTabpTab) ){ whereLoopAddVirtual(pBuilder, iTab, mExtra); }else{ @@ -5380,6 +5402,107 @@ static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ pBuilder->pNew = 0; } +/* +** Given the list of WhereLoop objects on pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo){ + const int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + double rCost; /* Cost of a path */ + double mxCost; /* Maximum cost of a set of paths */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + char *pSpace; /* Temporary memory used by this routine */ + + db = pWInfo->pParse->db; + nLoop = pWInfo->nLevel; + assert( nLoop<=pWInfo->pTabList->nSrc ); + + /* Allocate and initialize space for aTo and aFrom */ + ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + pSpace = sqlite3DbMallocRaw(db, ii); + if( pSpace==0 ) return SQLITE_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=0, pFrom=aTo; iiaLoop = pX; + } + + nFrom = 1; + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + Bitmask maskNew; + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + for(jj=0, pTo=aTo; jjmaskLoop!=maskNew; jj++){} + if( jj>=nTo ){ + if( nTo>=mxChoice && rCost>=mxCost ) continue; + if( nTo=mxCost; jj++){ assert(jj>0); } + } + pTo = &aTo[jj]; + } + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->nRow = pFrom->nRow * pWLoop->nOut; + pTo->rCost = rCost; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxCost = aTo[0].rCost; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost ) mxCost = pTo->rCost; + } + } + } + } + + /* Swap the roles of aFrom and aTo in preparation for the next + ** cycle. */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + /* TEMPORARY */ + if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } + assert( nFrom>0 ); + + /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + for(iLoop=0; iLoopa[iLoop].pWLoop = pFrom->aLoop[iLoop]; + } + + /* Free temporary memory and return success */ + sqlite3DbFree(db, pSpace); + return SQLITE_OK; +} /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -5621,28 +5744,28 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); whereLoopAddAll(&sWLB); + if( db->mallocFailed ) goto whereBeginError; /* Display all of the WhereLoop objects if wheretrace is enabled */ #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) if( sqlite3WhereTrace ){ WhereLoop *p; - int nb = 2*((nTabList+15)/16); for(p=pWInfo->pLoops; p; p=p->pNextLoop){ - struct SrcList_item *pItem = pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); - sqlite3DebugPrintf(" %6s", - pItem->zAlias ? pItem->zAlias : pTab->zName); - if( p->pIndex ){ - sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); - }else{ - sqlite3DebugPrintf("%12s",""); - } - sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", - p->wsFlags, p->iOb, p->nOb, p->nTerm); - sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", - p->prereq, p->rSetup, p->rRun, p->nOut); + whereLoopPrint(p, pTabList); + } + } +#endif + + wherePathSolver(pWInfo); + if( db->mallocFailed ) goto whereBeginError; +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) + if( sqlite3WhereTrace ){ + int ii; + sqlite3DebugPrintf("------------ Solution ----------------\n"); + for(ii=0; iia[ii].pWLoop, pTabList); } } #endif From f204dac1e33067f056ebb59634d771db53b3ebda Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 03:22:07 +0000 Subject: [PATCH 006/109] Bug fixes in the solver. FossilOrigin-Name: b36034bbd19bc5677b26a6f60ca96eb2b37db373 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 53780d9f27..7b52c55ce1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sNGQP\ssolver. -D 2013-05-08T03:05:41.388 +C Bug\sfixes\sin\sthe\ssolver. +D 2013-05-08T03:22:07.536 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c cac7c10d422ece684740285366285ef06529796f +F src/where.c 22da73b779e8892a59fcd03835ddf7c3e63a305c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 15cc8a16482777d8e138c4d0863faf8d54fef33a -R 6dc017dbdcd8ab4e7013a1e96308cbfe +P 5d37587c50d8932b6357bfd03152a851510a4317 +R 0e4e284cdf8cf22ac3d57735e7fe3f80 U drh -Z 6f2b95a5959d577b5b6c3e6985beb9a5 +Z bad1f86f1f7d261e6176b4a4726658c0 diff --git a/manifest.uuid b/manifest.uuid index 379ce7b07e..27876fbc73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d37587c50d8932b6357bfd03152a851510a4317 \ No newline at end of file +b36034bbd19bc5677b26a6f60ca96eb2b37db373 \ No newline at end of file diff --git a/src/where.c b/src/where.c index e87d2ae643..abd99f5a84 100644 --- a/src/where.c +++ b/src/where.c @@ -5443,6 +5443,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ pFrom->aLoop = pX; } + aFrom[0].nRow = (double)1; nFrom = 1; for(iLoop=0; iLoop=mxCost; jj++){ assert(jj>0); } } pTo = &aTo[jj]; + }else{ + if( pTo->rCost<=rCost ) continue; } pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; pTo->nRow = pFrom->nRow * pWLoop->nOut; @@ -5477,6 +5480,19 @@ static int wherePathSolver(WhereInfo *pWInfo){ } } +#if 0 + if( sqlite3WhereTrace ){ + sqlite3DebugPrintf("---- round %d ---- nTo=%d\n", iLoop, nTo); + for(ii=0; iipTabList); + } + } + } +#endif + /* Swap the roles of aFrom and aTo in preparation for the next ** cycle. */ pFrom = aTo; From a184fb87b8ba04e053454ab2591eb250beea2eb3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 04:22:59 +0000 Subject: [PATCH 007/109] More bug fixes to the WhereLoop generator and the solver in NGQP. Now finds the best plan for TPC-H Q8. This seems to prove the concept, but there is still much work to be done. FossilOrigin-Name: 8e5aad37529ec3042e3468acf15186f566e2df8a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++--------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 7b52c55ce1..10086d2469 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfixes\sin\sthe\ssolver. -D 2013-05-08T03:22:07.536 +C More\sbug\sfixes\sto\sthe\sWhereLoop\sgenerator\sand\sthe\ssolver\sin\sNGQP.\nNow\sfinds\sthe\sbest\splan\sfor\sTPC-H\sQ8.\s\sThis\sseems\sto\sprove\sthe\nconcept,\sbut\sthere\sis\sstill\smuch\swork\sto\sbe\sdone. +D 2013-05-08T04:22:59.354 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 22da73b779e8892a59fcd03835ddf7c3e63a305c +F src/where.c c02037efa641eb9f60f054a52bc4ef3c8230369a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5d37587c50d8932b6357bfd03152a851510a4317 -R 0e4e284cdf8cf22ac3d57735e7fe3f80 +P b36034bbd19bc5677b26a6f60ca96eb2b37db373 +R 133717a86de1df991cb6332a4c908d9a U drh -Z bad1f86f1f7d261e6176b4a4726658c0 +Z f247935a6bdcd4102bf6e24d9a3606bf diff --git a/manifest.uuid b/manifest.uuid index 27876fbc73..f6844eb427 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b36034bbd19bc5677b26a6f60ca96eb2b37db373 \ No newline at end of file +8e5aad37529ec3042e3468acf15186f566e2df8a \ No newline at end of file diff --git a/src/where.c b/src/where.c index abd99f5a84..1f198beab4 100644 --- a/src/where.c +++ b/src/where.c @@ -710,15 +710,6 @@ WhereTerm *whereScanNext(WhereScan *pScan){ iColumn = pScan->aEquiv[pScan->iEquiv-1]; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+pScan->k; pScan->knTerm; pScan->k++, pTerm++){ - if( pTerm->iParent>=0 ){ - WhereTerm *pParent = &pWC->a[pTerm->iParent]; - int j; - for(j=pScan->iEquiv-4; j>=0; j-=2 ){ - if( pParent->leftCursor==pScan->aEquiv[j] - && pParent->u.leftColumn==pScan->aEquiv[j+1] ) break; - } - if( j>=0 ) continue; - } if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaEquiv) @@ -754,6 +745,13 @@ WhereTerm *whereScanNext(WhereScan *pScan){ continue; } } + if( (pTerm->eOperator & WO_EQ)!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aEquiv[0] + && pX->iColumn==pScan->aEquiv[1] + ){ + continue; + } pScan->pCurrent = pTerm; pScan->k++; return pTerm; @@ -5058,13 +5056,14 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ int nb = 2*((pTabList->nSrc+15)/16); struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); - sqlite3DebugPrintf(" %6s", + sqlite3DebugPrintf("%2d.%0*llx.%0*llx", + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %8s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( p->pIndex ){ - sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); + sqlite3DebugPrintf(".%-12s %2d", p->pIndex->zName, p->nEq); }else{ - sqlite3DebugPrintf("%12s",""); + sqlite3DebugPrintf("%16s",""); } sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", p->wsFlags, p->iOb, p->nOb, p->nTerm); @@ -5282,7 +5281,7 @@ static void whereLoopAddBtree( pNew = pBuilder->pNew; db = pBuilder->db; pSrc = pBuilder->pTabList->a + iTab; - pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); + pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5460,7 +5459,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ if( nTo=mxCost; jj++){ assert(jj>0); } + for(jj=nTo-1; aTo[jj].rCost>=mxCost; jj--){ assert(jj>0); } } pTo = &aTo[jj]; }else{ From 4f58074a8083c751c46165e367601dc95b2e1d12 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 14:13:51 +0000 Subject: [PATCH 008/109] Fix the wholenumber virtual table so that it returns higher costs for unconstrained usage. FossilOrigin-Name: ceff8955020cd1314bf1ab0af7d075fe2c0863e5 --- ext/misc/wholenumber.c | 8 +++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ext/misc/wholenumber.c b/ext/misc/wholenumber.c index a6d79507c4..63369c6ac4 100644 --- a/ext/misc/wholenumber.c +++ b/ext/misc/wholenumber.c @@ -218,7 +218,13 @@ static int wholenumberBestIndex( ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)1; + if( (idxNum & 12)==0 ){ + pIdxInfo->estimatedCost = (double)100000000; + }else if( (idxNum & 3)==0 ){ + pIdxInfo->estimatedCost = (double)5; + }else{ + pIdxInfo->estimatedCost = (double)1; + } return SQLITE_OK; } diff --git a/manifest b/manifest index 10086d2469..1b4299d8c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sbug\sfixes\sto\sthe\sWhereLoop\sgenerator\sand\sthe\ssolver\sin\sNGQP.\nNow\sfinds\sthe\sbest\splan\sfor\sTPC-H\sQ8.\s\sThis\sseems\sto\sprove\sthe\nconcept,\sbut\sthere\sis\sstill\smuch\swork\sto\sbe\sdone. -D 2013-05-08T04:22:59.354 +C Fix\sthe\swholenumber\svirtual\stable\sso\sthat\sit\sreturns\shigher\scosts\sfor\nunconstrained\susage. +D 2013-05-08T14:13:51.196 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -90,7 +90,7 @@ F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 F ext/misc/spellfix.c f9d24a2b2617cee143b7841b453e4e1fd8f189cc -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 +F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b36034bbd19bc5677b26a6f60ca96eb2b37db373 -R 133717a86de1df991cb6332a4c908d9a +P 8e5aad37529ec3042e3468acf15186f566e2df8a +R b3d674751906663b092b90a1bf5d9075 U drh -Z f247935a6bdcd4102bf6e24d9a3606bf +Z ebd03db274e88f39387d4aac452cd0be diff --git a/manifest.uuid b/manifest.uuid index f6844eb427..3eb60dca2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e5aad37529ec3042e3468acf15186f566e2df8a \ No newline at end of file +ceff8955020cd1314bf1ab0af7d075fe2c0863e5 \ No newline at end of file From 5346e95ddaf7bcd3b71effe47bfdb22461fa8a12 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 14:14:26 +0000 Subject: [PATCH 009/109] NGQP working with virtualt tables, though many legacy tests fail and there are yet some memory leaks. FossilOrigin-Name: bd9327a9684b99978734ccd561eea1ad864ab13b --- manifest | 12 +- manifest.uuid | 2 +- src/where.c | 303 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 259 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index 1b4299d8c9..45446a4e34 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\swholenumber\svirtual\stable\sso\sthat\sit\sreturns\shigher\scosts\sfor\nunconstrained\susage. -D 2013-05-08T14:13:51.196 +C NGQP\sworking\swith\svirtualt\stables,\sthough\smany\slegacy\stests\sfail\sand\sthere\nare\syet\ssome\smemory\sleaks. +D 2013-05-08T14:14:26.339 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c02037efa641eb9f60f054a52bc4ef3c8230369a +F src/where.c 409691bea5b1442e7cfdeca004c3f38208585ca4 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8e5aad37529ec3042e3468acf15186f566e2df8a -R b3d674751906663b092b90a1bf5d9075 +P ceff8955020cd1314bf1ab0af7d075fe2c0863e5 +R 82aea8626061f3b7c0b8cf00cdb63138 U drh -Z ebd03db274e88f39387d4aac452cd0be +Z 1ad6961249f560a4ead02c9a1a345bbe diff --git a/manifest.uuid b/manifest.uuid index 3eb60dca2b..dff7e747b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ceff8955020cd1314bf1ab0af7d075fe2c0863e5 \ No newline at end of file +bd9327a9684b99978734ccd561eea1ad864ab13b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1f198beab4..41b0e7a488 100644 --- a/src/where.c +++ b/src/where.c @@ -44,6 +44,8 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; +typedef struct WhereVtabPlan WhereVtabPlan; + /* ** Each instance of this object represents a way of evaluating one @@ -54,15 +56,24 @@ typedef struct WhereScan WhereScan; struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ Bitmask maskSelf; /* Bitmask identifying table iTab */ - int iTab; /* Index of the table coded by this loop */ + u16 iTab; /* Index of the table coded by this loop */ + u16 nTerm; /* Number of entries in aTerm[] */ u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ double rSetup; /* One-time setup cost (ex: create transient index) */ double rRun; /* Cost of running each loop */ double nOut; /* Estimated number of output rows */ u32 wsFlags; /* WHERE_* flags describing the plan */ - u16 nEq; /* Number of equality constraints */ - u16 nTerm; /* Number of entries in aTerm[] */ - Index *pIndex; /* Index used */ + union { + struct { /* Information for internal btree tables */ + int nEq; /* Number of equality constraints */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtualt tables */ + int idxNum; /* Index number */ + int needFree; /* True if sqlite3_free(idxStr) is needed */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; WhereTerm **aTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ }; @@ -314,7 +325,7 @@ struct WhereLoopBuilder { */ #define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ #define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ -#define WHERE_IPK 0x00004000 /* x is the INTEGER PRIMARY KEY */ +#define WHERE_IPK 0x00008000 /* x is the INTEGER PRIMARY KEY */ #define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ #define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ #define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ @@ -333,6 +344,7 @@ struct WhereLoopBuilder { #define WHERE_OB_UNIQUE 0x00004000 /* Values in ORDER BY columns are ** different for every output row */ #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ +#define WHERE_FREEIDXSTR 0x04000000 /* neeed to free WhereLoop.u.idxStr */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ #define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ @@ -2218,11 +2230,12 @@ static void constructAutomaticIndex( ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to sqlite3_free(). */ -static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ - Parse *pParse = p->pParse; - WhereClause *pWC = p->pWC; - struct SrcList_item *pSrc = p->pSrc; - ExprList *pOrderBy = p->pOrderBy; +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ int i, j; int nTerm; struct sqlite3_index_constraint *pIdxCons; @@ -2411,7 +2424,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ */ pIdxInfo = *p->ppIdxInfo; if( pIdxInfo==0 ){ - *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(p); + *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse,pWC,pSrc,p->pOrderBy); } if( pIdxInfo==0 ){ return; @@ -5060,10 +5073,22 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ p->iTab, nb, p->maskSelf, nb, p->prereq); sqlite3DebugPrintf(" %8s", pItem->zAlias ? pItem->zAlias : pTab->zName); - if( p->pIndex ){ - sqlite3DebugPrintf(".%-12s %2d", p->pIndex->zName, p->nEq); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + if( p->u.btree.pIndex ){ + sqlite3DebugPrintf(".%-12s %2d", + p->u.btree.pIndex->zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%16s",""); + } }else{ - sqlite3DebugPrintf("%16s",""); + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\")", p->u.vtab.idxNum,p->u.vtab.idxStr); + }else{ + z = sqlite3_mprintf("(%d)", p->u.vtab.idxNum); + } + sqlite3DebugPrintf(" %-15s", z); + sqlite3_free(z); } sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", p->wsFlags, p->iOb, p->nOb, p->nTerm); @@ -5072,11 +5097,25 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ } #endif +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + sqlite3DbFree(db, p->aTerm); + p->aTerm = 0; + p->nTerm = 0; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + if( p->u.vtab.needFree ) sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + } +} + /* ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ - sqlite3DbFree(db, p->aTerm); + whereLoopClear(db, p); sqlite3DbFree(db, p); } @@ -5149,10 +5188,8 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ && p->rRun>=pTemplate->rRun ){ /* Overwrite an existing WhereLoop with a better one */ - sqlite3DbFree(db, p->aTerm); - p->aTerm = 0; - p->nTerm = 0; pNext = p->pNextLoop; + whereLoopClear(db, p); break; } } @@ -5176,21 +5213,25 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ p->pNextLoop = pNext; *ppPrev = p; p->aTerm = paTerm; - if( p->pIndex && p->pIndex->tnum==0 ) p->pIndex = 0; if( pTemplate->nTerm ){ memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); } + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ) p->u.btree.pIndex = 0; + }else{ + pTemplate->u.vtab.needFree = 0; + } return SQLITE_OK; } /* -** We have so far matched pBuilder->pNew->nEq terms of the index pIndex. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. ** Try to match one more. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. */ -static void whereLoopAddBtreeIndex( +static int whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ struct SrcList_item *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ @@ -5203,12 +5244,14 @@ static void whereLoopAddBtreeIndex( WhereScan scan; /* Iterator for WHERE terms */ WhereLoop savedLoop; /* Saved original content of pNew[] */ int iCol; /* Index of the column in the table */ + int rc = SQLITE_OK; /* Return code */ db = pBuilder->db; pNew = pBuilder->pNew; - if( db->mallocFailed ) return; + if( db->mallocFailed ) return SQLITE_NOMEM; - assert( pNew->nEqnColumn ); + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( pNew->u.btree.nEqnColumn ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; @@ -5218,14 +5261,14 @@ static void whereLoopAddBtreeIndex( opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } - iCol = pProbe->aiColumn[pNew->nEq]; + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, iCol>=0 ? pProbe : 0); savedLoop = *pNew; pNew->rSetup = (double)0; - for(; pTerm!=0; pTerm = whereScanNext(&scan)){ + for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 1; - pNew->nEq = savedLoop.nEq; + pNew->u.btree.nEq = savedLoop.u.btree.nEq; pNew->nTerm = savedLoop.nTerm; pNew->aTerm[pNew->nTerm++] = pTerm; pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; @@ -5239,12 +5282,12 @@ static void whereLoopAddBtreeIndex( /* "x IN (value, value, ...)" */ nIn = pExpr->x.pList->nExpr; } - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; + pNew->u.btree.nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->u.btree.nEq] * nInMul * nIn; }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; + pNew->u.btree.nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->u.btree.nEq] * nInMul; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pNew->nOut = savedLoop.nOut/3; @@ -5253,19 +5296,22 @@ static void whereLoopAddBtreeIndex( pNew->nOut = savedLoop.nOut/3; } pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; - whereLoopInsert(pBuilder->pWInfo, pNew); - if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ + rc = whereLoopInsert(pBuilder->pWInfo, pNew); + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEqnColumn + ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); } } *pNew = savedLoop; + return rc; } /* ** Add all WhereLoop objects for the iTab-th table of the join. That ** table is guaranteed to be a b-tree table, not a virtual table. */ -static void whereLoopAddBtree( +static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ int iTab, /* The table to process */ Bitmask mExtra /* Extra prerequesites for using this table */ @@ -5277,6 +5323,7 @@ static void whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ sqlite3 *db; /* The database connection */ WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE_OK; /* Return code */ pNew = pBuilder->pNew; db = pBuilder->db; @@ -5311,24 +5358,24 @@ static void whereLoopAddBtree( /* Insert a full table scan */ pNew->iTab = iTab; - pNew->nEq = 0; + pNew->u.btree.nEq = 0; pNew->nTerm = 0; pNew->rSetup = (double)0; pNew->prereq = 0; - pNew->pIndex = 0; + pNew->u.btree.pIndex = 0; pNew->wsFlags = 0; pNew->iOb = pNew->nOb = 0; pNew->rRun = (double)pSrc->pTab->nRowEst; pNew->nOut = (double)pSrc->pTab->nRowEst; - whereLoopInsert(pBuilder->pWInfo, pNew); + rc = whereLoopInsert(pBuilder->pWInfo, pNew); /* Loop over all indices */ - for(; pProbe; pProbe=pProbe->pNext){ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){ WhereTerm **paTerm; pNew->prereq = mExtra; pNew->iTab = iTab; - pNew->nEq = 0; + pNew->u.btree.nEq = 0; pNew->nTerm = 0; if( pProbe->tnum<=0 ){ /* Integer primary key index */ @@ -5345,34 +5392,185 @@ static void whereLoopAddBtree( pNew->wsFlags = m==0 ? WHERE_IDX_ONLY : 0; } paTerm = sqlite3DbRealloc(db, pNew->aTerm, - (pProbe->nColumn+1)*sizeof(pNew->aTerm[0])); - if( paTerm==0 ) break; + (pProbe->nColumn+2)*sizeof(pNew->aTerm[0])); + if( paTerm==0 ){ rc = SQLITE_NOMEM; break; } pNew->aTerm = paTerm; - pNew->pIndex = pProbe; + pNew->u.btree.pIndex = pProbe; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ if( pSrc->pIndex ) break; } + return rc; } /* ** Add all WhereLoop objects for the iTab-th table of the join. That ** table is guaranteed to be a virtual table. */ -static void whereLoopAddVirtual( +static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ int iTab, /* The table to process */ Bitmask mExtra /* Extra prerequesites for using this table */ ){ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Table *pTab; + sqlite3 *db; + sqlite3_index_info *pIdxInfo; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int iTerm, mxTerm; + int seenIn = 0; /* True if an IN operator is seen */ + int seenVar = 0; /* True if a non-constant constraint is seen */ + int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + WhereLoop *pNew; + WhereTerm **paTerm; + int rc = SQLITE_OK; + + pParse = pBuilder->pParse; + db = pParse->db; + pWC = pBuilder->pWC; + pSrc = &pBuilder->pTabList->a[iTab]; + pTab = pSrc->pTab; + pNew = pBuilder->pNew; + pIdxInfo = allocateIndexInfo(pParse,pWC,pSrc,pBuilder->pOrderBy); + if( pIdxInfo==0 ) return SQLITE_NOMEM; + paTerm = sqlite3DbRealloc(db, pNew->aTerm, + pIdxInfo->nConstraint*sizeof(pNew->aTerm[0])); + if( paTerm==0 ) return SQLITE_NOMEM; + pNew->aTerm = paTerm; + pNew->prereq = 0; + pNew->iTab = iTab; + pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); + pNew->iOb = 0; + pNew->nOb = 0; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nTerm = 0; + pNew->u.vtab.needFree = 0; + pUsage = pIdxInfo->aConstraintUsage; + + for(iPhase=0; iPhase<=2; iPhase++){ + if( !seenIn && (iPhase&1)!=0 ){ + iPhase++; + if( iPhase>3 ) break; + } + if( !seenVar && iPhase>1 ) break; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + switch( iPhase ){ + case 0: /* Constants without IN operator */ + pIdxCons->usable = 0; + if( (pTerm->eOperator & WO_IN)!=0 ){ + seenIn = 1; + }else if( pTerm->prereqRight!=0 ){ + seenVar = 1; + }else{ + pIdxCons->usable = 1; + } + break; + case 1: /* Constants with IN operators */ + assert( seenIn ); + pIdxCons->usable = (pTerm->prereqRight==0); + break; + case 2: /* Variables without IN */ + assert( seenVar ); + pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; + break; + default: /* Variables with IN */ + assert( seenVar && seenIn ); + pIdxCons->usable = 1; + break; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + rc = vtabBestIndex(pParse, pTab, pIdxInfo); + if( rc ) goto whereLoopAddVtab_exit; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pNew->prereq = 0; + for(i=0; inConstraint; i++) pNew->aTerm[i] = 0; + mxTerm = 0; + for(i=0; inConstraint; i++, pIdxCons++){ + if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ + j = pIdxCons->iTermOffset; + if( iTerm>=pIdxInfo->nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aTerm[iTerm]!=0 + ){ + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); + goto whereLoopAddVtab_exit; + } + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + pNew->aTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + if( (pTerm->eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=pIdxInfo->nConstraint ){ + pNew->nTerm = mxTerm+1; + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->iOb = 0; + if( pIdxInfo->orderByConsumed ){ + pNew->nOb = (u16)(pIdxInfo->nOrderBy&0xffff); + }else{ + pNew->nOb = 0; + } + pNew->rSetup = (double)0; + pNew->rRun = pIdxInfo->estimatedCost; + pNew->nOut = (double)25; + whereLoopInsert(pBuilder->pWInfo, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + } + } + +whereLoopAddVtab_exit: + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + sqlite3DbFree(db, pIdxInfo); + return rc; } /* ** Add all WhereLoop objects for all tables */ -static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ Bitmask mExtra = 0; Bitmask mPrior = 0; int iTab; @@ -5381,24 +5579,26 @@ static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ WhereClause *pWC = pBuilder->pWC; sqlite3 *db = pBuilder->db; int nTabList = pBuilder->pWInfo->nLevel; + int rc = SQLITE_OK; /* Loop over the tables in the join, from left to right */ pBuilder->pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); - if( pBuilder->pNew==0 ) return; + if( pBuilder->pNew==0 ) return SQLITE_NOMEM; for(iTab=0, pItem=pTabList->a; iTabpTab) ){ - whereLoopAddVirtual(pBuilder, iTab, mExtra); + rc = whereLoopAddVirtual(pBuilder, iTab, mExtra); }else{ - whereLoopAddBtree(pBuilder, iTab, mExtra); + rc = whereLoopAddBtree(pBuilder, iTab, mExtra); } mPrior |= getMask(pWC->pMaskSet, pItem->iCursor); if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ mExtra = mPrior; } - if( db->mallocFailed ) break; + if( rc || db->mallocFailed ) break; } whereLoopDelete(db, pBuilder->pNew); pBuilder->pNew = 0; + return rc; } /* @@ -5501,7 +5701,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ } /* TEMPORARY */ - if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } +// if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } assert( nFrom>0 ); /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ @@ -5629,6 +5829,7 @@ WhereInfo *sqlite3WhereBegin( int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ + int rc; /* Return code */ /* Variable initialization */ @@ -5758,8 +5959,8 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); - whereLoopAddAll(&sWLB); - if( db->mallocFailed ) goto whereBeginError; + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; /* Display all of the WhereLoop objects if wheretrace is enabled */ #if defined(SQLITE_DEBUG) \ From c718f1c894b0e0b0bd6d44e9a4e1412fa3e7abfa Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 20:05:58 +0000 Subject: [PATCH 010/109] Fix memory leaks in the NGQP logic for virtual tables. FossilOrigin-Name: 3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 45446a4e34..cead004f71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C NGQP\sworking\swith\svirtualt\stables,\sthough\smany\slegacy\stests\sfail\sand\sthere\nare\syet\ssome\smemory\sleaks. -D 2013-05-08T14:14:26.339 +C Fix\smemory\sleaks\sin\sthe\sNGQP\slogic\sfor\svirtual\stables. +D 2013-05-08T20:05:58.332 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 409691bea5b1442e7cfdeca004c3f38208585ca4 +F src/where.c e2e0ff816591684ff74bef65fae747cc57cee335 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ceff8955020cd1314bf1ab0af7d075fe2c0863e5 -R 82aea8626061f3b7c0b8cf00cdb63138 +P bd9327a9684b99978734ccd561eea1ad864ab13b +R 08bf850cd4f323af143e8919e5b89b44 U drh -Z 1ad6961249f560a4ead02c9a1a345bbe +Z 89b960e01fe84155abc15edf55ac2ba0 diff --git a/manifest.uuid b/manifest.uuid index dff7e747b9..6cc7d9d5c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd9327a9684b99978734ccd561eea1ad864ab13b \ No newline at end of file +3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 41b0e7a488..6d7805dfdd 100644 --- a/src/where.c +++ b/src/where.c @@ -5442,8 +5442,11 @@ static int whereLoopAddVirtual( pIdxInfo = allocateIndexInfo(pParse,pWC,pSrc,pBuilder->pOrderBy); if( pIdxInfo==0 ) return SQLITE_NOMEM; paTerm = sqlite3DbRealloc(db, pNew->aTerm, - pIdxInfo->nConstraint*sizeof(pNew->aTerm[0])); - if( paTerm==0 ) return SQLITE_NOMEM; + (pIdxInfo->nConstraint+1)*sizeof(pNew->aTerm[0])); + if( paTerm==0 ){ + sqlite3DbFree(db, pIdxInfo); + return SQLITE_NOMEM; + } pNew->aTerm = paTerm; pNew->prereq = 0; pNew->iTab = iTab; @@ -5504,7 +5507,7 @@ static int whereLoopAddVirtual( pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; pNew->prereq = 0; for(i=0; inConstraint; i++) pNew->aTerm[i] = 0; - mxTerm = 0; + mxTerm = -1; for(i=0; inConstraint; i++, pIdxCons++){ if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ j = pIdxCons->iTermOffset; @@ -5659,7 +5662,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ if( nTo=mxCost; jj--){ assert(jj>0); } + for(jj=nTo-1; aTo[jj].rCost0); } } pTo = &aTo[jj]; }else{ @@ -5701,7 +5704,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ } /* TEMPORARY */ -// if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } + if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } assert( nFrom>0 ); /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ From 6b7157bbd5183960c8dd6a6214b33163d9122bb7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 May 2013 02:00:35 +0000 Subject: [PATCH 011/109] Free up bits of wsFlags for reuse. Install the ORDER BY optimization infrastructure for the NGQP. FossilOrigin-Name: 82d50e198025a2fdb8ee733edb8419d388ee5362 --- manifest | 14 ++-- manifest.uuid | 2 +- src/sqliteInt.h | 3 + src/where.c | 190 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 139 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index cead004f71..70d136a298 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smemory\sleaks\sin\sthe\sNGQP\slogic\sfor\svirtual\stables. -D 2013-05-08T20:05:58.332 +C Free\sup\sbits\sof\swsFlags\sfor\sreuse.\s\sInstall\sthe\sORDER\sBY\soptimization\ninfrastructure\sfor\sthe\sNGQP. +D 2013-05-10T02:00:35.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h aee09cf5ee7d5e899246affdd8c3cf58f086ade5 +F src/sqliteInt.h b4411cfc1c7803cdf393975d5420b4da0d8dd1c4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e2e0ff816591684ff74bef65fae747cc57cee335 +F src/where.c 2be67c1a1a018b1de91f08e48baa02db13652e40 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bd9327a9684b99978734ccd561eea1ad864ab13b -R 08bf850cd4f323af143e8919e5b89b44 +P 3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 +R fe9c54bd236af22a2f9e27cec8708db5 U drh -Z 89b960e01fe84155abc15edf55ac2ba0 +Z c9a2e102e5146d5a293863d252ad4424 diff --git a/manifest.uuid b/manifest.uuid index 6cc7d9d5c8..5c1c65ebf9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 \ No newline at end of file +82d50e198025a2fdb8ee733edb8419d388ee5362 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 86b123749c..08ecfa78d1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2045,6 +2045,7 @@ struct WhereLevel { #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ +#define WHREE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ /* ** The WHERE clause processing routine has two halves. The @@ -2056,6 +2057,8 @@ struct WhereLevel { struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pDistinct; /* DISTINCT ON values, or NULL */ u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ diff --git a/src/where.c b/src/where.c index 6d7805dfdd..56ca3e75aa 100644 --- a/src/where.c +++ b/src/where.c @@ -50,7 +50,7 @@ typedef struct WhereVtabPlan WhereVtabPlan; /* ** Each instance of this object represents a way of evaluating one ** term of a join. The WhereClause object holds a table of these -** objects using (iTab,prereq,iOb,nOb) as the primary key. Note that the +** objects using (maskSelf,prereq,) as the primary key. Note that the ** same join term might have multiple associated WhereLoop objects. */ struct WhereLoop { @@ -58,19 +58,19 @@ struct WhereLoop { Bitmask maskSelf; /* Bitmask identifying table iTab */ u16 iTab; /* Index of the table coded by this loop */ u16 nTerm; /* Number of entries in aTerm[] */ - u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ + u32 wsFlags; /* WHERE_* flags describing the plan */ double rSetup; /* One-time setup cost (ex: create transient index) */ double rRun; /* Cost of running each loop */ double nOut; /* Estimated number of output rows */ - u32 wsFlags; /* WHERE_* flags describing the plan */ union { struct { /* Information for internal btree tables */ int nEq; /* Number of equality constraints */ Index *pIndex; /* Index used, or NULL */ } btree; - struct { /* Information for virtualt tables */ + struct { /* Information for virtual tables */ int idxNum; /* Index number */ - int needFree; /* True if sqlite3_free(idxStr) is needed */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + u8 isOrdered; /* True if satisfies ORDER BY */ char *idxStr; /* Index identifier string */ } vtab; } u; @@ -86,6 +86,8 @@ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ double nRow; /* Estimated number of rows generated by this path */ double rCost; /* Total cost of this path */ + u8 isOrdered; /* True if this path satisfies ORDER BY */ + u8 isOrderedValid; /* True if the isOrdered field is valid */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; @@ -314,41 +316,34 @@ struct WhereLoopBuilder { ** Value for wsFlags returned by bestIndex() and stored in ** WhereLevel.wsFlags. These flags determine which search ** strategies are appropriate. -** -** The least significant 12 bits is reserved as a mask for WO_ values above. -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. -** But if the table is the right table of a left join, WhereLevel.wsFlags -** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as -** the "op" parameter to findTerm when we are resolving equality constraints. -** ISNULL constraints will then not be used on the right table of a left -** join. Tickets #2177 and #2189. */ -#define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ -#define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ -#define WHERE_IPK 0x00008000 /* x is the INTEGER PRIMARY KEY */ -#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ -#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x080f1000 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xEXPR */ +#define WHERE_NULL_OK 0x00000004 /* Ok to use WO_ISNULL */ +#define WHERE_IPK 0x00000008 /* x is the INTEGER PRIMARY KEY */ +#define WHERE_COLUMN_EQ 0x00000010 /* x=EXPR or x IN (...) or x IS NULL */ +#define WHERE_COLUMN_RANGE 0x00000020 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00000040 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000080 /* x IS NULL */ +#define WHERE_INDEXED 0x000000f0 /* Anything that uses an index */ +#define WHERE_NOT_FULLSCAN 0x000200f3 /* Does not do a full table scan */ +#define WHERE_IN_ABLE 0x000100f1 /* Able to support an IN operator */ +#define WHERE_TOP_LIMIT 0x00000100 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000300 /* Both x>EXPR and xu.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; WhereTerm *pOrTerm; - int flags = WHERE_MULTI_OR; double rTotal = 0; double nRow = 0; Bitmask used = 0; @@ -1967,7 +1961,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ p->cost.used = used; p->cost.plan.nRow = nRow; p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; - p->cost.plan.wsFlags = flags; + p->cost.plan.wsFlags = WHERE_MULTI_OR; p->cost.plan.u.pTerm = pTerm; } } @@ -2149,7 +2143,7 @@ static void constructAutomaticIndex( if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ nColumn += pTable->nCol - BMS + 1; } - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ nByte = sizeof(Index); @@ -3815,7 +3809,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ bestOrClauseIndex(p); bestAutomaticIndex(p); - p->cost.plan.wsFlags |= eqTermMask; + if( eqTermMask & WO_ISNULL ) p->cost.plan.wsFlags |= WHERE_NULL_OK; } /* @@ -4066,6 +4060,7 @@ static int codeAllEqualityTerms( int regBase; /* Base register */ int nReg; /* Number of registers to allocate */ char *zAff; /* Affinity string to return */ + int eqFlags; /* WO_EQ|WO_IN and maybe also WO_ISNULL */ /* This module is only called on query plans that use an index. */ assert( pLevel->plan.wsFlags & WHERE_INDEXED ); @@ -4085,10 +4080,12 @@ static int codeAllEqualityTerms( /* Evaluate the equality constraints */ assert( pIdx->nColumn>=nEq ); + eqFlags = (pLevel->plan.wsFlags&WHERE_NULL_OK) ? (WO_EQ|WO_IN|WO_ISNULL) + : (WO_EQ|WO_IN); for(j=0; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); + pTerm = findTerm(pWC, iCur, k, notReady, eqFlags, pIdx); if( pTerm==0 ) break; /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ @@ -5090,8 +5087,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", - p->wsFlags, p->iOb, p->nOb, p->nTerm); + sqlite3DebugPrintf(" fg %08x N %2d", p->wsFlags, p->nTerm); sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } @@ -5172,8 +5168,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->prereq & pTemplate->prereq)==p->prereq - && p->nOb>=pTemplate->nOb - && p->iOb==pTemplate->iOb && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ @@ -5182,8 +5176,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ return SQLITE_OK; } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->nOb<=pTemplate->nOb - && p->iOb==pTemplate->iOb && p->rSetup>=pTemplate->rSetup && p->rRun>=pTemplate->rRun ){ @@ -5364,7 +5356,6 @@ static int whereLoopAddBtree( pNew->prereq = 0; pNew->u.btree.pIndex = 0; pNew->wsFlags = 0; - pNew->iOb = pNew->nOb = 0; pNew->rRun = (double)pSrc->pTab->nRowEst; pNew->nOut = (double)pSrc->pTab->nRowEst; rc = whereLoopInsert(pBuilder->pWInfo, pNew); @@ -5451,8 +5442,6 @@ static int whereLoopAddVirtual( pNew->prereq = 0; pNew->iTab = iTab; pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); - pNew->iOb = 0; - pNew->nOb = 0; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; pNew->nTerm = 0; @@ -5547,12 +5536,7 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->iOb = 0; - if( pIdxInfo->orderByConsumed ){ - pNew->nOb = (u16)(pIdxInfo->nOrderBy&0xffff); - }else{ - pNew->nOb = 0; - } + pNew->u.vtab.isOrdered = (u8)(pIdxInfo->nOrderBy!=0); pNew->rSetup = (double)0; pNew->rRun = pIdxInfo->estimatedCost; pNew->nOut = (double)25; @@ -5604,6 +5588,29 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } +/* +** Examine a WherePath to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate source operation. Return 1 +** if it does and 0 if it does not and -1 if we cannot tell. +*/ +static int wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + WherePath *pPath, /* The WherePath to check */ + int nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLoop /* Add this WhereLoop to the end of pPath->aLoop[] */ +){ + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + return nLoop==0 && pLoop->u.vtab.isOrdered; + }else{ + /* TBD: Check to see if pFrom + pWLoop satisfies the ORDER BY. + ** (1) If yes: set isOrderedValid and isOrdered to 1. + ** (2) If no: set isOrderedValid to 1 and isOrdered to 0. + ** (3) unknown: no-op */ + return 0; + } +} + + /* ** Given the list of WhereLoop objects on pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -5620,12 +5627,14 @@ static int wherePathSolver(WhereInfo *pWInfo){ int ii, jj; /* Loop counters */ double rCost; /* Cost of a path */ double mxCost; /* Maximum cost of a set of paths */ + double rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ WherePath *pFrom; /* An element of aFrom[] that we are working on */ WherePath *pTo; /* An element of aTo[] that we are working on */ WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop *pNext; /* Next loop */ WhereLoop **pX; /* Used to divy up the pSpace memory */ char *pSpace; /* Temporary memory used by this routine */ @@ -5645,18 +5654,67 @@ static int wherePathSolver(WhereInfo *pWInfo){ pFrom->aLoop = pX; } + /* Seed the search with a single WherePath containing zero WhereLoops */ aFrom[0].nRow = (double)1; nFrom = 1; + + /* Precompute the cost of sorting the final result set, if the caller + ** to sqlite3WhereBegin() was concerned about sorting */ + rSortCost = (double)0; + if( pWInfo->pOrderBy==0 ){ + aFrom[0].isOrderedValid = 1; + }else{ + /* Compute an estimate on the cost to sort the entire result set */ + rSortCost = (double)1; + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pNext){ + pNext = pWLoop->pNextLoop; + rCost = pWLoop->nOut; + while( pNext && pNext->iTab==pWLoop->iTab ){ + if( pNext->nOutnOut; + pNext = pNext->pNextLoop; + } + rSortCost *= rCost; + } + rSortCost *= estLog(rSortCost); + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; + u8 isOrderedValid = pFrom->isOrderedValid; + u8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; maskNew = pFrom->maskLoop | pWLoop->maskSelf; - for(jj=0, pTo=aTo; jjmaskLoop!=maskNew; jj++){} + if( !isOrderedValid ){ + switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, pWLoop) ){ + case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ + isOrdered = 1; + isOrderedValid = 1; + break; + case 0: /* No. pFrom+pWLoop will require a separate sort */ + isOrdered = 0; + isOrderedValid = 1; + rCost += rSortCost; + break; + default: /* Cannot tell yet. Try again on the next iteration */ + break; + } + } + /* Check to see if pWLoop should be added to the mxChoice best so far */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew && pTo->isOrderedValid==isOrderedValid ){ + break; + } + } if( jj>=nTo ){ if( nTo>=mxChoice && rCost>=mxCost ) continue; if( nTorCost<=rCost ) continue; } + /* pWLoop is a winner. Add it to the set of best so far */ pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; pTo->nRow = pFrom->nRow * pWLoop->nOut; pTo->rCost = rCost; + pTo->isOrderedValid = isOrderedValid; + pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ @@ -5695,8 +5756,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ } #endif - /* Swap the roles of aFrom and aTo in preparation for the next - ** cycle. */ + /* Swap the roles of aFrom and aTo for the next generation */ pFrom = aTo; aTo = aFrom; aFrom = pFrom; @@ -5707,15 +5767,19 @@ static int wherePathSolver(WhereInfo *pWInfo){ if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } assert( nFrom>0 ); - /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ + /* Find the lowest cost path. pFrom will be left pointing to that path */ pFrom = aFrom; for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; } assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoopa[iLoop].pWLoop = pFrom->aLoop[iLoop]; } + if( pFrom->isOrdered ){ + pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + } /* Free temporary memory and return success */ sqlite3DbFree(db, pSpace); @@ -5882,6 +5946,8 @@ WhereInfo *sqlite3WhereBegin( pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pDistinct = pDistinct; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; From b2a90f09b8a7af6d84184d90f3956ea0c884df8f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 May 2013 03:30:49 +0000 Subject: [PATCH 012/109] Factor out common operations into whereLoopAddAll(). Add stubs for missing features. FossilOrigin-Name: 0278e420614e02fb2d8474ed99b0904275882dfe --- manifest | 12 +++++----- manifest.uuid | 2 +- src/where.c | 65 +++++++++++++++++++++++---------------------------- 3 files changed, 36 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 57a0d63001..f81212a36c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sthe\slatest\strunk\schanges. -D 2013-05-10T02:11:40.008 +C Factor\sout\scommon\soperations\sinto\swhereLoopAddAll().\s\sAdd\sstubs\sfor\nmissing\sfeatures. +D 2013-05-10T03:30:49.076 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 4816e3ec624c96b5169f0cc83dbb5cf459ce3fd1 +F src/where.c 31160bacc36e1fa5818caf091d5559d55eae7a5d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 82d50e198025a2fdb8ee733edb8419d388ee5362 cf5c3642247fdd34d87f0368594cd7b8f081636a -R bed3d93321566d3c8fda15cb3370678a +P 5ed31c8279a4f465b982df5dc20cefc286928b93 +R 6a6b39040b040f90ce4281cc062a3caa U drh -Z 9d7433ee1472cd0008b2ab99b16fa998 +Z ca90367b6425b581381608ce8a76b4fa diff --git a/manifest.uuid b/manifest.uuid index 64be8db9eb..a2c17feaba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ed31c8279a4f465b982df5dc20cefc286928b93 \ No newline at end of file +0278e420614e02fb2d8474ed99b0904275882dfe \ No newline at end of file diff --git a/src/where.c b/src/where.c index 26db3b3edf..368081feae 100644 --- a/src/where.c +++ b/src/where.c @@ -5309,7 +5309,6 @@ static int whereLoopAddBtreeIndex( */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - int iTab, /* The table to process */ Bitmask mExtra /* Extra prerequesites for using this table */ ){ Index *pProbe; /* An index we are evaluating */ @@ -5317,14 +5316,11 @@ static int whereLoopAddBtree( tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ - sqlite3 *db; /* The database connection */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ pNew = pBuilder->pNew; - db = pBuilder->db; - pSrc = pBuilder->pTabList->a + iTab; - pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); + pSrc = pBuilder->pTabList->a + pNew->iTab; if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5353,23 +5349,21 @@ static int whereLoopAddBtree( } /* Insert a full table scan */ - pNew->iTab = iTab; pNew->u.btree.nEq = 0; pNew->nTerm = 0; pNew->rSetup = (double)0; - pNew->prereq = 0; + pNew->prereq = mExtra; pNew->u.btree.pIndex = 0; pNew->wsFlags = 0; pNew->rRun = (double)pSrc->pTab->nRowEst; pNew->nOut = (double)pSrc->pTab->nRowEst; rc = whereLoopInsert(pBuilder->pWInfo, pNew); + /* TBD: Insert automatic index opportunities */ + /* Loop over all indices */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){ - WhereTerm **paTerm; - pNew->prereq = mExtra; - pNew->iTab = iTab; pNew->u.btree.nEq = 0; pNew->nTerm = 0; if( pProbe->tnum<=0 ){ @@ -5386,10 +5380,6 @@ static int whereLoopAddBtree( } pNew->wsFlags = m==0 ? WHERE_IDX_ONLY : 0; } - paTerm = sqlite3DbRealloc(db, pNew->aTerm, - (pProbe->nColumn+2)*sizeof(pNew->aTerm[0])); - if( paTerm==0 ){ rc = SQLITE_NOMEM; break; } - pNew->aTerm = paTerm; pNew->u.btree.pIndex = pProbe; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); @@ -5407,7 +5397,6 @@ static int whereLoopAddBtree( */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - int iTab, /* The table to process */ Bitmask mExtra /* Extra prerequesites for using this table */ ){ Parse *pParse; /* The parsing context */ @@ -5425,27 +5414,17 @@ static int whereLoopAddVirtual( int seenVar = 0; /* True if a non-constant constraint is seen */ int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ WhereLoop *pNew; - WhereTerm **paTerm; int rc = SQLITE_OK; pParse = pBuilder->pParse; db = pParse->db; pWC = pBuilder->pWC; - pSrc = &pBuilder->pTabList->a[iTab]; - pTab = pSrc->pTab; pNew = pBuilder->pNew; - pIdxInfo = allocateIndexInfo(pParse,pWC,pSrc,pBuilder->pOrderBy); + pSrc = &pBuilder->pTabList->a[pNew->iTab]; + pTab = pSrc->pTab; + pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); if( pIdxInfo==0 ) return SQLITE_NOMEM; - paTerm = sqlite3DbRealloc(db, pNew->aTerm, - (pIdxInfo->nConstraint+1)*sizeof(pNew->aTerm[0])); - if( paTerm==0 ){ - sqlite3DbFree(db, pIdxInfo); - return SQLITE_NOMEM; - } - pNew->aTerm = paTerm; pNew->prereq = 0; - pNew->iTab = iTab; - pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; pNew->nTerm = 0; @@ -5571,22 +5550,36 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pBuilder->db; int nTabList = pBuilder->pWInfo->nLevel; int rc = SQLITE_OK; + WhereLoop *pNew; /* Loop over the tables in the join, from left to right */ - pBuilder->pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); - if( pBuilder->pNew==0 ) return SQLITE_NOMEM; + pBuilder->pNew = pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->aTerm = sqlite3DbMallocZero(db, (pWC->nTerm+1)*sizeof(pNew->aTerm[0])); + if( pNew->aTerm==0 ){ + rc = SQLITE_NOMEM; + goto whereLoopAddAll_end; + } for(iTab=0, pItem=pTabList->a; iTabpTab) ){ - rc = whereLoopAddVirtual(pBuilder, iTab, mExtra); - }else{ - rc = whereLoopAddBtree(pBuilder, iTab, mExtra); - } - mPrior |= getMask(pWC->pMaskSet, pItem->iCursor); + pNew->iTab = iTab; + pNew->maskSelf = getMask(pWC->pMaskSet, pItem->iCursor); if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ mExtra = mPrior; } + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(pBuilder, mExtra); + }else{ + rc = whereLoopAddBtree(pBuilder, mExtra); + } +#if 0 + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(pBuilder, mExtra); + } +#endif + mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } +whereLoopAddAll_end: whereLoopDelete(db, pBuilder->pNew); pBuilder->pNew = 0; return rc; From eb04de3224c0f6e5160d8c641bf4dd2c83bb63e8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 May 2013 15:16:30 +0000 Subject: [PATCH 013/109] Update the NGQP so that it can produce plans that include automatic indices. FossilOrigin-Name: 586b55d8d7722de1c0530b3b832bae0511e6d05c --- manifest | 12 ++++----- manifest.uuid | 2 +- src/where.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f81212a36c..180d5b5b51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\scommon\soperations\sinto\swhereLoopAddAll().\s\sAdd\sstubs\sfor\nmissing\sfeatures. -D 2013-05-10T03:30:49.076 +C Update\sthe\sNGQP\sso\sthat\sit\scan\sproduce\splans\sthat\sinclude\sautomatic\sindices. +D 2013-05-10T15:16:30.669 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 31160bacc36e1fa5818caf091d5559d55eae7a5d +F src/where.c f96b2602fd2da89873ed5a210608c58bf80d7c0e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5ed31c8279a4f465b982df5dc20cefc286928b93 -R 6a6b39040b040f90ce4281cc062a3caa +P 0278e420614e02fb2d8474ed99b0904275882dfe +R 69dec67361636bd352a6f82f3ef9f1cb U drh -Z ca90367b6425b581381608ce8a76b4fa +Z ef95f556fc74affef6b6df28ead69ff3 diff --git a/manifest.uuid b/manifest.uuid index a2c17feaba..0143fae8fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0278e420614e02fb2d8474ed99b0904275882dfe \ No newline at end of file +586b55d8d7722de1c0530b3b832bae0511e6d05c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 368081feae..40653d7c51 100644 --- a/src/where.c +++ b/src/where.c @@ -289,6 +289,7 @@ struct WhereLoopBuilder { SrcList *pTabList; /* FROM clause */ ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ + int mxTerm; /* Maximum number of aTerm[] entries on pNew */ }; /* @@ -5241,6 +5242,7 @@ static int whereLoopAddBtreeIndex( WhereLoop savedLoop; /* Saved original content of pNew[] */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + double rLogSize; /* Logarithm of table size */ db = pBuilder->db; pNew = pBuilder->pNew; @@ -5262,10 +5264,12 @@ static int whereLoopAddBtreeIndex( opMask, iCol>=0 ? pProbe : 0); savedLoop = *pNew; pNew->rSetup = (double)0; + rLogSize = estLog(pProbe->aiRowEst[0]); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 1; pNew->u.btree.nEq = savedLoop.u.btree.nEq; pNew->nTerm = savedLoop.nTerm; + if( pNew->nTerm>=pBuilder->mxTerm ) break; /* Repeated column in index */ pNew->aTerm[pNew->nTerm++] = pTerm; pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; if( pTerm->eOperator & WO_IN ){ @@ -5291,7 +5295,16 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pNew->nOut = savedLoop.nOut/3; } - pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; + pNew->rRun = rLogSize*nIn; /* Cost for nIn binary searches */ + if( pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK) ){ + pNew->rRun += pNew->nOut; /* Unit step cost to reach each row */ + }else{ + /* Each row involves a step of the index, then a binary search of + ** the main table */ + pNew->rRun += pNew->nOut*(1 + rLogSize); + } + /* TBD: Adjust nOut and rRun for STAT3 range values */ + /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder->pWInfo, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn @@ -5318,6 +5331,8 @@ static int whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ + double rSize; /* number of rows in the table */ + double rLogSize; /* Logarithm of the number of rows in the table */ pNew = pBuilder->pNew; pSrc = pBuilder->pTabList->a + pNew->iTab; @@ -5347,6 +5362,33 @@ static int whereLoopAddBtree( } pProbe = &sPk; } + rSize = (double)pSrc->pTab->nRowEst; + rLogSize = estLog(rSize); + + /* Automatic indexes */ + if( (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 + && !pSrc->viaCoroutine + && !pSrc->notIndexed + && !pSrc->isCorrelated + ){ + /* Generate auto-index WhereLoops */ + WhereClause *pWC = pBuilder->pWC; + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + for(pTerm=pWC->a; rc==SQLITE_OK && pTermu.btree.nEq = 1; + pNew->nTerm = 1; + pNew->aTerm[0] = pTerm; + pNew->rSetup = 2*rLogSize*pSrc->pTab->nRowEst; + pNew->nOut = (double)10; + pNew->rRun = rLogSize + pNew->nOut; + pNew->wsFlags = WHERE_TEMP_INDEX; + pNew->prereq = mExtra | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder->pWInfo, pNew); + } + } + } /* Insert a full table scan */ pNew->u.btree.nEq = 0; @@ -5355,12 +5397,11 @@ static int whereLoopAddBtree( pNew->prereq = mExtra; pNew->u.btree.pIndex = 0; pNew->wsFlags = 0; - pNew->rRun = (double)pSrc->pTab->nRowEst; - pNew->nOut = (double)pSrc->pTab->nRowEst; + pNew->nOut = rSize; + pNew->rRun = rSize + rLogSize; + /* TBD: Reduce nOut using constraints */ rc = whereLoopInsert(pBuilder->pWInfo, pNew); - /* TBD: Insert automatic index opportunities */ - /* Loop over all indices */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){ @@ -5478,6 +5519,7 @@ static int whereLoopAddVirtual( if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; pNew->prereq = 0; + assert( pIdxInfo->nConstraint<=pBuilder->mxTerm ); for(i=0; inConstraint; i++) pNew->aTerm[i] = 0; mxTerm = -1; for(i=0; inConstraint; i++, pIdxCons++){ @@ -5555,7 +5597,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pBuilder->pNew = pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); if( pNew==0 ) return SQLITE_NOMEM; - pNew->aTerm = sqlite3DbMallocZero(db, (pWC->nTerm+1)*sizeof(pNew->aTerm[0])); + pBuilder->mxTerm = pWC->nTerm+1; + while( pWC->pOuter ){ + pWC = pWC->pOuter; + pBuilder->mxTerm += pWC->nTerm; + } + pWC = pBuilder->pWC; + pNew->aTerm = sqlite3DbMallocZero(db,pBuilder->mxTerm*sizeof(pNew->aTerm[0])); if( pNew->aTerm==0 ){ rc = SQLITE_NOMEM; goto whereLoopAddAll_end; @@ -6331,6 +6379,17 @@ WhereInfo *sqlite3WhereBegin( pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; } +#if 0 + /* Scaffolding: Check the new query plan against the old. Report any + ** discrepencies */ + for(ii=0; iia[ii].iFrom!=pWInfo->a[ii].pWLoop->iTab ){ + sqlite3DebugPrintf("(QP-Mismatch)"); + break; + } + } +#endif + /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ From cf8fa7a65b5562d92c19cb4267cbd4ed9989a2b6 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 May 2013 20:26:22 +0000 Subject: [PATCH 014/109] Now generating OR-clause plans. FossilOrigin-Name: e17003fcfec0c0b524b1b9ff8e15e7ee83efa571 --- manifest | 12 ++--- manifest.uuid | 2 +- src/where.c | 121 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 180d5b5b51..5d695c8a7f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sNGQP\sso\sthat\sit\scan\sproduce\splans\sthat\sinclude\sautomatic\sindices. -D 2013-05-10T15:16:30.669 +C Now\sgenerating\sOR-clause\splans. +D 2013-05-10T20:26:22.071 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c f96b2602fd2da89873ed5a210608c58bf80d7c0e +F src/where.c 374f50ced988b23addabd3c351f1c18d8652d394 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0278e420614e02fb2d8474ed99b0904275882dfe -R 69dec67361636bd352a6f82f3ef9f1cb +P 586b55d8d7722de1c0530b3b832bae0511e6d05c +R a9f14f7930497f93900a124a55885f20 U drh -Z ef95f556fc74affef6b6df28ead69ff3 +Z 9615c6567bc86000a51d8e76292d6ef0 diff --git a/manifest.uuid b/manifest.uuid index 0143fae8fc..250be95162 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -586b55d8d7722de1c0530b3b832bae0511e6d05c \ No newline at end of file +e17003fcfec0c0b524b1b9ff8e15e7ee83efa571 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 40653d7c51..fdadc3e738 100644 --- a/src/where.c +++ b/src/where.c @@ -289,6 +289,7 @@ struct WhereLoopBuilder { SrcList *pTabList; /* FROM clause */ ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ + WhereLoop *pBest; /* If non-NULL, store single best loop here */ int mxTerm; /* Maximum number of aTerm[] entries on pNew */ }; @@ -5162,10 +5163,27 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** fewer dependencies than the template. Otherwise a new WhereLoop is ** added based no the template. */ -static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; WhereTerm **paTerm = 0; - sqlite3 *db = pWInfo->pParse->db; + sqlite3 *db = pBuilder->db; + WhereInfo *pWInfo = pBuilder->pWInfo; + + if( (p = pBuilder->pBest)!=0 ){ + if( p->maskSelf!=0 ){ + if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){ + return SQLITE_OK; + } + if( p->rRun+p->rSetup == pTemplate->rRun+pTemplate->rSetup + && p->prereq <= pTemplate->prereq ){ + return SQLITE_OK; + } + } + *p = *pTemplate; + p->aTerm = 0; + p->u.vtab.needFree = 0; + return SQLITE_OK; + } /* Search for an existing WhereLoop to overwrite, or which takes ** priority over pTemplate. @@ -5210,11 +5228,13 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ p->pNextLoop = pNext; *ppPrev = p; p->aTerm = paTerm; - if( pTemplate->nTerm ){ - memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); + if( p->nTerm ){ + memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0])); } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ) p->u.btree.pIndex = 0; + if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ + p->u.btree.pIndex = 0; + } }else{ pTemplate->u.vtab.needFree = 0; } @@ -5305,7 +5325,7 @@ static int whereLoopAddBtreeIndex( } /* TBD: Adjust nOut and rRun for STAT3 range values */ /* TBD: Adjust nOut for additional constraints */ - rc = whereLoopInsert(pBuilder->pWInfo, pNew); + rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn ){ @@ -5366,7 +5386,8 @@ static int whereLoopAddBtree( rLogSize = estLog(rSize); /* Automatic indexes */ - if( (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 + if( !pBuilder->pBest + && (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->viaCoroutine && !pSrc->notIndexed && !pSrc->isCorrelated @@ -5385,7 +5406,7 @@ static int whereLoopAddBtree( pNew->rRun = rLogSize + pNew->nOut; pNew->wsFlags = WHERE_TEMP_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; - rc = whereLoopInsert(pBuilder->pWInfo, pNew); + rc = whereLoopInsert(pBuilder, pNew); } } } @@ -5400,7 +5421,7 @@ static int whereLoopAddBtree( pNew->nOut = rSize; pNew->rRun = rSize + rLogSize; /* TBD: Reduce nOut using constraints */ - rc = whereLoopInsert(pBuilder->pWInfo, pNew); + rc = whereLoopInsert(pBuilder, pNew); /* Loop over all indices */ @@ -5565,7 +5586,7 @@ static int whereLoopAddVirtual( pNew->rSetup = (double)0; pNew->rRun = pIdxInfo->estimatedCost; pNew->nOut = (double)25; - whereLoopInsert(pBuilder->pWInfo, pNew); + whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; @@ -5579,6 +5600,84 @@ whereLoopAddVtab_exit: return rc; } +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereLoop sBest; + struct SrcList_item *pItem; + + + pWC = pBuilder->pWC; + if( pWC->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + pItem = pBuilder->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pBest = &sBest; + tempWC.pParse = pWC->pParse; + tempWC.pMaskSet = pWC->pMaskSet; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.wctrlFlags = 0; + tempWC.nTerm = 1; + + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + double rTotal = 0; + double nRow = 0; + Bitmask prereq = mExtra; + + + for(pOrTerm=pOrWC->a; pOrTermeOperator& WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sBest.maskSelf = 0; + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild, mExtra); + }else{ + rc = whereLoopAddBtree(&sSubBuild, mExtra); + } + if( sBest.maskSelf==0 ) break; + assert( sBest.rSetup==(double)0 ); + rTotal += sBest.rRun; + nRow += sBest.nOut; + prereq |= sBest.prereq; + } + pNew->nTerm = 1; + pNew->aTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = (double)0; + pNew->rRun = rTotal; + pNew->nOut = nRow; + pNew->prereq = prereq; + rc = whereLoopInsert(pBuilder, pNew); + } + } + return rc; +} + /* ** Add all WhereLoop objects for all tables */ @@ -5619,11 +5718,9 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ }else{ rc = whereLoopAddBtree(pBuilder, mExtra); } -#if 0 if( rc==SQLITE_OK ){ rc = whereLoopAddOr(pBuilder, mExtra); } -#endif mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } From 0823c89c9bc1190d479df006137fbbaa6c08bf00 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 May 2013 00:06:23 +0000 Subject: [PATCH 015/109] Minor fixes to the OR-clause processing in the NGQP. FossilOrigin-Name: d6946f33c7851aa7efb04b93ac2ae1ac50c26eec --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 17 ++++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 5d695c8a7f..53d49f8c0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Now\sgenerating\sOR-clause\splans. -D 2013-05-10T20:26:22.071 +C Minor\sfixes\sto\sthe\sOR-clause\sprocessing\sin\sthe\sNGQP. +D 2013-05-11T00:06:23.252 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 374f50ced988b23addabd3c351f1c18d8652d394 +F src/where.c 049e7011b90717768eed37cdbe912d85f66a395b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 586b55d8d7722de1c0530b3b832bae0511e6d05c -R a9f14f7930497f93900a124a55885f20 +P e17003fcfec0c0b524b1b9ff8e15e7ee83efa571 +R eb646fd2ea7876f6a2dea69031e80ca3 U drh -Z 9615c6567bc86000a51d8e76292d6ef0 +Z cf78dc58c7e86082569e8dd0f46801e5 diff --git a/manifest.uuid b/manifest.uuid index 250be95162..be800b411f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e17003fcfec0c0b524b1b9ff8e15e7ee83efa571 \ No newline at end of file +d6946f33c7851aa7efb04b93ac2ae1ac50c26eec \ No newline at end of file diff --git a/src/where.c b/src/where.c index fdadc3e738..d0a3fce772 100644 --- a/src/where.c +++ b/src/where.c @@ -5337,8 +5337,9 @@ static int whereLoopAddBtreeIndex( } /* -** Add all WhereLoop objects for the iTab-th table of the join. That -** table is guaranteed to be a b-tree table, not a virtual table. +** Add all WhereLoop objects a single table of the join were the table +** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ @@ -5356,6 +5357,7 @@ static int whereLoopAddBtree( pNew = pBuilder->pNew; pSrc = pBuilder->pTabList->a + pNew->iTab; + assert( !IsVirtual(pSrc->pTab) ); if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5454,8 +5456,8 @@ static int whereLoopAddBtree( } /* -** Add all WhereLoop objects for the iTab-th table of the join. That -** table is guaranteed to be a virtual table. +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ @@ -5484,6 +5486,7 @@ static int whereLoopAddVirtual( pNew = pBuilder->pNew; pSrc = &pBuilder->pTabList->a[pNew->iTab]; pTab = pSrc->pTab; + assert( IsVirtual(pTab) ); pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); if( pIdxInfo==0 ) return SQLITE_NOMEM; pNew->prereq = 0; @@ -5493,7 +5496,7 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = 0; pUsage = pIdxInfo->aConstraintUsage; - for(iPhase=0; iPhase<=2; iPhase++){ + for(iPhase=0; iPhase<=3; iPhase++){ if( !seenIn && (iPhase&1)!=0 ){ iPhase++; if( iPhase>3 ) break; @@ -5540,11 +5543,11 @@ static int whereLoopAddVirtual( if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; pNew->prereq = 0; - assert( pIdxInfo->nConstraint<=pBuilder->mxTerm ); - for(i=0; inConstraint; i++) pNew->aTerm[i] = 0; mxTerm = -1; + for(i=0; imxTerm; i++) pNew->aTerm[i] = 0; for(i=0; inConstraint; i++, pIdxCons++){ if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ + if( iTerm>=pBuilder->mxTerm ) break; j = pIdxCons->iTermOffset; if( iTerm>=pIdxInfo->nConstraint || j<0 From 319f677d57a72e49c0618753e1a588d6f5840cf8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 May 2013 15:31:07 +0000 Subject: [PATCH 016/109] First attempt to get ORDER BY optimization working in NGQP. FossilOrigin-Name: 9fe20292558bb9422de91e35648cb834cbf3b306 --- manifest | 14 ++--- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- src/where.c | 160 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 153 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 53d49f8c0c..316d75d659 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sto\sthe\sOR-clause\sprocessing\sin\sthe\sNGQP. -D 2013-05-11T00:06:23.252 +C First\sattempt\sto\sget\sORDER\sBY\soptimization\sworking\sin\sNGQP. +D 2013-05-14T15:31:07.121 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h fab5580c75a16dced398bdd249c0409c9441b466 +F src/sqliteInt.h b9b6a2e97254fb50b6977ec55ad0f4e4ce0510d2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 049e7011b90717768eed37cdbe912d85f66a395b +F src/where.c 51d935c6593748ed004006ee0f2147a860d56e90 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e17003fcfec0c0b524b1b9ff8e15e7ee83efa571 -R eb646fd2ea7876f6a2dea69031e80ca3 +P d6946f33c7851aa7efb04b93ac2ae1ac50c26eec +R fe984d1689a56a08a31662e67855ab37 U drh -Z cf78dc58c7e86082569e8dd0f46801e5 +Z 13c634b8e2044390ac483fb2d5137194 diff --git a/manifest.uuid b/manifest.uuid index be800b411f..7326bbb499 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6946f33c7851aa7efb04b93ac2ae1ac50c26eec \ No newline at end of file +9fe20292558bb9422de91e35648cb834cbf3b306 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 45ae796c44..3ba0a3597b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2045,7 +2045,7 @@ struct WhereLevel { #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ -#define WHREE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ +#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ /* ** The WHERE clause processing routine has two halves. The diff --git a/src/where.c b/src/where.c index d0a3fce772..2eff05ab39 100644 --- a/src/where.c +++ b/src/where.c @@ -84,6 +84,7 @@ struct WhereLoop { */ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ double nRow; /* Estimated number of rows generated by this path */ double rCost; /* Total cost of this path */ u8 isOrdered; /* True if this path satisfies ORDER BY */ @@ -5078,8 +5079,13 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ pItem->zAlias ? pItem->zAlias : pTab->zName); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ if( p->u.btree.pIndex ){ - sqlite3DebugPrintf(".%-12s %2d", - p->u.btree.pIndex->zName, p->u.btree.nEq); + const char *zName = p->u.btree.pIndex->zName; + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite3DebugPrintf(".%-12s %2d", zName, p->u.btree.nEq); }else{ sqlite3DebugPrintf("%16s",""); } @@ -5734,25 +5740,139 @@ whereLoopAddAll_end: } /* -** Examine a WherePath to see if it outputs rows in the requested ORDER BY -** (or GROUP BY) without requiring a separate source operation. Return 1 -** if it does and 0 if it does not and -1 if we cannot tell. +** Examine a WherePath (with the addition of the extra WhereLoop of the 4th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate source operation. Return: +** +** 0: ORDER BY is not satisfied. Sorting required +** 1: ORDER BY is satisfied. Omit sorting +** -1: Unknown at this time +** */ static int wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ WherePath *pPath, /* The WherePath to check */ int nLoop, /* Number of entries in pPath->aLoop[] */ - WhereLoop *pLoop /* Add this WhereLoop to the end of pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* Mask of WhereLoops to run in reverse order */ ){ - if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - return nLoop==0 && pLoop->u.vtab.isOrdered; - }else{ - /* TBD: Check to see if pFrom + pWLoop satisfies the ORDER BY. - ** (1) If yes: set isOrderedValid and isOrdered to 1. - ** (2) If no: set isOrderedValid to 1 and isOrdered to 0. - ** (3) unknown: no-op */ - return 0; + u8 revSet; + u8 rev; + u8 isUnique; + u8 requireUnique = 0; + u16 nColumn; + u16 nOrderBy; + int i, j; + int nUsed = 0; + int iCur; + int iColumn; + WhereLoop *pLoop; + ExprList *pOrderBy = pWInfo->pOrderBy; + Expr *pOBExpr; + CollSeq *pColl; + Index *pIndex; + sqlite3 *db = pWInfo->pParse->db; + Bitmask revMask = 0; + + /* + ** We say the WhereLoop is "one-row" if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** + ** General rules: (not an algorithm!) + ** + ** (1) If the current WhereLoop is one-row, then match over any and all + ** ORDER BY terms for the current WhereLoop and proceed to the next + ** WhereLoop. + ** + ** (2) If the current WhereLoop is not one-row, then all subsequent + ** WhereLoops must be one-row. + ** + ** (3) Optionally match any ORDER BY terms against the first nEq columns + ** of the index. + ** + ** (4) Index columns past nEq must match ORDER BY terms one-for-one. + */ + + assert( pOrderBy!=0 ); + + /* Sortability of virtual tables is determined by the xBestIndex method + ** of the virtual table itself */ + if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ + assert( nLoop==0 ); + return pLast->u.vtab.isOrdered; } + + /* Sorting is always required if any term of the ORDER BY is not a + ** column reference */ + nOrderBy = pOrderBy->nExpr; + for(i=0; ia[nUsed].pExpr); + if( pOBExpr->op!=TK_COLUMN ) return 0; + } + + for(i=0; i<=nLoop && nUsedaLoop[i] : pLast; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + isUnique = 1; + if( pLoop->wsFlags & WHERE_IPK ){ + if( (pLoop->wsFlags & WHERE_COLUMN_EQ)!=0 ) isUnique = 0; + pIndex = 0; + nColumn = 1; + }else if( pLoop->u.btree.pIndex==0 ){ + return 0; + }else{ + pIndex = pLoop->u.btree.pIndex; + nColumn = pIndex->nColumn; + if( pIndex->onError==OE_None ){ + isUnique = 0; + }else if( (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_RANGE + |WHERE_COLUMN_NULL))!=0 ){ + isUnique = 0; + }else if( pLoop->u.btree.nEq < pIndex->nColumn ){ + isUnique = 0; + } + } + if( !isUnique && requireUnique ) return 0; + requireUnique = !isUnique; + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + j = 0; + revSet = rev = 0; + for(j=0; ja[nUsed].pExpr); + assert( pOBExpr->op==TK_COLUMN ); + if( pOBExpr->iTable!=iCur ) break; + if( pIndex==0 ){ + if( pOBExpr->iColumn<0 && j==0 ){ + isUnique = 1; + rev = pOrderBy->a[nUsed].sortOrder; + }else if( isUnique ){ + continue; + }else{ + return 0; + } + } + if( isUnique ) continue; + iColumn = pIndex->aiColumn[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; + if( pOBExpr->iColumn!=iColumn ) return 0; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) return 0; + if( revSet ){ + if( pIndex->aSortOrder[j]!=rev ) return 0; + }else{ + rev = pIndex->aSortOrder[j]; + revSet = 1; + } + } + if( rev ) revMask |= ((Bitmask)1)<pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; + Bitmask revMask = 0; u8 isOrderedValid = pFrom->isOrderedValid; u8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; @@ -5840,7 +5961,8 @@ static int wherePathSolver(WhereInfo *pWInfo){ rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, pWLoop) ){ + switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, + pWLoop, &revMask) ){ case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ isOrdered = 1; isOrderedValid = 1; @@ -5873,6 +5995,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ } /* pWLoop is a winner. Add it to the set of best so far */ pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; pTo->nRow = pFrom->nRow * pWLoop->nOut; pTo->rCost = rCost; pTo->isOrderedValid = isOrderedValid; @@ -6193,7 +6316,12 @@ WhereInfo *sqlite3WhereBegin( && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) if( sqlite3WhereTrace ){ int ii; - sqlite3DebugPrintf("------------ Solution ----------------\n"); + sqlite3DebugPrintf("------------ Solution -------------"); + if( pWInfo->nOBSat ){ + sqlite3DebugPrintf(" ORDER BY omitted\n"); + }else{ + sqlite3DebugPrintf("\n"); + } for(ii=0; iia[ii].pWLoop, pTabList); } From 23f98daabb42f2f1def180b9bd2aa81c2f96b608 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 May 2013 15:52:07 +0000 Subject: [PATCH 017/109] Work toward improving the NGQP's ability to optimize out ORDER BY clauses. FossilOrigin-Name: 67367f1e1f0c3eb6be65eea9873910aa62b49884 --- manifest | 14 +++---- manifest.uuid | 2 +- src/sqliteInt.h | 4 +- src/where.c | 101 ++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 325be9d757..ac6a9e67b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sall\strunk\schanges\sup\sthrough\sthe\s3.7.17\srelease. -D 2013-05-20T15:14:42.009 +C Work\stoward\simproving\sthe\sNGQP's\sability\sto\soptimize\sout\sORDER\sBY\sclauses. +D 2013-05-21T15:52:07.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h b9b6a2e97254fb50b6977ec55ad0f4e4ce0510d2 +F src/sqliteInt.h 3b89d4517d984363a96f6f7b348440e01f13ced3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 51d935c6593748ed004006ee0f2147a860d56e90 +F src/where.c a01d93b37fbce377326849fe810dbcd2334d40c6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9fe20292558bb9422de91e35648cb834cbf3b306 118a3b35693b134d56ebd780123b7fd6f1497668 -R b3fec2668cd66935dfd81d00e243b60d +P 14ab6675e5eab3761256a06dad23d2b11220788a +R fc1fe5e0f9d1ee7dd6bc32ff7b604dd6 U drh -Z e02385c0316003dbdd17c2ecf0ba1968 +Z 0d777b81f154f68ed82e3cd424f83c9e diff --git a/manifest.uuid b/manifest.uuid index 52fb631158..75cf59ccb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14ab6675e5eab3761256a06dad23d2b11220788a \ No newline at end of file +67367f1e1f0c3eb6be65eea9873910aa62b49884 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ba0a3597b..176413c946 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1919,8 +1919,8 @@ typedef u64 Bitmask; ** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { - i16 nSrc; /* Number of tables or subqueries in the FROM clause */ - i16 nAlloc; /* Number of entries allocated in a[] below */ + u8 nSrc; /* Number of tables or subqueries in the FROM clause */ + u8 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ diff --git a/src/where.c b/src/where.c index 2eff05ab39..0f61f6ed24 100644 --- a/src/where.c +++ b/src/where.c @@ -56,7 +56,8 @@ typedef struct WhereVtabPlan WhereVtabPlan; struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ Bitmask maskSelf; /* Bitmask identifying table iTab */ - u16 iTab; /* Index of the table coded by this loop */ + u8 iTab; /* Position in FROM clause of table coded by this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ u16 nTerm; /* Number of entries in aTerm[] */ u32 wsFlags; /* WHERE_* flags describing the plan */ double rSetup; /* One-time setup cost (ex: create transient index) */ @@ -1987,6 +1988,7 @@ static int termCanDriveIndex( if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; return 1; @@ -5168,6 +5170,21 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** and no insert will occur if an existing WhereLoop is faster and has ** fewer dependencies than the template. Otherwise a new WhereLoop is ** added based no the template. +** +** If pBuilder->pBest is not NULL then we only care about the very +** best template and that template should be stored in pBuilder->pBest. +** If pBuilder->pBest is NULL then a list of the best templates are stored +** in pBuilder->pWInfo->pLoops. +** +** When accumulating multiple loops (when pBuilder->pBest is NULL) we +** still might overwrite similar loops with the new template if the +** template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; @@ -5175,6 +5192,11 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ sqlite3 *db = pBuilder->db; WhereInfo *pWInfo = pBuilder->pWInfo; + /* If pBuilder->pBest is defined, then only keep track of the single + ** best WhereLoop. pBuilder->pBest->maskSelf==0 indicates that no + ** prior WhereLoops have been evaluated and that the current pTemplate + ** is therefore the first and hence the best and should be retained. + */ if( (p = pBuilder->pBest)!=0 ){ if( p->maskSelf!=0 ){ if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){ @@ -5195,7 +5217,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** priority over pTemplate. */ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab ) continue; + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue; if( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun @@ -5238,7 +5260,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0])); } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ + if( p->u.btree.pIndex && p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } }else{ @@ -5342,6 +5364,36 @@ static int whereLoopAddBtreeIndex( return rc; } +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + int iCol; + int ii; + + if( (pOB = pBuilder->pOrderBy)==0 ) return 0; + iCol = pIndex->aiColumn[0]; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pOB->a[ii].pExpr; + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn==iCol ) return 1; + return 0; + } + } + return 0; +} + /* ** Add all WhereLoop objects a single table of the join were the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be @@ -5358,9 +5410,11 @@ static int whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 0; /* Index number */ + int b; /* A boolean value */ double rSize; /* number of rows in the table */ double rLogSize; /* Logarithm of the number of rows in the table */ - + pNew = pBuilder->pNew; pSrc = pBuilder->pTabList->a + pNew->iTab; assert( !IsVirtual(pSrc->pTab) ); @@ -5419,26 +5473,25 @@ static int whereLoopAddBtree( } } - /* Insert a full table scan */ - pNew->u.btree.nEq = 0; - pNew->nTerm = 0; - pNew->rSetup = (double)0; - pNew->prereq = mExtra; - pNew->u.btree.pIndex = 0; - pNew->wsFlags = 0; - pNew->nOut = rSize; - pNew->rRun = rSize + rLogSize; - /* TBD: Reduce nOut using constraints */ - rc = whereLoopInsert(pBuilder, pNew); - /* Loop over all indices */ - for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ pNew->u.btree.nEq = 0; pNew->nTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = (double)0; + pNew->prereq = mExtra; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); if( pProbe->tnum<=0 ){ /* Integer primary key index */ pNew->wsFlags = WHERE_IPK; + + /* Full table scan */ + pNew->nOut = rSize; + pNew->rRun = (rSize + rLogSize)*(3+b); /* 4x penalty for a full-scan */ + rc = whereLoopInsert(pBuilder, pNew); + if( rc ) break; }else{ Bitmask m = pSrc->colUsed; int j; @@ -5448,10 +5501,17 @@ static int whereLoopAddBtree( m &= ~(((Bitmask)1)<wsFlags = m==0 ? WHERE_IDX_ONLY : 0; - } - pNew->u.btree.pIndex = pProbe; + pNew->wsFlags = (m==0) ? WHERE_IDX_ONLY : 0; + /* Full scan via index */ + if( m==0 || b ){ + pNew->iSortIdx = b ? iSortIdx : 0; + pNew->nOut = rSize; + pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize); + rc = whereLoopInsert(pBuilder, pNew); + if( rc ) break; + } + } rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is @@ -5681,6 +5741,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->rRun = rTotal; pNew->nOut = nRow; pNew->prereq = prereq; + memset(&pNew->u, 0, sizeof(pNew->u)); rc = whereLoopInsert(pBuilder, pNew); } } From d15cb17174cc9c7649dc018701ab5a055ac74a6a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 May 2013 19:23:10 +0000 Subject: [PATCH 018/109] Enhanced "wheretrace" output in the NGQP solver routine. FossilOrigin-Name: 04dfb85a2a7025d4b5056b73fa8477691323919f --- manifest | 14 +++--- manifest.uuid | 2 +- src/where.c | 104 ++++++++++++++++++++++++++++++++++---------- test/boundary3.test | 1 + 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index ac6a9e67b4..785a029e3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\simproving\sthe\sNGQP's\sability\sto\soptimize\sout\sORDER\sBY\sclauses. -D 2013-05-21T15:52:07.219 +C Enhanced\s"wheretrace"\soutput\sin\sthe\sNGQP\ssolver\sroutine. +D 2013-05-21T19:23:10.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c a01d93b37fbce377326849fe810dbcd2334d40c6 +F src/where.c 759c34becf7a414b4bebddd30ad65109eb4510fc F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -324,7 +324,7 @@ F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb F test/boundary2.test 9ae758d7dab7e882c8b6cc4a6a10278385bff8fa F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 -F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 +F test/boundary3.test 04c269017471f1562da2dd6310359f16073a2a65 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/btreefault.test f52c593513bda80a506c848325c73c840590884d @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 14ab6675e5eab3761256a06dad23d2b11220788a -R fc1fe5e0f9d1ee7dd6bc32ff7b604dd6 +P 67367f1e1f0c3eb6be65eea9873910aa62b49884 +R 26b15d08d00e63218ca6971b3d46c9f6 U drh -Z 0d777b81f154f68ed82e3cd424f83c9e +Z 6ae4459022aa081e691ba708466a0da1 diff --git a/manifest.uuid b/manifest.uuid index 75cf59ccb0..d686a351db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67367f1e1f0c3eb6be65eea9873910aa62b49884 \ No newline at end of file +04dfb85a2a7025d4b5056b73fa8477691323919f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0f61f6ed24..a5a01ce071 100644 --- a/src/where.c +++ b/src/where.c @@ -28,6 +28,7 @@ #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) # define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X +# define WHERETRACE_ENABLED 1 #else # define WHERETRACE(X) #endif @@ -56,6 +57,9 @@ typedef struct WhereVtabPlan WhereVtabPlan; struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif u8 iTab; /* Position in FROM clause of table coded by this loop */ u8 iSortIdx; /* Sorting index number. 0==None */ u16 nTerm; /* Number of entries in aTerm[] */ @@ -1835,7 +1839,7 @@ static double estLog(double N){ ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines ** are no-ops. */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_DEBUG) +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ int i; if( !sqlite3WhereTrace ) return; @@ -5066,8 +5070,7 @@ static int nQPlan = 0; /* Next free slow in _query_plan[] */ #endif /* SQLITE_TEST */ -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +#ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes */ @@ -5075,7 +5078,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ int nb = 2*((pTabList->nSrc+15)/16); struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%2d.%0*llx.%0*llx", + sqlite3DebugPrintf("%c %2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq); sqlite3DebugPrintf(" %8s", pItem->zAlias ? pItem->zAlias : pTab->zName); @@ -5102,7 +5105,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3_free(z); } sqlite3DebugPrintf(" fg %08x N %2d", p->wsFlags, p->nTerm); - sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", + sqlite3DebugPrintf(" cost %.2g,%.2g,%.2g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } #endif @@ -5877,7 +5880,8 @@ static int wherePathSatisfiesOrderBy( assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); isUnique = 1; if( pLoop->wsFlags & WHERE_IPK ){ - if( (pLoop->wsFlags & WHERE_COLUMN_EQ)!=0 ) isUnique = 0; + if( (pLoop->wsFlags & WHERE_COLUMN_IN)!=0 ) isUnique = 0; + if( pLoop->u.btree.nEq!=1 ) isUnique = 0; pIndex = 0; nColumn = 1; }else if( pLoop->u.btree.pIndex==0 ){ @@ -5936,6 +5940,18 @@ static int wherePathSatisfiesOrderBy( return -1; } +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; +} +#endif + /* ** Given the list of WhereLoop objects on pWInfo->pLoops, this routine @@ -6044,15 +6060,59 @@ static int wherePathSolver(WhereInfo *pWInfo){ } } if( jj>=nTo ){ - if( nTo>=mxChoice && rCost>=mxCost ) continue; + if( nTo>=mxChoice && rCost>=mxCost ){ +#ifdef WHERETRACE_ENABLE + if( sqlite3WhereTrace>=3 ){ + sqlite3DebugPrintf("Skip %s cost=%-7.2g order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + } +#endif + continue; + } + /* Add a new Path to the aTo[] set */ if( nTo0); } } pTo = &aTo[jj]; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=3 ){ + sqlite3DebugPrintf("New %s cost=%-7.2g order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + } +#endif }else{ - if( pTo->rCost<=rCost ) continue; + if( pTo->rCost<=rCost ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=3 ){ + sqlite3DebugPrintf( + "Skip %s cost=%-7.2g order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + sqlite3DebugPrintf(" vs %s cost=%-7.2g order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + } +#endif + continue; + } + /* A new and better score for a previously created equivalent path */ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=3 ){ + sqlite3DebugPrintf( + "Update %s cost=%-7.2g order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + sqlite3DebugPrintf(" was %s cost=%-7.2g order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + } +#endif } /* pWLoop is a winner. Add it to the set of best so far */ pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; @@ -6072,15 +6132,13 @@ static int wherePathSolver(WhereInfo *pWInfo){ } } -#if 0 - if( sqlite3WhereTrace ){ - sqlite3DebugPrintf("---- round %d ---- nTo=%d\n", iLoop, nTo); - for(ii=0; iipTabList); - } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=2 ){ + sqlite3DebugPrintf("---- round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); } } #endif @@ -6361,11 +6419,14 @@ WhereInfo *sqlite3WhereBegin( if( rc ) goto whereBeginError; /* Display all of the WhereLoop objects if wheretrace is enabled */ -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +#ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ WhereLoop *p; + int i = 0; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; for(p=pWInfo->pLoops; p; p=p->pNextLoop){ + p->cId = zLabel[(i++)%sizeof(zLabel)]; whereLoopPrint(p, pTabList); } } @@ -6373,8 +6434,7 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo); if( db->mallocFailed ) goto whereBeginError; -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +#ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ int ii; sqlite3DebugPrintf("------------ Solution -------------"); @@ -6668,7 +6728,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; } -#if 0 +#if 1 /* Scaffolding: Check the new query plan against the old. Report any ** discrepencies */ for(ii=0; ii Date: Wed, 22 May 2013 02:06:59 +0000 Subject: [PATCH 019/109] Improvements to ORDER BY handling in the NGQP. Fix an "exit" mistakenly left in a test script during the previous check-in. FossilOrigin-Name: 12c709b4369c7d94d7fb743d0d0da7a9350a3d16 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 27 ++++++++++++++++++++------- test/boundary3.test | 1 - 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 785a029e3c..a0719128fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhanced\s"wheretrace"\soutput\sin\sthe\sNGQP\ssolver\sroutine. -D 2013-05-21T19:23:10.944 +C Improvements\sto\sORDER\sBY\shandling\sin\sthe\sNGQP.\s\sFix\san\s"exit"\smistakenly\nleft\sin\sa\stest\sscript\sduring\sthe\sprevious\scheck-in. +D 2013-05-22T02:06:59.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 759c34becf7a414b4bebddd30ad65109eb4510fc +F src/where.c b695db3fe80456a57461c43f1fa1596a38fb4652 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -324,7 +324,7 @@ F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb F test/boundary2.test 9ae758d7dab7e882c8b6cc4a6a10278385bff8fa F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 -F test/boundary3.test 04c269017471f1562da2dd6310359f16073a2a65 +F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/btreefault.test f52c593513bda80a506c848325c73c840590884d @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 67367f1e1f0c3eb6be65eea9873910aa62b49884 -R 26b15d08d00e63218ca6971b3d46c9f6 +P 04dfb85a2a7025d4b5056b73fa8477691323919f +R b7bf12d6215811537a2285f41fc9e19a U drh -Z 6ae4459022aa081e691ba708466a0da1 +Z 6465b78fdb5633463ec5d8b46a4226e0 diff --git a/manifest.uuid b/manifest.uuid index d686a351db..d7f05dad96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04dfb85a2a7025d4b5056b73fa8477691323919f \ No newline at end of file +12c709b4369c7d94d7fb743d0d0da7a9350a3d16 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a5a01ce071..ca9f9938b6 100644 --- a/src/where.c +++ b/src/where.c @@ -5961,7 +5961,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation ** error occurs. */ -static int wherePathSolver(WhereInfo *pWInfo){ +static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ const int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ sqlite3 *db; /* The database connection */ @@ -5976,7 +5976,6 @@ static int wherePathSolver(WhereInfo *pWInfo){ WherePath *pFrom; /* An element of aFrom[] that we are working on */ WherePath *pTo; /* An element of aTo[] that we are working on */ WhereLoop *pWLoop; /* One of the WhereLoop objects */ - WhereLoop *pNext; /* Next loop */ WhereLoop **pX; /* Used to divy up the pSpace memory */ char *pSpace; /* Temporary memory used by this routine */ @@ -6003,10 +6002,11 @@ static int wherePathSolver(WhereInfo *pWInfo){ /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ rSortCost = (double)0; - if( pWInfo->pOrderBy==0 ){ + if( pWInfo->pOrderBy==0 || nRowEst<0.0 ){ aFrom[0].isOrderedValid = 1; }else{ /* Compute an estimate on the cost to sort the entire result set */ +#if 0 rSortCost = (double)1; for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pNext){ pNext = pWLoop->pNextLoop; @@ -6018,6 +6018,14 @@ static int wherePathSolver(WhereInfo *pWInfo){ rSortCost *= rCost; } rSortCost *= estLog(rSortCost); +#else + rSortCost = nRowEst*estLog(nRowEst); +#endif +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=2 ){ + sqlite3DebugPrintf("--solver sort cost=%-7.2g\n", rSortCost); + } +#endif } /* Compute successively longer WherePaths using the previous generation @@ -6134,10 +6142,10 @@ static int wherePathSolver(WhereInfo *pWInfo){ #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace>=2 ){ - sqlite3DebugPrintf("---- round %d ----\n", iLoop); + sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iirCost, pTo->nRow, + sqlite3DebugPrintf(" %s cost=%-7.2g nrow=%-7.2g order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); } } @@ -6167,6 +6175,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ if( pFrom->isOrdered ){ pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; } + pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ sqlite3DbFree(db, pSpace); @@ -6432,8 +6441,12 @@ WhereInfo *sqlite3WhereBegin( } #endif - wherePathSolver(pWInfo); + wherePathSolver(pWInfo, -1); if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut); + if( db->mallocFailed ) goto whereBeginError; + } #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ int ii; diff --git a/test/boundary3.test b/test/boundary3.test index e1331111f9..dbe9abfa0a 100644 --- a/test/boundary3.test +++ b/test/boundary3.test @@ -123,7 +123,6 @@ do_test boundary3-2.1.3 { SELECT t1.rowid, x FROM t1 JOIN t2 ON t2.r=t1.rowid WHERE t2.a=17 } } {72057594037927935 00ffffffffffffff} -exit do_test boundary3-2.1.gt.1 { db eval { SELECT t2.a FROM t1 JOIN t2 USING(a) From 0f133a483278d8bae17f917b7962c3ad1ccefe80 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 May 2013 17:01:17 +0000 Subject: [PATCH 020/109] Allow the rowid at the end of an index to be used in a constraint on that index. FossilOrigin-Name: 9bf0524df7ca2e7fcd92b2878a8457264b3c7f6e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 22 +++++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a0719128fb..3b53419b6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sORDER\sBY\shandling\sin\sthe\sNGQP.\s\sFix\san\s"exit"\smistakenly\nleft\sin\sa\stest\sscript\sduring\sthe\sprevious\scheck-in. -D 2013-05-22T02:06:59.780 +C Allow\sthe\srowid\sat\sthe\send\sof\san\sindex\sto\sbe\sused\sin\sa\sconstraint\son\sthat\sindex. +D 2013-05-22T17:01:17.424 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c b695db3fe80456a57461c43f1fa1596a38fb4652 +F src/where.c 3ca12d203c0cca5bc36f0cd408a97ed5a9357047 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 04dfb85a2a7025d4b5056b73fa8477691323919f -R b7bf12d6215811537a2285f41fc9e19a +P 12c709b4369c7d94d7fb743d0d0da7a9350a3d16 +R 7322735243734361422a3e6e35778930 U drh -Z 6465b78fdb5633463ec5d8b46a4226e0 +Z fd0ab2669226a74d38406e095d929629 diff --git a/manifest.uuid b/manifest.uuid index d7f05dad96..98ddda464c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12c709b4369c7d94d7fb743d0d0da7a9350a3d16 \ No newline at end of file +9bf0524df7ca2e7fcd92b2878a8457264b3c7f6e \ No newline at end of file diff --git a/src/where.c b/src/where.c index ca9f9938b6..e28fe4d62c 100644 --- a/src/where.c +++ b/src/where.c @@ -5104,7 +5104,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %08x N %2d", p->wsFlags, p->nTerm); + sqlite3DebugPrintf(" fg %08x N %d", p->wsFlags, p->nTerm); sqlite3DebugPrintf(" cost %.2g,%.2g,%.2g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } @@ -5293,6 +5293,7 @@ static int whereLoopAddBtreeIndex( WhereLoop savedLoop; /* Saved original content of pNew[] */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + tRowcnt iRowEst; /* Estimated index selectivity */ double rLogSize; /* Logarithm of table size */ db = pBuilder->db; @@ -5300,7 +5301,7 @@ static int whereLoopAddBtreeIndex( if( db->mallocFailed ) return SQLITE_NOMEM; assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); - assert( pNew->u.btree.nEqnColumn ); + assert( pNew->u.btree.nEq<=pProbe->nColumn ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; @@ -5310,9 +5311,15 @@ static int whereLoopAddBtreeIndex( opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + if( pNew->u.btree.nEq < pProbe->nColumn ){ + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + iRowEst = pProbe->aiRowEst[pNew->u.btree.nEq+1]; + }else{ + iCol = -1; + iRowEst = 1; + } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, iCol>=0 ? pProbe : 0); + opMask, pProbe); savedLoop = *pNew; pNew->rSetup = (double)0; rLogSize = estLog(pProbe->aiRowEst[0]); @@ -5334,11 +5341,11 @@ static int whereLoopAddBtreeIndex( nIn = pExpr->x.pList->nExpr; } pNew->u.btree.nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->u.btree.nEq] * nInMul * nIn; + pNew->nOut = (double)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; pNew->u.btree.nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->u.btree.nEq] * nInMul; + pNew->nOut = (double)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pNew->nOut = savedLoop.nOut/3; @@ -5358,7 +5365,8 @@ static int whereLoopAddBtreeIndex( /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEqnColumn + && pNew->u.btree.nEq<=pProbe->nColumn + && pProbe->zName!=0 ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); } From ef8663765575762847a9a9f44b6f82817e87b793 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 May 2013 20:49:02 +0000 Subject: [PATCH 021/109] Use the NGQP plan for EXPLAIN QUERY PLAN output. This change causes 207 errors in veryquick.test, many of which are benign. FossilOrigin-Name: f783e8e6b10de44029c7c5f57e4648a4a677ca1b --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/where.c | 52 +++++++++++++++++++++++++++---------------------- test/tester.tcl | 25 +++++++++++++++++++++--- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 3b53419b6d..f6c1ed6e1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\srowid\sat\sthe\send\sof\san\sindex\sto\sbe\sused\sin\sa\sconstraint\son\sthat\sindex. -D 2013-05-22T17:01:17.424 +C Use\sthe\sNGQP\splan\sfor\sEXPLAIN\sQUERY\sPLAN\soutput.\s\sThis\schange\scauses\s207\nerrors\sin\sveryquick.test,\smany\sof\swhich\sare\sbenign. +D 2013-05-22T20:49:02.170 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3ca12d203c0cca5bc36f0cd408a97ed5a9357047 +F src/where.c 627bf1af3ab30e1efcc7ac362b15e7e945e9250d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -779,7 +779,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 3b08771e6d601612fe62d13787db0e50aac4cf7b +F test/tester.tcl 693700993b7e0a5b7b2171b291a54cdbad1d86ff F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 12c709b4369c7d94d7fb743d0d0da7a9350a3d16 -R 7322735243734361422a3e6e35778930 +P 9bf0524df7ca2e7fcd92b2878a8457264b3c7f6e +R 97f27b7ea9f3efcce127b9443ade87ed U drh -Z fd0ab2669226a74d38406e095d929629 +Z 96c17883fc9fb8d90fa5ada1b6415a79 diff --git a/manifest.uuid b/manifest.uuid index 98ddda464c..0de3a8524f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bf0524df7ca2e7fcd92b2878a8457264b3c7f6e \ No newline at end of file +f783e8e6b10de44029c7c5f57e4648a4a677ca1b \ No newline at end of file diff --git a/src/where.c b/src/where.c index e28fe4d62c..fa38b57383 100644 --- a/src/where.c +++ b/src/where.c @@ -4168,16 +4168,16 @@ static void explainAppendTerm( ** It is the responsibility of the caller to free the buffer when it is ** no longer required. */ -static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ - WherePlan *pPlan = &pLevel->plan; - Index *pIndex = pPlan->u.pIdx; - int nEq = pPlan->nEq; +static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ + Index *pIndex = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; int i, j; Column *aCol = pTab->aCol; int *aiColumn = pIndex->aiColumn; StrAccum txt; - if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + if( pIndex==0 ) return 0; + if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ return 0; } sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); @@ -4188,11 +4188,11 @@ static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ } j = i; - if( pPlan->wsFlags&WHERE_BTM_LIMIT ){ + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i++, z, ">"); } - if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i, z, "<"); } @@ -4215,7 +4215,6 @@ static void explainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ if( pParse->explain==2 ){ - u32 flags = pLevel->plan.wsFlags; struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ @@ -4223,12 +4222,16 @@ static void explainOneScan( sqlite3_int64 nRow; /* Expected number of rows visited by scan */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; - isSearch = (pLevel->plan.nEq>0) - || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); if( pItem->pSelect ){ @@ -4240,20 +4243,22 @@ static void explainOneScan( if( pItem->zAlias ){ zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); } - if( (flags & WHERE_INDEXED)!=0 ){ - char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 + && pLoop->u.btree.pIndex!=0 + ){ + char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), ((flags & WHERE_IDX_ONLY)?"COVERING ":""), ((flags & WHERE_TEMP_INDEX)?"":" "), - ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), + ((flags & WHERE_TEMP_INDEX)?"": pLoop->u.btree.pIndex->zName), zWhere ); sqlite3DbFree(db, zWhere); - }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_INDEXED)!=0 ){ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); - if( flags&WHERE_ROWID_EQ ){ + if( flags&WHERE_COLUMN_EQ ){ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowidplan.u.pVtabIdx; zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ testcase( wctrlFlags & WHERE_ORDERBY_MIN ); nRow = 1; }else{ - nRow = (sqlite3_int64)pLevel->plan.nRow; + nRow = (sqlite3_int64)pLoop->nOut; } zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); @@ -5263,7 +5267,8 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0])); } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - if( p->u.btree.pIndex && p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } }else{ @@ -5310,6 +5315,7 @@ static int whereLoopAddBtreeIndex( }else{ opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; @@ -5472,6 +5478,7 @@ static int whereLoopAddBtree( for(pTerm=pWC->a; rc==SQLITE_OK && pTermu.btree.nEq = 1; + pNew->u.btree.pIndex = 0; pNew->nTerm = 1; pNew->aTerm[0] = pTerm; pNew->rSetup = 2*rLogSize*pSrc->pTab->nRowEst; @@ -5515,7 +5522,7 @@ static int whereLoopAddBtree( pNew->wsFlags = (m==0) ? WHERE_IDX_ONLY : 0; /* Full scan via index */ - if( m==0 || b ){ + if( (m==0 || b) && pProbe->bUnordered==0 ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize); @@ -5892,10 +5899,9 @@ static int wherePathSatisfiesOrderBy( if( pLoop->u.btree.nEq!=1 ) isUnique = 0; pIndex = 0; nColumn = 1; - }else if( pLoop->u.btree.pIndex==0 ){ + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ - pIndex = pLoop->u.btree.pIndex; nColumn = pIndex->nColumn; if( pIndex->onError==OE_None ){ isUnique = 0; diff --git a/test/tester.tcl b/test/tester.tcl index 761a36e355..cad9807ed7 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -793,9 +793,28 @@ proc finalize_testing {} { set nTest [incr_ntest] set nErr [set_test_counter errors] - puts "$nErr errors out of $nTest tests" - if {$nErr>0} { - puts "Failures on these tests: [set_test_counter fail_list]" + set nKnown 0 + if {[file readable known-problems.txt]} { + set fd [open known-problems.txt] + set content [read $fd] + close $fd + foreach x $content {set known_error($x) 1} + foreach x [set_test_counter fail_list] { + if {[info exists known_error($x)]} {incr nKnown} + } + } + if {$nKnown>0} { + puts "[expr {$nErr-$nKnown}] new errors and $nKnown known errors\ + out of $nTest tests" + } else { + puts "$nErr errors out of $nTest tests" + } + if {$nErr>$nKnown} { + puts -nonewline "Failures on these tests:" + foreach x [set_test_counter fail_list] { + if {![info exists known_error($x)]} {puts -nonewline " $x"} + } + puts "" } foreach warning [set_test_counter warn_list] { puts "Warning: $warning" From 3bd26f0543ec42e0985ffbaa8f8d927fcb690de6 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 May 2013 14:52:03 +0000 Subject: [PATCH 022/109] Record in the WhereLoop object the set of virtual table constraints that need not be separately checked. FossilOrigin-Name: b49fa74561f38c945be6149693678fd6518c2de4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 579e939060..010739cba8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sNGQP\sbranch. -D 2013-05-24T13:55:23.561 +C Record\sin\sthe\sWhereLoop\sobject\sthe\sset\sof\svirtual\stable\sconstraints\sthat\nneed\snot\sbe\sseparately\schecked. +D 2013-05-24T14:52:03.958 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 627bf1af3ab30e1efcc7ac362b15e7e945e9250d +F src/where.c 2fd11c00ffc37a65b762c8b2bbf5710723eefdd7 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1066,7 +1066,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f783e8e6b10de44029c7c5f57e4648a4a677ca1b 61a10452399db28cd5ea4ba9d416b87a34c2eddb -R 5ae2bb9eca4144ac9f28ec145cae1f78 +P 7c8f992c04504a132686ad0652efdff19e43480d +R 91429a505370ccdd08f4d1fe64f3bf63 U drh -Z e63f19ec26c366b47598ade89ccbddd5 +Z 676fd5c7993f5cc12a1abed75be6a4c4 diff --git a/manifest.uuid b/manifest.uuid index 1dd79ce63e..bec1afe37d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c8f992c04504a132686ad0652efdff19e43480d \ No newline at end of file +b49fa74561f38c945be6149693678fd6518c2de4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index fa38b57383..6416dd9503 100644 --- a/src/where.c +++ b/src/where.c @@ -76,6 +76,7 @@ struct WhereLoop { int idxNum; /* Index number */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */ u8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ } vtab; } u; @@ -5101,9 +5102,10 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ }else{ char *z; if( p->u.vtab.idxStr ){ - z = sqlite3_mprintf("(%d,\"%s\")", p->u.vtab.idxNum,p->u.vtab.idxStr); + z = sqlite3_mprintf("(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); }else{ - z = sqlite3_mprintf("(%d)", p->u.vtab.idxNum); + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); } sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); @@ -5629,6 +5631,7 @@ static int whereLoopAddVirtual( pNew->prereq = 0; mxTerm = -1; for(i=0; imxTerm; i++) pNew->aTerm[i] = 0; + pNew->u.vtab.omitMask = 0; for(i=0; inConstraint; i++, pIdxCons++){ if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ if( iTerm>=pBuilder->mxTerm ) break; @@ -5646,6 +5649,7 @@ static int whereLoopAddVirtual( pNew->prereq |= pTerm->prereqRight; pNew->aTerm[iTerm] = pTerm; if( iTerm>mxTerm ) mxTerm = iTerm; + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ if( pUsage[i].omit==0 ){ /* Do not attempt to use an IN constraint if the virtual table From 88da644f2cf2f5991e1875c3c41494e82360e057 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 May 2013 17:59:37 +0000 Subject: [PATCH 023/109] Update the NGQP to record which loops need be run in reverse order to satisfy ORDER BY clauses. FossilOrigin-Name: 211f7a5374fe20a02535edc8b799a8a7136ff6b3 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 82 +++++++++++++++++++++++-------------------------- 4 files changed, 48 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 010739cba8..8b6a0ac134 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Record\sin\sthe\sWhereLoop\sobject\sthe\sset\sof\svirtual\stable\sconstraints\sthat\nneed\snot\sbe\sseparately\schecked. -D 2013-05-24T14:52:03.958 +C Update\sthe\sNGQP\sto\srecord\swhich\sloops\sneed\sbe\srun\sin\sreverse\sorder\sto\nsatisfy\sORDER\sBY\sclauses. +D 2013-05-27T17:59:37.917 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/shell.c 9a18124ff209ca308d786c99a466e8e270193ff3 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 0b0ba16c1449d0e7ef6ff78039ffef38f58f585a +F src/sqliteInt.h 9e12a56e05420b5c7aa40aca4d44ee4d9535b68a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2fd11c00ffc37a65b762c8b2bbf5710723eefdd7 +F src/where.c 3f4f25df651e8cea5b02d94aeee28838e9c5e2c3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1066,7 +1066,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7c8f992c04504a132686ad0652efdff19e43480d -R 91429a505370ccdd08f4d1fe64f3bf63 +P b49fa74561f38c945be6149693678fd6518c2de4 +R dbfb376dc6b826c13c37ea187ec4a9c4 U drh -Z 676fd5c7993f5cc12a1abed75be6a4c4 +Z 716f8aec8e11bc86fa358274cd806e0b diff --git a/manifest.uuid b/manifest.uuid index bec1afe37d..a1b72955fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b49fa74561f38c945be6149693678fd6518c2de4 \ No newline at end of file +211f7a5374fe20a02535edc8b799a8a7136ff6b3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c1a4f766c8..79f5d23492 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2051,6 +2051,7 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pDistinct; /* DISTINCT ON values, or NULL */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ diff --git a/src/where.c b/src/where.c index 6416dd9503..b0b953af30 100644 --- a/src/where.c +++ b/src/where.c @@ -5839,8 +5839,9 @@ static int wherePathSatisfiesOrderBy( WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* Mask of WhereLoops to run in reverse order */ ){ - u8 revSet; - u8 rev; + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ u8 isUnique; u8 requireUnique = 0; u16 nColumn; @@ -5890,7 +5891,7 @@ static int wherePathSatisfiesOrderBy( ** column reference */ nOrderBy = pOrderBy->nExpr; for(i=0; ia[nUsed].pExpr); + pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) return 0; } @@ -5902,7 +5903,7 @@ static int wherePathSatisfiesOrderBy( if( (pLoop->wsFlags & WHERE_COLUMN_IN)!=0 ) isUnique = 0; if( pLoop->u.btree.nEq!=1 ) isUnique = 0; pIndex = 0; - nColumn = 1; + nColumn = 0; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -5921,31 +5922,31 @@ static int wherePathSatisfiesOrderBy( iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; j = 0; revSet = rev = 0; - for(j=0; ja[nUsed].pExpr); assert( pOBExpr->op==TK_COLUMN ); if( pOBExpr->iTable!=iCur ) break; - if( pIndex==0 ){ - if( pOBExpr->iColumn<0 && j==0 ){ - isUnique = 1; - rev = pOrderBy->a[nUsed].sortOrder; - }else if( isUnique ){ - continue; - }else{ - return 0; - } - } if( isUnique ) continue; - iColumn = pIndex->aiColumn[j]; - if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; - if( pOBExpr->iColumn!=iColumn ) return 0; - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); - if( !pColl ) pColl = db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) return 0; - if( revSet ){ - if( pIndex->aSortOrder[j]!=rev ) return 0; + if( jaiColumn[j]; + revIdx = pIndex->aSortOrder[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - rev = pIndex->aSortOrder[j]; + /* The ROWID column at the end */ + iColumn = -1; + revIdx = 0; + } + if( pOBExpr->iColumn!=iColumn ) return 0; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) return 0; + } + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[nUsed].sortOrder ) return 0; + }else{ + rev = revIdx ^ pOrderBy->a[nUsed].sortOrder; revSet = 1; } } @@ -6000,6 +6001,9 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ db = pWInfo->pParse->db; nLoop = pWInfo->nLevel; assert( nLoop<=pWInfo->pTabList->nSrc ); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace>=2 ) sqlite3DebugPrintf("---- begin solver\n"); +#endif /* Allocate and initialize space for aTo and aFrom */ ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; @@ -6020,28 +6024,14 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ rSortCost = (double)0; - if( pWInfo->pOrderBy==0 || nRowEst<0.0 ){ + if( pWInfo->pOrderBy==0 || nRowEst<=0.0 ){ aFrom[0].isOrderedValid = 1; }else{ /* Compute an estimate on the cost to sort the entire result set */ -#if 0 - rSortCost = (double)1; - for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pNext){ - pNext = pWLoop->pNextLoop; - rCost = pWLoop->nOut; - while( pNext && pNext->iTab==pWLoop->iTab ){ - if( pNext->nOutnOut; - pNext = pNext->pNextLoop; - } - rSortCost *= rCost; - } - rSortCost *= estLog(rSortCost); -#else rSortCost = nRowEst*estLog(nRowEst); -#endif #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace>=2 ){ - sqlite3DebugPrintf("--solver sort cost=%-7.2g\n", rSortCost); + sqlite3DebugPrintf("---- sort cost=%-7.2g\n", rSortCost); } #endif } @@ -6162,9 +6152,14 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ if( sqlite3WhereTrace>=2 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iirCost, pTo->nRow, pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + if( pTo->isOrderedValid && pTo->isOrdered ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } } } #endif @@ -6192,6 +6187,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ } if( pFrom->isOrdered ){ pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + pWInfo->revMask = pFrom->revLoop; } pWInfo->nRowOut = pFrom->nRow; @@ -6468,9 +6464,9 @@ WhereInfo *sqlite3WhereBegin( #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ int ii; - sqlite3DebugPrintf("------------ Solution -------------"); + sqlite3DebugPrintf("---- Solution"); if( pWInfo->nOBSat ){ - sqlite3DebugPrintf(" ORDER BY omitted\n"); + sqlite3DebugPrintf(" ORDER BY omitted rev=0x%llx\n", pWInfo->revMask); }else{ sqlite3DebugPrintf("\n"); } From 7ba39a921a326294e679c755aab2172a700e695b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 May 2013 17:43:19 +0000 Subject: [PATCH 024/109] Cut over the NGQP query planner. Remove lots of legacy code. This check-in compiles but does not work. The test suite gets incorrect answers and crashes. FossilOrigin-Name: 001539df4b74dc1cbceb010a91407003ab4d8735 --- manifest | 14 +- manifest.uuid | 2 +- src/sqliteInt.h | 12 +- src/where.c | 1847 ++++------------------------------------------- 4 files changed, 135 insertions(+), 1740 deletions(-) diff --git a/manifest b/manifest index 39a8c39490..7344d1b38d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sNGQP\sbranch. -D 2013-05-30T11:48:36.654 +C Cut\sover\sthe\sNGQP\squery\splanner.\s\sRemove\slots\sof\slegacy\scode.\s\s\nThis\scheck-in\scompiles\sbut\sdoes\snot\swork.\s\sThe\stest\ssuite\sgets\sincorrect\nanswers\sand\scrashes. +D 2013-05-30T17:43:19.293 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/shell.c 9a18124ff209ca308d786c99a466e8e270193ff3 F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 55f43f10f165c8c17334e33a11b2d58a557b4a2c +F src/sqliteInt.h c386fb7c61c03a0a65d78e326016449767be5b93 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3f4f25df651e8cea5b02d94aeee28838e9c5e2c3 +F src/where.c 296baae1cbfd99527d8c87f9773a6c98116f6b8a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 211f7a5374fe20a02535edc8b799a8a7136ff6b3 d5b084e9d8cfe9c0c339aca076d472bb50aa764c -R 6c4c190a12a33f58ef606f4f8676d338 +P aebe1f2603ee04d792af73aaf59625bda99d5cd1 +R cdb9a6b0104d69931bfa04869e682c85 U drh -Z e692c34c627e293c83ac7dc20bf238bc +Z 36d07876c5b50e19bee990a15e1431a7 diff --git a/manifest.uuid b/manifest.uuid index eeb2053b55..ea56fcb1ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aebe1f2603ee04d792af73aaf59625bda99d5cd1 \ No newline at end of file +001539df4b74dc1cbceb010a91407003ab4d8735 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 45a587780b..a0e7bba7d6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1989,7 +1989,6 @@ struct WherePlan { ** after FROM clause ordering. */ struct WhereLevel { - WherePlan plan; /* query plan for this element of the FROM clause */ int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ int iTabCur; /* The VDBE cursor used to access the table */ int iIdxCur; /* The VDBE cursor used to access pIdx */ @@ -1997,7 +1996,7 @@ struct WhereLevel { int addrNxt; /* Jump here to start the next IN combination */ int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ - u8 iFrom; /* Which entry in the FROM clause */ + u8 iFrom; /* FIXME: Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ union { /* Information that depends on plan.wsFlags */ @@ -2011,16 +2010,7 @@ struct WhereLevel { } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ } u; - double rOptCost; /* "Optimal" cost for this level */ struct WhereLoop *pWLoop; /* The selected WhereLoop object */ - - /* The following field is really not part of the current level. But - ** we need a place to cache virtual table index information for each - ** virtual table in the FROM clause and the WhereLevel structure is - ** a convenient place since there is one WhereLevel for each FROM clause - ** element. - */ - sqlite3_index_info *pIdxInfo; /* Index info for n-th source table */ }; /* diff --git a/src/where.c b/src/where.c index b0b953af30..a9a431ba9a 100644 --- a/src/where.c +++ b/src/where.c @@ -60,7 +60,7 @@ struct WhereLoop { #ifdef SQLITE_DEBUG char cId; /* Symbolic ID of this loop for debugging use */ #endif - u8 iTab; /* Position in FROM clause of table coded by this loop */ + u8 iTab; /* Position in FROM clause of table for this loop */ u8 iSortIdx; /* Sorting index number. 0==None */ u16 nTerm; /* Number of entries in aTerm[] */ u32 wsFlags; /* WHERE_* flags describing the plan */ @@ -190,7 +190,7 @@ struct WhereScan { WhereTerm *pCurrent; /* Most recent match */ WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ - char *zCollName; /* Must have this collating sequence, if not NULL */ + char *zCollName; /* Required collating sequence, if not NULL */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ @@ -376,17 +376,6 @@ struct WhereBestIdx { WhereCost cost; /* Lowest cost query plan */ }; -/* -** Return TRUE if the probe cost is less than the baseline cost -*/ -static int compareCost(const WhereCost *pProbe, const WhereCost *pBaseline){ - if( pProbe->rCostrCost ) return 1; - if( pProbe->rCost>pBaseline->rCost ) return 0; - if( pProbe->plan.nOBSat>pBaseline->plan.nOBSat ) return 1; - if( pProbe->plan.nRowplan.nRow ) return 1; - return 0; -} - /* ** Initialize a preallocated WhereClause structure. */ @@ -1691,66 +1680,6 @@ static int findIndexCol( return -1; } -/* -** This routine determines if pIdx can be used to assist in processing a -** DISTINCT qualifier. In other words, it tests whether or not using this -** index for the outer loop guarantees that rows with equal values for -** all expressions in the pDistinct list are delivered grouped together. -** -** For example, the query -** -** SELECT DISTINCT a, b, c FROM tbl WHERE a = ? -** -** can benefit from any index on columns "b" and "c". -*/ -static int isDistinctIndex( - Parse *pParse, /* Parsing context */ - WhereClause *pWC, /* The WHERE clause */ - Index *pIdx, /* The index being considered */ - int base, /* Cursor number for the table pIdx is on */ - ExprList *pDistinct, /* The DISTINCT expressions */ - int nEqCol /* Number of index columns with == */ -){ - Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ - int i; /* Iterator variable */ - - assert( pDistinct!=0 ); - if( pIdx->zName==0 || pDistinct->nExpr>=BMS ) return 0; - testcase( pDistinct->nExpr==BMS-1 ); - - /* Loop through all the expressions in the distinct list. If any of them - ** are not simple column references, return early. Otherwise, test if the - ** WHERE clause contains a "col=X" clause. If it does, the expression - ** can be ignored. If it does not, and the column does not belong to the - ** same table as index pIdx, return early. Finally, if there is no - ** matching "col=X" expression and the column is on the same table as pIdx, - ** set the corresponding bit in variable mask. - */ - for(i=0; inExpr; i++){ - WhereTerm *pTerm; - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); - if( p->op!=TK_COLUMN ) return 0; - pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); - if( pTerm ){ - Expr *pX = pTerm->pExpr; - CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - CollSeq *p2 = sqlite3ExprCollSeq(pParse, p); - if( p1==p2 ) continue; - } - if( p->iTable!=base ) return 0; - mask |= (((Bitmask)1) << i); - } - - for(i=nEqCol; mask && inColumn; i++){ - int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i); - if( iExpr<0 ) break; - mask &= ~(((Bitmask)1) << iExpr); - } - - return (mask==0); -} - - /* ** Return true if the DISTINCT expression-list passed as the third argument ** is redundant. A DISTINCT list is redundant if the database contains a @@ -1878,106 +1807,6 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ #define TRACE_IDX_OUTPUTS(A) #endif -/* -** Required because bestIndex() is called by bestOrClauseIndex() -*/ -static void bestIndex(WhereBestIdx*); - -/* -** This routine attempts to find an scanning strategy that can be used -** to optimize an 'OR' expression that is part of a WHERE clause. -** -** The table associated with FROM clause term pSrc may be either a -** regular B-Tree table or a virtual table. -*/ -static void bestOrClauseIndex(WhereBestIdx *p){ -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - WhereClause *pWC = p->pWC; /* The WHERE clause */ - struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ - const int iCur = pSrc->iCursor; /* The cursor of the table */ - const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ - WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - - /* The OR-clause optimization is disallowed if the INDEXED BY or - ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ - if( pSrc->notIndexed || pSrc->pIndex!=0 ){ - return; - } - if( pWC->wctrlFlags & WHERE_AND_ONLY ){ - return; - } - - /* Search the WHERE clause terms for a usable WO_OR term. */ - for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 - && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 - && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 - ){ - WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; - WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; - WhereTerm *pOrTerm; - double rTotal = 0; - double nRow = 0; - Bitmask used = 0; - WhereBestIdx sBOI; - - sBOI = *p; - sBOI.pOrderBy = 0; - sBOI.pDistinct = 0; - sBOI.ppIdxInfo = 0; - for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) - ));*/ - if( (pOrTerm->eOperator& WO_AND)!=0 ){ - sBOI.pWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(&sBOI); - }else if( pOrTerm->leftCursor==iCur ){ - WhereClause tempWC; - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; - tempWC.pOuter = pWC; - tempWC.op = TK_AND; - tempWC.a = pOrTerm; - tempWC.wctrlFlags = 0; - tempWC.nTerm = 1; - sBOI.pWC = &tempWC; - bestIndex(&sBOI); - }else{ - continue; - } - rTotal += sBOI.cost.rCost; - nRow += sBOI.cost.plan.nRow; - used |= sBOI.cost.used; - if( rTotal>=p->cost.rCost ) break; - } - - /* If there is an ORDER BY clause, increase the scan cost to account - ** for the cost of the sort. */ - if( p->pOrderBy!=0 ){ - /*WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", - rTotal, rTotal+nRow*estLog(nRow)));*/ - rTotal += nRow*estLog(nRow); - } - - /* If the cost of scanning using this OR term for optimization is - ** less than the current cost stored in pCost, replace the contents - ** of pCost. */ - /*WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));*/ - if( rTotalcost.rCost ){ - p->cost.rCost = rTotal; - p->cost.used = used; - p->cost.plan.nRow = nRow; - p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; - p->cost.plan.wsFlags = WHERE_MULTI_OR; - p->cost.plan.u.pTerm = pTerm; - } - } - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ -} - #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -2000,82 +1829,6 @@ static int termCanDriveIndex( } #endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** If the query plan for pSrc specified in pCost is a full table scan -** and indexing is allows (if there is no NOT INDEXED clause) and it -** possible to construct a transient index that would perform better -** than a full table scan even when the cost of constructing the index -** is taken into account, then alter the query plan to use the -** transient index. -*/ -static void bestAutomaticIndex(WhereBestIdx *p){ - Parse *pParse = p->pParse; /* The parsing context */ - WhereClause *pWC = p->pWC; /* The WHERE clause */ - struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ - double costTempIdx; /* per-query cost of the transient index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Table *pTable; /* Table tht might be indexed */ - - if( pParse->nQueryLoop<=(double)1 ){ - /* There is no point in building an automatic index for a single scan */ - return; - } - if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ - /* Automatic indices are disabled at run-time */ - return; - } - if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 - && (p->cost.plan.wsFlags & WHERE_COVER_SCAN)==0 - ){ - /* We already have some kind of index in use for this query. */ - return; - } - if( pSrc->viaCoroutine ){ - /* Cannot index a co-routine */ - return; - } - if( pSrc->notIndexed ){ - /* The NOT INDEXED clause appears in the SQL. */ - return; - } - if( pSrc->isCorrelated ){ - /* The source is a correlated sub-query. No point in indexing it. */ - return; - } - - assert( pParse->nQueryLoop >= (double)1 ); - pTable = pSrc->pTab; - nTableRow = pTable->nRowEst; - logN = estLog(nTableRow); - costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=p->cost.rCost ){ - /* The cost of creating the transient table would be greater than - ** doing the full table scan */ - return; - } - - /* Search for any equality comparison term */ - pWCEnd = &pWC->a[pWC->nTerm]; - for(pTerm=pWC->a; pTermnotReady) ){ - /*WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", - p->cost.rCost, costTempIdx));*/ - p->cost.rCost = costTempIdx; - p->cost.plan.nRow = logN + 1; - p->cost.plan.wsFlags = WHERE_TEMP_INDEX; - p->cost.used = pTerm->prereqRight; - break; - } - } -} -#else -# define bestAutomaticIndex(A) /* no-op */ -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* @@ -2105,6 +1858,7 @@ static void constructAutomaticIndex( int i; /* Loop counter */ int mxBitCol; /* Maximum column in pSrc->colUsed */ CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ @@ -2119,6 +1873,7 @@ static void constructAutomaticIndex( nColumn = 0; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTerm0 ); - pLevel->plan.nEq = nColumn; + pLoop->u.btree.nEq = nColumn; /* Count the number of additional columns needed to create a ** covering index. A "covering index" is an index that contains all @@ -2153,7 +1908,7 @@ static void constructAutomaticIndex( if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ nColumn += pTable->nCol - BMS + 1; } - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; + pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ nByte = sizeof(Index); @@ -2162,7 +1917,7 @@ static void constructAutomaticIndex( nByte += nColumn; /* Index.aSortOrder */ pIdx = sqlite3DbMallocZero(pParse->db, nByte); if( pIdx==0 ) return; - pLevel->plan.u.pIdx = pIdx; + pLoop->u.btree.pIndex = pIdx; pIdx->azColl = (char**)&pIdx[1]; pIdx->aiColumn = (int*)&pIdx->azColl[nColumn]; pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn]; @@ -2185,7 +1940,7 @@ static void constructAutomaticIndex( } } } - assert( (u32)n==pLevel->plan.nEq ); + assert( (u32)n==pLoop->u.btree.nEq ); /* Add additional columns needed to make the automatic index into ** a covering index */ @@ -2384,208 +2139,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ return pParse->nErr; } +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -/* -** Compute the best index for a virtual table. -** -** The best index is computed by the xBestIndex method of the virtual -** table module. This routine is really just a wrapper that sets up -** the sqlite3_index_info structure that is used to communicate with -** xBestIndex. -** -** In a join, this routine might be called multiple times for the -** same virtual table. The sqlite3_index_info structure is created -** and initialized on the first invocation and reused on all subsequent -** invocations. The sqlite3_index_info structure is also used when -** code is generated to access the virtual table. The whereInfoDelete() -** routine takes care of freeing the sqlite3_index_info structure after -** everybody has finished with it. -*/ -static void bestVirtualIndex(WhereBestIdx *p){ - Parse *pParse = p->pParse; /* The parsing context */ - WhereClause *pWC = p->pWC; /* The WHERE clause */ - struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ - Table *pTab = pSrc->pTab; - sqlite3_index_info *pIdxInfo; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int nOrderBy; - int bAllowIN; /* Allow IN optimizations */ - double rCost; - - /* Make sure wsFlags is initialized to some sane value. Otherwise, if the - ** malloc in allocateIndexInfo() fails and this function returns leaving - ** wsFlags in an uninitialized state, the caller may behave unpredictably. - */ - memset(&p->cost, 0, sizeof(p->cost)); - p->cost.plan.wsFlags = WHERE_VIRTUALTABLE; - - /* If the sqlite3_index_info structure has not been previously - ** allocated and initialized, then allocate and initialize it now. - */ - pIdxInfo = *p->ppIdxInfo; - if( pIdxInfo==0 ){ - *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse,pWC,pSrc,p->pOrderBy); - } - if( pIdxInfo==0 ){ - return; - } - - /* At this point, the sqlite3_index_info structure that pIdxInfo points - ** to will have been initialized, either during the current invocation or - ** during some prior invocation. Now we just have to customize the - ** details of pIdxInfo for the current invocation and pass it to - ** xBestIndex. - */ - - /* The module name must be defined. Also, by this point there must - ** be a pointer to an sqlite3_vtab structure. Otherwise - ** sqlite3ViewGetColumnNames() would have picked up the error. - */ - assert( pTab->azModuleArg && pTab->azModuleArg[0] ); - assert( sqlite3GetVTable(pParse->db, pTab) ); - - /* Try once or twice. On the first attempt, allow IN optimizations. - ** If an IN optimization is accepted by the virtual table xBestIndex - ** method, but the pInfo->aConstrainUsage.omit flag is not set, then - ** the query will not work because it might allow duplicate rows in - ** output. In that case, run the xBestIndex method a second time - ** without the IN constraints. Usually this loop only runs once. - ** The loop will exit using a "break" statement. - */ - for(bAllowIN=1; 1; bAllowIN--){ - assert( bAllowIN==0 || bAllowIN==1 ); - - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. - */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - if( (pTerm->prereqRight&p->notReady)==0 - && (bAllowIN || (pTerm->eOperator & WO_IN)==0) - ){ - pIdxCons->usable = 1; - }else{ - pIdxCons->usable = 0; - } - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !p->pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } - - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++, pIdxCons++){ - if( pUsage[i].argvIndex>0 ){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - p->cost.used |= pTerm->prereqRight; - if( (pTerm->eOperator & WO_IN)!=0 ){ - if( pUsage[i].omit==0 ){ - /* Do not attempt to use an IN constraint if the virtual table - ** says that the equivalent EQ constraint cannot be safely omitted. - ** If we do attempt to use such a constraint, some rows might be - ** repeated in the output. */ - break; - } - /* A virtual table that is constrained by an IN clause may not - ** consume the ORDER BY clause because (1) the order of IN terms - ** is not necessarily related to the order of output terms and - ** (2) Multiple outputs from a single IN value will not merge - ** together. */ - pIdxInfo->orderByConsumed = 0; - } - } - } - if( i>=pIdxInfo->nConstraint ) break; - } - - /* The orderByConsumed signal is only valid if all outer loops collectively - ** generate just a single row of output. - */ - if( pIdxInfo->orderByConsumed ){ - for(i=0; ii; i++){ - if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ - pIdxInfo->orderByConsumed = 0; - } - } - } - - /* If there is an ORDER BY clause, and the selected virtual table index - ** does not satisfy it, increase the cost of the scan accordingly. This - ** matches the processing for non-virtual tables in bestBtreeIndex(). - */ - rCost = pIdxInfo->estimatedCost; - if( p->pOrderBy && pIdxInfo->orderByConsumed==0 ){ - rCost += estLog(rCost)*rCost; - } - - /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the - ** inital value of lowestCost in this loop. If it is, then the - ** (costcost.rCost = (SQLITE_BIG_DBL/((double)2)); - }else{ - p->cost.rCost = rCost; - } - p->cost.plan.u.pVtabIdx = pIdxInfo; - if( pIdxInfo->orderByConsumed ){ - p->cost.plan.wsFlags |= WHERE_ORDERED; - p->cost.plan.nOBSat = nOrderBy; - }else{ - p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; - } - p->cost.plan.nEq = 0; - pIdxInfo->nOrderBy = nOrderBy; - - /* Try to find a more efficient access pattern by using multiple indexes - ** to optimize an OR expression within the WHERE clause. - */ - bestOrClauseIndex(p); -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - #ifdef SQLITE_ENABLE_STAT3 /* ** Estimate the location of a particular key among all keys in an @@ -2979,879 +2535,6 @@ static int whereInScanEst( } #endif /* defined(SQLITE_ENABLE_STAT3) */ -/* -** Check to see if column iCol of the table with cursor iTab will appear -** in sorted order according to the current query plan. -** -** Return values: -** -** 0 iCol is not ordered -** 1 iCol has only a single value -** 2 iCol is in ASC order -** 3 iCol is in DESC order -*/ -static int isOrderedColumn( - WhereBestIdx *p, - int iTab, - int iCol -){ - int i, j; - WhereLevel *pLevel = &p->aLevel[p->i-1]; - Index *pIdx; - u8 sortOrder; - for(i=p->i-1; i>=0; i--, pLevel--){ - if( pLevel->iTabCur!=iTab ) continue; - if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ - return 1; - } - assert( (pLevel->plan.wsFlags & WHERE_ORDERED)!=0 ); - if( (pIdx = pLevel->plan.u.pIdx)!=0 ){ - if( iCol<0 ){ - sortOrder = 0; - testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); - }else{ - int n = pIdx->nColumn; - for(j=0; jaiColumn[j] ) break; - } - if( j>=n ) return 0; - sortOrder = pIdx->aSortOrder[j]; - testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); - } - }else{ - if( iCol!=(-1) ) return 0; - sortOrder = 0; - testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); - } - if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ){ - assert( sortOrder==0 || sortOrder==1 ); - testcase( sortOrder==1 ); - sortOrder = 1 - sortOrder; - } - return sortOrder+2; - } - return 0; -} - -/* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause, either in whole or in part. The return value is the -** cumulative number of terms in the ORDER BY clause that are satisfied -** by the index pIdx and other indices in outer loops. -** -** The table being queried has a cursor number of "base". pIdx is the -** index that is postulated for use to access the table. -** -** The *pbRev value is set to 0 order 1 depending on whether or not -** pIdx should be run in the forward order or in reverse order. -*/ -static int isSortingIndex( - WhereBestIdx *p, /* Best index search context */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - int *pbRev, /* Set to 1 for reverse-order scan of pIdx */ - int *pbObUnique /* ORDER BY column values will different in every row */ -){ - int i; /* Number of pIdx terms used */ - int j; /* Number of ORDER BY terms satisfied */ - int sortOrder = 2; /* 0: forward. 1: backward. 2: unknown */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pOBItem;/* A term of the ORDER BY clause */ - Table *pTab = pIdx->pTable; /* Table that owns index pIdx */ - ExprList *pOrderBy; /* The ORDER BY clause */ - Parse *pParse = p->pParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - int nPriorSat; /* ORDER BY terms satisfied by outer loops */ - int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ - int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ - int outerObUnique; /* Outer loops generate different values in - ** every row for the ORDER BY columns */ - - if( p->i==0 ){ - nPriorSat = 0; - outerObUnique = 1; - }else{ - u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags; - nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - if( (wsFlags & WHERE_ORDERED)==0 ){ - /* This loop cannot be ordered unless the next outer loop is - ** also ordered */ - return nPriorSat; - } - if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ){ - /* Only look at the outer-most loop if the OrderByIdxJoin - ** optimization is disabled */ - return nPriorSat; - } - testcase( wsFlags & WHERE_OB_UNIQUE ); - testcase( wsFlags & WHERE_ALL_UNIQUE ); - outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0; - } - pOrderBy = p->pOrderBy; - assert( pOrderBy!=0 ); - if( pIdx->bUnordered ){ - /* Hash indices (indicated by the "unordered" tag on sqlite_stat1) cannot - ** be used for sorting */ - return nPriorSat; - } - nTerm = pOrderBy->nExpr; - uniqueNotNull = pIdx->onError!=OE_None; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - j = nPriorSat; - for(i=0,pOBItem=&pOrderBy->a[j]; jnColumn; i++){ - Expr *pOBExpr; /* The expression of the ORDER BY pOBItem */ - CollSeq *pColl; /* The collating sequence of pOBExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - int isEq; /* Subject to an == or IS NULL constraint */ - int isMatch; /* ORDER BY term matches the index term */ - const char *zColl; /* Name of collating sequence for i-th index term */ - WhereTerm *pConstraint; /* A constraint in the WHERE clause */ - - /* If the next term of the ORDER BY clause refers to anything other than - ** a column in the "base" table, then this index will not be of any - ** further use in handling the ORDER BY. */ - pOBExpr = sqlite3ExprSkipCollate(pOBItem->pExpr); - if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){ - break; - } - - /* Find column number and collating sequence for the next entry - ** in the index */ - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - assert( zColl!=0 ); - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = 0; - } - - /* Check to see if the column number and collating sequence of the - ** index match the column number and collating sequence of the ORDER BY - ** clause entry. Set isMatch to 1 if they both match. */ - if( pOBExpr->iColumn==iColumn ){ - if( zColl ){ - pColl = sqlite3ExprCollSeq(pParse, pOBItem->pExpr); - if( !pColl ) pColl = db->pDfltColl; - isMatch = sqlite3StrICmp(pColl->zName, zColl)==0; - }else{ - isMatch = 1; - } - }else{ - isMatch = 0; - } - - /* termSortOrder is 0 or 1 for whether or not the access loop should - ** run forward or backwards (respectively) in order to satisfy this - ** term of the ORDER BY clause. */ - assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pOBItem->sortOrder; - - /* If X is the column in the index and ORDER BY clause, check to see - ** if there are any X= or X IS NULL constraints in the WHERE clause. */ - pConstraint = findTerm(p->pWC, base, iColumn, p->notReady, - WO_EQ|WO_ISNULL|WO_IN, pIdx); - if( pConstraint==0 ){ - isEq = 0; - }else if( (pConstraint->eOperator & WO_IN)!=0 ){ - isEq = 0; - }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ - uniqueNotNull = 0; - isEq = 1; /* "X IS NULL" means X has only a single value */ - }else if( pConstraint->prereqRight==0 ){ - isEq = 1; /* Constraint "X=constant" means X has only a single value */ - }else{ - Expr *pRight = pConstraint->pExpr->pRight; - if( pRight->op==TK_COLUMN ){ - /*WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", - pRight->iTable, pRight->iColumn));*/ - isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); - /*WHERETRACE((" -> isEq=%d\n", isEq));*/ - - /* If the constraint is of the form X=Y where Y is an ordered value - ** in an outer loop, then make sure the sort order of Y matches the - ** sort order required for X. */ - if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ - testcase( isEq==2 ); - testcase( isEq==3 ); - break; - } - }else{ - isEq = 0; /* "X=expr" places no ordering constraints on X */ - } - } - if( !isMatch ){ - if( isEq==0 ){ - break; - }else{ - continue; - } - }else if( isEq!=1 ){ - if( sortOrder==2 ){ - sortOrder = termSortOrder; - }else if( termSortOrder!=sortOrder ){ - break; - } - } - j++; - pOBItem++; - if( iColumn<0 ){ - seenRowid = 1; - break; - }else if( pTab->aCol[iColumn].notNull==0 && isEq!=1 ){ - testcase( isEq==0 ); - testcase( isEq==2 ); - testcase( isEq==3 ); - uniqueNotNull = 0; - } - } - if( seenRowid ){ - uniqueNotNull = 1; - }else if( uniqueNotNull==0 || inColumn ){ - uniqueNotNull = 0; - } - - /* If we have not found at least one ORDER BY term that matches the - ** index, then show no progress. */ - if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; - - /* Either the outer queries must generate rows where there are no two - ** rows with the same values in all ORDER BY columns, or else this - ** loop must generate just a single row of output. Example: Suppose - ** the outer loops generate A=1 and A=1, and this loop generates B=3 - ** and B=4. Then without the following test, ORDER BY A,B would - ** generate the wrong order output: 1,3 1,4 1,3 1,4 - */ - if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat; - *pbObUnique = uniqueNotNull; - - /* Return the necessary scan order back to the caller */ - *pbRev = sortOrder & 1; - - /* If there was an "ORDER BY rowid" term that matched, or it is only - ** possible for a single row from this table to match, then skip over - ** any additional ORDER BY terms dealing with this table. - */ - if( uniqueNotNull ){ - /* Advance j over additional ORDER BY terms associated with base */ - WhereMaskSet *pMS = p->pWC->pMaskSet; - Bitmask m = ~getMask(pMS, base); - while( ja[j].pExpr)&m)==0 ){ - j++; - } - } - return j; -} - -/* -** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the p->cost. -** -** The lowest cost plan wins. The cost is an estimate of the amount of -** CPU and disk I/O needed to process the requested result. -** Factors that influence cost include: -** -** * The estimated number of rows that will be retrieved. (The -** fewer the better.) -** -** * Whether or not sorting must occur. -** -** * Whether or not there must be separate lookups in the -** index and in the main table. -** -** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in -** the SQL statement, then this function only considers plans using the -** named index. If no such plan is found, then the returned cost is -** SQLITE_BIG_DBL. If a plan is found that uses the named index, -** then the cost is calculated in the usual way. -** -** If a NOT INDEXED clause was attached to the table -** in the SELECT statement, then no indexes are considered. However, the -** selected plan may still take advantage of the built-in rowid primary key -** index. -*/ -static void bestBtreeIndex(WhereBestIdx *p){ - Parse *pParse = p->pParse; /* The parsing context */ - WhereClause *pWC = p->pWC; /* The WHERE clause */ - struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ - int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - Index *pProbe; /* An index we are evaluating */ - Index *pIdx; /* Copy of pProbe, or zero for IPK index */ - int eqTermMask; /* Current mask of valid equality operators */ - int idxEqTermMask; /* Index mask of valid equality operators */ - Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ - int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in p->cost.plan.wsFlag */ - int nPriorSat; /* ORDER BY terms satisfied by outer loops */ - int nOrderBy; /* Number of ORDER BY terms */ - char bSortInit; /* Initializer for bSort in inner loop */ - char bDistInit; /* Initializer for bDist in inner loop */ - - - /* Initialize the cost to a worst-case value */ - memset(&p->cost, 0, sizeof(p->cost)); - p->cost.rCost = SQLITE_BIG_DBL; - - /* If the pSrc table is the right table of a LEFT JOIN then we may not - ** use an index to satisfy IS NULL constraints on that table. This is - ** because columns might end up being NULL if the table does not match - - ** a circumstance which the index cannot help us discover. Ticket #2177. - */ - if( pSrc->jointype & JT_LEFT ){ - idxEqTermMask = WO_EQ|WO_IN; - }else{ - idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL; - } - - if( pSrc->pIndex ){ - /* An INDEXED BY clause specifies a particular index to use */ - pIdx = pProbe = pSrc->pIndex; - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; - }else{ - /* There is no INDEXED BY clause. Create a fake Index object in local - ** variable sPk to represent the rowid primary key index. Make this - ** fake index the first in a chain of Index objects with all of the real - ** indices to follow */ - Index *pFirst; /* First of real indices on the table */ - memset(&sPk, 0, sizeof(Index)); - sPk.nColumn = 1; - sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; - sPk.onError = OE_Replace; - sPk.pTable = pSrc->pTab; - aiRowEstPk[0] = pSrc->pTab->nRowEst; - aiRowEstPk[1] = 1; - pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ - /* The real indices of the table are only considered if the - ** NOT INDEXED qualifier is omitted from the FROM clause */ - sPk.pNext = pFirst; - } - pProbe = &sPk; - wsFlagMask = ~( - WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE - ); - eqTermMask = WO_EQ|WO_IN; - pIdx = 0; - } - - nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; - if( p->i ){ - nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - bSortInit = nPriorSat0; - bDistInit = p->pDistinct!=0; - } - - /* Loop over all indices looking for the best one to use - */ - for(; pProbe; pIdx=pProbe=pProbe->pNext){ - const tRowcnt * const aiRowEst = pProbe->aiRowEst; - WhereCost pc; /* Cost of using pProbe */ - double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - - /* The following variables are populated based on the properties of - ** index being evaluated. They are then used to determine the expected - ** cost and number of rows returned. - ** - ** pc.plan.nEq: - ** Number of equality terms that can be implemented using the index. - ** In other words, the number of initial fields in the index that - ** are used in == or IN or NOT NULL constraints of the WHERE clause. - ** - ** nInMul: - ** The "in-multiplier". This is an estimate of how many seek operations - ** SQLite must perform on the index in question. For example, if the - ** WHERE clause is: - ** - ** WHERE a IN (1, 2, 3) AND b IN (4, 5, 6) - ** - ** SQLite must perform 9 lookups on an index on (a, b), so nInMul is - ** set to 9. Given the same schema and either of the following WHERE - ** clauses: - ** - ** WHERE a = 1 - ** WHERE a >= 2 - ** - ** nInMul is set to 1. - ** - ** If there exists a WHERE term of the form "x IN (SELECT ...)", then - ** the sub-select is assumed to return 25 rows for the purposes of - ** determining nInMul. - ** - ** bInEst: - ** Set to true if there was at least one "x IN (SELECT ...)" term used - ** in determining the value of nInMul. Note that the RHS of the - ** IN operator must be a SELECT, not a value list, for this variable - ** to be true. - ** - ** rangeDiv: - ** An estimate of a divisor by which to reduce the search space due - ** to inequality constraints. In the absence of sqlite_stat3 ANALYZE - ** data, a single inequality reduces the search space to 1/4rd its - ** original size (rangeDiv==4). Two inequalities reduce the search - ** space to 1/16th of its original size (rangeDiv==16). - ** - ** bSort: - ** Boolean. True if there is an ORDER BY clause that will require an - ** external sort (i.e. scanning the index being evaluated will not - ** correctly order records). - ** - ** bDist: - ** Boolean. True if there is a DISTINCT clause that will require an - ** external btree. - ** - ** bLookup: - ** Boolean. True if a table lookup is required for each index entry - ** visited. In other words, true if this is not a covering index. - ** This is always false for the rowid primary key index of a table. - ** For other indexes, it is true unless all the columns of the table - ** used by the SELECT statement are present in the index (such an - ** index is sometimes described as a covering index). - ** For example, given the index on (a, b), the second of the following - ** two queries requires table b-tree lookups in order to find the value - ** of column c, but the first does not because columns a and b are - ** both available in the index. - ** - ** SELECT a, b FROM tbl WHERE a = 1; - ** SELECT a, b, c FROM tbl WHERE a = 1; - */ - int bInEst = 0; /* True if "x IN (SELECT...)" seen */ - int nInMul = 1; /* Number of distinct equalities to lookup */ - double rangeDiv = (double)1; /* Estimated reduction in search space */ - int nBound = 0; /* Number of range constraints seen */ - char bSort = bSortInit; /* True if external sort required */ - char bDist = bDistInit; /* True if index cannot help with DISTINCT */ - char bLookup = 0; /* True if not a covering index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ -#ifdef SQLITE_ENABLE_STAT3 - WhereTerm *pFirstTerm = 0; /* First term matching the index */ -#endif - - /*WHERETRACE(( - " %s(%s):\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") - ));*/ - memset(&pc, 0, sizeof(pc)); - pc.plan.nOBSat = nPriorSat; - - /* Determine the values of pc.plan.nEq and nInMul */ - for(pc.plan.nEq=0; pc.plan.nEqnColumn; pc.plan.nEq++){ - int j = pProbe->aiColumn[pc.plan.nEq]; - pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); - if( pTerm==0 ) break; - pc.plan.wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); - testcase( pTerm->pWC!=pWC ); - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - pc.plan.wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nInMul *= 25; - bInEst = 1; - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nInMul *= pExpr->x.pList->nExpr; - } - }else if( pTerm->eOperator & WO_ISNULL ){ - pc.plan.wsFlags |= WHERE_COLUMN_NULL; - } -#ifdef SQLITE_ENABLE_STAT3 - if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; -#endif - pc.used |= pTerm->prereqRight; - } - - /* If the index being considered is UNIQUE, and there is an equality - ** constraint for all columns in the index, then this search will find - ** at most a single row. In this case set the WHERE_UNIQUE flag to - ** indicate this to the caller. - ** - ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (pc.plan.nEq+1) that - ** can be optimized using the index. - */ - if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ - testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); - testcase( pc.plan.wsFlags & WHERE_COLUMN_NULL ); - if( (pc.plan.wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - pc.plan.wsFlags |= WHERE_UNIQUE; - if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ - pc.plan.wsFlags |= WHERE_ALL_UNIQUE; - } - } - }else if( pProbe->bUnordered==0 ){ - int j; - j = (pc.plan.nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[pc.plan.nEq]); - if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop, *pBtm; - pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx); - pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, pc.plan.nEq, pBtm, pTop, &rangeDiv); - if( pTop ){ - nBound = 1; - pc.plan.wsFlags |= WHERE_TOP_LIMIT; - pc.used |= pTop->prereqRight; - testcase( pTop->pWC!=pWC ); - } - if( pBtm ){ - nBound++; - pc.plan.wsFlags |= WHERE_BTM_LIMIT; - pc.used |= pBtm->prereqRight; - testcase( pBtm->pWC!=pWC ); - } - pc.plan.wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); - } - } - - /* If there is an ORDER BY clause and the index being considered will - ** naturally scan rows in the required order, set the appropriate flags - ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but - ** the index will scan rows in a different order, set the bSort - ** variable. */ - if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ - int bRev = 2; - int bObUnique = 0; - /*WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat));*/ - pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); - /*WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", - bRev, bObUnique, pc.plan.nOBSat));*/ - if( nPriorSatpDistinct, pc.plan.nEq) - && (pc.plan.wsFlags & WHERE_COLUMN_IN)==0 - ){ - bDist = 0; - pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; - } - - /* If currently calculating the cost of using an index (not the IPK - ** index), determine if all required column data may be obtained without - ** using the main table (i.e. if the index is a covering - ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** pc.plan.wsFlags. Otherwise, set the bLookup variable to true. */ - if( pIdx ){ - Bitmask m = pSrc->colUsed; - int j; - for(j=0; jnColumn; j++){ - int x = pIdx->aiColumn[j]; - if( xaiRowEst[0] ){ - pc.plan.nRow = aiRowEst[0]/2; - nInMul = (int)(pc.plan.nRow / aiRowEst[pc.plan.nEq]); - } - -#ifdef SQLITE_ENABLE_STAT3 - /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) - ** and we do not think that values of x are unique and if histogram - ** data is available for column x, then it might be possible - ** to get a better estimate on the number of rows based on - ** VALUE and how common that value is according to the histogram. - */ - if( pc.plan.nRow>(double)1 && pc.plan.nEq==1 - && pFirstTerm!=0 && aiRowEst[1]>1 ){ - assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); - if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator & WO_EQ ); - testcase( pFirstTerm->eOperator & WO_EQUIV ); - testcase( pFirstTerm->eOperator & WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, - &pc.plan.nRow); - }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator & WO_IN ); - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, - &pc.plan.nRow); - } - } -#endif /* SQLITE_ENABLE_STAT3 */ - - /* Adjust the number of output rows and downward to reflect rows - ** that are excluded by range constraints. - */ - pc.plan.nRow = pc.plan.nRow/rangeDiv; - if( pc.plan.nRow<1 ) pc.plan.nRow = 1; - - /* Experiments run on real SQLite databases show that the time needed - ** to do a binary search to locate a row in a table or index is roughly - ** log10(N) times the time to move from one row to the next row within - ** a table or index. The actual times can vary, with the size of - ** records being an important factor. Both moves and searches are - ** slower with larger records, presumably because fewer records fit - ** on one page and hence more pages have to be fetched. - ** - ** The ANALYZE command and the sqlite_stat1 and sqlite_stat3 tables do - ** not give us data on the relative sizes of table and index records. - ** So this computation assumes table records are about twice as big - ** as index records - */ - if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE)) - ==WHERE_IDX_ONLY - && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 - && sqlite3GlobalConfig.bUseCis - && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) - ){ - /* This index is not useful for indexing, but it is a covering index. - ** A full-scan of the index might be a little faster than a full-scan - ** of the table, so give this case a cost slightly less than a table - ** scan. */ - pc.rCost = aiRowEst[0]*3 + pProbe->nColumn; - pc.plan.wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE; - }else if( (pc.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - /* The cost of a full table scan is a number of move operations equal - ** to the number of rows in the table. - ** - ** We add an additional 4x penalty to full table scans. This causes - ** the cost function to err on the side of choosing an index over - ** choosing a full scan. This 4x full-scan penalty is an arguable - ** decision and one which we expect to revisit in the future. But - ** it seems to be working well enough at the moment. - */ - pc.rCost = aiRowEst[0]*4; - pc.plan.wsFlags &= ~WHERE_IDX_ONLY; - if( pIdx ){ - pc.plan.wsFlags &= ~WHERE_ORDERED; - pc.plan.nOBSat = nPriorSat; - } - }else{ - log10N = estLog(aiRowEst[0]); - pc.rCost = pc.plan.nRow; - if( pIdx ){ - if( bLookup ){ - /* For an index lookup followed by a table lookup: - ** nInMul index searches to find the start of each index range - ** + nRow steps through the index - ** + nRow table searches to lookup the table entry using the rowid - */ - pc.rCost += (nInMul + pc.plan.nRow)*log10N; - }else{ - /* For a covering index: - ** nInMul index searches to find the initial entry - ** + nRow steps through the index - */ - pc.rCost += nInMul*log10N; - } - }else{ - /* For a rowid primary key lookup: - ** nInMult table searches to find the initial entry for each range - ** + nRow steps through the table - */ - pc.rCost += nInMul*log10N; - } - } - - /* Add in the estimated cost of sorting the result. Actual experimental - ** measurements of sorting performance in SQLite show that sorting time - ** adds C*N*log10(N) to the cost, where N is the number of rows to be - ** sorted and C is a factor between 1.95 and 4.3. We will split the - ** difference and select C of 3.0. - */ - if( bSort ){ - double m = estLog(pc.plan.nRow*(nOrderBy - pc.plan.nOBSat)/nOrderBy); - m *= (double)(pc.plan.nOBSat ? 2 : 3); - pc.rCost += pc.plan.nRow*m; - } - if( bDist ){ - pc.rCost += pc.plan.nRow*estLog(pc.plan.nRow)*3; - } - - /**** Cost of using this index has now been computed ****/ - - /* If there are additional constraints on this table that cannot - ** be used with the current index, but which might lower the number - ** of output rows, adjust the nRow value accordingly. This only - ** matters if the current index is the least costly, so do not bother - ** with this step if we already know this index will not be chosen. - ** Also, never reduce the output row count below 2 using this step. - ** - ** It is critical that the notValid mask be used here instead of - ** the notReady mask. When computing an "optimal" index, the notReady - ** mask will only have one bit set - the bit for the current table. - ** The notValid mask, on the other hand, always has all bits set for - ** tables that are not in outer loops. If notReady is used here instead - ** of notValid, then a optimal index that depends on inner joins loops - ** might be selected even when there exists an optimal index that has - ** no such dependency. - */ - if( pc.plan.nRow>2 && pc.rCost<=p->cost.rCost ){ - int k; /* Loop counter */ - int nSkipEq = pc.plan.nEq; /* Number of == constraints to skip */ - int nSkipRange = nBound; /* Number of < constraints to skip */ - Bitmask thisTab; /* Bitmap for pSrc */ - - thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; pc.plan.nRow>2 && k; k--, pTerm++){ - if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue; - if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ - if( nSkipEq ){ - /* Ignore the first pc.plan.nEq equality matches since the index - ** has already accounted for these */ - nSkipEq--; - }else{ - /* Assume each additional equality match reduces the result - ** set size by a factor of 10 */ - pc.plan.nRow /= 10; - } - }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ - if( nSkipRange ){ - /* Ignore the first nSkipRange range constraints since the index - ** has already accounted for these */ - nSkipRange--; - }else{ - /* Assume each additional range constraint reduces the result - ** set size by a factor of 3. Indexed range constraints reduce - ** the search space by a larger factor: 4. We make indexed range - ** more selective intentionally because of the subjective - ** observation that indexed range constraints really are more - ** selective in practice, on average. */ - pc.plan.nRow /= 3; - } - }else if( (pTerm->eOperator & WO_NOOP)==0 ){ - /* Any other expression lowers the output row count by half */ - pc.plan.nRow /= 2; - } - } - if( pc.plan.nRow<2 ) pc.plan.nRow = 2; - } - - - /*WHERETRACE(( - " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" - " used=0x%llx nOBSat=%d\n", - pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, - p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, - pc.plan.nOBSat - ));*/ - - /* If this index is the best we have seen so far, then record this - ** index and its cost in the p->cost structure. - */ - if( (!pIdx || pc.plan.wsFlags) && compareCost(&pc, &p->cost) ){ - p->cost = pc; - p->cost.plan.wsFlags &= wsFlagMask; - p->cost.plan.u.pIdx = pIdx; - } - - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIndex ) break; - - /* Reset masks for the next index in the loop */ - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; - } - - /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag - ** is set, then reverse the order that the index will be scanned - ** in. This is used for application testing, to help find cases - ** where application behavior depends on the (undefined) order that - ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - p->cost.plan.wsFlags |= WHERE_REVERSE; - } - - assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERED)==0 ); - assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 ); - assert( pSrc->pIndex==0 - || p->cost.plan.u.pIdx==0 - || p->cost.plan.u.pIdx==pSrc->pIndex - ); - - /*WHERETRACE((" best index is %s cost=%.1f\n", - p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk", - p->cost.rCost));*/ - - bestOrClauseIndex(p); - bestAutomaticIndex(p); - if( eqTermMask & WO_ISNULL ) p->cost.plan.wsFlags |= WHERE_NULL_OK; -} - -/* -** Find the query plan for accessing table pSrc->pTab. Write the -** best query plan and its cost into the WhereCost object supplied -** as the last parameter. This function may calculate the cost of -** both real and virtual table scans. -** -** This function does not take ORDER BY or DISTINCT into account. Nor -** does it remember the virtual table query plan. All it does is compute -** the cost while determining if an OR optimization is applicable. The -** details will be reconsidered later if the optimization is found to be -** applicable. -*/ -static void bestIndex(WhereBestIdx *p){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(p->pSrc->pTab) ){ - sqlite3_index_info *pIdxInfo = 0; - p->ppIdxInfo = &pIdxInfo; - bestVirtualIndex(p); - assert( pIdxInfo!=0 || p->pParse->db->mallocFailed ); - if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - sqlite3DbFree(p->pParse->db, pIdxInfo); - }else -#endif - { - bestBtreeIndex(p); - } -} - /* ** Disable a term in the WHERE clause. Except, do not disable the term ** if it controls a LEFT OUTER JOIN and it did not originate in the ON @@ -3949,6 +2632,7 @@ static int codeEqualityTerm( WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ WhereLevel *pLevel, /* The level of the FROM clause we are working on */ int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; @@ -3966,10 +2650,11 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; - u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + WhereLoop *pLoop = pLevel->pWLoop; - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 - && pLevel->plan.u.pIdx->aSortOrder[iEq] + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] ){ testcase( iEq==0 ); testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 ); @@ -3986,7 +2671,7 @@ static int codeEqualityTerm( } iTab = pX->iTable; sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); + assert( pLoop->wsFlags & WHERE_IN_ABLE ); if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); } @@ -4058,28 +2743,31 @@ static int codeAllEqualityTerms( WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ WhereClause *pWC, /* The WHERE clause */ Bitmask notReady, /* Which parts of FROM have not yet been coded */ + int bRev, /* Reverse the order of IN operators */ int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ ){ - int nEq = pLevel->plan.nEq; /* The number of == or IN constraints to code */ + int nEq; /* The number of == or IN constraints to code */ Vdbe *v = pParse->pVdbe; /* The vm under construction */ Index *pIdx; /* The index being used for this loop */ - int iCur = pLevel->iTabCur; /* The cursor of the table */ WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ int j; /* Loop counter */ int regBase; /* Base register */ int nReg; /* Number of registers to allocate */ char *zAff; /* Affinity string to return */ - int eqFlags; /* WO_EQ|WO_IN and maybe also WO_ISNULL */ /* This module is only called on query plans that use an index. */ - assert( pLevel->plan.wsFlags & WHERE_INDEXED ); - pIdx = pLevel->plan.u.pIdx; + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLevel->plan.nEq + nExtraReg; + nReg = pLoop->u.btree.nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); @@ -4090,18 +2778,15 @@ static int codeAllEqualityTerms( /* Evaluate the equality constraints */ assert( pIdx->nColumn>=nEq ); - eqFlags = (pLevel->plan.wsFlags&WHERE_NULL_OK) ? (WO_EQ|WO_IN|WO_ISNULL) - : (WO_EQ|WO_IN); for(j=0; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, eqFlags, pIdx); - if( pTerm==0 ) break; + pTerm = pLoop->aTerm[j]; + assert( pTerm!=0 ); /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, regBase+j); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); @@ -4306,6 +2991,7 @@ static Bitmask codeOneLoopStart( int omitTable; /* True if we use the index only */ int bRev; /* True if we need to scan in reverse order */ WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ WhereTerm *pTerm; /* A WHERE clause term */ Parse *pParse; /* Parsing context */ @@ -4321,10 +3007,11 @@ static Bitmask codeOneLoopStart( v = pParse->pVdbe; pWC = pWInfo->pWC; pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; - bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; - omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); @@ -4362,47 +3049,36 @@ static Bitmask codeOneLoopStart( }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 0: The table is a virtual-table. Use the VFilter and VNext + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; - sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - int nConstraint = pVtabIdx->nConstraint; - struct sqlite3_index_constraint_usage *aUsage = - pVtabIdx->aConstraintUsage; - const struct sqlite3_index_constraint *aConstraint = - pVtabIdx->aConstraint; + int nConstraint = pLoop->nTerm; sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; - for(j=1; j<=nConstraint; j++){ - for(k=0; ka[aConstraint[k].iTermOffset]; - if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, k, iTarget); - addrNotFound = pLevel->addrNxt; - }else{ - sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); - } - break; - } - } - if( k==nConstraint ) break; - } - sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pVtabIdx->idxStr, - pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); - pVtabIdx->needToFreeIdxStr = 0; for(j=0; ja[iTerm]); + int iTarget = iReg+j+1; + pTerm = pLoop->aTerm[j]; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aTerm[j]); } } pLevel->op = OP_VNext; @@ -4413,19 +3089,22 @@ static Bitmask codeOneLoopStart( }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ - if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){ - /* Case 1: We can directly reference a single row using an + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an ** equality comparison against the ROWID field. Or ** we reference multiple rows using a "rowid IN (...)" ** construct. */ + assert( pLoop->u.btree.nEq==1 ); iReleaseReg = sqlite3GetTempReg(pParse); - pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); + pTerm = pLoop->aTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, iReleaseReg); + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); @@ -4433,8 +3112,10 @@ static Bitmask codeOneLoopStart( sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); pLevel->op = OP_Noop; - }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){ - /* Case 2: We have an inequality comparison against the ROWID field. + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. */ int testOp = OP_Noop; int start; @@ -4442,8 +3123,10 @@ static Bitmask codeOneLoopStart( WhereTerm *pStart, *pEnd; assert( omitTable==0 ); - pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0); - pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aTerm[j++]; if( bRev ){ pTerm = pStart; pStart = pEnd; @@ -4510,8 +3193,8 @@ static Bitmask codeOneLoopStart( sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } - }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){ - /* Case 3: A scan using an index. + }else if( pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ|WHERE_IDX_ONLY) ){ + /* Case 4: A scan using an index. ** ** The WHERE clause may contain zero or more equality ** terms ("==" or "IN" operators) that refer to the N @@ -4557,8 +3240,8 @@ static Bitmask codeOneLoopStart( OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + int nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ @@ -4574,7 +3257,7 @@ static Bitmask codeOneLoopStart( char *zStartAff; /* Affinity for start of range constraint */ char *zEndAff; /* Affinity for end of range constraint */ - pIdx = pLevel->plan.u.pIdx; + pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; k = (nEq==pIdx->nColumn ? -1 : pIdx->aiColumn[nEq]); @@ -4587,7 +3270,7 @@ static Bitmask codeOneLoopStart( ** this requires some special handling. */ if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERED) + && (pWInfo->nOBSat>0) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ @@ -4599,12 +3282,13 @@ static Bitmask codeOneLoopStart( /* Find any inequality constraint terms for the start and end ** of the range. */ - if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = findTerm(pWC, iCur, k, notReady, (WO_LT|WO_LE), pIdx); + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aTerm[j++]; nExtraReg = 1; } - if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = findTerm(pWC, iCur, k, notReady, (WO_GT|WO_GE), pIdx); + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aTerm[j++]; nExtraReg = 1; } @@ -4613,7 +3297,7 @@ static Bitmask codeOneLoopStart( ** starting at regBase. */ regBase = codeAllEqualityTerms( - pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff + pParse, pLevel, pWC, notReady, bRev, nExtraReg, &zStartAff ); zEndAff = sqlite3DbStrDup(pParse->db, zStartAff); addrNxt = pLevel->addrNxt; @@ -4723,7 +3407,7 @@ static Bitmask codeOneLoopStart( r1 = sqlite3GetTempReg(pParse); testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ + if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); } @@ -4742,7 +3426,7 @@ static Bitmask codeOneLoopStart( /* Record the instruction used to terminate the loop. Disable ** WHERE clause terms made redundant by the index range scan. */ - if( pLevel->plan.wsFlags & WHERE_UNIQUE ){ + if( pLoop->wsFlags & WHERE_UNIQUE ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -4750,7 +3434,7 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - if( pLevel->plan.wsFlags & WHERE_COVER_SCAN ){ + if( pLoop->wsFlags & WHERE_COVER_SCAN ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ assert( pLevel->p5==0 ); @@ -4758,8 +3442,8 @@ static Bitmask codeOneLoopStart( }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - /* Case 4: Two or more separately indexed terms connected by OR + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR ** ** Example: ** @@ -4812,7 +3496,7 @@ static Bitmask codeOneLoopStart( int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - pTerm = pLevel->plan.u.pTerm; + pTerm = pLoop->aTerm[0]; assert( pTerm!=0 ); assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); @@ -4904,7 +3588,7 @@ static Bitmask codeOneLoopStart( WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); assert( pSubWInfo || pParse->nErr || pParse->db->mallocFailed ); if( pSubWInfo ){ - WhereLevel *pLvl; + WhereLoop *pSubLoop; explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); @@ -4937,13 +3621,13 @@ static Bitmask codeOneLoopStart( ** pCov to NULL to indicate that no candidate covering index will ** be available. */ - pLvl = &pSubWInfo->a[0]; - if( (pLvl->plan.wsFlags & WHERE_INDEXED)!=0 - && (pLvl->plan.wsFlags & WHERE_TEMP_INDEX)==0 - && (ii==0 || pLvl->plan.u.pIdx==pCov) + pSubLoop = pSubWInfo->a[0].pWLoop; + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (pSubLoop->wsFlags & WHERE_TEMP_INDEX)==0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) ){ - assert( pLvl->iIdxCur==iCovCur ); - pCov = pLvl->plan.u.pIdx; + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pLoop->u.btree.pIndex; }else{ pCov = 0; } @@ -4969,13 +3653,12 @@ static Bitmask codeOneLoopStart( #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ { - /* Case 5: There is no usable index. We must do a complete + /* Case 6: There is no usable index. We must do a complete ** scan of the entire table. */ static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - assert( omitTable==0 ); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); @@ -5145,16 +3828,8 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ int i; for(i=0; inLevel; i++){ - sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo; - if( pInfo ){ - /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */ - if( pInfo->needToFreeIdxStr ){ - sqlite3_free(pInfo->idxStr); - } - sqlite3DbFree(db, pInfo); - } - if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){ - Index *pIdx = pWInfo->a[i].plan.u.pIdx; + if( pWInfo->a[i].pWLoop->wsFlags & WHERE_TEMP_INDEX ){ + Index *pIdx = pWInfo->a[i].pWLoop->u.btree.pIndex; if( pIdx ){ sqlite3DbFree(db, pIdx->zColAff); sqlite3DbFree(db, pIdx); @@ -5333,6 +4008,7 @@ static int whereLoopAddBtreeIndex( rLogSize = estLog(pProbe->aiRowEst[0]); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 1; + pNew->wsFlags = savedLoop.wsFlags; pNew->u.btree.nEq = savedLoop.u.btree.nEq; pNew->nTerm = savedLoop.nTerm; if( pNew->nTerm>=pBuilder->mxTerm ) break; /* Repeated column in index */ @@ -6183,7 +4859,10 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ assert( pWInfo->nLevel==nLoop ); /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoopa[iLoop].pWLoop = pFrom->aLoop[iLoop]; + WhereLevel *pLevel = pWInfo->a + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; /* FIXME: Omit the iFrom field */ + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; } if( pFrom->isOrdered ){ pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; @@ -6302,8 +4981,6 @@ WhereInfo *sqlite3WhereBegin( WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ WhereLevel *pLevel; /* A single level in pWInfo->a[] */ - int iFrom; /* First unused FROM clause element */ - int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ int rc; /* Return code */ @@ -6476,274 +5153,7 @@ WhereInfo *sqlite3WhereBegin( } #endif - /* Chose the best index to use for each table in the FROM clause. - ** - ** This loop fills in the following fields: - ** - ** pWInfo->a[].pIdx The index to use for this level of the loop. - ** pWInfo->a[].wsFlags WHERE_xxx flags associated with pIdx - ** pWInfo->a[].nEq The number of == and IN constraints - ** pWInfo->a[].iFrom Which term of the FROM clause is being coded - ** pWInfo->a[].iTabCur The VDBE cursor for the database table - ** pWInfo->a[].iIdxCur The VDBE cursor for the index - ** pWInfo->a[].pTerm When wsFlags==WO_OR, the OR-clause term - ** - ** This loop also figures out the nesting order of tables in the FROM - ** clause. - */ - sWBI.notValid = ~(Bitmask)0; - sWBI.pOrderBy = pOrderBy; - sWBI.n = nTabList; - sWBI.pDistinct = pDistinct; - andFlags = ~0; - for(sWBI.i=iFrom=0, pLevel=pWInfo->a; sWBI.ia[j]; jiCursor); - if( (m & sWBI.notValid)==0 ){ - if( j==iFrom ) iFrom++; - continue; - } - if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ) break; - if( ++ckOptimal ) break; - if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; - } - } - assert( ckOptimal==0 || ckOptimal==1 ); - - for(isOptimal=ckOptimal; isOptimal>=0 && bestJ<0; isOptimal--){ - for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; jiFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ){ - /* This break and one like it in the ckOptimal computation loop - ** above prevent table reordering across LEFT and CROSS JOINs. - ** The LEFT JOIN case is necessary for correctness. The prohibition - ** against reordering across a CROSS JOIN is an SQLite feature that - ** allows the developer to control table reordering */ - break; - } - m = getMask(pMaskSet, sWBI.pSrc->iCursor); - if( (m & sWBI.notValid)==0 ){ - assert( j>iFrom ); - continue; - } - sWBI.notReady = (isOptimal ? m : sWBI.notValid); - if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; - - /*WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", - j, sWBI.pSrc->pTab->zName, isOptimal));*/ - assert( sWBI.pSrc->pTab ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(sWBI.pSrc->pTab) ){ - sWBI.ppIdxInfo = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(&sWBI); - }else -#endif - { - bestBtreeIndex(&sWBI); - } - assert( isOptimal || (sWBI.cost.used&sWBI.notValid)==0 ); - - /* If an INDEXED BY clause is present, then the plan must use that - ** index if it uses any index at all */ - assert( sWBI.pSrc->pIndex==0 - || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sWBI.cost.plan.u.pIdx==sWBI.pSrc->pIndex ); - - if( isOptimal && (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - notIndexed |= m; - } - if( isOptimal ){ - pWInfo->a[j].rOptCost = sWBI.cost.rCost; - }else if( ckOptimal ){ - /* If two or more tables have nearly the same outer loop cost, but - ** very different inner loop (optimal) cost, we want to choose - ** for the outer loop that table which benefits the least from - ** being in the inner loop. The following code scales the - ** outer loop cost estimate to accomplish that. */ - /*WHERETRACE((" scaling cost from %.1f to %.1f\n", - sWBI.cost.rCost, - sWBI.cost.rCost/pWInfo->a[j].rOptCost));*/ - sWBI.cost.rCost /= pWInfo->a[j].rOptCost; - } - - /* Conditions under which this table becomes the best so far: - ** - ** (1) The table must not depend on other tables that have not - ** yet run. (In other words, it must not depend on tables - ** in inner loops.) - ** - ** (2) (This rule was removed on 2012-11-09. The scaling of the - ** cost using the optimal scan cost made this rule obsolete.) - ** - ** (3) All tables have an INDEXED BY clause or this table lacks an - ** INDEXED BY clause or this table uses the specific - ** index specified by its INDEXED BY clause. This rule ensures - ** that a best-so-far is always selected even if an impossible - ** combination of INDEXED BY clauses are given. The error - ** will be detected and relayed back to the application later. - ** The NEVER() comes about because rule (2) above prevents - ** An indexable full-table-scan from reaching rule (3). - ** - ** (4) The plan cost must be lower than prior plans, where "cost" - ** is defined by the compareCost() function above. - */ - if( (sWBI.cost.used&sWBI.notValid)==0 /* (1) */ - && (nUnconstrained==0 || sWBI.pSrc->pIndex==0 /* (3) */ - || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || compareCost(&sWBI.cost, &bestPlan)) /* (4) */ - ){ - /*WHERETRACE((" === table %d (%s) is best so far\n" - " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", - j, sWBI.pSrc->pTab->zName, - sWBI.cost.rCost, sWBI.cost.plan.nRow, - sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags));*/ - bestPlan = sWBI.cost; - bestJ = j; - } - - /* In a join like "w JOIN x LEFT JOIN y JOIN z" make sure that - ** table y (and not table z) is always the next inner loop inside - ** of table x. */ - if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; - } - } - assert( bestJ>=0 ); - assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - assert( bestJ==iFrom || (pTabList->a[iFrom].jointype & JT_LEFT)==0 ); - testcase( bestJ>iFrom && (pTabList->a[iFrom].jointype & JT_CROSS)!=0 ); - testcase( bestJ>iFrom && bestJa[bestJ+1].jointype & JT_LEFT)!=0 ); - /*WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" - " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", - bestJ, pTabList->a[bestJ].pTab->zName, - pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, - bestPlan.plan.nOBSat, bestPlan.plan.wsFlags));*/ - if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ - assert( pWInfo->eDistinct==0 ); - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; - } - andFlags &= bestPlan.plan.wsFlags; - pLevel->plan = bestPlan.plan; - pLevel->iTabCur = pTabList->a[bestJ].iCursor; - testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); - testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); - if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ - if( (wctrlFlags & WHERE_ONETABLE_ONLY) - && (bestPlan.plan.wsFlags & WHERE_TEMP_INDEX)==0 - ){ - pLevel->iIdxCur = iIdxCur; - }else{ - pLevel->iIdxCur = pParse->nTab++; - } - }else{ - pLevel->iIdxCur = -1; - } - sWBI.notValid &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); - pLevel->iFrom = (u8)bestJ; - if( bestPlan.plan.nRow>=(double)1 ){ - pParse->nQueryLoop *= bestPlan.plan.nRow; - } - - /* Check that if the table scanned by this loop iteration had an - ** INDEXED BY clause attached to it, that the named index is being - ** used for the scan. If not, then query compilation has failed. - ** Return an error. - */ - pIdx = pTabList->a[bestJ].pIndex; - if( pIdx ){ - if( (bestPlan.plan.wsFlags & WHERE_INDEXED)==0 ){ - sqlite3ErrorMsg(pParse, "cannot use index: %s", pIdx->zName); - goto whereBeginError; - }else{ - /* If an INDEXED BY clause is used, the bestIndex() function is - ** guaranteed to find the index specified in the INDEXED BY clause - ** if it find an index at all. */ - assert( bestPlan.plan.u.pIdx==pIdx ); - } - } - } - WHERETRACE(("*** Optimizer Finished ***\n")); - if( pParse->nErr || db->mallocFailed ){ - goto whereBeginError; - } - if( nTabList ){ - pLevel--; - pWInfo->nOBSat = pLevel->plan.nOBSat; - }else{ - pWInfo->nOBSat = 0; - } - - /* If the total query only selects a single row, then the ORDER BY - ** clause is irrelevant. - */ - if( (andFlags & WHERE_UNIQUE)!=0 && pOrderBy ){ - assert( nTabList==0 || (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ); - pWInfo->nOBSat = pOrderBy->nExpr; - } - +#if 0 /* FIXME: Add this back in? */ /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. ** The one-pass algorithm only works if the WHERE clause constraints @@ -6754,16 +5164,6 @@ WhereInfo *sqlite3WhereBegin( pWInfo->okOnePass = 1; pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; } - -#if 1 - /* Scaffolding: Check the new query plan against the old. Report any - ** discrepencies */ - for(ii=0; iia[ii].iFrom!=pWInfo->a[ii].pWLoop->iTab ){ - sqlite3DebugPrintf("(QP-Mismatch)"); - break; - } - } #endif /* Open all tables in the pTabList and any indices selected for @@ -6776,16 +5176,17 @@ WhereInfo *sqlite3WhereBegin( Table *pTab; /* Table to open */ int iDb; /* Index of database containing table/index */ struct SrcList_item *pTabItem; + WhereLoop *pLoop; pTabItem = &pTabList->a[pLevel->iFrom]; pTab = pTabItem->pTab; - pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ /* Do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); int iCur = pTabItem->iCursor; sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); @@ -6793,7 +5194,7 @@ WhereInfo *sqlite3WhereBegin( /* noop */ }else #endif - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); @@ -6811,14 +5212,16 @@ WhereInfo *sqlite3WhereBegin( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ + if( (pLoop->wsFlags & WHERE_TEMP_INDEX)!=0 ){ constructAutomaticIndex(pParse, sWBI.pWC, pTabItem, notReady, pLevel); }else #endif - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - Index *pIx = pLevel->plan.u.pIdx; + if( pLoop->u.btree.pIndex!=0 ){ + Index *pIx = pLoop->u.btree.pIndex; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); - int iIndexCur = pLevel->iIdxCur; + /* FIXME: Might need to be the iIdxCur parameter. As an optimization + ** use pTabItem->iCursor if WHERE_IDX_ONLY */ + int iIndexCur = pLevel->iIdxCur = pParse->nTab++; assert( pIx->pSchema==pTab->pSchema ); assert( iIndexCur>=0 ); sqlite3VdbeAddOp4(v, OP_OpenRead, iIndexCur, pIx->tnum, iDb, @@ -6843,7 +5246,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->iContinue = pLevel->addrCont; } -#ifdef SQLITE_TEST /* For testing and debugging use only */ +#if defined(SQLITE_TEST) && 0 /* For testing and debugging use only */ /* Record in the query plan information about the current table ** and the index used to access it (if any). If the table itself ** is not used, its name is just '{}'. If no index is used @@ -6919,6 +5322,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ Vdbe *v = pParse->pVdbe; int i; WhereLevel *pLevel; + WhereLoop *pLoop; SrcList *pTabList = pWInfo->pTabList; sqlite3 *db = pParse->db; @@ -6927,12 +5331,13 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); sqlite3VdbeChangeP5(v, pLevel->p5); } - if( pLevel->plan.wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite3VdbeResolveLabel(v, pLevel->addrNxt); @@ -6947,9 +5352,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLevel->iLeftJoin ){ int addr; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); } if( pLevel->iIdxCur>=0 ){ @@ -6977,15 +5382,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; Table *pTab = pTabItem->pTab; assert( pTab!=0 ); + pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)==0 && pTab->pSelect==0 && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ - int ws = pLevel->plan.wsFlags; + int ws = pLoop->wsFlags; if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } - if( (ws & WHERE_INDEXED)!=0 && (ws & WHERE_TEMP_INDEX)==0 ){ + if( (ws & WHERE_INDEXED)!=0 && (ws & (WHERE_IPK|WHERE_TEMP_INDEX))==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); } } @@ -7003,12 +5409,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** that reference the table and converts them into opcodes that ** reference the index. */ - if( pLevel->plan.wsFlags & WHERE_INDEXED ){ - pIdx = pLevel->plan.u.pIdx; - }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ + if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ + pIdx = pLoop->u.btree.pIndex; + }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed){ + if( pIdx && !db->mallocFailed ){ int k, j, last; VdbeOp *pOp; @@ -7024,8 +5430,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ break; } } - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || jnColumn ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || jnColumn ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; From 70bdcc738e6cf06cd68bdfdf7322f8412703d325 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 May 2013 19:28:34 +0000 Subject: [PATCH 025/109] The expected result in a test case can be of the form "*glob*" or "~*glob*" to match or not match the GLOB pattern. This is useful for matching EXPLAIN QUERY PLAN output that contains regular expression syntax characters like "?", "(", and ")". FossilOrigin-Name: a3b4e261bd7e278f150872cce7b020af5ad8d2ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/tester.tcl | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7344d1b38d..e5e6cf6db7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cut\sover\sthe\sNGQP\squery\splanner.\s\sRemove\slots\sof\slegacy\scode.\s\s\nThis\scheck-in\scompiles\sbut\sdoes\snot\swork.\s\sThe\stest\ssuite\sgets\sincorrect\nanswers\sand\scrashes. -D 2013-05-30T17:43:19.293 +C The\sexpected\sresult\sin\sa\stest\scase\scan\sbe\sof\sthe\sform\s"*glob*"\sor\s"~*glob*"\sto\s\nmatch\sor\snot\smatch\sthe\sGLOB\spattern.\s\sThis\sis\suseful\sfor\smatching\nEXPLAIN\sQUERY\sPLAN\soutput\sthat\scontains\sregular\sexpression\ssyntax\scharacters\nlike\s"?",\s"(",\sand\s")". +D 2013-05-30T19:28:34.431 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -806,7 +806,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 693700993b7e0a5b7b2171b291a54cdbad1d86ff +F test/tester.tcl 63b24679c75a952c51f924de2802b2b57cddd22d F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aebe1f2603ee04d792af73aaf59625bda99d5cd1 -R cdb9a6b0104d69931bfa04869e682c85 +P 001539df4b74dc1cbceb010a91407003ab4d8735 +R ae73b581603fe5cb9a5fdce5dbec1c53 U drh -Z 36d07876c5b50e19bee990a15e1431a7 +Z 68529a7f58c35b9a5dca06d2c9973260 diff --git a/manifest.uuid b/manifest.uuid index ea56fcb1ab..08e57409dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -001539df4b74dc1cbceb010a91407003ab4d8735 \ No newline at end of file +a3b4e261bd7e278f150872cce7b020af5ad8d2ed \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index cad9807ed7..32dca4cb78 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -551,6 +551,9 @@ proc do_test {name cmd expected} { fail_test $name } else { if {[regexp {^~?/.*/$} $expected]} { + # "expected" is of the form "/PATTERN/" then the result if correct if + # regular expression PATTERN matches the result. "~/PATTERN/" means + # the regular expression must not match. if {[string index $expected 0]=="~"} { set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] set ok [expr {![regexp $re $result]}] @@ -558,6 +561,16 @@ proc do_test {name cmd expected} { set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] set ok [regexp $re $result] } + } elseif {[regexp {^~?\*.*\*$} $expected]} { + # "expected" is of the form "*GLOB*" then the result if correct if + # glob pattern GLOB matches the result. "~/GLOB/" means + # the glob must not match. + if {[string index $expected 0]=="~"} { + set e [string range $expected 1 end] + set ok [expr {![string match $e $result]}] + } else { + set ok [string match $expected $result] + } } else { set ok [expr {[string compare $result $expected]==0}] } From 6fa978dade88b77ca42657b6b433f1fe55e8f798 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 May 2013 19:29:19 +0000 Subject: [PATCH 026/109] Futher simplifications to the NGQP. Fix some test cases to use EXPLAIN QUERY PLAN rather than the (now obsolete) sqlite_query_plan global variable. FossilOrigin-Name: ae985db4fa08b5efbef5a834e852f0b05101264b --- manifest | 16 +++--- manifest.uuid | 2 +- src/test1.c | 9 +-- src/where.c | 142 +++++++++++------------------------------------- test/where.test | 49 ++++++++--------- 5 files changed, 70 insertions(+), 148 deletions(-) diff --git a/manifest b/manifest index e5e6cf6db7..b536417138 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sexpected\sresult\sin\sa\stest\scase\scan\sbe\sof\sthe\sform\s"*glob*"\sor\s"~*glob*"\sto\s\nmatch\sor\snot\smatch\sthe\sGLOB\spattern.\s\sThis\sis\suseful\sfor\smatching\nEXPLAIN\sQUERY\sPLAN\soutput\sthat\scontains\sregular\sexpression\ssyntax\scharacters\nlike\s"?",\s"(",\sand\s")". -D 2013-05-30T19:28:34.431 +C Futher\ssimplifications\sto\sthe\sNGQP.\s\sFix\ssome\stest\scases\sto\suse\nEXPLAIN\sQUERY\sPLAN\srather\sthan\sthe\s(now\sobsolete)\ssqlite_query_plan\nglobal\svariable. +D 2013-05-30T19:29:19.963 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,7 +225,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c 6d2a340eea1d866bf7059894491652a69a7ee802 +F src/test1.c 5f9837aee1a0708ab2dbf67ce0b9c18e658cd36b F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 296baae1cbfd99527d8c87f9773a6c98116f6b8a +F src/where.c 3e9b7b3323f78ec3056c5e3aa3443661bc590054 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1028,7 +1028,7 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 15ac8611c9439a2c5f4a6ac10cfe4c1ec9854c24 +F test/where.test 054a6b6f7933c5a5f50d0bcd650b5eccb450cc81 F test/where2.test 399b3178289925a0aa976b3d60ef139740540ecd F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 001539df4b74dc1cbceb010a91407003ab4d8735 -R ae73b581603fe5cb9a5fdce5dbec1c53 +P a3b4e261bd7e278f150872cce7b020af5ad8d2ed +R b028bf2657ab598aee5d4698a0488fdc U drh -Z 68529a7f58c35b9a5dca06d2c9973260 +Z 3432943259d0f5aa9c669ba0e80caf85 diff --git a/manifest.uuid b/manifest.uuid index 08e57409dd..bdeb520d01 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3b4e261bd7e278f150872cce7b020af5ad8d2ed \ No newline at end of file +ae985db4fa08b5efbef5a834e852f0b05101264b \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index bd6d4a4cfd..8edaedab8a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6302,8 +6302,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite3WalTrace; #endif #ifdef SQLITE_TEST - extern char sqlite3_query_plan[]; - static char *query_plan = sqlite3_query_plan; #ifdef SQLITE_ENABLE_FTS3 extern int sqlite3_fts3_enable_parentheses; #endif @@ -6357,8 +6355,11 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&sqlite3_os_type, TCL_LINK_INT); #endif #ifdef SQLITE_TEST - Tcl_LinkVar(interp, "sqlite_query_plan", - (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); + { + static const char *query_plan = "*** OBSOLETE VARIABLE ***"; + Tcl_LinkVar(interp, "sqlite_query_plan", + (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); + } #endif #ifdef SQLITE_DEBUG Tcl_LinkVar(interp, "sqlite_where_trace", diff --git a/src/where.c b/src/where.c index a9a431ba9a..8ddbd5d72b 100644 --- a/src/where.c +++ b/src/where.c @@ -326,33 +326,22 @@ struct WhereLoopBuilder { ** WhereLevel.wsFlags. These flags determine which search ** strategies are appropriate. */ -#define WHERE_ROWID_EQ 0x00000001 /* rowid=EXPR or rowid IN (...) */ -#define WHERE_ROWID_RANGE 0x00000002 /* rowidEXPR */ -#define WHERE_NULL_OK 0x00000004 /* Ok to use WO_ISNULL */ -#define WHERE_IPK 0x00000008 /* x is the INTEGER PRIMARY KEY */ -#define WHERE_COLUMN_EQ 0x00000010 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00000020 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00000040 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00000080 /* x IS NULL */ -#define WHERE_INDEXED 0x000000f0 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x000200f3 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000100f1 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00000100 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00000300 /* Both x>EXPR and xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and xiTable; sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - assert( pLoop->wsFlags & WHERE_IN_ABLE ); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); } @@ -3193,7 +3183,8 @@ static Bitmask codeOneLoopStart( sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } - }else if( pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ|WHERE_IDX_ONLY) ){ + }else if( pLoop->wsFlags & (WHERE_COLUMN_RANGE | WHERE_COLUMN_NULL | + WHERE_COLUMN_EQ | WHERE_IDX_ONLY) ){ /* Case 4: A scan using an index. ** ** The WHERE clause may contain zero or more equality @@ -3746,18 +3737,6 @@ static Bitmask codeOneLoopStart( return newNotReady; } -#if defined(SQLITE_TEST) -/* -** The following variable holds a text description of query plan generated -** by the most recent call to sqlite3WhereBegin(). Each call to WhereBegin -** overwrites the previous. This information is used for testing and -** analysis only. -*/ -char sqlite3_query_plan[BMS*2*40]; /* Text of the join */ -static int nQPlan = 0; /* Next free slow in _query_plan[] */ - -#endif /* SQLITE_TEST */ - #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes @@ -3793,7 +3772,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %08x N %d", p->wsFlags, p->nTerm); + sqlite3DebugPrintf(" fg %05x N %d", p->wsFlags, p->nTerm); sqlite3DebugPrintf(" cost %.2g,%.2g,%.2g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } @@ -3810,6 +3789,10 @@ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ if( p->u.vtab.needFree ) sqlite3_free(p->u.vtab.idxStr); p->u.vtab.needFree = 0; p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_TEMP_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; } } @@ -3826,16 +3809,6 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - if( pWInfo->a[i].pWLoop->wsFlags & WHERE_TEMP_INDEX ){ - Index *pIdx = pWInfo->a[i].pWLoop->u.btree.pIndex; - if( pIdx ){ - sqlite3DbFree(db, pIdx->zColAff); - sqlite3DbFree(db, pIdx); - } - } - } whereClauseClear(pWInfo->pWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; @@ -4026,10 +3999,14 @@ static int whereLoopAddBtreeIndex( } pNew->u.btree.nEq++; pNew->nOut = (double)iRowEst * nInMul * nIn; - }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ + }else if( pTerm->eOperator & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; pNew->u.btree.nEq++; pNew->nOut = (double)iRowEst * nInMul; + }else if( pTerm->eOperator & (WO_ISNULL) ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + pNew->u.btree.nEq++; + pNew->nOut = (double)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pNew->nOut = savedLoop.nOut/3; @@ -4197,7 +4174,7 @@ static int whereLoopAddBtree( m &= ~(((Bitmask)1)<wsFlags = (m==0) ? WHERE_IDX_ONLY : 0; + pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; /* Full scan via index */ if( (m==0 || b) && pProbe->bUnordered==0 ){ @@ -5246,62 +5223,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->iContinue = pLevel->addrCont; } -#if defined(SQLITE_TEST) && 0 /* For testing and debugging use only */ - /* Record in the query plan information about the current table - ** and the index used to access it (if any). If the table itself - ** is not used, its name is just '{}'. If no index is used - ** the index is listed as "{}". If the primary key is used the - ** index name is '*'. - */ - for(ii=0; iia[ii]; - w = pLevel->plan.wsFlags; - pTabItem = &pTabList->a[pLevel->iFrom]; - z = pTabItem->zAlias; - if( z==0 ) z = pTabItem->pTab->zName; - n = sqlite3Strlen30(z); - if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ - if( (w & WHERE_IDX_ONLY)!=0 && (w & WHERE_COVER_SCAN)==0 ){ - memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); - nQPlan += 2; - }else{ - memcpy(&sqlite3_query_plan[nQPlan], z, n); - nQPlan += n; - } - sqlite3_query_plan[nQPlan++] = ' '; - } - testcase( w & WHERE_ROWID_EQ ); - testcase( w & WHERE_ROWID_RANGE ); - if( w & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); - nQPlan += 2; - }else if( (w & WHERE_INDEXED)!=0 && (w & WHERE_COVER_SCAN)==0 ){ - n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); - if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ - memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); - nQPlan += n; - sqlite3_query_plan[nQPlan++] = ' '; - } - }else{ - memcpy(&sqlite3_query_plan[nQPlan], "{} ", 3); - nQPlan += 3; - } - } - while( nQPlan>0 && sqlite3_query_plan[nQPlan-1]==' ' ){ - sqlite3_query_plan[--nQPlan] = 0; - } - sqlite3_query_plan[nQPlan] = 0; - nQPlan = 0; -#endif /* SQLITE_TEST // Testing and debugging use only */ - - /* Record the continuation address in the WhereInfo structure. Then - ** clean up and return. - */ + /* Done. */ return pWInfo; /* Jump here if malloc fails */ diff --git a/test/where.test b/test/where.test index 2dbc283417..8610fa6f3d 100644 --- a/test/where.test +++ b/test/where.test @@ -65,9 +65,9 @@ proc count sql { do_test where-1.1.1 { count {SELECT x, y, w FROM t1 WHERE w=10} } {3 121 10 3} -do_test where-1.1.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.1.2 { + SELECT x, y, w FROM t1 WHERE w=10 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} do_test where-1.1.3 { db status step } {0} @@ -77,15 +77,15 @@ do_test where-1.1.4 { do_test where-1.1.5 { db status step } {99} -do_test where-1.1.6 { - set sqlite_query_plan -} {t1 {}} +do_eqp_test where-1.1.6 { + SELECT x, y, w FROM t1 WHERE +w=10 +} {*SCAN TABLE t1 *} do_test where-1.1.7 { count {SELECT x, y, w AS abc FROM t1 WHERE abc=10} } {3 121 10 3} -do_test where-1.1.8 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.1.8 { + SELECT x, y, w AS abc FROM t1 WHERE abc=10 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} do_test where-1.1.9 { db status step } {0} @@ -104,21 +104,21 @@ do_test where-1.3.2 { do_test where-1.4.1 { count {SELECT w, x, y FROM t1 WHERE 11=w AND x>2} } {11 3 144 3} -do_test where-1.4.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.4.2 { + SELECT w, x, y FROM t1 WHERE 11=w AND x>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} do_test where-1.4.3 { count {SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2} } {11 3 144 3} -do_test where-1.4.4 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.4.4 { + SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} do_test where-1.5 { count {SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2} } {3 144 3} -do_test where-1.5.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.5.2 { + SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} do_test where-1.6 { count {SELECT x, y FROM t1 WHERE y<200 AND x>2 AND w=11} } {3 144 3} @@ -128,13 +128,12 @@ do_test where-1.7 { do_test where-1.8 { count {SELECT x, y FROM t1 WHERE w>10 AND y=144 AND x=3} } {3 144 3} -do_test where-1.8.2 { - set sqlite_query_plan -} {t1 i1xy} -do_test where-1.8.3 { - count {SELECT x, y FROM t1 WHERE y=144 AND x=3} - set sqlite_query_plan -} {{} i1xy} +do_eqp_test where-1.8.2 { + SELECT x, y FROM t1 WHERE w>10 AND y=144 AND x=3 +} {*SEARCH TABLE t1 USING INDEX i1xy (x=? AND y=?) *} +do_eqp_test where-1.8.3 { + SELECT x, y FROM t1 WHERE y=144 AND x=3 +} {*SEARCH TABLE t1 USING COVERING INDEX i1xy (x=? AND y=?) *} do_test where-1.9 { count {SELECT x, y FROM t1 WHERE y=144 AND w>10 AND x=3} } {3 144 3} From 1b0f026dbb0fe56141cb198a2722d42c7be75cb2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 May 2013 22:27:09 +0000 Subject: [PATCH 027/109] Incremental check-in with various NGQP fixes. Many tests still fail. FossilOrigin-Name: a51d8c92496436488e1a6eabd85785e8fedf2736 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 +++++++++++++++++++---- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b536417138..e2d80171bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Futher\ssimplifications\sto\sthe\sNGQP.\s\sFix\ssome\stest\scases\sto\suse\nEXPLAIN\sQUERY\sPLAN\srather\sthan\sthe\s(now\sobsolete)\ssqlite_query_plan\nglobal\svariable. -D 2013-05-30T19:29:19.963 +C Incremental\scheck-in\swith\svarious\sNGQP\sfixes.\s\sMany\stests\sstill\sfail. +D 2013-05-30T22:27:09.004 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3e9b7b3323f78ec3056c5e3aa3443661bc590054 +F src/where.c 28e0cefe34698b6de4e6f5484cd987ae993c49f2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a3b4e261bd7e278f150872cce7b020af5ad8d2ed -R b028bf2657ab598aee5d4698a0488fdc +P ae985db4fa08b5efbef5a834e852f0b05101264b +R 5e6d0ab71af5f0e6d4eac06636af5d1b U drh -Z 3432943259d0f5aa9c669ba0e80caf85 +Z 130160475bcddf72f5447461f586642a diff --git a/manifest.uuid b/manifest.uuid index bdeb520d01..22be55a88a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae985db4fa08b5efbef5a834e852f0b05101264b \ No newline at end of file +a51d8c92496436488e1a6eabd85785e8fedf2736 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8ddbd5d72b..5663e68491 100644 --- a/src/where.c +++ b/src/where.c @@ -3183,8 +3183,7 @@ static Bitmask codeOneLoopStart( sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } - }else if( pLoop->wsFlags & (WHERE_COLUMN_RANGE | WHERE_COLUMN_NULL | - WHERE_COLUMN_EQ | WHERE_IDX_ONLY) ){ + }else if( pLoop->wsFlags & WHERE_INDEXED ){ /* Case 4: A scan using an index. ** ** The WHERE clause may contain zero or more equality @@ -3425,7 +3424,8 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - if( pLoop->wsFlags & WHERE_COVER_SCAN ){ + if( (pLoop->wsFlags & (WHERE_COLUMN_EQ | WHERE_COLUMN_RANGE | + WHERE_COLUMN_NULL | WHERE_COLUMN_IN))==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ assert( pLevel->p5==0 ); @@ -3875,6 +3875,18 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ */ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue; + if( p->nTermnTerm + && (p->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && p->u.btree.pIndex==pTemplate->u.btree.pIndex + && p->prereq==pTemplate->prereq + ){ + /* Overwrite an existing WhereLoop with an similar one that uses + ** more terms of the index */ + pNext = p->pNextLoop; + whereLoopClear(db, p); + break; + } if( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun @@ -4590,7 +4602,10 @@ static int wherePathSatisfiesOrderBy( iColumn = -1; revIdx = 0; } - if( pOBExpr->iColumn!=iColumn ) return 0; + if( pOBExpr->iColumn!=iColumn ){ + if( ju.btree.nEq ){ nUsed--; continue; } + return 0; + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); if( !pColl ) pColl = db->pDfltColl; From dc3cd4b0208efa1f0b2ac78bd417cf511b510e2d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 May 2013 23:21:20 +0000 Subject: [PATCH 028/109] Improvements to the ORDER BY suppressor in the NGQP. FossilOrigin-Name: 24a2e9ddcecd3926817d77abbb75d068ee7140ad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 20 +++++++++++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e2d80171bf..f227274658 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\scheck-in\swith\svarious\sNGQP\sfixes.\s\sMany\stests\sstill\sfail. -D 2013-05-30T22:27:09.004 +C Improvements\sto\sthe\sORDER\sBY\ssuppressor\sin\sthe\sNGQP. +D 2013-05-30T23:21:20.132 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 28e0cefe34698b6de4e6f5484cd987ae993c49f2 +F src/where.c 38beadcfdf477b0f9a07c4d0a479ab522843b14c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ae985db4fa08b5efbef5a834e852f0b05101264b -R 5e6d0ab71af5f0e6d4eac06636af5d1b +P a51d8c92496436488e1a6eabd85785e8fedf2736 +R 405146591851ecbfbac166e3c1b2db18 U drh -Z 130160475bcddf72f5447461f586642a +Z 4964f74ab7b0cf98172fb87fb30a1333 diff --git a/manifest.uuid b/manifest.uuid index 22be55a88a..ee070b19f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a51d8c92496436488e1a6eabd85785e8fedf2736 \ No newline at end of file +24a2e9ddcecd3926817d77abbb75d068ee7140ad \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5663e68491..9d454bc80c 100644 --- a/src/where.c +++ b/src/where.c @@ -4588,6 +4588,7 @@ static int wherePathSatisfiesOrderBy( j = 0; revSet = rev = 0; for(j=0; j<=nColumn && nUseda[nUsed].pExpr); assert( pOBExpr->op==TK_COLUMN ); if( pOBExpr->iTable!=iCur ) break; @@ -4602,20 +4603,25 @@ static int wherePathSatisfiesOrderBy( iColumn = -1; revIdx = 0; } + skipable = ju.btree.nEq && pLoop->aTerm[j]->eOperator!=WO_IN; if( pOBExpr->iColumn!=iColumn ){ - if( ju.btree.nEq ){ nUsed--; continue; } + if( skipable ){ nUsed--; continue; } return 0; } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); if( !pColl ) pColl = db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) return 0; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ){ + return 0; + } } - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[nUsed].sortOrder ) return 0; - }else{ - rev = revIdx ^ pOrderBy->a[nUsed].sortOrder; - revSet = 1; + if( !skipable ){ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[nUsed].sortOrder ) return 0; + }else{ + rev = revIdx ^ pOrderBy->a[nUsed].sortOrder; + revSet = 1; + } } } if( rev ) revMask |= ((Bitmask)1)< Date: Fri, 31 May 2013 11:57:39 +0000 Subject: [PATCH 029/109] Fix the constructAutomaticIndex() routine so that it works with NGQP. FossilOrigin-Name: 5e1e61399513b4a95fd93df2377a2603f1670063 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 12 +++++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f227274658..dbb58ea9b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sORDER\sBY\ssuppressor\sin\sthe\sNGQP. -D 2013-05-30T23:21:20.132 +C Fix\sthe\sconstructAutomaticIndex()\sroutine\sso\sthat\sit\sworks\swith\sNGQP. +D 2013-05-31T11:57:39.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 38beadcfdf477b0f9a07c4d0a479ab522843b14c +F src/where.c 530dd22d4f803747e964edcb052d699c4c92e1a1 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a51d8c92496436488e1a6eabd85785e8fedf2736 -R 405146591851ecbfbac166e3c1b2db18 +P 24a2e9ddcecd3926817d77abbb75d068ee7140ad +R 92deaa75392a3f586cae6bde8f65a470 U drh -Z 4964f74ab7b0cf98172fb87fb30a1333 +Z 293d3cdab228aa747c73a2994c326e63 diff --git a/manifest.uuid b/manifest.uuid index ee070b19f5..57b2c0e51a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24a2e9ddcecd3926817d77abbb75d068ee7140ad \ No newline at end of file +5e1e61399513b4a95fd93df2377a2603f1670063 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9d454bc80c..dea0ff2fdd 100644 --- a/src/where.c +++ b/src/where.c @@ -1850,6 +1850,7 @@ static void constructAutomaticIndex( WhereLoop *pLoop; /* The Loop object */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ + const int mxConstraint = 10; /* Maximum number of constraints */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1864,20 +1865,25 @@ static void constructAutomaticIndex( pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; - for(pTerm=pWC->a; pTermaTerm = sqlite3DbRealloc(pParse->db, pLoop->aTerm, + mxConstraint*sizeof(pLoop->aTerm[0])); + if( pLoop->aTerm==0 ) return; + for(pTerm=pWC->a; pTermnTermu.leftColumn; Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<aTerm[nColumn++] = pTerm; idxCols |= cMask; } } } assert( nColumn>0 ); - pLoop->u.btree.nEq = nColumn; + pLoop->u.btree.nEq = pLoop->nTerm = nColumn; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_TEMP_INDEX; /* Count the number of additional columns needed to create a ** covering index. A "covering index" is an index that contains all From d044d209f3535e4a514dd9bbad7000e4432e685e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 12:43:55 +0000 Subject: [PATCH 030/109] Improved detection of unnecessary ORDER BY clauses. FossilOrigin-Name: 58805eb36b9975706e2c4e382689519454e9a504 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 35 ++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index dbb58ea9b3..edf228f872 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sconstructAutomaticIndex()\sroutine\sso\sthat\sit\sworks\swith\sNGQP. -D 2013-05-31T11:57:39.405 +C Improved\sdetection\sof\sunnecessary\sORDER\sBY\sclauses. +D 2013-05-31T12:43:55.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 530dd22d4f803747e964edcb052d699c4c92e1a1 +F src/where.c 0e1c41804f89dbfd48fbe8dfa68adc6b6d682a75 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 24a2e9ddcecd3926817d77abbb75d068ee7140ad -R 92deaa75392a3f586cae6bde8f65a470 +P 5e1e61399513b4a95fd93df2377a2603f1670063 +R 17288f799b200fcc6199a08359305634 U drh -Z 293d3cdab228aa747c73a2994c326e63 +Z 9542ccfa5fa5ec11248b1f41f8b389ee diff --git a/manifest.uuid b/manifest.uuid index 57b2c0e51a..ec86401b40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e1e61399513b4a95fd93df2377a2603f1670063 \ No newline at end of file +58805eb36b9975706e2c4e382689519454e9a504 \ No newline at end of file diff --git a/src/where.c b/src/where.c index dea0ff2fdd..249cc724b1 100644 --- a/src/where.c +++ b/src/where.c @@ -3832,7 +3832,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** is better and has fewer dependencies. Or the template will be ignored ** and no insert will occur if an existing WhereLoop is faster and has ** fewer dependencies than the template. Otherwise a new WhereLoop is -** added based no the template. +** added based on the template. ** ** If pBuilder->pBest is not NULL then we only care about the very ** best template and that template should be stored in pBuilder->pBest. @@ -3848,6 +3848,8 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** (2) They have the same iSortIdx. ** (3) The template has same or fewer dependencies than the current loop ** (4) The template has the same or lower cost than the current loop +** (5) The template uses more terms of the same index but has no additional +** dependencies */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; @@ -4100,7 +4102,7 @@ static int whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ - int iSortIdx = 0; /* Index number */ + int iSortIdx = 1; /* Index number */ int b; /* A boolean value */ double rSize; /* number of rows in the table */ double rLogSize; /* Logarithm of the number of rows in the table */ @@ -4179,6 +4181,7 @@ static int whereLoopAddBtree( pNew->wsFlags = WHERE_IPK; /* Full table scan */ + pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; pNew->rRun = (rSize + rLogSize)*(3+b); /* 4x penalty for a full-scan */ rc = whereLoopInsert(pBuilder, pNew); @@ -4513,8 +4516,9 @@ static int wherePathSatisfiesOrderBy( u8 revSet; /* True if rev is known */ u8 rev; /* Composite sort order */ u8 revIdx; /* Index sort order */ - u8 isUnique; - u8 requireUnique = 0; + u8 isOneRow; /* Current WhereLoop is a one-row loop */ + u8 requireOneRow = 0; /* All subsequent loops must be one-row */ + u8 isUniqueIdx; /* Current WhereLoop uses a unique index */ u16 nColumn; u16 nOrderBy; int i, j; @@ -4547,6 +4551,10 @@ static int wherePathSatisfiesOrderBy( ** of the index. ** ** (4) Index columns past nEq must match ORDER BY terms one-for-one. + ** + ** (5) If all columns of a UNIQUE index have been matched against ORDER BY + ** terms, then any subsequent entries in the ORDER BY clause against the + ** same table can be skipped. */ assert( pOrderBy!=0 ); @@ -4569,10 +4577,10 @@ static int wherePathSatisfiesOrderBy( for(i=0; i<=nLoop && nUsedaLoop[i] : pLast; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - isUnique = 1; + isOneRow = isUniqueIdx = 1; if( pLoop->wsFlags & WHERE_IPK ){ - if( (pLoop->wsFlags & WHERE_COLUMN_IN)!=0 ) isUnique = 0; - if( pLoop->u.btree.nEq!=1 ) isUnique = 0; + if( (pLoop->wsFlags & WHERE_COLUMN_IN)!=0 ) isOneRow = 0; + if( pLoop->u.btree.nEq!=1 ) isOneRow = 0; pIndex = 0; nColumn = 0; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ @@ -4580,16 +4588,16 @@ static int wherePathSatisfiesOrderBy( }else{ nColumn = pIndex->nColumn; if( pIndex->onError==OE_None ){ - isUnique = 0; + isOneRow = isUniqueIdx = 0; }else if( (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_RANGE |WHERE_COLUMN_NULL))!=0 ){ - isUnique = 0; + isOneRow = 0; }else if( pLoop->u.btree.nEq < pIndex->nColumn ){ - isUnique = 0; + isOneRow = 0; } } - if( !isUnique && requireUnique ) return 0; - requireUnique = !isUnique; + if( !isOneRow && requireOneRow ) return 0; + requireOneRow = !isOneRow; iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; j = 0; revSet = rev = 0; @@ -4598,7 +4606,7 @@ static int wherePathSatisfiesOrderBy( pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[nUsed].pExpr); assert( pOBExpr->op==TK_COLUMN ); if( pOBExpr->iTable!=iCur ) break; - if( isUnique ) continue; + if( isOneRow ){ j--; continue; } if( jaiColumn[j]; @@ -4629,6 +4637,7 @@ static int wherePathSatisfiesOrderBy( revSet = 1; } } + if( j>=nColumn-1 && isUniqueIdx ){ j--; isOneRow = 1; } } if( rev ) revMask |= ((Bitmask)1)< Date: Fri, 31 May 2013 13:36:32 +0000 Subject: [PATCH 031/109] Futher enhancements to the ORDER BY optimizer. FossilOrigin-Name: d8efa5f8b60bc4c8df8bfad077f87f76f7ee9bf6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index edf228f872..24f702b1fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sof\sunnecessary\sORDER\sBY\sclauses. -D 2013-05-31T12:43:55.143 +C Futher\senhancements\sto\sthe\sORDER\sBY\soptimizer. +D 2013-05-31T13:36:32.993 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0e1c41804f89dbfd48fbe8dfa68adc6b6d682a75 +F src/where.c 8e9922815b07290982cfc826497d128115ce358c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5e1e61399513b4a95fd93df2377a2603f1670063 -R 17288f799b200fcc6199a08359305634 +P 58805eb36b9975706e2c4e382689519454e9a504 +R d753e9381f16048231d3d54f23e54ee3 U drh -Z 9542ccfa5fa5ec11248b1f41f8b389ee +Z 2bef9deff008e88de89f5b60b513bb1c diff --git a/manifest.uuid b/manifest.uuid index ec86401b40..0e1ff8e72f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58805eb36b9975706e2c4e382689519454e9a504 \ No newline at end of file +d8efa5f8b60bc4c8df8bfad077f87f76f7ee9bf6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 249cc724b1..d4f9a71441 100644 --- a/src/where.c +++ b/src/where.c @@ -4510,6 +4510,7 @@ static int wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ WherePath *pPath, /* The WherePath to check */ int nLoop, /* Number of entries in pPath->aLoop[] */ + int isLastLoop, /* True for the very last loop */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* Mask of WhereLoops to run in reverse order */ ){ @@ -4569,10 +4570,12 @@ static int wherePathSatisfiesOrderBy( /* Sorting is always required if any term of the ORDER BY is not a ** column reference */ nOrderBy = pOrderBy->nExpr; +#if 0 for(i=0; ia[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) return 0; } +#endif for(i=0; i<=nLoop && nUsedaLoop[i] : pLast; @@ -4604,7 +4607,7 @@ static int wherePathSatisfiesOrderBy( for(j=0; j<=nColumn && nUseda[nUsed].pExpr); - assert( pOBExpr->op==TK_COLUMN ); + if( pOBExpr->op!=TK_COLUMN ) return 0; if( pOBExpr->iTable!=iCur ) break; if( isOneRow ){ j--; continue; } if( j=nColumn-1 && isUniqueIdx ){ j--; isOneRow = 1; } + if( j>=nColumn-1 && isUniqueIdx ){ + if( isLastLoop && i==nLoop ) break; + j--; + isOneRow = 1; + } } if( rev ) revMask |= ((Bitmask)1)<rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, + switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, iLoop==nLoop-1, pWLoop, &revMask) ){ case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ isOrdered = 1; From 84f1414b1a5b89e33a31fb988e309f1ae906b0fb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 14:31:11 +0000 Subject: [PATCH 032/109] Enhance the shell to provide more flexibility when entering numeric arguments on dot-commands. In particular, allow hex arguments to .wheretrace. FossilOrigin-Name: b9578c371ee569dca6a0964019959a93407c8ef9 --- manifest | 12 +++---- manifest.uuid | 2 +- src/shell.c | 88 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 73 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 24f702b1fb..b3d407069e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Futher\senhancements\sto\sthe\sORDER\sBY\soptimizer. -D 2013-05-31T13:36:32.993 +C Enhance\sthe\sshell\sto\sprovide\smore\sflexibility\swhen\sentering\snumeric\sarguments\non\sdot-commands.\s\sIn\sparticular,\sallow\shex\sarguments\sto\s.wheretrace. +D 2013-05-31T14:31:11.784 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 22ee971346a736ddefdc4497d07c92f2e9978afc -F src/shell.c 9a18124ff209ca308d786c99a466e8e270193ff3 +F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 58805eb36b9975706e2c4e382689519454e9a504 -R d753e9381f16048231d3d54f23e54ee3 +P d8efa5f8b60bc4c8df8bfad077f87f76f7ee9bf6 +R cab66f9edd3249c76db6815f19ee5773 U drh -Z 2bef9deff008e88de89f5b60b513bb1c +Z 93073f371b16237bef628d122f206ede diff --git a/manifest.uuid b/manifest.uuid index 0e1ff8e72f..277db05743 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8efa5f8b60bc4c8df8bfad077f87f76f7ee9bf6 \ No newline at end of file +b9578c371ee569dca6a0964019959a93407c8ef9 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index d5fa9c6963..9f2f46bd18 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1523,21 +1523,14 @@ static void resolve_backslashes(char *z){ } /* -** Interpret zArg as a boolean value. Return either 0 or 1. +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. */ -static int booleanValue(char *zArg){ - int i; - for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} - if( i>0 && zArg[i]==0 ) return atoi(zArg); - if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ - return 1; - } - if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ - return 0; - } - fprintf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", - zArg); - return 0; +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; } /* @@ -1564,9 +1557,18 @@ static sqlite3_int64 integerValue(const char *zArg){ }else if( zArg[0]=='+' ){ zArg++; } - while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; - zArg++; + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } } for(i=0; i=0; i++){} + }else{ + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + } + if( i>0 && zArg[i]==0 ) return (int)(integerValue(zArg) & 0xffffffff); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; + } + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + fprintf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", + zArg); + return 0; +} + /* ** Close an output file, assuming it is not stderr or stdout */ @@ -1808,7 +1833,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ - if( nArg>1 && (rc = atoi(azArg[1]))!=0 ) exit(rc); + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; }else @@ -2305,6 +2330,25 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + /* Undocumented commands for internal testing. Subject to change + ** without notice. */ + if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ + if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + int i, v; + for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); + } + } + if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + int i; sqlite3_int64 v; + for(i=1; iout, "%s: %lld 0x%llx\n", azArg[i], v, v); + } + } + }else + if( c=='s' && strncmp(azArg[0], "separator", n)==0 && nArg==2 ){ sqlite3_snprintf(sizeof(p->separator), p->separator, "%.*s", (int)sizeof(p->separator)-1, azArg[1]); @@ -2458,7 +2502,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } } } - if( testctrl<0 ) testctrl = atoi(azArg[1]); + if( testctrl<0 ) testctrl = (int)integerValue(azArg[1]); if( (testctrlSQLITE_TESTCTRL_LAST) ){ fprintf(stderr,"Error: invalid testctrl option: %s\n", azArg[1]); }else{ @@ -2505,7 +2549,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ case SQLITE_TESTCTRL_ASSERT: case SQLITE_TESTCTRL_ALWAYS: if( nArg==3 ){ - int opt = atoi(azArg[2]); + int opt = booleanValue(azArg[2]); rc = sqlite3_test_control(testctrl, opt); fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { @@ -2542,7 +2586,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 && nArg==2 ){ open_db(p); - sqlite3_busy_timeout(p->db, atoi(azArg[1])); + sqlite3_busy_timeout(p->db, (int)integerValue(azArg[1])); }else if( HAS_TIMER && c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 @@ -2592,7 +2636,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ int j; assert( nArg<=ArraySize(azArg) ); for(j=1; jcolWidth); j++){ - p->colWidth[j-1] = atoi(azArg[j]); + p->colWidth[j-1] = (int)integerValue(azArg[j]); } }else From ae70cf1816a537c0bbeee508a3b61accb28a2795 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 15:18:46 +0000 Subject: [PATCH 033/109] Allow tracing of whereLoopInsert() when the 0x8 bit is set on ".wheretrace". Remove the use of sqlite_query_plan from where2.test. Fix a bug in the code generator for OR clause optimizations. FossilOrigin-Name: 707f0323264c35be14847a6adc49a0dc5eaf4ad2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 50 ++++++++++++++++++++++++++++++++++++------------ test/where2.test | 13 ++++++++++++- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index b3d407069e..535b2266fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sshell\sto\sprovide\smore\sflexibility\swhen\sentering\snumeric\sarguments\non\sdot-commands.\s\sIn\sparticular,\sallow\shex\sarguments\sto\s.wheretrace. -D 2013-05-31T14:31:11.784 +C Allow\stracing\sof\swhereLoopInsert()\swhen\sthe\s0x8\sbit\sis\sset\son\s".wheretrace".\nRemove\sthe\suse\sof\ssqlite_query_plan\sfrom\swhere2.test.\s\sFix\sa\sbug\sin\sthe\ncode\sgenerator\sfor\sOR\sclause\soptimizations. +D 2013-05-31T15:18:46.709 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 8e9922815b07290982cfc826497d128115ce358c +F src/where.c e120e28cb55cea658589ec2eca5764dcfa0396c2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1029,7 +1029,7 @@ F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 054a6b6f7933c5a5f50d0bcd650b5eccb450cc81 -F test/where2.test 399b3178289925a0aa976b3d60ef139740540ecd +F test/where2.test f902f62d60f20b0257dd700f0c4a2945527fca8e F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d8efa5f8b60bc4c8df8bfad077f87f76f7ee9bf6 -R cab66f9edd3249c76db6815f19ee5773 +P b9578c371ee569dca6a0964019959a93407c8ef9 +R 10e84b9c96bf0dc4179bf10aeea4849d U drh -Z 93073f371b16237bef628d122f206ede +Z 1f5041366a1e7c65f32184b30b5b84bc diff --git a/manifest.uuid b/manifest.uuid index 277db05743..f519953e67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9578c371ee569dca6a0964019959a93407c8ef9 \ No newline at end of file +707f0323264c35be14847a6adc49a0dc5eaf4ad2 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d4f9a71441..6b4521b40a 100644 --- a/src/where.c +++ b/src/where.c @@ -3758,6 +3758,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ if( p->u.btree.pIndex ){ const char *zName = p->u.btree.pIndex->zName; + if( zName==0 ) zName = "ipk"; if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ int i = sqlite3Strlen30(zName) - 1; while( zName[i]!='_' ) i--; @@ -3865,16 +3866,22 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ if( (p = pBuilder->pBest)!=0 ){ if( p->maskSelf!=0 ){ if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){ - return SQLITE_OK; + goto whereLoopInsert_noop; } if( p->rRun+p->rSetup == pTemplate->rRun+pTemplate->rSetup && p->prereq <= pTemplate->prereq ){ - return SQLITE_OK; + goto whereLoopInsert_noop; } } *p = *pTemplate; p->aTerm = 0; p->u.vtab.needFree = 0; +#if WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-best: "); + whereLoopPrint(pTemplate, pBuilder->pTabList); + } +#endif return SQLITE_OK; } @@ -3901,7 +3908,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ){ /* Already holding an equal or better WhereLoop. ** Return without changing or adding anything */ - return SQLITE_OK; + goto whereLoopInsert_noop; } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq && p->rSetup>=pTemplate->rSetup @@ -3918,6 +3925,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new ** WhereLoop and insert it. */ +#if WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(p, pBuilder->pTabList); + } + sqlite3DebugPrintf("ins-new: "); + whereLoopPrint(pTemplate, pBuilder->pTabList); + } +#endif if( p==0 ){ p = pToFree = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; @@ -3945,6 +3962,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ pTemplate->u.vtab.needFree = 0; } return SQLITE_OK; + + /* Jump here if the insert is a no-op */ +whereLoopInsert_noop: +#if WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-noop: "); + whereLoopPrint(pTemplate, pBuilder->pTabList); + } +#endif + return SQLITE_OK; } /* @@ -4156,7 +4183,7 @@ static int whereLoopAddBtree( pNew->u.btree.pIndex = 0; pNew->nTerm = 1; pNew->aTerm[0] = pTerm; - pNew->rSetup = 2*rLogSize*pSrc->pTab->nRowEst; + pNew->rSetup = 20*rLogSize*pSrc->pTab->nRowEst; pNew->nOut = (double)10; pNew->rRun = rLogSize + pNew->nOut; pNew->wsFlags = WHERE_TEMP_INDEX; @@ -4773,8 +4800,8 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ } if( jj>=nTo ){ if( nTo>=mxChoice && rCost>=mxCost ){ -#ifdef WHERETRACE_ENABLE - if( sqlite3WhereTrace>=3 ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-7.2g order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); @@ -4792,7 +4819,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ } pTo = &aTo[jj]; #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace>=3 ){ + if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("New %s cost=%-7.2g order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); @@ -4801,7 +4828,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ }else{ if( pTo->rCost<=rCost ){ #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace>=3 ){ + if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( "Skip %s cost=%-7.2g order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, @@ -4815,7 +4842,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ } /* A new and better score for a previously created equivalent path */ #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace>=3 ){ + if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( "Update %s cost=%-7.2g order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, @@ -5239,9 +5266,8 @@ WhereInfo *sqlite3WhereBegin( if( pLoop->u.btree.pIndex!=0 ){ Index *pIx = pLoop->u.btree.pIndex; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); - /* FIXME: Might need to be the iIdxCur parameter. As an optimization - ** use pTabItem->iCursor if WHERE_IDX_ONLY */ - int iIndexCur = pLevel->iIdxCur = pParse->nTab++; + /* FIXME: As an optimization use pTabItem->iCursor if WHERE_IDX_ONLY */ + int iIndexCur = pLevel->iIdxCur = iIdxCur ? iIdxCur : pParse->nTab++; assert( pIx->pSchema==pTab->pSchema ); assert( iIndexCur>=0 ); sqlite3VdbeAddOp4(v, OP_OpenRead, iIndexCur, pIx->tnum, iDb, diff --git a/test/where2.test b/test/where2.test index e8c2f3613f..60937adcc5 100644 --- a/test/where2.test +++ b/test/where2.test @@ -73,7 +73,18 @@ proc queryplan {sql} { set data [execsql $sql] if {$::sqlite_sort_count} {set x sort} {set x nosort} lappend data $x - return [concat $data $::sqlite_query_plan] + set eqp [execsql "EXPLAIN QUERY PLAN $sql"] + # puts eqp=$eqp + foreach {a b c x} $eqp { + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + $x all as tab idx]} { + lappend data $tab $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + lappend data $tab * + } + } + return $data + # return [concat $data $::sqlite_query_plan] } From 8030dc7b1f36e2f0f56c0ae6fc71596996a5105d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 15:50:39 +0000 Subject: [PATCH 034/109] Remove the definitions of objects that are no longer used: WhereCost, WherePlan, and WhereBestIdx. FossilOrigin-Name: 816f8add7e60de2ef8df4fdac090c244f8dbda39 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 27 ------------------------- src/where.c | 53 ++++++++----------------------------------------- 4 files changed, 16 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 535b2266fe..1dc7a47d76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\stracing\sof\swhereLoopInsert()\swhen\sthe\s0x8\sbit\sis\sset\son\s".wheretrace".\nRemove\sthe\suse\sof\ssqlite_query_plan\sfrom\swhere2.test.\s\sFix\sa\sbug\sin\sthe\ncode\sgenerator\sfor\sOR\sclause\soptimizations. -D 2013-05-31T15:18:46.709 +C Remove\sthe\sdefinitions\sof\sobjects\sthat\sare\sno\slonger\sused:\s\sWhereCost,\s\nWherePlan,\sand\sWhereBestIdx. +D 2013-05-31T15:50:39.999 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h c386fb7c61c03a0a65d78e326016449767be5b93 +F src/sqliteInt.h 3ddccdf8ef912119da26945f2d8dff98f59e1d58 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e120e28cb55cea658589ec2eca5764dcfa0396c2 +F src/where.c 586b6c3628b75619542468575706ab3192aad236 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b9578c371ee569dca6a0964019959a93407c8ef9 -R 10e84b9c96bf0dc4179bf10aeea4849d +P 707f0323264c35be14847a6adc49a0dc5eaf4ad2 +R 427cbfc4828d3a50415e243b6cfd8031 U drh -Z 1f5041366a1e7c65f32184b30b5b84bc +Z 69d89052012f1e53f4573be3e4c8c80b diff --git a/manifest.uuid b/manifest.uuid index f519953e67..e1da51f18b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -707f0323264c35be14847a6adc49a0dc5eaf4ad2 \ No newline at end of file +816f8add7e60de2ef8df4fdac090c244f8dbda39 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a0e7bba7d6..e99098cd0a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -720,7 +720,6 @@ typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; -typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; typedef struct WhereLevel WhereLevel; @@ -1949,32 +1948,6 @@ struct SrcList { #define JT_ERROR 0x0040 /* unknown or unsupported join type */ -/* -** A WherePlan object holds information that describes a lookup -** strategy. -** -** This object is intended to be opaque outside of the where.c module. -** It is included here only so that that compiler will know how big it -** is. None of the fields in this object should be used outside of -** the where.c module. -** -** Within the union, pIdx is only used when wsFlags&WHERE_INDEXED is true. -** pTerm is only used when wsFlags&WHERE_MULTI_OR is true. And pVtabIdx -** is only used when wsFlags&WHERE_VIRTUALTABLE is true. It is never the -** case that more than one of these conditions is true. -*/ -struct WherePlan { - u32 wsFlags; /* WHERE_* flags that describe the strategy */ - u16 nEq; /* Number of == constraints */ - u16 nOBSat; /* Number of ORDER BY terms satisfied */ - double nRow; /* Estimated number of rows (for EQP) */ - union { - Index *pIdx; /* Index when WHERE_INDEXED is true */ - struct WhereTerm *pTerm; /* WHERE clause term for OR-search */ - sqlite3_index_info *pVtabIdx; /* Virtual table index to use */ - } u; -}; - /* ** For each nested loop in a WHERE clause implementation, the WhereInfo ** structure contains a single instance of this structure. This structure diff --git a/src/where.c b/src/where.c index 6b4521b40a..eaef66cc8e 100644 --- a/src/where.c +++ b/src/where.c @@ -39,7 +39,6 @@ typedef struct WhereClause WhereClause; typedef struct WhereMaskSet WhereMaskSet; typedef struct WhereOrInfo WhereOrInfo; typedef struct WhereAndInfo WhereAndInfo; -typedef struct WhereCost WhereCost; typedef struct WhereLoop WhereLoop; typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; @@ -275,16 +274,6 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; -/* -** A WhereCost object records a lookup strategy and the estimated -** cost of pursuing that strategy. -*/ -struct WhereCost { - WherePlan plan; /* The lookup strategy */ - double rCost; /* Overall cost of pursuing this search strategy */ - Bitmask used; /* Bitmask of cursors used by this plan */ -}; - /* ** This object is a factory for WhereLoop objects for a particular query. */ @@ -343,28 +332,6 @@ struct WhereLoopBuilder { #define WHERE_TEMP_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_COVER_SCAN 0x00008000 /* Full scan of a covering index */ -/* -** This module contains many separate subroutines that work together to -** find the best indices to use for accessing a particular table in a query. -** An instance of the following structure holds context information about the -** index search so that it can be more easily passed between the various -** routines. -*/ -typedef struct WhereBestIdx WhereBestIdx; -struct WhereBestIdx { - Parse *pParse; /* Parser context */ - WhereClause *pWC; /* The WHERE clause */ - struct SrcList_item *pSrc; /* The FROM clause term to search */ - Bitmask notReady; /* Mask of cursors not available */ - Bitmask notValid; /* Cursors not available for any purpose */ - ExprList *pOrderBy; /* The ORDER BY clause */ - ExprList *pDistinct; /* The select-list if query is DISTINCT */ - sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */ - int i, n; /* Which loop is being coded; # of loops */ - WhereLevel *aLevel; /* Info about outer loops */ - WhereCost cost; /* Lowest cost query plan */ -}; - /* ** Initialize a preallocated WhereClause structure. */ @@ -5024,7 +4991,6 @@ WhereInfo *sqlite3WhereBegin( WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ - WhereBestIdx sWBI; /* Best index search context */ WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ WhereLevel *pLevel; /* A single level in pWInfo->a[] */ @@ -5034,8 +5000,6 @@ WhereInfo *sqlite3WhereBegin( /* Variable initialization */ - memset(&sWBI, 0, sizeof(sWBI)); - sWBI.pParse = pParse; memset(&sWLB, 0, sizeof(sWLB)); sWLB.pParse = pParse; sWLB.db = pParse->db; @@ -5083,11 +5047,10 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pOrderBy = pOrderBy; pWInfo->pDistinct = pDistinct; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; - sWBI.aLevel = pWInfo->a; + pMaskSet = (WhereMaskSet*)&pWInfo->pWC[1]; sWLB.pWInfo = pWInfo; sWLB.pWC = pWInfo->pWC; @@ -5099,9 +5062,9 @@ WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(sWBI.pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(pWInfo->pWC, pParse, pMaskSet, wctrlFlags); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(sWBI.pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(pWInfo->pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -5146,7 +5109,7 @@ WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, sWBI.pWC); + exprAnalyzeAll(pTabList, pWInfo->pWC); if( db->mallocFailed ){ goto whereBeginError; } @@ -5155,7 +5118,7 @@ WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse, pTabList, sWBI.pWC, pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse,pTabList,pWInfo->pWC,pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5260,7 +5223,7 @@ WhereInfo *sqlite3WhereBegin( } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLoop->wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, sWBI.pWC, pTabItem, notReady, pLevel); + constructAutomaticIndex(pParse, pWInfo->pWC, pTabItem, notReady, pLevel); }else #endif if( pLoop->u.btree.pIndex!=0 ){ @@ -5275,7 +5238,7 @@ WhereInfo *sqlite3WhereBegin( VdbeComment((v, "%s", pIx->zName)); } sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(sWBI.pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(pWInfo->pWC->pMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; From 7e47cb8b7595835f1695aa01645036a25ddc7887 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 17:55:27 +0000 Subject: [PATCH 035/109] Fix problems in the virtual table logic for NGQP. FossilOrigin-Name: 23af28e24b8f7ffacd006978b25bab990a43b8c5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1dc7a47d76..dca4b55f11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sdefinitions\sof\sobjects\sthat\sare\sno\slonger\sused:\s\sWhereCost,\s\nWherePlan,\sand\sWhereBestIdx. -D 2013-05-31T15:50:39.999 +C Fix\sproblems\sin\sthe\svirtual\stable\slogic\sfor\sNGQP. +D 2013-05-31T17:55:27.487 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 586b6c3628b75619542468575706ab3192aad236 +F src/where.c 103ccfa82fa177f755d35f0d48932713857516a1 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 707f0323264c35be14847a6adc49a0dc5eaf4ad2 -R 427cbfc4828d3a50415e243b6cfd8031 +P 816f8add7e60de2ef8df4fdac090c244f8dbda39 +R 876fcc7116fcba2a9fe2cf6848606247 U drh -Z 69d89052012f1e53f4573be3e4c8c80b +Z 5ce875f360b3d8cc74f8b2b339e3e8e8 diff --git a/manifest.uuid b/manifest.uuid index e1da51f18b..3c1f169293 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -816f8add7e60de2ef8df4fdac090c244f8dbda39 \ No newline at end of file +23af28e24b8f7ffacd006978b25bab990a43b8c5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index eaef66cc8e..833077497b 100644 --- a/src/where.c +++ b/src/where.c @@ -3034,7 +3034,7 @@ static Bitmask codeOneLoopStart( } } sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pLoop->u.vtab.idxStr, pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); @@ -5226,7 +5226,7 @@ WhereInfo *sqlite3WhereBegin( constructAutomaticIndex(pParse, pWInfo->pWC, pTabItem, notReady, pLevel); }else #endif - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->wsFlags & WHERE_INDEXED ){ Index *pIx = pLoop->u.btree.pIndex; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); /* FIXME: As an optimization use pTabItem->iCursor if WHERE_IDX_ONLY */ From e22501721f72ed3eb4c3eeaf64abd5e1148ee354 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 18:13:50 +0000 Subject: [PATCH 036/109] Another virtual table fix in NGQP. FossilOrigin-Name: 2f2ce4167728399cc30bef045aab500940af978c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dca4b55f11..731f7bf161 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\sthe\svirtual\stable\slogic\sfor\sNGQP. -D 2013-05-31T17:55:27.487 +C Another\svirtual\stable\sfix\sin\sNGQP. +D 2013-05-31T18:13:50.522 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 103ccfa82fa177f755d35f0d48932713857516a1 +F src/where.c e6d80ad378cfdf98b0477248307872dab302bd0b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 816f8add7e60de2ef8df4fdac090c244f8dbda39 -R 876fcc7116fcba2a9fe2cf6848606247 +P 23af28e24b8f7ffacd006978b25bab990a43b8c5 +R 77968cfca458e219a6b5161f66e2b4eb U drh -Z 5ce875f360b3d8cc74f8b2b339e3e8e8 +Z 390707269c04791307b06380f248a89b diff --git a/manifest.uuid b/manifest.uuid index 3c1f169293..6937c337ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23af28e24b8f7ffacd006978b25bab990a43b8c5 \ No newline at end of file +2f2ce4167728399cc30bef045aab500940af978c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 833077497b..a83dbae9dc 100644 --- a/src/where.c +++ b/src/where.c @@ -3024,7 +3024,7 @@ static Bitmask codeOneLoopStart( iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; jaTerm[j]; if( pTerm->eOperator & WO_IN ){ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); From 76f4cfb2e6098e822cdd7f86557f7211f46f265f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 18:20:52 +0000 Subject: [PATCH 037/109] Fix a problem with code generation on LEFT JOIN of tables without an index. FossilOrigin-Name: 0b1bee92a0d25f409d5a95f484b05ddf10ae945f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 731f7bf161..e78a656654 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\svirtual\stable\sfix\sin\sNGQP. -D 2013-05-31T18:13:50.522 +C Fix\sa\sproblem\swith\scode\sgeneration\son\sLEFT\sJOIN\sof\stables\swithout\san\sindex. +D 2013-05-31T18:20:52.849 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e6d80ad378cfdf98b0477248307872dab302bd0b +F src/where.c a8bb7f25f61b354bc26fc09df0d1367ba3389f12 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 23af28e24b8f7ffacd006978b25bab990a43b8c5 -R 77968cfca458e219a6b5161f66e2b4eb +P 2f2ce4167728399cc30bef045aab500940af978c +R 0f759a90cbe05948f904b521254c6d0c U drh -Z 390707269c04791307b06380f248a89b +Z 66b770a6b254520a8bdad94186b27ffc diff --git a/manifest.uuid b/manifest.uuid index 6937c337ca..d9b24b220e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f2ce4167728399cc30bef045aab500940af978c \ No newline at end of file +0b1bee92a0d25f409d5a95f484b05ddf10ae945f \ No newline at end of file diff --git a/src/where.c b/src/where.c index a83dbae9dc..4b1d407159 100644 --- a/src/where.c +++ b/src/where.c @@ -5311,7 +5311,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); } - if( pLevel->iIdxCur>=0 ){ + if( pLoop->wsFlags & WHERE_INDEXED ){ sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } if( pLevel->op==OP_Return ){ From 0edc94dcc71e1f4009b497e42b993387a4ce17e4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 19:14:56 +0000 Subject: [PATCH 038/109] Do not attempt to generate code in the NGQP if there have been prior errors, since with prior errors some of the expressions might not be fully named resolved. FossilOrigin-Name: 665e4291c6e78f11d7181c18c5f2418d1adfcb1c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e78a656654..e546c5f58b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\scode\sgeneration\son\sLEFT\sJOIN\sof\stables\swithout\san\sindex. -D 2013-05-31T18:20:52.849 +C Do\snot\sattempt\sto\sgenerate\scode\sin\sthe\sNGQP\sif\sthere\shave\sbeen\sprior\serrors,\nsince\swith\sprior\serrors\ssome\sof\sthe\sexpressions\smight\snot\sbe\sfully\s\nnamed\sresolved. +D 2013-05-31T19:14:56.877 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c a8bb7f25f61b354bc26fc09df0d1367ba3389f12 +F src/where.c 0c1d6feb0c2ffcc3e37b2d0b5ce044773daa3d44 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2f2ce4167728399cc30bef045aab500940af978c -R 0f759a90cbe05948f904b521254c6d0c +P 0b1bee92a0d25f409d5a95f484b05ddf10ae945f +R 3775b32a7236de56bac48d67633687ab U drh -Z 66b770a6b254520a8bdad94186b27ffc +Z ab504bcb48497c438de08e5ddd0f5c9c diff --git a/manifest.uuid b/manifest.uuid index d9b24b220e..ca22f8fcd8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b1bee92a0d25f409d5a95f484b05ddf10ae945f \ No newline at end of file +665e4291c6e78f11d7181c18c5f2418d1adfcb1c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4b1d407159..aee574a23a 100644 --- a/src/where.c +++ b/src/where.c @@ -5162,6 +5162,10 @@ WhereInfo *sqlite3WhereBegin( } } #endif + WHERETRACE(("*** Optimizer Finished ***\n")); + if( pParse->nErr || db->mallocFailed ){ + goto whereBeginError; + } #if 0 /* FIXME: Add this back in? */ /* If the caller is an UPDATE or DELETE statement that is requesting From a1f4124cf88a444a5d355d42dbff781dd1fe570f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 20:00:58 +0000 Subject: [PATCH 039/109] Make sure a unique cursor number is allocated for automatic indices. FossilOrigin-Name: 433d1aecd399590b767ebf07b32023fb718a6574 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e546c5f58b..b32322793d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sattempt\sto\sgenerate\scode\sin\sthe\sNGQP\sif\sthere\shave\sbeen\sprior\serrors,\nsince\swith\sprior\serrors\ssome\sof\sthe\sexpressions\smight\snot\sbe\sfully\s\nnamed\sresolved. -D 2013-05-31T19:14:56.877 +C Make\ssure\sa\sunique\scursor\snumber\sis\sallocated\sfor\sautomatic\sindices. +D 2013-05-31T20:00:58.768 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0c1d6feb0c2ffcc3e37b2d0b5ce044773daa3d44 +F src/where.c 3d93b77c4acc97f0dc018d251f2e2833f8df5663 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0b1bee92a0d25f409d5a95f484b05ddf10ae945f -R 3775b32a7236de56bac48d67633687ab +P 665e4291c6e78f11d7181c18c5f2418d1adfcb1c +R 2b43f41e454a1bbd459d18838a33d5bf U drh -Z ab504bcb48497c438de08e5ddd0f5c9c +Z 9f4cea41d946c49ff071caa4f1da93db diff --git a/manifest.uuid b/manifest.uuid index ca22f8fcd8..9e292ef1c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -665e4291c6e78f11d7181c18c5f2418d1adfcb1c \ No newline at end of file +433d1aecd399590b767ebf07b32023fb718a6574 \ No newline at end of file diff --git a/src/where.c b/src/where.c index aee574a23a..bd0d410cd2 100644 --- a/src/where.c +++ b/src/where.c @@ -1925,6 +1925,7 @@ static void constructAutomaticIndex( /* Create the automatic index */ pKeyinfo = sqlite3IndexKeyinfo(pParse, pIdx); assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0, (char*)pKeyinfo, P4_KEYINFO_HANDOFF); VdbeComment((v, "for %s", pTable->zName)); From 79a13bfd1eb920f187ee1cacad0406cf97d1bae3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 20:28:28 +0000 Subject: [PATCH 040/109] Do not search using ON clause constraints of LEFT JOINs to the right of the table. FossilOrigin-Name: 723f901a63313b64a9f10516711fb6136526b79b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b32322793d..5aec48fb0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sa\sunique\scursor\snumber\sis\sallocated\sfor\sautomatic\sindices. -D 2013-05-31T20:00:58.768 +C Do\snot\ssearch\susing\sON\sclause\sconstraints\sof\sLEFT\sJOINs\sto\sthe\sright\sof\nthe\stable. +D 2013-05-31T20:28:28.662 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3d93b77c4acc97f0dc018d251f2e2833f8df5663 +F src/where.c 3e4ca21a9b34ab25b2ec13704e8659581d57878d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 665e4291c6e78f11d7181c18c5f2418d1adfcb1c -R 2b43f41e454a1bbd459d18838a33d5bf +P 433d1aecd399590b767ebf07b32023fb718a6574 +R bd0568ec5c6f80c0ba5870a23aeeb2df U drh -Z 9f4cea41d946c49ff071caa4f1da93db +Z 6a19f7c88688bac3b6839a73befa2b33 diff --git a/manifest.uuid b/manifest.uuid index 9e292ef1c6..6cebeafdb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -433d1aecd399590b767ebf07b32023fb718a6574 \ No newline at end of file +723f901a63313b64a9f10516711fb6136526b79b \ No newline at end of file diff --git a/src/where.c b/src/where.c index bd0d410cd2..c4a8c22243 100644 --- a/src/where.c +++ b/src/where.c @@ -3996,6 +3996,7 @@ static int whereLoopAddBtreeIndex( rLogSize = estLog(pProbe->aiRowEst[0]); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 1; + if( pTerm->prereqRight & pNew->maskSelf ) continue; pNew->wsFlags = savedLoop.wsFlags; pNew->u.btree.nEq = savedLoop.u.btree.nEq; pNew->nTerm = savedLoop.nTerm; @@ -4146,6 +4147,7 @@ static int whereLoopAddBtree( WhereTerm *pTerm; WhereTerm *pWCEnd = pWC->a + pWC->nTerm; for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; pNew->u.btree.pIndex = 0; From 75b9340520fbbd3f7c660537acdba55c42c5e40b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 May 2013 20:43:57 +0000 Subject: [PATCH 041/109] The NGQP might not find a solution if INDEXED BY constraints on the query are set wrong. If that is the case, print a "no query solution" error and abort before trying to use the non-existant solution. FossilOrigin-Name: 42511a7e12a70cf3436d25e4d069e5d309dfd678 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 5aec48fb0e..e85a435ce4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssearch\susing\sON\sclause\sconstraints\sof\sLEFT\sJOINs\sto\sthe\sright\sof\nthe\stable. -D 2013-05-31T20:28:28.662 +C The\sNGQP\smight\snot\sfind\sa\ssolution\sif\sINDEXED\sBY\sconstraints\son\sthe\squery\nare\sset\swrong.\s\sIf\sthat\sis\sthe\scase,\sprint\sa\s"no\squery\ssolution"\serror\nand\sabort\sbefore\strying\sto\suse\sthe\snon-existant\ssolution. +D 2013-05-31T20:43:57.656 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3e4ca21a9b34ab25b2ec13704e8659581d57878d +F src/where.c 7c931bf930fcf77d9a446e6d60692ab7a6251d00 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 433d1aecd399590b767ebf07b32023fb718a6574 -R bd0568ec5c6f80c0ba5870a23aeeb2df +P 723f901a63313b64a9f10516711fb6136526b79b +R f0026590576c82bf02c9ed7b1fd9cc97 U drh -Z 6a19f7c88688bac3b6839a73befa2b33 +Z 4e2d6d22eda85e326fefafc64c333d74 diff --git a/manifest.uuid b/manifest.uuid index 6cebeafdb2..1a1fa457c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -723f901a63313b64a9f10516711fb6136526b79b \ No newline at end of file +42511a7e12a70cf3436d25e4d069e5d309dfd678 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c4a8c22243..4e78f1ebde 100644 --- a/src/where.c +++ b/src/where.c @@ -4864,9 +4864,11 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ nFrom = nTo; } - /* TEMPORARY */ - if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } - assert( nFrom>0 ); + if( nFrom==0 ){ + sqlite3ErrorMsg(pWInfo->pParse, "no query solution"); + sqlite3DbFree(db, pSpace); + return SQLITE_ERROR; + } /* Find the lowest cost path. pFrom will be left pointing to that path */ pFrom = aFrom; @@ -5151,6 +5153,9 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, pWInfo->nRowOut); if( db->mallocFailed ) goto whereBeginError; } + if( pParse->nErr || db->mallocFailed ){ + goto whereBeginError; + } #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ int ii; @@ -5166,9 +5171,6 @@ WhereInfo *sqlite3WhereBegin( } #endif WHERETRACE(("*** Optimizer Finished ***\n")); - if( pParse->nErr || db->mallocFailed ){ - goto whereBeginError; - } #if 0 /* FIXME: Add this back in? */ /* If the caller is an UPDATE or DELETE statement that is requesting From 6186b309019d818bd0946856752fa241d8cc7943 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 14:15:34 +0000 Subject: [PATCH 042/109] Add test cases to verify that ticket [bc1aea7b725f2761] has been fixed in the NGQP. FossilOrigin-Name: 9b97af94ae7b1624ef1f5adee58010593fbb8b1c --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/whereD.test | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e85a435ce4..fd53c9cffe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sNGQP\smight\snot\sfind\sa\ssolution\sif\sINDEXED\sBY\sconstraints\son\sthe\squery\nare\sset\swrong.\s\sIf\sthat\sis\sthe\scase,\sprint\sa\s"no\squery\ssolution"\serror\nand\sabort\sbefore\strying\sto\suse\sthe\snon-existant\ssolution. -D 2013-05-31T20:43:57.656 +C Add\stest\scases\sto\sverify\sthat\sticket\s[bc1aea7b725f2761]\shas\sbeen\sfixed\sin\nthe\sNGQP. +D 2013-06-03T14:15:34.631 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1041,7 +1041,7 @@ F test/where9.test 1b4387c6eacc9a32b28b4d837c27f857c785d0d8 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test 13ff5ec0dba407c0e0c075980c75b3275a6774e5 -F test/whereD.test 3f3ee93825c94804f1fc91eef2de0d365981759a +F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 723f901a63313b64a9f10516711fb6136526b79b -R f0026590576c82bf02c9ed7b1fd9cc97 +P 42511a7e12a70cf3436d25e4d069e5d309dfd678 +R 9315575e95aec71de6da1203a12a0f54 U drh -Z 4e2d6d22eda85e326fefafc64c333d74 +Z bf9c6fc8107d350958f4a9bdd4f10f03 diff --git a/manifest.uuid b/manifest.uuid index 1a1fa457c3..824fa2dabf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42511a7e12a70cf3436d25e4d069e5d309dfd678 \ No newline at end of file +9b97af94ae7b1624ef1f5adee58010593fbb8b1c \ No newline at end of file diff --git a/test/whereD.test b/test/whereD.test index 9ac5a68097..eb0f864025 100644 --- a/test/whereD.test +++ b/test/whereD.test @@ -186,4 +186,37 @@ do_test 4.3 { } } {1 2 3 3 6 9 4 5 6 {} {} {}} +# Ticket [bc1aea7b725f276177] +# Incorrect result on LEFT JOIN with OR constraints and an ORDER BY clause. +# +do_execsql_test 4.4 { + CREATE TABLE t44(a INTEGER, b INTEGER); + INSERT INTO t44 VALUES(1,2); + INSERT INTO t44 VALUES(3,4); + SELECT * + FROM t44 AS x + LEFT JOIN (SELECT a AS c, b AS d FROM t44) AS y ON a=c + WHERE d=4 OR d IS NULL; +} {3 4 3 4} +do_execsql_test 4.5 { + SELECT * + FROM t44 AS x + LEFT JOIN (SELECT a AS c, b AS d FROM t44) AS y ON a=c + WHERE d=4 OR d IS NULL + ORDER BY a; +} {3 4 3 4} +do_execsql_test 4.6 { + CREATE TABLE t46(c INTEGER, d INTEGER); + INSERT INTO t46 SELECT a, b FROM t44; + SELECT * FROM t44 LEFT JOIN t46 ON a=c + WHERE d=4 OR d IS NULL; +} {3 4 3 4} +do_execsql_test 4.7 { + SELECT * FROM t44 LEFT JOIN t46 ON a=c + WHERE d=4 OR d IS NULL + ORDER BY a; +} {3 4 3 4} + + + finish_test From 21f7ff7d22ae722ae3955cc61a1020d98b2b959c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 15:07:23 +0000 Subject: [PATCH 043/109] Set the WHERE_UNIQUE flag on loops that can only run once. FossilOrigin-Name: 510f4d8ecf6eb284f14b91951e7236505bd71203 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fd53c9cffe..4456115495 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\sverify\sthat\sticket\s[bc1aea7b725f2761]\shas\sbeen\sfixed\sin\nthe\sNGQP. -D 2013-06-03T14:15:34.631 +C Set\sthe\sWHERE_UNIQUE\sflag\son\sloops\sthat\scan\sonly\srun\sonce. +D 2013-06-03T15:07:23.693 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 7c931bf930fcf77d9a446e6d60692ab7a6251d00 +F src/where.c 7e34bb26affb2bf0eee8a8c5ab955cf80e7f9580 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 42511a7e12a70cf3436d25e4d069e5d309dfd678 -R 9315575e95aec71de6da1203a12a0f54 +P 9b97af94ae7b1624ef1f5adee58010593fbb8b1c +R c10d3b1c9e2b7a04d0ad05024b58cd34 U drh -Z bf9c6fc8107d350958f4a9bdd4f10f03 +Z 86153958e4f1f3df668821cb8ff6a748 diff --git a/manifest.uuid b/manifest.uuid index 824fa2dabf..950d83a47f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b97af94ae7b1624ef1f5adee58010593fbb8b1c \ No newline at end of file +510f4d8ecf6eb284f14b91951e7236505bd71203 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4e78f1ebde..1efa409942 100644 --- a/src/where.c +++ b/src/where.c @@ -4017,6 +4017,12 @@ static int whereLoopAddBtreeIndex( pNew->nOut = (double)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; + if( iCol<0 + || (pProbe->onError==OE_Abort && nInMul==1 + && pNew->u.btree.nEq==pProbe->nColumn-1) + ){ + pNew->wsFlags |= WHERE_UNIQUE; + } pNew->u.btree.nEq++; pNew->nOut = (double)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_ISNULL) ){ From d108d74d48c9863eddaa01b150a6e802de862261 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 15:24:11 +0000 Subject: [PATCH 044/109] Virtual tables now always report 25 rows instead of 0 rows in the EXPLAIN QUERY PLAN output. Adjust tests accordingly. FossilOrigin-Name: 7d91f688815597ff65ec04b7daa21cb9e5d6bf98 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3aux1.test | 27 +++++++++++++-------------- test/fts3query.test | 5 ++--- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 4456115495..ba3a006024 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sthe\sWHERE_UNIQUE\sflag\son\sloops\sthat\scan\sonly\srun\sonce. -D 2013-06-03T15:07:23.693 +C Virtual\stables\snow\salways\sreport\s25\srows\sinstead\sof\s0\srows\sin\sthe\s\nEXPLAIN\sQUERY\sPLAN\soutput.\s\sAdjust\stests\saccordingly. +D 2013-06-03T15:24:11.404 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -511,7 +511,7 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test fb398ab50aa232489e2a17f9b29d7ad3a3885f36 F test/fts3auto.test 74315a7377403a57ba82a652a33704197fe1e4be -F test/fts3aux1.test 03cec2dea379931c219dd4406817485caa69d1d8 +F test/fts3aux1.test 52cc50162e5e0c27fc08d12c8fe41b8df20af555 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c @@ -535,7 +535,7 @@ F test/fts3matchinfo.test ecb08f586d027eb03941bcfcded6cb9d8ccb3a66 F test/fts3near.test 12895557870b0f9af7cc0be81a0171abb2d12f12 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5 -F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 +F test/fts3query.test 00b519001b42ea0bb65314103a395604880dd7cf F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test c2f60e152e8554549eb25f0a7593ea01389c5037 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9b97af94ae7b1624ef1f5adee58010593fbb8b1c -R c10d3b1c9e2b7a04d0ad05024b58cd34 +P 510f4d8ecf6eb284f14b91951e7236505bd71203 +R b46e673a11585c0adf84b26ffd5c9ca9 U drh -Z 86153958e4f1f3df668821cb8ff6a748 +Z 24bffcc0033c051bdc90970a04c0eaf7 diff --git a/manifest.uuid b/manifest.uuid index 950d83a47f..f196f60bd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -510f4d8ecf6eb284f14b91951e7236505bd71203 \ No newline at end of file +7d91f688815597ff65ec04b7daa21cb9e5d6bf98 \ No newline at end of file diff --git a/test/fts3aux1.test b/test/fts3aux1.test index ef17949fd9..f421b7f6ef 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -105,10 +105,10 @@ db func rec rec # do_execsql_test 2.1.1.1 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term='braid' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} } do_execsql_test 2.1.1.2 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term='braid' -} {0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)}} +} {0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)}} # Now show that using "term='braid'" means the virtual table returns # only 1 row to SQLite, but "+term='braid'" means all 19 are returned. @@ -154,24 +154,24 @@ do_execsql_test 2.1.5 { SELECT * FROM terms WHERE term=NULL } {} do_execsql_test 2.2.1.1 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term>'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 2: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 2: (~25 rows)} } do_execsql_test 2.2.1.2 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term>'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } do_execsql_test 2.2.1.3 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term<'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 4: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 4: (~25 rows)} } do_execsql_test 2.2.1.4 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term<'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } do_execsql_test 2.2.1.5 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term BETWEEN 'brags' AND 'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 6: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 6: (~25 rows)} } do_execsql_test 2.2.1.6 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term BETWEEN 'brags' AND 'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } do_test 2.2.2.1 { set cnt 0 @@ -335,7 +335,7 @@ foreach {tn sort orderby} { 9 1 "ORDER BY occurrences DESC" } { - set res [list 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)}] + set res [list 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)}] if {$sort} { lappend res 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } set sql "SELECT * FROM terms $orderby" @@ -411,7 +411,7 @@ do_plansql_test 4.2 { SELECT y FROM x2, terms WHERE y = term AND col = '*' } { 0 0 0 {SCAN TABLE x2 (~1000000 rows)} - 0 1 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~0 rows)} + 0 1 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} a b c d e f g h i j k l } @@ -419,14 +419,14 @@ do_plansql_test 4.3 { SELECT y FROM terms, x2 WHERE y = term AND col = '*' } { 0 0 1 {SCAN TABLE x2 (~1000000 rows)} - 0 1 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~0 rows)} + 0 1 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} a b c d e f g h i j k l } do_plansql_test 4.4 { SELECT y FROM x3, terms WHERE y = term AND col = '*' } { - 0 0 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)} + 0 0 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} 0 1 0 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?) (~10 rows)} a b c d e f g h i j k l } @@ -434,7 +434,7 @@ do_plansql_test 4.4 { do_plansql_test 4.5 { SELECT y FROM terms, x3 WHERE y = term AND occurrences>1 AND col = '*' } { - 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~0 rows)} + 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} 0 1 1 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?) (~10 rows)} a k l } @@ -519,4 +519,3 @@ do_test 8.2 { } {1 {SQL logic error or missing database}} finish_test - diff --git a/test/fts3query.test b/test/fts3query.test index c8f8686bff..51b84be906 100644 --- a/test/fts3query.test +++ b/test/fts3query.test @@ -119,13 +119,13 @@ do_eqp_test fts3query-4.2 { SELECT t1.number FROM t1, ft WHERE t1.number=ft.rowid ORDER BY t1.date } { 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 1 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~0 rows)} + 0 1 1 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~25 rows)} } do_eqp_test fts3query-4.3 { SELECT t1.number FROM ft, t1 WHERE t1.number=ft.rowid ORDER BY t1.date } { 0 0 1 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 0 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~0 rows)} + 0 1 0 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~25 rows)} } do_eqp_test fts3query-4.4 { SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date @@ -210,4 +210,3 @@ do_select_tests 6.2 { finish_test - From 3a5ba8b1b77e26e4b1a8eb4b538c0b7bf25b6717 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 15:34:48 +0000 Subject: [PATCH 045/109] Fix an issue that was causing ORDER BY DESC to come out in ascending order. FossilOrigin-Name: 029840124022387a8f6d2e1a685a003688d0ef3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ba3a006024..34d692d6a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Virtual\stables\snow\salways\sreport\s25\srows\sinstead\sof\s0\srows\sin\sthe\s\nEXPLAIN\sQUERY\sPLAN\soutput.\s\sAdjust\stests\saccordingly. -D 2013-06-03T15:24:11.404 +C Fix\san\sissue\sthat\swas\scausing\sORDER\sBY\sDESC\sto\scome\sout\sin\sascending\sorder. +D 2013-06-03T15:34:48.695 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 7e34bb26affb2bf0eee8a8c5ab955cf80e7f9580 +F src/where.c 078c3aeafacf8544bdc733bf6b60d087d9e6346c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 510f4d8ecf6eb284f14b91951e7236505bd71203 -R b46e673a11585c0adf84b26ffd5c9ca9 +P 7d91f688815597ff65ec04b7daa21cb9e5d6bf98 +R 47cac5b8ea3311240b582844f809143b U drh -Z 24bffcc0033c051bdc90970a04c0eaf7 +Z d1b03d3fcac2015c05c39b44e5780b67 diff --git a/manifest.uuid b/manifest.uuid index f196f60bd2..fa405a84b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d91f688815597ff65ec04b7daa21cb9e5d6bf98 \ No newline at end of file +029840124022387a8f6d2e1a685a003688d0ef3a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1efa409942..d9e33d6058 100644 --- a/src/where.c +++ b/src/where.c @@ -4767,6 +4767,8 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ default: /* Cannot tell yet. Try again on the next iteration */ break; } + }else{ + revMask = pFrom->revLoop; } /* Check to see if pWLoop should be added to the mxChoice best so far */ for(jj=0, pTo=aTo; jj Date: Mon, 3 Jun 2013 16:03:16 +0000 Subject: [PATCH 046/109] Fix to the logic that disables constraints on virtual tables. FossilOrigin-Name: 62d382406b28c1347e13114f42215939ddfd7a9d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 34d692d6a5..432c1a0ab8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sthat\swas\scausing\sORDER\sBY\sDESC\sto\scome\sout\sin\sascending\sorder. -D 2013-06-03T15:34:48.695 +C Fix\sto\sthe\slogic\sthat\sdisables\sconstraints\son\svirtual\stables. +D 2013-06-03T16:03:16.168 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 078c3aeafacf8544bdc733bf6b60d087d9e6346c +F src/where.c b8a41bcd6a61e560d9c45e4a066d3858a06657d8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7d91f688815597ff65ec04b7daa21cb9e5d6bf98 -R 47cac5b8ea3311240b582844f809143b +P 029840124022387a8f6d2e1a685a003688d0ef3a +R 74a79b344cf1d282ed20a762beed972e U drh -Z d1b03d3fcac2015c05c39b44e5780b67 +Z 4d150a08a0845f1055b728ec84c21984 diff --git a/manifest.uuid b/manifest.uuid index fa405a84b7..8704c41e00 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -029840124022387a8f6d2e1a685a003688d0ef3a \ No newline at end of file +62d382406b28c1347e13114f42215939ddfd7a9d \ No newline at end of file diff --git a/src/where.c b/src/where.c index d9e33d6058..3c4b7140cd 100644 --- a/src/where.c +++ b/src/where.c @@ -4326,7 +4326,7 @@ static int whereLoopAddVirtual( pNew->prereq |= pTerm->prereqRight; pNew->aTerm[iTerm] = pTerm; if( iTerm>mxTerm ) mxTerm = iTerm; - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ if( pUsage[i].omit==0 ){ /* Do not attempt to use an IN constraint if the virtual table From 3b1d808fab432b8547ad8d39c36cacb32a30a2b9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 16:56:37 +0000 Subject: [PATCH 047/109] Honor the orderByConsumed boolean returned from virtual table query planner. FossilOrigin-Name: aaf7f5896d3523531e1a9a1b90c4ad326f0c8fc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 432c1a0ab8..3a94412d02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\sthe\slogic\sthat\sdisables\sconstraints\son\svirtual\stables. -D 2013-06-03T16:03:16.168 +C Honor\sthe\sorderByConsumed\sboolean\sreturned\sfrom\svirtual\stable\squery\splanner. +D 2013-06-03T16:56:37.438 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c b8a41bcd6a61e560d9c45e4a066d3858a06657d8 +F src/where.c 82c6fd16a6344108eac33642414958dc8db20d32 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 029840124022387a8f6d2e1a685a003688d0ef3a -R 74a79b344cf1d282ed20a762beed972e +P 62d382406b28c1347e13114f42215939ddfd7a9d +R 6374db7e82d972462ef1160c4988e05a U drh -Z 4d150a08a0845f1055b728ec84c21984 +Z 8802671c91d46932119ceee1b71a0d50 diff --git a/manifest.uuid b/manifest.uuid index 8704c41e00..ee3e9a8cbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62d382406b28c1347e13114f42215939ddfd7a9d \ No newline at end of file +aaf7f5896d3523531e1a9a1b90c4ad326f0c8fc7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3c4b7140cd..2d416f99d0 100644 --- a/src/where.c +++ b/src/where.c @@ -4350,7 +4350,8 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (u8)(pIdxInfo->nOrderBy!=0); + pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) + && pIdxInfo->orderByConsumed); pNew->rSetup = (double)0; pNew->rRun = pIdxInfo->estimatedCost; pNew->nOut = (double)25; From 6f2bfad24bb797b12e090b6981fe01dbe5fa3faf Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 17:35:22 +0000 Subject: [PATCH 048/109] Update the NGQP to make use of STAT3 information if it is available. FossilOrigin-Name: ff134e6ee95d41b0e59e03bba7e94bc15b04ff8c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++++++++++++++--- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3a94412d02..3195663c77 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Honor\sthe\sorderByConsumed\sboolean\sreturned\sfrom\svirtual\stable\squery\splanner. -D 2013-06-03T16:56:37.438 +C Update\sthe\sNGQP\sto\smake\suse\sof\sSTAT3\sinformation\sif\sit\sis\savailable. +D 2013-06-03T17:35:22.501 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 82c6fd16a6344108eac33642414958dc8db20d32 +F src/where.c a60b27296226bc073ae87988f7cec222d9523d7d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 62d382406b28c1347e13114f42215939ddfd7a9d -R 6374db7e82d972462ef1160c4988e05a +P aaf7f5896d3523531e1a9a1b90c4ad326f0c8fc7 +R 821d5edba86d3e2ff0b1efea7de5e5d1 U drh -Z 8802671c91d46932119ceee1b71a0d50 +Z 967f742fe27d5833ce434e13ad275aee diff --git a/manifest.uuid b/manifest.uuid index ee3e9a8cbf..f4885e9b76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aaf7f5896d3523531e1a9a1b90c4ad326f0c8fc7 \ No newline at end of file +ff134e6ee95d41b0e59e03bba7e94bc15b04ff8c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2d416f99d0..92aa607fc8 100644 --- a/src/where.c +++ b/src/where.c @@ -3965,6 +3965,7 @@ static int whereLoopAddBtreeIndex( int rc = SQLITE_OK; /* Return code */ tRowcnt iRowEst; /* Estimated index selectivity */ double rLogSize; /* Logarithm of table size */ + WhereTerm *pTop, *pBtm; /* Top and bottom range constraints */ db = pBuilder->db; pNew = pBuilder->pNew; @@ -4031,12 +4032,35 @@ static int whereLoopAddBtreeIndex( pNew->nOut = (double)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->nOut = savedLoop.nOut/3; + pBtm = pTerm; + pTop = 0; }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->nOut = savedLoop.nOut/3; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aTerm[pNew->nTerm-2] : 0; } pNew->rRun = rLogSize*nIn; /* Cost for nIn binary searches */ + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut and rRun for STAT3 range values */ + double rDiv; + whereRangeScanEst(pBuilder->pParse, pProbe, pNew->u.btree.nEq, + pBtm, pTop, &rDiv); + pNew->nOut = savedLoop.nOut/rDiv; + } +#ifdef SQLITE_ENABLE_STAT3 + if( pNew->u.btree.nEq==1 && pProbe->nSample ){ + if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ + rc = whereEqualScanEst(pBuilder->pParse, pProbe, pTerm->pExpr->pRight, + &pNew->nOut); + }else if( (pTerm->eOperator & WO_IN) + && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ + rc = whereInScanEst(pBuilder->pParse, pProbe, pTerm->pExpr->x.pList, + &pNew->nOut); + + } + } +#endif if( pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK) ){ pNew->rRun += pNew->nOut; /* Unit step cost to reach each row */ }else{ @@ -4044,7 +4068,6 @@ static int whereLoopAddBtreeIndex( ** the main table */ pNew->rRun += pNew->nOut*(1 + rLogSize); } - /* TBD: Adjust nOut and rRun for STAT3 range values */ /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 From 44dc8e822b990e477c0d1ff7b50f3f1d478740e8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 18:14:31 +0000 Subject: [PATCH 049/109] Fix EXPLAIN QUERY PLAN row counts for the rtree virtual table tests. FossilOrigin-Name: 15328987cc876d7a4e34561b6b0e08937aefe4be --- ext/rtree/rtree6.test | 10 +++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index 92edc8d104..320b993368 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -74,35 +74,35 @@ do_test rtree6-1.5 { do_eqp_test rtree6.2.1 { SELECT * FROM t1,t2 WHERE k=+ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~25 rows)} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} } do_eqp_test rtree6.2.2 { SELECT * FROM t1,t2 WHERE k=ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~25 rows)} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} } do_eqp_test rtree6.2.3 { SELECT * FROM t1,t2 WHERE k=ii } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~25 rows)} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} } do_eqp_test rtree6.2.4 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~0 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~25 rows)} 0 1 1 {SCAN TABLE t2 (~100000 rows)} } do_eqp_test rtree6.2.5 { SELECT * FROM t1,t2 WHERE k=ii AND x1 Date: Mon, 3 Jun 2013 18:50:15 +0000 Subject: [PATCH 050/109] Fix a bug in the transitive_closure virtual table that NGQP exposes. Update between.test module to avoid using sqlite_query_plan. FossilOrigin-Name: aa66b864204b4b8717d845499132b987eb8ef924 --- ext/misc/closure.c | 6 +++++- manifest | 14 +++++++------- manifest.uuid | 2 +- test/between.test | 22 ++++++++++++++++------ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/ext/misc/closure.c b/ext/misc/closure.c index 213b7633e6..665cc7ce44 100644 --- a/ext/misc/closure.c +++ b/ext/misc/closure.c @@ -828,6 +828,7 @@ static int closureBestIndex( int idx = 1; const struct sqlite3_index_constraint *pConstraint; closure_vtab *pVtab = (closure_vtab*)pTab; + double rCost = 10000000.0; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ @@ -839,6 +840,7 @@ static int closureBestIndex( iPlan |= 1; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 100.0; } if( (iPlan & 0x0000f0)==0 && pConstraint->iColumn==CLOSURE_COL_DEPTH @@ -849,6 +851,7 @@ static int closureBestIndex( iPlan |= idx<<4; pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + rCost /= 5.0; } if( (iPlan & 0x000f00)==0 && pConstraint->iColumn==CLOSURE_COL_TABLENAME @@ -857,6 +860,7 @@ static int closureBestIndex( iPlan |= idx<<8; pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 5.0; } if( (iPlan & 0x00f000)==0 && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN @@ -891,7 +895,7 @@ static int closureBestIndex( ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)10000; + pIdxInfo->estimatedCost = rCost; return SQLITE_OK; } diff --git a/manifest b/manifest index cb6fd82118..78c40ea581 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sEXPLAIN\sQUERY\sPLAN\srow\scounts\sfor\sthe\srtree\svirtual\stable\stests. -D 2013-06-03T18:14:31.081 +C Fix\sa\sbug\sin\sthe\stransitive_closure\svirtual\stable\sthat\sNGQP\sexposes.\nUpdate\sbetween.test\smodule\sto\savoid\susing\ssqlite_query_plan. +D 2013-06-03T18:50:15.467 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -107,7 +107,7 @@ F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c 7538f98eab2854cf17fa5f7797bffa6c76e3863b F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c eae8454cd9dcb287b2a3ec2e65a865a4ac5f0d06 -F ext/misc/closure.c 40788c54c59190a1f52f6492a260d8894a246fe9 +F ext/misc/closure.c b88aa95fbe32edc74e16ed5bb6646d249a2bf6ad F ext/misc/fuzzer.c 51bd96960b6b077d41d6f3cedefbcb57f29efaa2 F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e @@ -335,7 +335,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa -F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 +F test/between.test bec144cd75698402bbda8ad8e23bd8588e426145 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 7c79f1ef0c6c2c2bc1e7bd895596fab32bfb4796 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ff134e6ee95d41b0e59e03bba7e94bc15b04ff8c -R e905d61d613190200a15251273731837 +P 15328987cc876d7a4e34561b6b0e08937aefe4be +R 0bfbc2c20668c92dcdb991d487aaa95a U drh -Z 152c7a47705b2201e9c25505c9ef2b93 +Z d205bcca63c72f2d174184c36108bd89 diff --git a/manifest.uuid b/manifest.uuid index c02031bee7..3a34f12991 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15328987cc876d7a4e34561b6b0e08937aefe4be \ No newline at end of file +aa66b864204b4b8717d845499132b987eb8ef924 \ No newline at end of file diff --git a/test/between.test b/test/between.test index 45436756aa..98be29aea7 100644 --- a/test/between.test +++ b/test/between.test @@ -55,7 +55,17 @@ proc queryplan {sql} { set data [execsql $sql] if {$::sqlite_sort_count} {set x sort} {set x nosort} lappend data $x - return [concat $data $::sqlite_query_plan] + set eqp [execsql "EXPLAIN QUERY PLAN $sql"] + # puts eqp=$eqp + foreach {a b c x} $eqp { + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + $x all as tab idx]} { + lappend data $tab $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + lappend data $tab * + } + } + return $data } do_test between-1.1.1 { @@ -67,7 +77,7 @@ do_test between-1.1.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 5 AND 6 ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 *} do_test between-1.2.1 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 5 AND 65-y ORDER BY +w @@ -77,7 +87,7 @@ do_test between-1.2.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 5 AND 65-y ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 *} do_test between-1.3.1 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 41-y AND 6 ORDER BY +w @@ -87,12 +97,12 @@ do_test between-1.3.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 41-y AND 6 ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 *} do_test between-1.4 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 41-y AND 65-y ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 *} do_test between-1.5.1 { queryplan { SELECT * FROM t1 WHERE 26 BETWEEN y AND z ORDER BY +w @@ -107,7 +117,7 @@ do_test between-1.5.3 { queryplan { SELECT * FROM t1 WHERE 26 BETWEEN y AND +z ORDER BY +w } -} {4 2 25 27 sort t1 {}} +} {4 2 25 27 sort t1 *} finish_test From 74990dce64a98940db8bd437a5dda5fafa4caca0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 19:01:26 +0000 Subject: [PATCH 051/109] Resolve ambiguities in test cases in collate2.test. FossilOrigin-Name: dcbbcb2e6c85780276cb3a392549f70e21e94408 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/collate2.test | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 78c40ea581..58036befe9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\stransitive_closure\svirtual\stable\sthat\sNGQP\sexposes.\nUpdate\sbetween.test\smodule\sto\savoid\susing\ssqlite_query_plan. -D 2013-06-03T18:50:15.467 +C Resolve\sambiguities\sin\stest\scases\sin\scollate2.test. +D 2013-06-03T19:01:26.660 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -366,7 +366,7 @@ F test/close.test e37610d60e9c9b9979a981f3f50071d7dff28616 F test/closure01.test dbb28f1ea9eeaf0a53ec5bc0fed352e479def8c7 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce -F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 +F test/collate2.test 285cef1b58ce2b3cf074a386f763ce753c81c97f F test/collate3.test 79558a286362cb9ed603c6fa543f1cda7f563f0f F test/collate4.test 031f7265c13308b724ba3c49f41cc04612bd92b1 F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 15328987cc876d7a4e34561b6b0e08937aefe4be -R 0bfbc2c20668c92dcdb991d487aaa95a +P aa66b864204b4b8717d845499132b987eb8ef924 +R 62f1f59de9bf7b607ff3cad8a2a9dba4 U drh -Z d205bcca63c72f2d174184c36108bd89 +Z 7508c3c33af72ab568a1feb83056a8cb diff --git a/manifest.uuid b/manifest.uuid index 3a34f12991..f007fe30fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa66b864204b4b8717d845499132b987eb8ef924 \ No newline at end of file +dcbbcb2e6c85780276cb3a392549f70e21e94408 \ No newline at end of file diff --git a/test/collate2.test b/test/collate2.test index bf619231da..c758b0a6db 100644 --- a/test/collate2.test +++ b/test/collate2.test @@ -636,13 +636,15 @@ do_test collate2-4.2 { do_test collate2-4.3 { execsql { SELECT collate2t1.a FROM collate2t1, collate2t3 - WHERE collate2t1.b = collate2t3.b||''; + WHERE collate2t1.b = collate2t3.b||'' + ORDER BY +collate2t1.a DESC; } } {aa aA Aa AA} do_test collate2-4.4 { execsql { SELECT collate2t1.a FROM collate2t1, collate2t3 - WHERE collate2t3.b||'' = collate2t1.b; + WHERE collate2t3.b||'' = collate2t1.b + ORDER BY +collate2t1.a DESC; } } {aa aA Aa AA} From e3b7c9216cf2c5a5b8170b6024426447ebac8f9f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 19:17:40 +0000 Subject: [PATCH 052/109] Do not use an index fullscan for an UPDATE or DELETE or if disabled by sqlite3_test_control() or sqlite3_config(). FossilOrigin-Name: fabb21854e662b1d8e5631e79f828d5322ceb595 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58036befe9..0a83808515 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\sambiguities\sin\stest\scases\sin\scollate2.test. -D 2013-06-03T19:01:26.660 +C Do\snot\suse\san\sindex\sfullscan\sfor\san\sUPDATE\sor\sDELETE\sor\sif\sdisabled\sby\nsqlite3_test_control()\sor\ssqlite3_config(). +D 2013-06-03T19:17:40.684 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c a60b27296226bc073ae87988f7cec222d9523d7d +F src/where.c 6226d9912f494a92b51b06c58b5c0eed9b7eb84e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aa66b864204b4b8717d845499132b987eb8ef924 -R 62f1f59de9bf7b607ff3cad8a2a9dba4 +P dcbbcb2e6c85780276cb3a392549f70e21e94408 +R 5e83ed1a81fa16225d238fe289659759 U drh -Z 7508c3c33af72ab568a1feb83056a8cb +Z 8c74536a41fa8c42d526bd323c460f6c diff --git a/manifest.uuid b/manifest.uuid index f007fe30fa..c94f0e1371 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcbbcb2e6c85780276cb3a392549f70e21e94408 \ No newline at end of file +fabb21854e662b1d8e5631e79f828d5322ceb595 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 92aa607fc8..ee604f0713 100644 --- a/src/where.c +++ b/src/where.c @@ -4224,7 +4224,12 @@ static int whereLoopAddBtree( pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; /* Full scan via index */ - if( (m==0 || b) && pProbe->bUnordered==0 ){ + if( (m==0 || b) + && pProbe->bUnordered==0 + && (pBuilder->pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pBuilder->pParse->db, SQLITE_CoverIdxScan) + ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize); From a3855653ed06222b8c0f0329a4ab23ea8a8adbbb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 21:25:28 +0000 Subject: [PATCH 053/109] Adjust the xBestIndex methods on both the fuzzer and transitive_closure virtual tables so that an unused MATCH operator gets a really large cost. Remove ambiguities from the fuzzer test cases. FossilOrigin-Name: e2c1af78b65a8ace976fa6c035db212e1ffc79b8 --- ext/misc/closure.c | 6 ++++++ ext/misc/fuzzer.c | 13 ++++++++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzer1.test | 17 +++++++++++------ 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/ext/misc/closure.c b/ext/misc/closure.c index 665cc7ce44..260a783042 100644 --- a/ext/misc/closure.c +++ b/ext/misc/closure.c @@ -826,12 +826,17 @@ static int closureBestIndex( int iPlan = 0; int i; int idx = 1; + int seenMatch = 0; const struct sqlite3_index_constraint *pConstraint; closure_vtab *pVtab = (closure_vtab*)pTab; double rCost = 10000000.0; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + seenMatch = 1; + } if( pConstraint->usable==0 ) continue; if( (iPlan & 1)==0 && pConstraint->iColumn==CLOSURE_COL_ROOT @@ -895,6 +900,7 @@ static int closureBestIndex( ){ pIdxInfo->orderByConsumed = 1; } + if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30; pIdxInfo->estimatedCost = rCost; return SQLITE_OK; diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 642b8f9e92..023bdb1d09 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -1077,9 +1077,16 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iDistTerm = -1; int iRulesetTerm = -1; int i; + int seenMatch = 0; const struct sqlite3_index_constraint *pConstraint; + double rCost = 100000; + pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + seenMatch = 1; + } if( pConstraint->usable==0 ) continue; if( (iPlan & 1)==0 && pConstraint->iColumn==0 @@ -1088,6 +1095,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 1; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 1000000.0; } if( (iPlan & 2)==0 && pConstraint->iColumn==1 @@ -1096,6 +1104,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ iPlan |= 2; iDistTerm = i; + rCost /= 10.0; } if( (iPlan & 4)==0 && pConstraint->iColumn==2 @@ -1104,6 +1113,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 4; pIdxInfo->aConstraintUsage[i].omit = 1; iRulesetTerm = i; + rCost /= 10.0; } } if( iPlan & 2 ){ @@ -1122,7 +1132,8 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)10000; + if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30; + pIdxInfo->estimatedCost = rCost; return SQLITE_OK; } diff --git a/manifest b/manifest index dba927beb0..75f057b5a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pull\sin\srecent\strunk\schanges.\s\sFix\sthe\sORDER\sBY\soptimizer\sso\sthat\sit\sis\sbetter\nable\sto\sdeal\swith\sCOLLATE\sclauses.\s\sClean\sup\sambiguities\sin\sthe\sdescidx1.test\nscript. -D 2013-06-03T20:46:35.011 +C Adjust\sthe\sxBestIndex\smethods\son\sboth\sthe\sfuzzer\sand\stransitive_closure\nvirtual\stables\sso\sthat\san\sunused\sMATCH\soperator\sgets\sa\sreally\slarge\scost.\nRemove\sambiguities\sfrom\sthe\sfuzzer\stest\scases. +D 2013-06-03T21:25:28.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -107,8 +107,8 @@ F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c 7538f98eab2854cf17fa5f7797bffa6c76e3863b F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c eae8454cd9dcb287b2a3ec2e65a865a4ac5f0d06 -F ext/misc/closure.c b88aa95fbe32edc74e16ed5bb6646d249a2bf6ad -F ext/misc/fuzzer.c 51bd96960b6b077d41d6f3cedefbcb57f29efaa2 +F ext/misc/closure.c 997c20ddf35f85ab399f4a02a557a9baa822ec32 +F ext/misc/fuzzer.c 79ac20b68794dfe03626f3a31681e6c240583052 F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/percentile.c 4fb5e46c4312b0be74e8e497ac18f805f0e3e6c5 @@ -561,7 +561,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 41bd5aa6ae0cf18d06342a4476e3cad98604ae48 +F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/hook.test 45cb22b940c3cc0af616ba7430f666e245711a48 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fabb21854e662b1d8e5631e79f828d5322ceb595 4d74fccf02134a998a84097b021ba9d501e34ff0 -R b746dc544342dfc07463de0423275bdc +P 6bc71dfcf0ef757c5c2b426dd8fddc1e5ae0f598 +R 9f95af129af308d556c3b0bed3929212 U drh -Z 5182eac11ae0820ee1af6bccc6719c26 +Z 30d36dfc5ff4e44e38a2e90ed964881e diff --git a/manifest.uuid b/manifest.uuid index 1e5db89b77..5fac46b81c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bc71dfcf0ef757c5c2b426dd8fddc1e5ae0f598 \ No newline at end of file +e2c1af78b65a8ace976fa6c035db212e1ffc79b8 \ No newline at end of file diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 473d0e1868..4ee5730807 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1728,36 +1728,41 @@ do_execsql_test 8.1 { do_execsql_test 8.2.1 { SELECT cFrom, cTo, word FROM x3_rules CROSS JOIN x3 - WHERE word MATCH 'a' AND cost=distance AND ruleset=2; + WHERE word MATCH 'a' AND cost=distance AND ruleset=2 + ORDER BY +cTo; } {a x x a y y a z z} do_execsql_test 8.2.2 { SELECT cFrom, cTo, word FROM x3 CROSS JOIN x3_rules - WHERE word MATCH 'a' AND cost=distance AND ruleset=2; + WHERE word MATCH 'a' AND cost=distance AND ruleset=2 + ORDER BY +cTo DESC } {a z z a y y a x x} do_execsql_test 8.2.3 { SELECT cFrom, cTo, word FROM x3_rules, x3 - WHERE word MATCH 'a' AND cost=distance AND ruleset=2; + WHERE word MATCH 'a' AND cost=distance AND ruleset=2 + ORDER BY +cTo DESC; } {a z z a y y a x x} do_execsql_test 8.2.4 { SELECT cFrom, cTo, word FROM x3, x3_rules - WHERE word MATCH 'a' AND cost=distance AND ruleset=2; + WHERE word MATCH 'a' AND cost=distance AND ruleset=2 + ORDER BY +cTo DESC; } {a z z a y y a x x} do_execsql_test 8.2.5 { CREATE INDEX i1 ON x3_rules(cost); SELECT cFrom, cTo, word FROM x3_rules, x3 - WHERE word MATCH 'a' AND cost=distance AND ruleset=2; + WHERE word MATCH 'a' AND cost=distance AND ruleset=2 + ORDER BY +cTo DESC; } {a z z a y y a x x} do_execsql_test 8.2.5 { - SELECT word FROM x3_rules, x3 WHERE word MATCH x3_rules.cFrom AND ruleset=2; + SELECT word FROM x3_rules, x3 WHERE word MATCH x3_rules.cFrom AND ruleset=2 } {a z y x a z y x a z y x} do_execsql_test 8.2.6 { From 7c171098b7adec43ea43022e58e1b8acafec7d28 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Jun 2013 22:08:20 +0000 Subject: [PATCH 054/109] Remove more vestiges of sqlite_query_plan from the test cases. FossilOrigin-Name: eb27086e8a8a4d5fcb2ea358256a555e34339423 --- manifest | 22 +++++++++--------- manifest.uuid | 2 +- test/between.test | 2 +- test/intpkey.test | 7 ++++-- test/like.test | 57 +++++++++++++++++++++++++++++++--------------- test/subquery.test | 7 ++++-- test/where2.test | 3 +-- test/where3.test | 26 ++++++++++++++------- 8 files changed, 81 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 75f057b5a8..e5312f3930 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\sxBestIndex\smethods\son\sboth\sthe\sfuzzer\sand\stransitive_closure\nvirtual\stables\sso\sthat\san\sunused\sMATCH\soperator\sgets\sa\sreally\slarge\scost.\nRemove\sambiguities\sfrom\sthe\sfuzzer\stest\scases. -D 2013-06-03T21:25:28.624 +C Remove\smore\svestiges\sof\ssqlite_query_plan\sfrom\sthe\stest\scases. +D 2013-06-03T22:08:20.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -335,7 +335,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa -F test/between.test bec144cd75698402bbda8ad8e23bd8588e426145 +F test/between.test e6896728a636e3e610bcb2f87c9305120c809bc5 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 7c79f1ef0c6c2c2bc1e7bd895596fab32bfb4796 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 @@ -596,7 +596,7 @@ F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc -F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 +F test/intpkey.test a9674fc6195e0952e4e6105a9981ce1e48e7f215 F test/io.test ecf44cc81664ad54d8253e2d88fc705b6554abe3 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d @@ -619,7 +619,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 F test/lastinsert.test 474d519c68cb79d07ecae56a763aa7f322c72f51 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test 0e5412f4dac4a849f613e1ef8b529d56a6e31d08 +F test/like.test 52e650adfa208325f928b847571d77b647af07c3 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 @@ -792,7 +792,7 @@ F test/spellfix.test bea537caf587df30d430c2c6a8fe9f64b8712834 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 -F test/subquery.test 869562de9e8c5d8147e0451a2ce5b58cf55ce389 +F test/subquery.test 3b97763ada8b3a4092d2c80db3ebc0e7870e7eaf F test/subquery2.test 91e1e364072aeff431d1f9689b15147e421d88c7 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a @@ -1029,8 +1029,8 @@ F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 054a6b6f7933c5a5f50d0bcd650b5eccb450cc81 -F test/where2.test f902f62d60f20b0257dd700f0c4a2945527fca8e -F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 +F test/where2.test 116fb0d6e98a423d12eb9a65906218ce09936674 +F test/where3.test ae3054e1216ecc0afbcb83674310ebc5de25ba09 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6bc71dfcf0ef757c5c2b426dd8fddc1e5ae0f598 -R 9f95af129af308d556c3b0bed3929212 +P e2c1af78b65a8ace976fa6c035db212e1ffc79b8 +R 85129ff0b9a881f15c06fc2c70cadd93 U drh -Z 30d36dfc5ff4e44e38a2e90ed964881e +Z 30b7be3ba3d79e592e3b359a6ec5ea92 diff --git a/manifest.uuid b/manifest.uuid index 5fac46b81c..387facd0c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2c1af78b65a8ace976fa6c035db212e1ffc79b8 \ No newline at end of file +eb27086e8a8a4d5fcb2ea358256a555e34339423 \ No newline at end of file diff --git a/test/between.test b/test/between.test index 98be29aea7..f089cd95d3 100644 --- a/test/between.test +++ b/test/between.test @@ -48,7 +48,7 @@ do_test between-1.0 { # This procedure executes the SQL. Then it appends to the result the # "sort" or "nosort" keyword depending on whether or not any sorting -# is done. Then it appends the ::sqlite_query_plan variable. +# is done. Then it appends the names of the table and index used. # proc queryplan {sql} { set ::sqlite_sort_count 0 diff --git a/test/intpkey.test b/test/intpkey.test index db39421284..7ed25e4a13 100644 --- a/test/intpkey.test +++ b/test/intpkey.test @@ -125,8 +125,11 @@ do_test intpkey-1.12.1 { } } {4 one two} do_test intpkey-1.12.2 { - set sqlite_query_plan -} {t1 *} + execsql { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a==4; + } +} {/SEARCH TABLE t1 /} # Try to insert a non-integer value into the primary key field. This # should result in a data type mismatch. diff --git a/test/like.test b/test/like.test index 80ba418588..f68a93537b 100644 --- a/test/like.test +++ b/test/like.test @@ -156,14 +156,27 @@ ifcapable !like_opt { # This procedure executes the SQL. Then it appends to the result the # "sort" or "nosort" keyword (as in the cksort procedure above) then -# it appends the ::sqlite_query_plan variable. +# it appends the names of the table and index used. # proc queryplan {sql} { set ::sqlite_sort_count 0 set data [execsql $sql] if {$::sqlite_sort_count} {set x sort} {set x nosort} lappend data $x - return [concat $data $::sqlite_query_plan] + set eqp [execsql "EXPLAIN QUERY PLAN $sql"] + # puts eqp=$eqp + foreach {a b c x} $eqp { + if {[regexp { TABLE (\w+ AS )?(\w+) USING COVERING INDEX (\w+)\W} \ + $x all as tab idx]} { + lappend data {} $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + $x all as tab idx]} { + lappend data $tab $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + lappend data $tab * + } + } + return $data } # Perform tests on the like optimization. @@ -176,7 +189,7 @@ do_test like-3.1 { queryplan { SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } -} {ABC {ABC abc xyz} abc abcd sort t1 {}} +} {ABC {ABC abc xyz} abc abcd sort t1 *} do_test like-3.2 { set sqlite_like_count } {12} @@ -269,8 +282,8 @@ do_test like-3.12 { # do_test like-3.13 { set sqlite_like_count 0 + db eval {PRAGMA case_sensitive_like=off;} queryplan { - PRAGMA case_sensitive_like=off; SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } } {ABC {ABC abc xyz} abc abcd nosort {} i1} @@ -282,12 +295,14 @@ do_test like-3.14 { # do_test like-3.15 { set sqlite_like_count 0 - queryplan { + db eval { PRAGMA case_sensitive_like=on; DROP INDEX i1; + } + queryplan { SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } -} {abc abcd sort t1 {}} +} {abc abcd sort t1 *} do_test like-3.16 { set sqlite_like_count } 12 @@ -299,7 +314,7 @@ do_test like-3.17 { queryplan { SELECT x FROM t1 WHERE x GLOB 'abc*' ORDER BY 1; } -} {abc abcd sort t1 {}} +} {abc abcd sort t1 *} do_test like-3.18 { set sqlite_like_count } 12 @@ -318,8 +333,8 @@ do_test like-3.20 { } 0 do_test like-3.21 { set sqlite_like_count 0 + db eval {PRAGMA case_sensitive_like=on;} queryplan { - PRAGMA case_sensitive_like=on; SELECT x FROM t1 WHERE x GLOB 'abc*' ORDER BY 1; } } {abc abcd nosort {} i1} @@ -328,8 +343,8 @@ do_test like-3.22 { } 0 do_test like-3.23 { set sqlite_like_count 0 + db eval {PRAGMA case_sensitive_like=off;} queryplan { - PRAGMA case_sensitive_like=off; SELECT x FROM t1 WHERE x GLOB 'a[bc]d' ORDER BY 1; } } {abd acd nosort {} i1} @@ -809,60 +824,66 @@ do_test like-11.0 { } } {12} do_test like-11.1 { + db eval {PRAGMA case_sensitive_like=OFF;} queryplan { - PRAGMA case_sensitive_like=OFF; SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } } {abc abcd ABC ABCD nosort t11 *} do_test like-11.2 { + db eval {PRAGMA case_sensitive_like=ON;} queryplan { - PRAGMA case_sensitive_like=ON; SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } } {abc abcd nosort t11 *} do_test like-11.3 { - queryplan { + db eval { PRAGMA case_sensitive_like=OFF; CREATE INDEX t11b ON t11(b); + } + queryplan { SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY +a; } } {abc abcd ABC ABCD sort {} t11b} do_test like-11.4 { + db eval {PRAGMA case_sensitive_like=ON;} queryplan { - PRAGMA case_sensitive_like=ON; SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } } {abc abcd nosort t11 *} do_test like-11.5 { - queryplan { + db eval { PRAGMA case_sensitive_like=OFF; DROP INDEX t11b; CREATE INDEX t11bnc ON t11(b COLLATE nocase); + } + queryplan { SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY +a; } } {abc abcd ABC ABCD sort {} t11bnc} do_test like-11.6 { + db eval {CREATE INDEX t11bb ON t11(b COLLATE binary);} queryplan { - CREATE INDEX t11bb ON t11(b COLLATE binary); SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY +a; } } {abc abcd ABC ABCD sort {} t11bnc} do_test like-11.7 { + db eval {PRAGMA case_sensitive_like=ON;} queryplan { - PRAGMA case_sensitive_like=ON; SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY +a; } } {abc abcd sort {} t11bb} do_test like-11.8 { + db eval {PRAGMA case_sensitive_like=OFF;} queryplan { - PRAGMA case_sensitive_like=OFF; SELECT b FROM t11 WHERE b GLOB 'abc*' ORDER BY +a; } } {abc abcd sort {} t11bb} do_test like-11.9 { - queryplan { + db eval { CREATE INDEX t11cnc ON t11(c COLLATE nocase); CREATE INDEX t11cb ON t11(c COLLATE binary); + } + queryplan { SELECT c FROM t11 WHERE c LIKE 'abc%' ORDER BY +a; } } {abc abcd ABC ABCD sort {} t11cnc} diff --git a/test/subquery.test b/test/subquery.test index f601d3f80b..da0a44cc5c 100644 --- a/test/subquery.test +++ b/test/subquery.test @@ -241,8 +241,11 @@ do_test subquery-2.5.3.1 { } {10.0} do_test subquery-2.5.3.2 { # Verify that the t4i index was not used in the previous query - set ::sqlite_query_plan -} {t4 {}} + execsql { + EXPLAIN QUERY PLAN + SELECT * FROM t4 WHERE x IN (SELECT a FROM t3); + } +} {/SCAN TABLE t4 /} do_test subquery-2.5.4 { execsql { DROP TABLE t3; diff --git a/test/where2.test b/test/where2.test index 60937adcc5..d7fb514a87 100644 --- a/test/where2.test +++ b/test/where2.test @@ -66,7 +66,7 @@ proc cksort {sql} { # This procedure executes the SQL. Then it appends to the result the # "sort" or "nosort" keyword (as in the cksort procedure above) then -# it appends the ::sqlite_query_plan variable. +# it appends the name of the table and index used. # proc queryplan {sql} { set ::sqlite_sort_count 0 @@ -84,7 +84,6 @@ proc queryplan {sql} { } } return $data - # return [concat $data $::sqlite_query_plan] } diff --git a/test/where3.test b/test/where3.test index e08f9051e2..c8ec8eb169 100644 --- a/test/where3.test +++ b/test/where3.test @@ -103,12 +103,22 @@ ifcapable explain { } # This procedure executes the SQL. Then it appends -# the ::sqlite_query_plan variable. +# the names of the table and index used # proc queryplan {sql} { set ::sqlite_sort_count 0 set data [execsql $sql] - return [concat $data $::sqlite_query_plan] + set eqp [execsql "EXPLAIN QUERY PLAN $sql"] + # puts eqp=$eqp + foreach {a b c x} $eqp { + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + $x all as tab idx]} { + lappend data $tab $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + lappend data $tab * + } + } + return $data } @@ -144,37 +154,37 @@ do_test where3-2.1 { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=bx AND bpk=ax } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.1.1 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON cx=dpk WHERE cpk=bx AND bpk=ax } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.1.2 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON cx=dpk WHERE bx=cpk AND bpk=ax } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.1.3 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON cx=dpk WHERE bx=cpk AND ax=bpk } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.1.4 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE bx=cpk AND ax=bpk } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.1.5 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=bx AND ax=bpk } -} {tA {} tB * tC * tD *} +} {tA * tB * tC * tD *} do_test where3-2.2 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx From 7699d1c4e5061ec35dc36c4a755de79869b8b990 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 12:42:29 +0000 Subject: [PATCH 055/109] Refactor the ORDER BY optimizer in the NGQP so that it is easier to maintain and so that it can support optimizing out GROUP BY and DISTINCT clauses. FossilOrigin-Name: e605c468e3a1163167831c4a6220825c0b5d083b --- manifest | 18 +-- manifest.uuid | 2 +- src/build.c | 2 + src/sqliteInt.h | 6 + src/where.c | 312 +++++++++++++++++++++++---------------- test/tkt-2a5629202f.test | 7 +- 6 files changed, 210 insertions(+), 137 deletions(-) diff --git a/manifest b/manifest index e5312f3930..3b65c600a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\smore\svestiges\sof\ssqlite_query_plan\sfrom\sthe\stest\scases. -D 2013-06-03T22:08:20.442 +C Refactor\sthe\sORDER\sBY\soptimizer\sin\sthe\sNGQP\sso\sthat\sit\sis\seasier\sto\smaintain\nand\sso\sthat\sit\scan\ssupport\soptimizing\sout\sGROUP\sBY\sand\sDISTINCT\sclauses. +D 2013-06-04T12:42:29.293 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0 F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 -F src/build.c 92ef9483189389828966153c5950f2e5a49c1182 +F src/build.c 838cbdcbf18de2fd5723ad5864a78cc806f6c75b F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 3ddccdf8ef912119da26945f2d8dff98f59e1d58 +F src/sqliteInt.h 259d999abebf4498615e9237a18c0815257a1f2f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0082ef59948fc8c28b35226f44e1718cb1b0300a +F src/where.c cca3284b915ee9c51dd9188e3a74e648ddf3db47 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -820,7 +820,7 @@ F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 -F test/tkt-2a5629202f.test 1ab32e084e9fc3d36be6dee2617530846a0eb0b6 +F test/tkt-2a5629202f.test befaa77b90426a5ca8025724279e0bc26336610e F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e2c1af78b65a8ace976fa6c035db212e1ffc79b8 -R 85129ff0b9a881f15c06fc2c70cadd93 +P eb27086e8a8a4d5fcb2ea358256a555e34339423 +R 3dd8dcdfd7f56eca682f5b0b9f200abc U drh -Z 30b7be3ba3d79e592e3b359a6ec5ea92 +Z 6cf52a3b39b332b4094772bd0e2b3942 diff --git a/manifest.uuid b/manifest.uuid index 387facd0c5..7dcbb18dc0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb27086e8a8a4d5fcb2ea358256a555e34339423 \ No newline at end of file +e605c468e3a1163167831c4a6220825c0b5d083b \ No newline at end of file diff --git a/src/build.c b/src/build.c index 3c91cdcfdb..255fd37a01 100644 --- a/src/build.c +++ b/src/build.c @@ -2695,6 +2695,7 @@ Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->nColumn = pList->nExpr; pIndex->onError = (u8)onError; + pIndex->uniqNotNull = onError==OE_Abort; pIndex->autoIndex = (u8)(pName==0); pIndex->pSchema = db->aDb[iDb].pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); @@ -2753,6 +2754,7 @@ Index *sqlite3CreateIndex( pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; + if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } sqlite3DefaultRowEst(pIndex); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e99098cd0a..2b07d2ef02 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1544,6 +1544,7 @@ struct Index { u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ + unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -1889,6 +1890,11 @@ typedef u64 Bitmask; */ #define BMS ((int)(sizeof(Bitmask)*8)) +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) + /* ** The following structure describes the FROM clause of a SELECT statement. ** Each table or subquery in the FROM clause is a separate element of diff --git a/src/where.c b/src/where.c index 57dc7e8aef..7577acbfb3 100644 --- a/src/where.c +++ b/src/where.c @@ -327,7 +327,7 @@ struct WhereLoopBuilder { #define WHERE_INDEXED 0x00000200 /* WhereLoop.u.btree.pIndex is valid */ #define WHERE_VIRTUALTABLE 0x00000400 /* WhereLoop.u.vtab is valid */ #define WHERE_IN_ABLE 0x00000800 /* Able to support an IN operator */ -#define WHERE_UNIQUE 0x00001000 /* Selects no more than one row */ +#define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_COVER_SCAN 0x00008000 /* Full scan of a covering index */ @@ -482,7 +482,7 @@ static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); for(i=0; in; i++){ if( pMaskSet->ix[i]==iCursor ){ - return ((Bitmask)1)<a; pTermnTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS ); testcase( iCol==BMS-1 ); if( (idxCols & cMask)==0 ){ @@ -1860,14 +1860,14 @@ static void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | (((Bitmask)1)<<(BMS-1))); + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & (((Bitmask)1)<<(BMS-1)) ){ + if( pSrc->colUsed & MASKBIT(BMS-1) ){ nColumn += pTable->nCol - BMS + 1; } pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; @@ -1891,7 +1891,7 @@ static void constructAutomaticIndex( for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); if( (idxCols & cMask)==0 ){ Expr *pX = pTerm->pExpr; idxCols |= cMask; @@ -1907,13 +1907,13 @@ static void constructAutomaticIndex( /* Add additional columns needed to make the automatic index into ** a covering index */ for(i=0; iaiColumn[n] = i; pIdx->azColl[n] = "BINARY"; n++; } } - if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ + if( pSrc->colUsed & MASKBIT(BMS-1) ){ for(i=BMS-1; inCol; i++){ pIdx->aiColumn[n] = i; pIdx->azColl[n] = "BINARY"; @@ -3390,7 +3390,7 @@ static Bitmask codeOneLoopStart( /* Record the instruction used to terminate the loop. Disable ** WHERE clause terms made redundant by the index range scan. */ - if( pLoop->wsFlags & WHERE_UNIQUE ){ + if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -4004,6 +4004,7 @@ static int whereLoopAddBtreeIndex( if( pNew->nTerm>=pBuilder->mxTerm ) break; /* Repeated column in index */ pNew->aTerm[pNew->nTerm++] = pTerm; pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; + pNew->rRun = rLogSize; if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; @@ -4014,22 +4015,27 @@ static int whereLoopAddBtreeIndex( /* "x IN (value, value, ...)" */ nIn = pExpr->x.pList->nExpr; } + pNew->rRun *= nIn; pNew->u.btree.nEq++; pNew->nOut = (double)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_EQ) ){ + assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 + || nInMul==1 ); pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 + if( iCol<0 || (pProbe->onError==OE_Abort && nInMul==1 && pNew->u.btree.nEq==pProbe->nColumn-1) ){ - pNew->wsFlags |= WHERE_UNIQUE; + testcase( pNew->wsFlags & WHERE_COLUMN_IN ); + pNew->wsFlags |= WHERE_ONEROW; } pNew->u.btree.nEq++; pNew->nOut = (double)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; - pNew->nOut = (double)iRowEst * nInMul; + nIn = 2; /* Assume IS NULL matches two rows */ + pNew->nOut = (double)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; @@ -4040,7 +4046,6 @@ static int whereLoopAddBtreeIndex( pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aTerm[pNew->nTerm-2] : 0; } - pNew->rRun = rLogSize*nIn; /* Cost for nIn binary searches */ if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ double rDiv; @@ -4218,7 +4223,7 @@ static int whereLoopAddBtree( for(j=pProbe->nColumn-1; j>=0; j--){ int x = pProbe->aiColumn[j]; if( xwsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; @@ -4529,7 +4534,7 @@ whereLoopAddAll_end: } /* -** Examine a WherePath (with the addition of the extra WhereLoop of the 4th +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th ** parameters) to see if it outputs rows in the requested ORDER BY ** (or GROUP BY) without requiring a separate source operation. Return: ** @@ -4542,52 +4547,50 @@ static int wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ WherePath *pPath, /* The WherePath to check */ int nLoop, /* Number of entries in pPath->aLoop[] */ - int isLastLoop, /* True for the very last loop */ + int isLastLoop, /* True if pLast is the inner-most loop */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* Mask of WhereLoops to run in reverse order */ ){ u8 revSet; /* True if rev is known */ u8 rev; /* Composite sort order */ u8 revIdx; /* Index sort order */ - u8 isOneRow; /* Current WhereLoop is a one-row loop */ - u8 requireOneRow = 0; /* All subsequent loops must be one-row */ - u8 isUniqueIdx; /* Current WhereLoop uses a unique index */ - u16 nColumn; - u16 nOrderBy; - int i, j; - int nUsed = 0; - int iCur; - int iColumn; - WhereLoop *pLoop; - ExprList *pOrderBy = pWInfo->pOrderBy; - Expr *pOBExpr; - CollSeq *pColl; - Index *pIndex; - sqlite3 *db = pWInfo->pParse->db; - Bitmask revMask = 0; + u8 isWellOrdered; /* All WhereLoops are well-ordered so far */ + u16 nColumn; /* Number of columns in pIndex */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop; /* Current WhereLoop being processed. */ + ExprList *pOrderBy = pWInfo->pOrderBy; /* the ORDER BY clause */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite3 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderedMask; /* Mask of all well-ordered loops */ + WhereMaskSet *pMaskSet; /* WhereMaskSet object for this where clause */ + /* - ** We say the WhereLoop is "one-row" if all of the following are true: + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: ** (a) All index columns match with WHERE_COLUMN_EQ. ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. ** - ** General rules: (not an algorithm!) - ** - ** (1) If the current WhereLoop is one-row, then match over any and all - ** ORDER BY terms for the current WhereLoop and proceed to the next - ** WhereLoop. - ** - ** (2) If the current WhereLoop is not one-row, then all subsequent - ** WhereLoops must be one-row. - ** - ** (3) Optionally match any ORDER BY terms against the first nEq columns - ** of the index. - ** - ** (4) Index columns past nEq must match ORDER BY terms one-for-one. - ** - ** (5) If all columns of a UNIQUE index have been matched against ORDER BY - ** terms, then any subsequent entries in the ORDER BY clause against the - ** same table can be skipped. + ** We say the WhereLoop is "well-ordered" if + ** (i) it satisfies at least one term of the ORDER BY clause, and + ** (ii) every row output is distinct over the terms that match the + ** ORDER BY clause. + ** Every one-row WhereLoop is automatically well-ordered, even if it + ** does not match any terms of the ORDER BY clause. + ** For condition (ii), be mindful that a UNIQUE column can have multiple + ** rows that are NULL and so it not necessarily distinct. The column + ** must be UNIQUE and NOT NULL. in order to be well-ordered. */ assert( pOrderBy!=0 ); @@ -4595,95 +4598,152 @@ static int wherePathSatisfiesOrderBy( /* Sortability of virtual tables is determined by the xBestIndex method ** of the virtual table itself */ if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ - assert( nLoop==0 ); + testcase( nLoop>0 ); /* True when outer loops are one-row and match + ** no ORDER BY terms */ return pLast->u.vtab.isOrdered; } + if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; - /* Sorting is always required if any term of the ORDER BY is not a - ** column reference */ nOrderBy = pOrderBy->nExpr; -#if 0 - for(i=0; ia[i].pExpr); - if( pOBExpr->op!=TK_COLUMN ) return 0; - } -#endif - - for(i=0; i<=nLoop && nUsedaLoop[i] : pLast; + if( nOrderBy>60 ) return 0; + isWellOrdered = 1; + obDone = MASKBIT(nOrderBy)-1; + orderedMask = 0; + pMaskSet = pWInfo->pWC->pMaskSet; + for(iLoop=0; isWellOrdered && obSataLoop[iLoop] : pLast; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - isOneRow = isUniqueIdx = 1; - if( pLoop->wsFlags & WHERE_IPK ){ - if( (pLoop->wsFlags & WHERE_COLUMN_IN)!=0 ) isOneRow = 0; - if( pLoop->u.btree.nEq!=1 ) isOneRow = 0; - pIndex = 0; - nColumn = 0; - }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ - return 0; - }else{ - nColumn = pIndex->nColumn; - if( pIndex->onError==OE_None ){ - isOneRow = isUniqueIdx = 0; - }else if( (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_RANGE - |WHERE_COLUMN_NULL))!=0 ){ - isOneRow = 0; - }else if( pLoop->u.btree.nEq < pIndex->nColumn ){ - isOneRow = 0; - } - } - if( !isOneRow && requireOneRow ) return 0; - requireOneRow = !isOneRow; iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; - j = 0; - revSet = rev = 0; - for(j=0; j<=nColumn && nUseda[nUsed].pExpr); - if( pOBExpr->op!=TK_COLUMN ) return 0; - if( pOBExpr->iTable!=iCur ) break; - if( isOneRow ){ j--; continue; } - if( jaiColumn[j]; - revIdx = pIndex->aSortOrder[j]; - if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; - }else{ - /* The ROWID column at the end */ - iColumn = -1; - revIdx = 0; - } - skipable = ju.btree.nEq && pLoop->aTerm[j]->eOperator!=WO_IN; - if( pOBExpr->iColumn!=iColumn ){ - if( skipable ){ nUsed--; continue; } + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nColumn = 0; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; + }else{ + nColumn = pIndex->nColumn; } - if( iColumn>=0 ){ - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[nUsed].pExpr); - if( !pColl ) pColl = db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ){ - return 0; + + /* For every term of the index that is constrained by == or IS NULL + ** mark off corresponding ORDER BY terms wherever they occur + ** in the ORDER BY clause. + */ + for(i=0; iu.btree.nEq; i++){ + pTerm = pLoop->aTerm[i]; + if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))==0 ) continue; + iColumn = pTerm->u.leftColumn; + for(j=0; ja[j].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[j].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[i])!=0 ) continue; + } + obSat |= MASKBIT(j); } + if( obSat==obDone ) return 1; } - if( !skipable ){ - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[nUsed].sortOrder ) return 0; + + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + for(j=0; j<=nColumn; j++){ + u8 bOnce; /* True to run the ORDER BY search loop */ + + if( ju.btree.nEq + && (pLoop->aTerm[j]->eOperator & (WO_EQ|WO_ISNULL))!=0 + ){ + continue; /* Skip == and IS NULL terms already processed */ + } + + /* Get the column number in the table and sort order for the + ** j-th column of the index for this WhereLoop + */ + if( jaiColumn[j]; + revIdx = pIndex->aSortOrder[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - rev = revIdx ^ pOrderBy->a[nUsed].sortOrder; - revSet = 1; + /* The ROWID column at the end */ + iColumn = -1; + revIdx = 0; + } + + /* An unconstrained column that might be NULL means that this + ** WhereLoop is not well-ordered + */ + if( iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isWellOrdered = 0; + } + + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and and mark that ORDER BY term off + */ + bOnce = 1; + for(i=0; bOnce && ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ) bOnce = 0; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; + } + bOnce = 1; + break; + } + if( bOnce && iwctrlFlags & WHERE_GROUPBY)==0 ){ + /* If we have an ORDER BY clause, we must match the next available + ** column of the ORDER BY */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + }else{ + /* No match found */ + if( jonError!=OE_Abort ){ + isWellOrdered = 0; + } + break; + } + } /* end Loop over all index columns */ + } /* end-if not one-row */ + + /* Mark off any other ORDER BY terms that reference pLoop */ + if( isWellOrdered ){ + orderedMask |= pLoop->maskSelf; + for(i=0; ia[i].pExpr; + if( (exprTableUsage(pMaskSet, p)&~orderedMask)==0 ){ + obSat |= MASKBIT(i); } - } - if( j>=nColumn-1 && isUniqueIdx ){ - if( isLastLoop && i==nLoop ) break; - j--; - isOneRow = 1; } } - if( rev ) revMask |= ((Bitmask)1)<nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_UNIQUE)!=0 ){ + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_ONEROW)!=0 ){ pWInfo->okOnePass = 1; pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; } diff --git a/test/tkt-2a5629202f.test b/test/tkt-2a5629202f.test index 037f100f65..5f31d1ecc0 100644 --- a/test/tkt-2a5629202f.test +++ b/test/tkt-2a5629202f.test @@ -46,6 +46,12 @@ do_execsql_test 1.3 { SELECT coalesce(b, 'null') || '/' || c FROM t8 x ORDER BY x.b, x.c } {null/four null/three a/one b/two} +do_execsql_test 1.4 { + DROP INDEX t8; + CREATE UNIQUE INDEX i1 ON t8(b, c); + SELECT coalesce(b, 'null') || '/' || c FROM t8 x ORDER BY x.b, x.c +} {null/four null/three a/one b/two} + #------------------------------------------------------------------------- # @@ -68,4 +74,3 @@ do_test 2.4 { } {sort} finish_test - From ef71c1f0904e1e3b245ef9a26dc8bcdcb4064ccb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 12:58:02 +0000 Subject: [PATCH 056/109] Fix a display issue with EXPLAIN QUERY PLAN. FossilOrigin-Name: ff2fa407558360f2499b6df0392ab3cdb4788dec --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3b65c600a9..5e757202a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sORDER\sBY\soptimizer\sin\sthe\sNGQP\sso\sthat\sit\sis\seasier\sto\smaintain\nand\sso\sthat\sit\scan\ssupport\soptimizing\sout\sGROUP\sBY\sand\sDISTINCT\sclauses. -D 2013-06-04T12:42:29.293 +C Fix\sa\sdisplay\sissue\swith\sEXPLAIN\sQUERY\sPLAN. +D 2013-06-04T12:58:02.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c cca3284b915ee9c51dd9188e3a74e648ddf3db47 +F src/where.c 89e9e0180fb061529c2335483e698461020008b3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P eb27086e8a8a4d5fcb2ea358256a555e34339423 -R 3dd8dcdfd7f56eca682f5b0b9f200abc +P e605c468e3a1163167831c4a6220825c0b5d083b +R 0c3dd4aecffc3c33e948e0ad3535455c U drh -Z 6cf52a3b39b332b4094772bd0e2b3942 +Z fde826e1fa4a5f4defa852132d2e9311 diff --git a/manifest.uuid b/manifest.uuid index 7dcbb18dc0..b2ed6c1c78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e605c468e3a1163167831c4a6220825c0b5d083b \ No newline at end of file +ff2fa407558360f2499b6df0392ab3cdb4788dec \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7577acbfb3..fa06259d67 100644 --- a/src/where.c +++ b/src/where.c @@ -319,6 +319,7 @@ struct WhereLoopBuilder { #define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ #define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ #define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ #define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and x Date: Tue, 4 Jun 2013 13:07:44 +0000 Subject: [PATCH 057/109] Increase the estimated full-scan cost for an FTS3 table. FossilOrigin-Name: 8d9f29f847f38612447c79a5fb366a9143ef6205 --- ext/fts3/fts3.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 374d690688..e3a11b301a 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1423,7 +1423,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** strategy is possible. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 500000; + pInfo->estimatedCost = 5000000; for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; if( pCons->usable==0 ) continue; diff --git a/manifest b/manifest index 5e757202a7..1046753ec0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdisplay\sissue\swith\sEXPLAIN\sQUERY\sPLAN. -D 2013-06-04T12:58:02.050 +C Increase\sthe\sestimated\sfull-scan\scost\sfor\san\sFTS3\stable. +D 2013-06-04T13:07:44.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 931b3c83abdd1ab3bb389b2130431c2a9ff73b91 +F ext/fts3/fts3.c 760fecd100598dc22f9733d472b7cf2e9375c620 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 0b167bed9e63151635620a4f639bc62ac6012cba F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e605c468e3a1163167831c4a6220825c0b5d083b -R 0c3dd4aecffc3c33e948e0ad3535455c +P ff2fa407558360f2499b6df0392ab3cdb4788dec +R 5715088460a26a1980fd1c02022e33fe U drh -Z fde826e1fa4a5f4defa852132d2e9311 +Z e8fb494e500ca7853631ddff96a44d93 diff --git a/manifest.uuid b/manifest.uuid index b2ed6c1c78..c6152c2bac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff2fa407558360f2499b6df0392ab3cdb4788dec \ No newline at end of file +8d9f29f847f38612447c79a5fb366a9143ef6205 \ No newline at end of file From ee73b87181bbbda3e00b172c42613bccbf1bf6d7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 13:37:26 +0000 Subject: [PATCH 058/109] Better determination of when an index is UNIQUE. FossilOrigin-Name: 63fd025ad98c33826342824c51436ac92e2bb579 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1046753ec0..7cdae30fd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sestimated\sfull-scan\scost\sfor\san\sFTS3\stable. -D 2013-06-04T13:07:44.387 +C Better\sdetermination\sof\swhen\san\sindex\sis\sUNIQUE. +D 2013-06-04T13:37:26.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 89e9e0180fb061529c2335483e698461020008b3 +F src/where.c 07d741484ae3f3758ec7e9a607bebc9ca3d04830 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ff2fa407558360f2499b6df0392ab3cdb4788dec -R 5715088460a26a1980fd1c02022e33fe +P 8d9f29f847f38612447c79a5fb366a9143ef6205 +R 816acabe38521333f8c2b91e9413c2fb U drh -Z e8fb494e500ca7853631ddff96a44d93 +Z 7aae80e9f7f08a338e021a4d83d77e82 diff --git a/manifest.uuid b/manifest.uuid index c6152c2bac..603eff6cc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d9f29f847f38612447c79a5fb366a9143ef6205 \ No newline at end of file +63fd025ad98c33826342824c51436ac92e2bb579 \ No newline at end of file diff --git a/src/where.c b/src/where.c index fa06259d67..4c062435c6 100644 --- a/src/where.c +++ b/src/where.c @@ -4024,7 +4024,7 @@ static int whereLoopAddBtreeIndex( || nInMul==1 ); pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 - || (pProbe->onError==OE_Abort && nInMul==1 + || (pProbe->onError!=OE_None && nInMul==1 && pNew->u.btree.nEq==pProbe->nColumn-1) ){ testcase( pNew->wsFlags & WHERE_COLUMN_IN ); @@ -4721,7 +4721,7 @@ static int wherePathSatisfiesOrderBy( } }else{ /* No match found */ - if( jonError!=OE_Abort ){ + if( jonError==OE_None ){ isWellOrdered = 0; } break; From aec93a1f2d37aa86e8174a4ef5be065c9b4dc3b3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 15:31:16 +0000 Subject: [PATCH 059/109] Test case updates. FossilOrigin-Name: 0360fec7c093870269211447e9642b5ee34ff778 --- manifest | 12 ++--- manifest.uuid | 2 +- test/orderby1.test | 125 ++++++++++++++++++++++++++------------------- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 7cdae30fd3..076d503589 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\sdetermination\sof\swhen\san\sindex\sis\sUNIQUE. -D 2013-06-04T13:37:26.983 +C Test\scase\supdates. +D 2013-06-04T15:31:16.980 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -690,7 +690,7 @@ F test/notnull.test 2afad748d18fd66d01f66463de73b3e2501fb226 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test f33968647da5c546528fe4d2bf86c6a6a2e5a7ae +F test/orderby1.test 3c5ff32c23364fb3d567ad53b945837736e701ce F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8d9f29f847f38612447c79a5fb366a9143ef6205 -R 816acabe38521333f8c2b91e9413c2fb +P 63fd025ad98c33826342824c51436ac92e2bb579 +R 67e11291890a37d4a1f5e5249ecb5cb8 U drh -Z 7aae80e9f7f08a338e021a4d83d77e82 +Z 7007403ff7385bdec3d57bd5a78d418a diff --git a/manifest.uuid b/manifest.uuid index 603eff6cc4..314ab804b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63fd025ad98c33826342824c51436ac92e2bb579 \ No newline at end of file +0360fec7c093870269211447e9642b5ee34ff778 \ No newline at end of file diff --git a/test/orderby1.test b/test/orderby1.test index f459fc8195..73ecb67dc7 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -48,7 +48,7 @@ do_test 1.0 { } {} do_test 1.1a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {one-a one-c two-a two-b three-a three-c} @@ -66,7 +66,7 @@ do_test 1.1b { # do_test 1.2a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn } } {one-a one-c two-a two-b three-a three-c} @@ -75,7 +75,7 @@ do_test 1.2a { do_test 1.2b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn } } {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms @@ -85,13 +85,13 @@ do_test 1.3a { optimization_control db order-by-idx-join 0 db cache flush db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {one-a one-c two-a two-b three-a three-c} do_test 1.3b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {/ORDER BY/} ;# separate sorting pass due to disabled optimization optimization_control db all 1 @@ -101,55 +101,58 @@ db cache flush # do_test 1.4a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } } {three-a three-c two-a two-b one-a one-c} do_test 1.4b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn } } {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting do_test 1.4c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } -} {~/ORDER BY/} ;# optimized out +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC do_test 1.5a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } } {one-c one-a two-b two-a three-c three-a} do_test 1.5b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC } } {one-c one-a two-b two-a three-c three-a} ;# verify same order after sorting do_test 1.5c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } -} {~/ORDER BY/} ;# optimized out +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC do_test 1.6a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album CROSS JOIN track USING (aid) + ORDER BY title DESC, tn DESC } } {three-c three-a two-b two-a one-c one-a} do_test 1.6b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + SELECT name FROM album CROSS JOIN track USING (aid) + ORDER BY +title DESC, +tn DESC } } {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting do_test 1.6c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album CROSS JOIN track USING (aid) + ORDER BY title DESC, tn DESC } -} {~/ORDER BY/} ;# ORDER BY optimized-out +} {~/ORDER BY/} ;# ORDER BY # Reconstruct the test data to use indices rather than integer primary keys. @@ -183,7 +186,7 @@ do_test 2.0 { } {} do_test 2.1a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {one-a one-c two-a two-b three-a three-c} @@ -192,28 +195,28 @@ do_test 2.1a { do_test 2.1b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } -} {~/ORDER BY/} ;# ORDER BY optimized out +} {/ORDER BY/} ;# ORDER BY required because of missing aid term in ORDER BY do_test 2.1c { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, aid, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, aid, tn } } {one-a one-c two-a two-b three-a three-c} do_test 2.1d { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, aid, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, aid, tn } -} {~/ORDER BY/} ;# ORDER BY optimized out +} {/ORDER BY/} ;# ORDER BY required in this case # The same query with ORDER BY clause optimization disabled via + operators # should give exactly the same answer. # do_test 2.2a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn } } {one-a one-c two-a two-b three-a three-c} @@ -222,7 +225,7 @@ do_test 2.2a { do_test 2.2b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn } } {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms @@ -232,13 +235,13 @@ do_test 2.3a { optimization_control db order-by-idx-join 0 db cache flush db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {one-a one-c two-a two-b three-a three-c} do_test 2.3b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {/ORDER BY/} ;# separate sorting pass due to disabled optimization optimization_control db all 1 @@ -248,55 +251,55 @@ db cache flush # do_test 2.4a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } } {three-a three-c two-a two-b one-a one-c} do_test 2.4b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn } } {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting do_test 2.4c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } -} {~/ORDER BY/} ;# optimized out +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC do_test 2.5a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } } {one-c one-a two-b two-a three-c three-a} do_test 2.5b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC } } {one-c one-a two-b two-a three-c three-a} ;# verify same order after sorting do_test 2.5c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } -} {~/ORDER BY/} ;# optimized out +} {/ORDER BY/} ;# separate sorting pass due to mixed ASC/DESC do_test 2.6a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } } {three-c three-a two-b two-a one-c one-a} do_test 2.6b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn DESC } } {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting do_test 2.6c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } -} {~/ORDER BY/} ;# ORDER BY optimized out +} {/ORDER BY/} ;# ORDER BY required # Generate another test dataset, but this time using mixed ASC/DESC indices. @@ -348,7 +351,7 @@ do_test 3.1b { # do_test 3.2a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC } } {one-c one-a two-b two-a three-c three-a} @@ -357,7 +360,7 @@ do_test 3.2a { do_test 3.2b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC } } {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms @@ -367,13 +370,13 @@ do_test 3.3a { optimization_control db order-by-idx-join 0 db cache flush db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } } {one-c one-a two-b two-a three-c three-a} do_test 3.3b { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } } {/ORDER BY/} ;# separate sorting pass due to disabled optimization optimization_control db all 1 @@ -383,38 +386,38 @@ db cache flush # do_test 3.4a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } } {one-a one-c two-a two-b three-a three-c} do_test 3.4b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title, +tn + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn } } {one-a one-c two-a two-b three-a three-c} ;# verify same order after sorting do_test 3.4c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn } -} {~/ORDER BY/} ;# optimized out +} {/ORDER BY/} ;# separate sorting pass due to mismatched DESC/ASC do_test 3.5a { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } } {three-c three-a two-b two-a one-c one-a} do_test 3.5b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn DESC } } {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting do_test 3.5c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY title DESC, tn DESC + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC } -} {~/ORDER BY/} ;# optimzed out +} {/ORDER BY/} ;# separate sorting pass due to mismatched ASC/DESC do_test 3.6a { @@ -424,7 +427,8 @@ do_test 3.6a { } {three-a three-c two-a two-b one-a one-c} do_test 3.6b { db eval { - SELECT name FROM album CROSS JOIN track USING (aid) ORDER BY +title DESC, +tn + SELECT name FROM album CROSS JOIN track USING (aid) + ORDER BY +title DESC, +tn } } {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting do_test 3.6c { @@ -434,5 +438,22 @@ do_test 3.6c { } } {~/ORDER BY/} ;# inverted ASC/DESC is optimized out +# Ticket 5ed1772895bf3deeab78c5e3519b1da9165c541b (2013-06-04) +# Incorrect ORDER BY on an indexed JOIN +# +do_test 4.0 { + db eval { + CREATE TABLE t41(a INT UNIQUE NOT NULL, b INT NOT NULL); + CREATE INDEX t41ba ON t41(b,a); + CREATE TABLE t42(x INT NOT NULL REFERENCES t41(a), y INT NOT NULL); + CREATE UNIQUE INDEX t42xy ON t42(x,y); + INSERT INTO t41 VALUES(1,1),(3,1); + INSERT INTO t42 VALUES(1,13),(1,15),(3,14),(3,16); + + SELECT b, y FROM t41 CROSS JOIN t42 ON x=a ORDER BY b, y; + } +} {1 13 1 14 1 15 1 16} + + finish_test From 907717fe074d5f5b44b40b7f868b0b2447d3e470 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 18:03:22 +0000 Subject: [PATCH 060/109] Get the index-only optimization working for OR queries. FossilOrigin-Name: 774d5ff857cfad3471401ed518ed0959eb912e6c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 076d503589..7dd0d0fc27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scase\supdates. -D 2013-06-04T15:31:16.980 +C Get\sthe\sindex-only\soptimization\sworking\sfor\sOR\squeries. +D 2013-06-04T18:03:22.637 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 07d741484ae3f3758ec7e9a607bebc9ca3d04830 +F src/where.c e46c09e1fb10d9b9822441cbc0e92712d68b193c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 63fd025ad98c33826342824c51436ac92e2bb579 -R 67e11291890a37d4a1f5e5249ecb5cb8 +P 0360fec7c093870269211447e9642b5ee34ff778 +R 0af0bd237ed8aa13c84924b3ff155245 U drh -Z 7007403ff7385bdec3d57bd5a78d418a +Z c037722d2012f6d1382fb3f047d6db42 diff --git a/manifest.uuid b/manifest.uuid index 314ab804b9..07874c82c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0360fec7c093870269211447e9642b5ee34ff778 \ No newline at end of file +774d5ff857cfad3471401ed518ed0959eb912e6c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c062435c6..885a15df35 100644 --- a/src/where.c +++ b/src/where.c @@ -3593,7 +3593,7 @@ static Bitmask codeOneLoopStart( && (ii==0 || pSubLoop->u.btree.pIndex==pCov) ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); - pCov = pLoop->u.btree.pIndex; + pCov = pSubLoop->u.btree.pIndex; }else{ pCov = 0; } From d84ce35bfdae2a4bb25eeb2e808d94f9ba0286a3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 18:27:41 +0000 Subject: [PATCH 061/109] Restore the PRAGMA reverse_unordered_selects behavior. FossilOrigin-Name: f49cd6c4e752e39801f9d5de0bc370d26f43837c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7dd0d0fc27..487bb8cc30 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\sindex-only\soptimization\sworking\sfor\sOR\squeries. -D 2013-06-04T18:03:22.637 +C Restore\sthe\sPRAGMA\sreverse_unordered_selects\sbehavior. +D 2013-06-04T18:27:41.462 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e46c09e1fb10d9b9822441cbc0e92712d68b193c +F src/where.c cc1e1a7255b6722a419d6a5016d89f227e7a9ce7 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0360fec7c093870269211447e9642b5ee34ff778 -R 0af0bd237ed8aa13c84924b3ff155245 +P 774d5ff857cfad3471401ed518ed0959eb912e6c +R a1a3ee93fd22dda060fc640b96157ac6 U drh -Z c037722d2012f6d1382fb3f047d6db42 +Z ec57b78c26a34e23e063493e6b075b85 diff --git a/manifest.uuid b/manifest.uuid index 07874c82c4..dd954ef445 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -774d5ff857cfad3471401ed518ed0959eb912e6c \ No newline at end of file +f49cd6c4e752e39801f9d5de0bc370d26f43837c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 885a15df35..3bb15bdc96 100644 --- a/src/where.c +++ b/src/where.c @@ -5250,6 +5250,8 @@ WhereInfo *sqlite3WhereBegin( if( pWInfo->pOrderBy ){ wherePathSolver(pWInfo, pWInfo->nRowOut); if( db->mallocFailed ) goto whereBeginError; + }else if( db->flags & SQLITE_ReverseOrder ){ + pWInfo->revMask = (Bitmask)(-1); } if( pParse->nErr || db->mallocFailed ){ goto whereBeginError; From e353ee3898ce329271e8b4dd726307e730907601 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Jun 2013 23:40:53 +0000 Subject: [PATCH 062/109] Update some variable names and comments in the ORDER BY optimizer. Fix a bug in the ORDER BY optimizer dealing with IS NULL constraints. Updates to test cases. FossilOrigin-Name: cf96eb5945a9bab71104cb1581ee13ab30022566 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/where.c | 82 +++++++++++++++++++++++++++--------------------- test/where3.test | 12 +++---- 4 files changed, 60 insertions(+), 50 deletions(-) diff --git a/manifest b/manifest index 487bb8cc30..7b304b267b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restore\sthe\sPRAGMA\sreverse_unordered_selects\sbehavior. -D 2013-06-04T18:27:41.462 +C Update\ssome\svariable\snames\sand\scomments\sin\sthe\sORDER\sBY\soptimizer.\s\sFix\sa\nbug\sin\sthe\sORDER\sBY\soptimizer\sdealing\swith\sIS\sNULL\sconstraints.\s\sUpdates\nto\stest\scases. +D 2013-06-04T23:40:53.563 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c cc1e1a7255b6722a419d6a5016d89f227e7a9ce7 +F src/where.c 0b8fd61de3c71df0ba084d13177c384bb50e4d22 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1030,7 +1030,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 054a6b6f7933c5a5f50d0bcd650b5eccb450cc81 F test/where2.test 116fb0d6e98a423d12eb9a65906218ce09936674 -F test/where3.test ae3054e1216ecc0afbcb83674310ebc5de25ba09 +F test/where3.test 311c04e16f72816616d05d96dd354db7bce545eb F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 774d5ff857cfad3471401ed518ed0959eb912e6c -R a1a3ee93fd22dda060fc640b96157ac6 +P f49cd6c4e752e39801f9d5de0bc370d26f43837c +R e9f7dc193466abcd06681335597fcecc U drh -Z ec57b78c26a34e23e063493e6b075b85 +Z dc06d4669e35a15ef353fefc6b023d96 diff --git a/manifest.uuid b/manifest.uuid index dd954ef445..269a302e79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49cd6c4e752e39801f9d5de0bc370d26f43837c \ No newline at end of file +cf96eb5945a9bab71104cb1581ee13ab30022566 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3bb15bdc96..d9dccf7541 100644 --- a/src/where.c +++ b/src/where.c @@ -4555,7 +4555,9 @@ static int wherePathSatisfiesOrderBy( u8 revSet; /* True if rev is known */ u8 rev; /* Composite sort order */ u8 revIdx; /* Index sort order */ - u8 isWellOrdered; /* All WhereLoops are well-ordered so far */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ u16 nColumn; /* Number of columns in pIndex */ u16 nOrderBy; /* Number terms in the ORDER BY clause */ int iLoop; /* Index of WhereLoop in pPath being processed */ @@ -4571,8 +4573,8 @@ static int wherePathSatisfiesOrderBy( sqlite3 *db = pWInfo->pParse->db; /* Database connection */ Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ Bitmask obDone; /* Mask of all ORDER BY terms */ - Bitmask orderedMask; /* Mask of all well-ordered loops */ - WhereMaskSet *pMaskSet; /* WhereMaskSet object for this where clause */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + WhereMaskSet *pMaskSet; /* WhereMaskSet object for this where clause */ /* @@ -4583,15 +4585,18 @@ static int wherePathSatisfiesOrderBy( ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. ** - ** We say the WhereLoop is "well-ordered" if - ** (i) it satisfies at least one term of the ORDER BY clause, and - ** (ii) every row output is distinct over the terms that match the - ** ORDER BY clause. - ** Every one-row WhereLoop is automatically well-ordered, even if it - ** does not match any terms of the ORDER BY clause. - ** For condition (ii), be mindful that a UNIQUE column can have multiple - ** rows that are NULL and so it not necessarily distinct. The column - ** must be UNIQUE and NOT NULL. in order to be well-ordered. + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. */ assert( pOrderBy!=0 ); @@ -4606,12 +4611,12 @@ static int wherePathSatisfiesOrderBy( if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nOrderBy = pOrderBy->nExpr; - if( nOrderBy>60 ) return 0; - isWellOrdered = 1; + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; obDone = MASKBIT(nOrderBy)-1; - orderedMask = 0; + orderDistinctMask = 0; pMaskSet = pWInfo->pWC->pMaskSet; - for(iLoop=0; isWellOrdered && obSataLoop[iLoop] : pLast; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -4623,9 +4628,10 @@ static int wherePathSatisfiesOrderBy( return 0; }else{ nColumn = pIndex->nColumn; + isOrderDistinct = pIndex->onError!=OE_None; } - /* For every term of the index that is constrained by == or IS NULL + /* For every term of the index that is constrained by == or IS NULL, ** mark off corresponding ORDER BY terms wherever they occur ** in the ORDER BY clause. */ @@ -4653,17 +4659,20 @@ static int wherePathSatisfiesOrderBy( ** that are not constrained by == or IN. */ rev = revSet = 0; + distinctColumns = 0; for(j=0; j<=nColumn; j++){ u8 bOnce; /* True to run the ORDER BY search loop */ + /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && (pLoop->aTerm[j]->eOperator & (WO_EQ|WO_ISNULL))!=0 + && ((i = pLoop->aTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ - continue; /* Skip == and IS NULL terms already processed */ + if( i & WO_ISNULL ) isOrderDistinct = 0; + continue; } - /* Get the column number in the table and sort order for the - ** j-th column of the index for this WhereLoop + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. */ if( j=0 + if( isOrderDistinct + && iColumn>=0 && j>=pLoop->u.btree.nEq && pIndex->pTable->aCol[iColumn].notNull==0 ){ - isWellOrdered = 0; + isOrderDistinct = 0; } /* Find the ORDER BY term that corresponds to the j-th column ** of the index and and mark that ORDER BY term off */ bOnce = 1; + isMatch = 0; for(i=0; bOnce && ia[i].pExpr); - if( pOBExpr->op!=TK_COLUMN ) continue; if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ) bOnce = 0; + if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; if( iColumn>=0 ){ @@ -4702,15 +4713,15 @@ static int wherePathSatisfiesOrderBy( if( !pColl ) pColl = db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } - bOnce = 1; + isMatch = 1; break; } - if( bOnce && iwctrlFlags & WHERE_GROUPBY)==0 ){ - /* If we have an ORDER BY clause, we must match the next available - ** column of the ORDER BY */ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; }else{ @@ -4721,29 +4732,28 @@ static int wherePathSatisfiesOrderBy( } }else{ /* No match found */ - if( jonError==OE_None ){ - isWellOrdered = 0; - } + if( j==0 || jmaskSelf; + if( isOrderDistinct ){ + orderDistinctMask |= pLoop->maskSelf; for(i=0; ia[i].pExpr; - if( (exprTableUsage(pMaskSet, p)&~orderedMask)==0 ){ + if( (exprTableUsage(pMaskSet, p)&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } } } } if( obSat==obDone ) return 1; - if( !isWellOrdered ) return 0; + if( !isOrderDistinct ) return 0; if( isLastLoop ) return 1; return -1; } diff --git a/test/where3.test b/test/where3.test index c8ec8eb169..842f683749 100644 --- a/test/where3.test +++ b/test/where3.test @@ -190,37 +190,37 @@ do_test where3-2.2 { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=bx AND apk=bx } -} {tB {} tA * tC * tD *} +} {tB * tA * tC * tD *} do_test where3-2.3 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=bx AND apk=bx } -} {tB {} tA * tC * tD *} +} {tB * tA * tC * tD *} do_test where3-2.4 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE apk=cx AND bpk=ax } -} {tC {} tA * tB * tD *} +} {tC * tA * tB * tD *} do_test where3-2.5 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=ax AND bpk=cx } -} {tA {} tC * tB * tD *} +} {tA * tC * tB * tD *} do_test where3-2.6 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE bpk=cx AND apk=bx } -} {tC {} tB * tA * tD *} +} {tC * tB * tA * tD *} do_test where3-2.7 { queryplan { SELECT * FROM tA, tB, tC LEFT JOIN tD ON dpk=cx WHERE cpk=bx AND apk=cx } -} {tB {} tC * tA * tD *} +} {tB * tC * tA * tD *} # Ticket [13f033c865f878953] # If the outer loop must be a full table scan, do not let ANALYZE trick From 6f403ca45093a939d867b3e1e6f4e07af375b2e7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Jun 2013 12:18:15 +0000 Subject: [PATCH 063/109] Add a test case contributed on the mailing list that works in NGQP but fails in legacy. FossilOrigin-Name: 96afe50866a08ed14e911c3c955030f7ab83a3ed --- manifest | 11 +++--- manifest.uuid | 2 +- test/contrib01.test | 90 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 test/contrib01.test diff --git a/manifest b/manifest index 7b304b267b..ed506f992c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\svariable\snames\sand\scomments\sin\sthe\sORDER\sBY\soptimizer.\s\sFix\sa\nbug\sin\sthe\sORDER\sBY\soptimizer\sdealing\swith\sIS\sNULL\sconstraints.\s\sUpdates\nto\stest\scases. -D 2013-06-04T23:40:53.563 +C Add\sa\stest\scase\scontributed\son\sthe\smailing\slist\nthat\sworks\sin\sNGQP\sbut\sfails\sin\slegacy. +D 2013-06-05T12:18:15.188 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -378,6 +378,7 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 087c42997754b8c648819832241daf724f813322 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 0b3922d2304a14a47e3ccd61bbd6824327af659b +F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test 4aabd06cff3fe759e3e658bcc17b71789710665e F test/corrupt2.test 9c0ab4becd50e9050bc1ebb8675456a4e5587bf0 F test/corrupt3.test 889d7cdb811800303aa722d7813fe8a4299cf726 @@ -1093,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f49cd6c4e752e39801f9d5de0bc370d26f43837c -R e9f7dc193466abcd06681335597fcecc +P cf96eb5945a9bab71104cb1581ee13ab30022566 +R 373c05c6780cec478b9924edf49e0658 U drh -Z dc06d4669e35a15ef353fefc6b023d96 +Z c785a25f44da51b36475ec1e8ccfb14e diff --git a/manifest.uuid b/manifest.uuid index 269a302e79..bafc3efa21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf96eb5945a9bab71104cb1581ee13ab30022566 \ No newline at end of file +96afe50866a08ed14e911c3c955030f7ab83a3ed \ No newline at end of file diff --git a/test/contrib01.test b/test/contrib01.test new file mode 100644 index 0000000000..43ea47c3e4 --- /dev/null +++ b/test/contrib01.test @@ -0,0 +1,90 @@ +# 2013-06-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file contains test cases that were contributed on the sqlite-users +# mailing list on 2013-06-05 by Mi Chen at mi.chen@echostar.com. +# +# At the time it was contributed, this test failed on trunk, but +# worked on the NGQP. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Build some test data +# +do_test contrib01-1.0 { + db eval { + CREATE TABLE T1 (B INTEGER NOT NULL, + C INTEGER NOT NULL, + D INTEGER NOT NULL, + E INTEGER NOT NULL, + F INTEGER NOT NULL, + G INTEGER NOT NULL, + H INTEGER NOT NULL, + PRIMARY KEY (B, C, D)); + + CREATE TABLE T2 (A INTEGER NOT NULL, + B INTEGER NOT NULL, + C INTEGER NOT NULL, + PRIMARY KEY (A, B, C)); + + INSERT INTO T2(A, B, C) VALUES(702118,16183,15527); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15560); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15561); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15563); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15564); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15566); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15567); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15569); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15612); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15613); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15638); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15681); + INSERT INTO T2(A, B, C) VALUES(702118,16183,15682); + + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15527,6,0,5,5,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15560,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15561,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15563,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15564,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15566,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15567,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15569,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15612,6,0,5,5,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15613,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15638,6,0,5,2,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15681,6,0,5,5,0); + INSERT INTO T1(B, C, D, E, F, G, H) VALUES(16183,15682,6,0,5,2,0); + } +} {} +do_test contrib01-1.1 { + db eval { + SELECT T2.A, T2.B, T1.D, T1.E, T1.F, T1.G, T1.H, MAX(T1.C), '^' + FROM T1, T2 + WHERE T1.B = T2.B + AND T1.C = T2.C + GROUP BY T2.A, T2.B, T1.D, T1.E, T1.F, T1.G, T1.H + ORDER BY +max(t1.c); + } +} {702118 16183 6 0 5 5 0 15681 ^ 702118 16183 6 0 5 2 0 15682 ^} +do_test contrib01-1.2 { + db eval { + SELECT T2.A, T2.B, T1.D, T1.E, T1.F, T1.G, T1.H, MAX(T1.C), '^' + FROM T1, T2 + WHERE T1.B = T2.B + AND T1.C = T2.C + GROUP BY T2.A, T2.B, T1.F, T1.D, T1.E, T1.G, T1.H + ORDER BY +max(t1.c); + } +} {702118 16183 6 0 5 5 0 15681 ^ 702118 16183 6 0 5 2 0 15682 ^} + +finish_test From cd0f407c3f9ab27cfac6be88e53ce0036d2208b2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Jun 2013 12:47:59 +0000 Subject: [PATCH 064/109] Performance tweak to whereLoopInsert(). FossilOrigin-Name: 1c4a78807b7a28c9976ca8c5722ff4e2e8755b7e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 32 +++++++++++++++++--------------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ed506f992c..ae6211e19d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\scontributed\son\sthe\smailing\slist\nthat\sworks\sin\sNGQP\sbut\sfails\sin\slegacy. -D 2013-06-05T12:18:15.188 +C Performance\stweak\sto\swhereLoopInsert(). +D 2013-06-05T12:47:59.358 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0b8fd61de3c71df0ba084d13177c384bb50e4d22 +F src/where.c 3b175d731c15e30bd8893b0309f9e7e8d0f11121 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P cf96eb5945a9bab71104cb1581ee13ab30022566 -R 373c05c6780cec478b9924edf49e0658 +P 96afe50866a08ed14e911c3c955030f7ab83a3ed +R 70441548922262718992448fc2500d0d U drh -Z c785a25f44da51b36475ec1e8ccfb14e +Z 415f390706a3e5c8723b2ce592b50c3b diff --git a/manifest.uuid b/manifest.uuid index bafc3efa21..ce1f9dfb25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96afe50866a08ed14e911c3c955030f7ab83a3ed \ No newline at end of file +1c4a78807b7a28c9976ca8c5722ff4e2e8755b7e \ No newline at end of file diff --git a/src/where.c b/src/where.c index d9dccf7541..3af73198e8 100644 --- a/src/where.c +++ b/src/where.c @@ -3859,25 +3859,27 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ */ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue; - if( p->nTermnTerm - && (p->wsFlags & WHERE_INDEXED)!=0 - && (pTemplate->wsFlags & WHERE_INDEXED)!=0 - && p->u.btree.pIndex==pTemplate->u.btree.pIndex - && p->prereq==pTemplate->prereq - ){ - /* Overwrite an existing WhereLoop with an similar one that uses - ** more terms of the index */ - pNext = p->pNextLoop; - whereLoopClear(db, p); - break; - } if( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ - /* Already holding an equal or better WhereLoop. - ** Return without changing or adding anything */ - goto whereLoopInsert_noop; + /* p is equal or better than pTemplate */ + if( p->nTermnTerm + && (p->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && p->u.btree.pIndex==pTemplate->u.btree.pIndex + && p->prereq==pTemplate->prereq + ){ + /* Overwrite an existing WhereLoop with an similar one that uses + ** more terms of the index */ + pNext = p->pNextLoop; + whereLoopClear(db, p); + break; + }else{ + /* pTemplate is not helpful. + ** Return without changing or adding anything */ + goto whereLoopInsert_noop; + } } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq && p->rSetup>=pTemplate->rSetup From e9d935a8aa0dcef214933cd14c2c454674f36431 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Jun 2013 16:19:59 +0000 Subject: [PATCH 065/109] Minor performance tuning of the NGQP. FossilOrigin-Name: cbef38c2d123e7d5a02c2a2450e8b329e3e96ee9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 12 +++++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ae6211e19d..454d643033 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\stweak\sto\swhereLoopInsert(). -D 2013-06-05T12:47:59.358 +C Minor\sperformance\stuning\sof\sthe\sNGQP. +D 2013-06-05T16:19:59.536 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3b175d731c15e30bd8893b0309f9e7e8d0f11121 +F src/where.c 7d8a020b2d60ff2b536e6325af18ed11c24f81e5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 96afe50866a08ed14e911c3c955030f7ab83a3ed -R 70441548922262718992448fc2500d0d +P 1c4a78807b7a28c9976ca8c5722ff4e2e8755b7e +R 4c15e208b04c5a33c6528133849a04db U drh -Z 415f390706a3e5c8723b2ce592b50c3b +Z c639596f7b6bcfbd4cf032fe7e7c5fd1 diff --git a/manifest.uuid b/manifest.uuid index ce1f9dfb25..2673f09571 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c4a78807b7a28c9976ca8c5722ff4e2e8755b7e \ No newline at end of file +cbef38c2d123e7d5a02c2a2450e8b329e3e96ee9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3af73198e8..4229ddc7ec 100644 --- a/src/where.c +++ b/src/where.c @@ -752,7 +752,8 @@ WhereTerm *whereScanInit( ){ int j; - memset(pScan, 0, sizeof(*pScan)); + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pCurrent = 0; pScan->pOrigWC = pWC; pScan->pWC = pWC; if( pIdx && iColumn>=0 ){ @@ -761,8 +762,12 @@ WhereTerm *whereScanInit( if( NEVER(j>=pIdx->nColumn) ) return 0; } pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; } pScan->opMask = opMask; + pScan->k = 0; pScan->aEquiv[0] = iCur; pScan->aEquiv[1] = iColumn; pScan->nEquiv = 2; @@ -4782,7 +4787,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** error occurs. */ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ - const int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ + int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ @@ -4801,6 +4806,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ db = pWInfo->pParse->db; nLoop = pWInfo->nLevel; + mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace>=2 ) sqlite3DebugPrintf("---- begin solver\n"); @@ -4814,7 +4820,7 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ aFrom = aTo+mxChoice; memset(aFrom, 0, sizeof(aFrom[0])); pX = (WhereLoop**)(aFrom+mxChoice); - for(ii=0, pFrom=aTo; ii0; ii--, pFrom++, pX += nLoop){ pFrom->aLoop = pX; } From 783dece5d67533c564215c295f66e8dcb3feb006 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Jun 2013 17:53:43 +0000 Subject: [PATCH 066/109] Performance improvement for the OR-clause analysis in the NGQP. FossilOrigin-Name: 9b1c4954e468d0acfb5787e6bff56d50a3e7bc1a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 28 ++++++++++++++-------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 454d643033..68794b6938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\stuning\sof\sthe\sNGQP. -D 2013-06-05T16:19:59.536 +C Performance\simprovement\sfor\sthe\sOR-clause\sanalysis\sin\sthe\sNGQP. +D 2013-06-05T17:53:43.355 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 7d8a020b2d60ff2b536e6325af18ed11c24f81e5 +F src/where.c 35e510bf0de4925316934e0ae989db9d062069d5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1c4a78807b7a28c9976ca8c5722ff4e2e8755b7e -R 4c15e208b04c5a33c6528133849a04db +P cbef38c2d123e7d5a02c2a2450e8b329e3e96ee9 +R 0e9737367b7d6db3cdf59c7691a34aed U drh -Z c639596f7b6bcfbd4cf032fe7e7c5fd1 +Z 4606f36521508d3eb4fac531dd74f101 diff --git a/manifest.uuid b/manifest.uuid index 2673f09571..0a8e044566 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbef38c2d123e7d5a02c2a2450e8b329e3e96ee9 \ No newline at end of file +9b1c4954e468d0acfb5787e6bff56d50a3e7bc1a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4229ddc7ec..342c9ac721 100644 --- a/src/where.c +++ b/src/where.c @@ -4430,17 +4430,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ if( pWC->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; - pItem = pBuilder->pTabList->a + pNew->iTab; - iCur = pItem->iCursor; - sSubBuild = *pBuilder; - sSubBuild.pOrderBy = 0; - sSubBuild.pBest = &sBest; - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; - tempWC.pOuter = pWC; - tempWC.op = TK_AND; - tempWC.wctrlFlags = 0; - tempWC.nTerm = 1; for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 @@ -4452,12 +4441,23 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ double rTotal = 0; double nRow = 0; Bitmask prereq = mExtra; - + + pItem = pBuilder->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pBest = &sBest; for(pOrTerm=pOrWC->a; pOrTermeOperator& WO_AND)!=0 ){ + if( (pOrTerm->eOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pParse = pWC->pParse; + tempWC.pMaskSet = pWC->pMaskSet; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.wctrlFlags = 0; + tempWC.nTerm = 1; tempWC.a = pOrTerm; sSubBuild.pWC = &tempWC; }else{ @@ -4787,7 +4787,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** error occurs. */ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ - int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ + int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ From 6f32848d619dbcd0ca213b2c815bc39bb6587986 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Jun 2013 23:39:34 +0000 Subject: [PATCH 067/109] Move the definitions of the WhereLevel and WhereInfo objects out of sqliteInt.h and into where.c. This will facilitate future refactoring of the internal query planner data structures. FossilOrigin-Name: 1574653b9b4522b489d4c62d9cf70166bb3bddfd --- manifest | 18 ++++---- manifest.uuid | 2 +- src/select.c | 19 +++++--- src/sqliteInt.h | 74 +++--------------------------- src/update.c | 2 +- src/where.c | 117 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+), 86 deletions(-) diff --git a/manifest b/manifest index 68794b6938..3602c781f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sfor\sthe\sOR-clause\sanalysis\sin\sthe\sNGQP. -D 2013-06-05T17:53:43.355 +C Move\sthe\sdefinitions\sof\sthe\sWhereLevel\sand\sWhereInfo\sobjects\sout\sof\nsqliteInt.h\sand\sinto\swhere.c.\s\sThis\swill\sfacilitate\sfuture\srefactoring\nof\sthe\sinternal\squery\splanner\sdata\sstructures. +D 2013-06-05T23:39:34.316 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,12 +215,12 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 22ee971346a736ddefdc4497d07c92f2e9978afc +F src/select.c 5e8fe15b1c5b2cc592880d3f039c6a6da55bf253 F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 259d999abebf4498615e9237a18c0815257a1f2f +F src/sqliteInt.h 7b85b09d746cde295c7a0f60711421a28f5c4905 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -272,7 +272,7 @@ F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c 5c0ea9b8755e7c5e1a700f3e27ac4f8d92dd221e -F src/update.c 4c0c6864c4349ba292042e984a56d15985b57f4e +F src/update.c 8e76c3d03e4b7b21cb250bd2df0c05e12993e577 F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c ddf21cc9577c4cb459d08bee9863a78ec000c5bb @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 35e510bf0de4925316934e0ae989db9d062069d5 +F src/where.c 83c6fce38c5c22a1573bc3acbc0d1241a140331a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P cbef38c2d123e7d5a02c2a2450e8b329e3e96ee9 -R 0e9737367b7d6db3cdf59c7691a34aed +P 9b1c4954e468d0acfb5787e6bff56d50a3e7bc1a +R 2a5a6c03912e176e1d2592bfb6917650 U drh -Z 4606f36521508d3eb4fac531dd74f101 +Z 58f6e83bb7528a16d0e3d3a98418299b diff --git a/manifest.uuid b/manifest.uuid index 0a8e044566..abc266545e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b1c4954e468d0acfb5787e6bff56d50a3e7bc1a \ No newline at end of file +1574653b9b4522b489d4c62d9cf70166bb3bddfd \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9705c19058..d160513bd3 100644 --- a/src/select.c +++ b/src/select.c @@ -4267,9 +4267,13 @@ int sqlite3Select( /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0); if( pWInfo==0 ) goto select_end; - if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; - if( pWInfo->eDistinct ) sDistinct.eTnctType = pWInfo->eDistinct; - if( pOrderBy && pWInfo->nOBSat==pOrderBy->nExpr ) pOrderBy = 0; + if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ + p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + } + if( sqlite3WhereIsDistinct(pWInfo) ){ + sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); + } + if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -4282,7 +4286,8 @@ int sqlite3Select( /* Use the standard inner loop. */ selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, - pWInfo->iContinue, pWInfo->iBreak); + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ @@ -4399,7 +4404,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); if( pWInfo==0 ) goto select_end; - if( pWInfo->nOBSat==pGroupBy->nExpr ){ + if( sqlite3WhereIsOrdered(pWInfo) ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo @@ -4680,8 +4685,8 @@ int sqlite3Select( } updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); - if( pWInfo->nOBSat>0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); + if( sqlite3WhereIsOrdered(pWInfo) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2b07d2ef02..76835385fb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -721,7 +721,6 @@ typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; -typedef struct WhereLevel WhereLevel; /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and @@ -1954,44 +1953,6 @@ struct SrcList { #define JT_ERROR 0x0040 /* unknown or unsupported join type */ -/* -** For each nested loop in a WHERE clause implementation, the WhereInfo -** structure contains a single instance of this structure. This structure -** is intended to be private to the where.c module and should not be -** access or modified by other modules. -** -** The pIdxInfo field is used to help pick the best index on a -** virtual table. The pIdxInfo pointer contains indexing -** information for the i-th table in the FROM clause before reordering. -** All the pIdxInfo pointers are freed by whereInfoFree() in where.c. -** All other information in the i-th WhereLevel object for the i-th table -** after FROM clause ordering. -*/ -struct WhereLevel { - int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ - int iTabCur; /* The VDBE cursor used to access the table */ - int iIdxCur; /* The VDBE cursor used to access pIdx */ - int addrBrk; /* Jump here to break out of the loop */ - int addrNxt; /* Jump here to start the next IN combination */ - int addrCont; /* Jump here to continue with the next loop cycle */ - int addrFirst; /* First instruction of interior of the loop */ - u8 iFrom; /* FIXME: Which entry in the FROM clause */ - u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ - union { /* Information that depends on plan.wsFlags */ - struct { - int nIn; /* Number of entries in aInLoop[] */ - struct InLoop { - int iCur; /* The VDBE cursor used by this IN operator */ - int addrInTop; /* Top of the IN loop */ - u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ - } *aInLoop; /* Information about each nested IN operator */ - } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ - Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ - } u; - struct WhereLoop *pWLoop; /* The selected WhereLoop object */ -}; - /* ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() ** and the WhereInfo.wctrlFlags member. @@ -2007,35 +1968,6 @@ struct WhereLevel { #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ -/* -** The WHERE clause processing routine has two halves. The -** first part does the start of the WHERE loop and the second -** half does the tail of the WHERE loop. An instance of -** this structure is returned by the first half and passed -** into the second half to give some continuity. -*/ -struct WhereInfo { - Parse *pParse; /* Parsing and code generating context */ - SrcList *pTabList; /* List of tables in the join */ - ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinct; /* DISTINCT ON values, or NULL */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ - int iTop; /* The very beginning of the WHERE loop */ - int iContinue; /* Jump here to continue with next record */ - int iBreak; /* Jump here to break out of the loop */ - int nLevel; /* Number of nested loop */ - struct WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct WhereLoop *pLoops; /* List of all WhereLoop objects */ - double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - double nRowOut; /* Estimated number of output rows */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ -}; - /* Allowed values for WhereInfo.eDistinct and DistinctCtx.eTnctType */ #define WHERE_DISTINCT_NOOP 0 /* DISTINCT keyword not used */ #define WHERE_DISTINCT_UNIQUE 1 /* No duplicates */ @@ -2864,6 +2796,12 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); +double sqlite3WhereOutputRowCount(WhereInfo*); +int sqlite3WhereIsDistinct(WhereInfo*); +int sqlite3WhereIsOrdered(WhereInfo*); +int sqlite3WhereContinueLabel(WhereInfo*); +int sqlite3WhereBreakLabel(WhereInfo*); +int sqlite3WhereOkOnePass(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); diff --git a/src/update.c b/src/update.c index 3ab1ab2a4b..5b18ebd6bf 100644 --- a/src/update.c +++ b/src/update.c @@ -318,7 +318,7 @@ void sqlite3Update( pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, 0 ); if( pWInfo==0 ) goto update_cleanup; - okOnePass = pWInfo->okOnePass; + okOnePass = sqlite3WhereOkOnePass(pWInfo); /* Remember the rowid of every item to be updated. */ diff --git a/src/where.c b/src/where.c index 342c9ac721..4d03776bf2 100644 --- a/src/where.c +++ b/src/where.c @@ -39,6 +39,7 @@ typedef struct WhereClause WhereClause; typedef struct WhereMaskSet WhereMaskSet; typedef struct WhereOrInfo WhereOrInfo; typedef struct WhereAndInfo WhereAndInfo; +typedef struct WhereLevel WhereLevel; typedef struct WhereLoop WhereLoop; typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; @@ -47,6 +48,73 @@ typedef struct WhereScan WhereScan; typedef struct WhereVtabPlan WhereVtabPlan; +/* +** For each nested loop in a WHERE clause implementation, the WhereInfo +** structure contains a single instance of this structure. This structure +** is intended to be private to the where.c module and should not be +** access or modified by other modules. +** +** The pIdxInfo field is used to help pick the best index on a +** virtual table. The pIdxInfo pointer contains indexing +** information for the i-th table in the FROM clause before reordering. +** All the pIdxInfo pointers are freed by whereInfoFree() in where.c. +** All other information in the i-th WhereLevel object for the i-th table +** after FROM clause ordering. +*/ +struct WhereLevel { + int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ + int iTabCur; /* The VDBE cursor used to access the table */ + int iIdxCur; /* The VDBE cursor used to access pIdx */ + int addrBrk; /* Jump here to break out of the loop */ + int addrNxt; /* Jump here to start the next IN combination */ + int addrCont; /* Jump here to continue with the next loop cycle */ + int addrFirst; /* First instruction of interior of the loop */ + u8 iFrom; /* FIXME: Which entry in the FROM clause */ + u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + int p1, p2; /* Operands of the opcode used to ends the loop */ + union { /* Information that depends on plan.wsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ +}; + +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pDistinct; /* DISTINCT ON values, or NULL */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int nLevel; /* Number of nested loop */ + struct WhereClause *pWC; /* Decomposition of the WHERE clause */ + struct WhereLoop *pLoops; /* List of all WhereLoop objects */ + double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + double nRowOut; /* Estimated number of output rows */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + /* ** Each instance of this object represents a way of evaluating one ** term of a join. The WhereClause object holds a table of these @@ -333,6 +401,54 @@ struct WhereLoopBuilder { #define WHERE_TEMP_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_COVER_SCAN 0x00008000 /* Full scan of a covering index */ +/* +** Return the estimated number of output rows from a WHERE clause +*/ +double sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return pWInfo->nRowOut; +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat>0; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return TRUE if an UPDATE or DELETE statement can operate directly on +** the rowids returned by a WHERE clause. Return FALSE if doing an +** UPDATE or DELETE might change subsequent WHERE clause results. +*/ +int sqlite3WhereOkOnePass(WhereInfo *pWInfo){ + return pWInfo->okOnePass; +} + /* ** Initialize a preallocated WhereClause structure. */ @@ -4179,6 +4295,7 @@ static int whereLoopAddBtree( /* Automatic indexes */ if( !pBuilder->pBest + && pBuilder->pTabList->nSrc>1 && (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->viaCoroutine && !pSrc->notIndexed From 70d18344831f0ed6a0fe82041a263eb269d4ad97 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Jun 2013 19:16:33 +0000 Subject: [PATCH 068/109] Clean up and reorganize the elements of the various objects in the analysis tree for the NGQP. FossilOrigin-Name: bfc76ae1e94fc5b7fd2c9484a36a8dfe6655d165 --- manifest | 12 +-- manifest.uuid | 2 +- src/where.c | 222 ++++++++++++++++++++++++-------------------------- 3 files changed, 113 insertions(+), 123 deletions(-) diff --git a/manifest b/manifest index 3602c781f7..9f0a3553bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\sdefinitions\sof\sthe\sWhereLevel\sand\sWhereInfo\sobjects\sout\sof\nsqliteInt.h\sand\sinto\swhere.c.\s\sThis\swill\sfacilitate\sfuture\srefactoring\nof\sthe\sinternal\squery\splanner\sdata\sstructures. -D 2013-06-05T23:39:34.316 +C Clean\sup\sand\sreorganize\sthe\selements\sof\sthe\svarious\sobjects\sin\sthe\nanalysis\stree\sfor\sthe\sNGQP. +D 2013-06-06T19:16:33.565 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 83c6fce38c5c22a1573bc3acbc0d1241a140331a +F src/where.c e3b7d7ef7aebae993b5ee37b7fc104f1dcea4bc0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9b1c4954e468d0acfb5787e6bff56d50a3e7bc1a -R 2a5a6c03912e176e1d2592bfb6917650 +P 1574653b9b4522b489d4c62d9cf70166bb3bddfd +R 5e3fae80c355d69a47b06cb86e66bb8c U drh -Z 58f6e83bb7528a16d0e3d3a98418299b +Z 38b156b13260a994e987fde6c73313cc diff --git a/manifest.uuid b/manifest.uuid index abc266545e..5ddbbffbb4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1574653b9b4522b489d4c62d9cf70166bb3bddfd \ No newline at end of file +bfc76ae1e94fc5b7fd2c9484a36a8dfe6655d165 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4d03776bf2..d86e3c6464 100644 --- a/src/where.c +++ b/src/where.c @@ -45,8 +45,6 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; -typedef struct WhereVtabPlan WhereVtabPlan; - /* ** For each nested loop in a WHERE clause implementation, the WhereInfo @@ -86,35 +84,6 @@ struct WhereLevel { struct WhereLoop *pWLoop; /* The selected WhereLoop object */ }; -/* -** The WHERE clause processing routine has two halves. The -** first part does the start of the WHERE loop and the second -** half does the tail of the WHERE loop. An instance of -** this structure is returned by the first half and passed -** into the second half to give some continuity. -*/ -struct WhereInfo { - Parse *pParse; /* Parsing and code generating context */ - SrcList *pTabList; /* List of tables in the join */ - ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinct; /* DISTINCT ON values, or NULL */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ - int iTop; /* The very beginning of the WHERE loop */ - int iContinue; /* Jump here to continue with next record */ - int iBreak; /* Jump here to break out of the loop */ - int nLevel; /* Number of nested loop */ - struct WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct WhereLoop *pLoops; /* List of all WhereLoop objects */ - double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - double nRowOut; /* Estimated number of output rows */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ -}; - /* ** Each instance of this object represents a way of evaluating one ** term of a join. The WhereClause object holds a table of these @@ -279,11 +248,9 @@ struct WhereScan { ** subclauses points to the WhereClause object for the whole clause. */ struct WhereClause { - Parse *pParse; /* The parser context */ - WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ + WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ - u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ @@ -347,16 +314,46 @@ struct WhereMaskSet { */ struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ - sqlite3 *db; /* Database connection */ - Parse *pParse; /* Parsing context */ +// sqlite3 *db; /* Database connection */ +// Parse *pParse; /* Parsing context */ +// SrcList *pTabList; /* FROM clause */ WhereClause *pWC; /* WHERE clause terms */ - SrcList *pTabList; /* FROM clause */ ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereLoop *pBest; /* If non-NULL, store single best loop here */ int mxTerm; /* Maximum number of aTerm[] entries on pNew */ }; +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pDistinct; /* DISTINCT ON values, or NULL */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int nLevel; /* Number of nested loop */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + double nRowOut; /* Estimated number of output rows */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + /* ** Bitmasks for the operators that indices are able to exploit. An ** OR-ed combination of these values can be used when searching for @@ -454,17 +451,13 @@ int sqlite3WhereOkOnePass(WhereInfo *pWInfo){ */ static void whereClauseInit( WhereClause *pWC, /* The WhereClause to be initialized */ - Parse *pParse, /* The parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmasks */ - u16 wctrlFlags /* Might include WHERE_AND_ONLY */ + WhereInfo *pWInfo /* The WHERE processing context */ ){ - pWC->pParse = pParse; - pWC->pMaskSet = pMaskSet; + pWC->pWInfo = pWInfo; pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->wctrlFlags = wctrlFlags; } /* Forward reference */ @@ -493,7 +486,7 @@ static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ static void whereClauseClear(WhereClause *pWC){ int i; WhereTerm *a; - sqlite3 *db = pWC->pParse->db; + sqlite3 *db = pWC->pWInfo->pParse->db; for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ if( a->wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, a->pExpr); @@ -534,7 +527,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pParse->db; + sqlite3 *db = pWC->pWInfo->pParse->db; pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ @@ -811,14 +804,15 @@ WhereTerm *whereScanNext(WhereScan *pScan){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ continue; } assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pWC->pParse, + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pWC->pParse->db->pDfltColl; + if( pColl==0 ) pColl = pParse->db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ continue; } @@ -1177,11 +1171,11 @@ static void exprAnalyzeOrTerm( WhereClause *pWC, /* the complete WHERE clause */ int idxTerm /* Index of the OR-term to be analyzed */ ){ - Parse *pParse = pWC->pParse; /* Parser context */ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ sqlite3 *db = pParse->db; /* Database connection */ WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */ int i; /* Loop counters */ WhereClause *pOrWc; /* Breakup of pTerm into subterms */ WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ @@ -1200,7 +1194,7 @@ static void exprAnalyzeOrTerm( if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWC->pParse, pMaskSet, pWC->wctrlFlags); + whereClauseInit(pOrWc, pWInfo); whereSplit(pOrWc, pExpr, TK_OR); exprAnalyzeAll(pSrc, pOrWc); if( db->mallocFailed ) return; @@ -1226,7 +1220,7 @@ static void exprAnalyzeOrTerm( pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pParse, pMaskSet, pWC->wctrlFlags); + whereClauseInit(pAndWC, pWC->pWInfo); whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); exprAnalyzeAll(pSrc, pAndWC); pAndWC->pOuter = pWC; @@ -1235,7 +1229,7 @@ static void exprAnalyzeOrTerm( for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(pMaskSet, pAndTerm->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); } } } @@ -1246,10 +1240,10 @@ static void exprAnalyzeOrTerm( ** corresponding TERM_VIRTUAL term */ }else{ Bitmask b; - b = getMask(pMaskSet, pOrTerm->leftCursor); + b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); if( pOrTerm->wtFlags & TERM_VIRTUAL ){ WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(pMaskSet, pOther->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); } indexable &= b; if( (pOrTerm->eOperator & WO_EQ)==0 ){ @@ -1311,7 +1305,7 @@ static void exprAnalyzeOrTerm( assert( j==1 ); continue; } - if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){ + if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the ** chngToIN set but t1 is not. This term will be either preceeded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term @@ -1330,7 +1324,7 @@ static void exprAnalyzeOrTerm( ** on the second iteration */ assert( j==1 ); assert( IsPowerOfTwo(chngToIN) ); - assert( chngToIN==getMask(pMaskSet, iCursor) ); + assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); break; } testcase( j==1 ); @@ -1379,7 +1373,7 @@ static void exprAnalyzeOrTerm( assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); pLeft = pOrTerm->pExpr->pLeft; } assert( pLeft!=0 ); @@ -1428,6 +1422,7 @@ static void exprAnalyze( WhereClause *pWC, /* the WHERE clause */ int idxTerm /* Index of the term to be analyzed */ ){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ @@ -1438,14 +1433,14 @@ static void exprAnalyze( int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ int noCase = 0; /* LIKE/GLOB distinguishes case */ int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWC->pParse; /* Parsing context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ if( db->mallocFailed ){ return; } pTerm = &pWC->a[idxTerm]; - pMaskSet = pWC->pMaskSet; + pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); @@ -3068,7 +3063,6 @@ static void explainOneScan( static Bitmask codeOneLoopStart( WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ - u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ @@ -3091,14 +3085,14 @@ static Bitmask codeOneLoopStart( pParse = pWInfo->pParse; v = pParse->pVdbe; - pWC = pWInfo->pWC; + pWC = &pWInfo->sWC; pLevel = &pWInfo->a[iLevel]; pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; bRev = (pWInfo->revMask>>iLevel)&1; omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (wctrlFlags & WHERE_FORCE_TABLE)==0; + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions @@ -3355,7 +3349,7 @@ static Bitmask codeOneLoopStart( ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ - if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 && (pWInfo->nOBSat>0) && (pIdx->nColumn>nEq) ){ @@ -3624,7 +3618,7 @@ static Bitmask codeOneLoopStart( ** fall through to the next instruction, just as an OP_Next does if ** called on an uninitialized cursor. */ - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ regRowset = ++pParse->nMem; regRowid = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); @@ -3679,7 +3673,7 @@ static Bitmask codeOneLoopStart( explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, @@ -3751,7 +3745,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -3906,7 +3900,7 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - whereClauseClear(pWInfo->pWC); + whereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; @@ -3945,8 +3939,8 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; WhereTerm **paTerm = 0; - sqlite3 *db = pBuilder->db; WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; /* If pBuilder->pBest is defined, then only keep track of the single ** best WhereLoop. pBuilder->pBest->maskSelf==0 indicates that no @@ -3969,7 +3963,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf("ins-best: "); - whereLoopPrint(pTemplate, pBuilder->pTabList); + whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif return SQLITE_OK; @@ -4021,10 +4015,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ sqlite3DebugPrintf("ins-del: "); - whereLoopPrint(p, pBuilder->pTabList); + whereLoopPrint(p, pWInfo->pTabList); } sqlite3DebugPrintf("ins-new: "); - whereLoopPrint(pTemplate, pBuilder->pTabList); + whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif if( p==0 ){ @@ -4060,7 +4054,7 @@ whereLoopInsert_noop: #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf("ins-noop: "); - whereLoopPrint(pTemplate, pBuilder->pTabList); + whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif return SQLITE_OK; @@ -4079,7 +4073,9 @@ static int whereLoopAddBtreeIndex( Index *pProbe, /* An index on pSrc */ int nInMul /* Number of iterations due to IN */ ){ - sqlite3 *db; /* Database connection malloc context */ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection malloc context */ WhereLoop *pNew; /* Template WhereLoop under construction */ WhereTerm *pTerm; /* A WhereTerm under consideration */ int opMask; /* Valid operators for constraints */ @@ -4091,7 +4087,6 @@ static int whereLoopAddBtreeIndex( double rLogSize; /* Logarithm of table size */ WhereTerm *pTop, *pBtm; /* Top and bottom range constraints */ - db = pBuilder->db; pNew = pBuilder->pNew; if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4173,18 +4168,18 @@ static int whereLoopAddBtreeIndex( if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ double rDiv; - whereRangeScanEst(pBuilder->pParse, pProbe, pNew->u.btree.nEq, + whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, pBtm, pTop, &rDiv); pNew->nOut = savedLoop.nOut/rDiv; } #ifdef SQLITE_ENABLE_STAT3 if( pNew->u.btree.nEq==1 && pProbe->nSample ){ if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - rc = whereEqualScanEst(pBuilder->pParse, pProbe, pTerm->pExpr->pRight, + rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &pNew->nOut); }else if( (pTerm->eOperator & WO_IN) && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pBuilder->pParse, pProbe, pTerm->pExpr->x.pList, + rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &pNew->nOut); } @@ -4227,7 +4222,7 @@ static int indexMightHelpWithOrderBy( int iCol; int ii; - if( (pOB = pBuilder->pOrderBy)==0 ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; iCol = pIndex->aiColumn[0]; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); @@ -4249,10 +4244,12 @@ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ Bitmask mExtra /* Extra prerequesites for using this table */ ){ + WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ @@ -4262,7 +4259,9 @@ static int whereLoopAddBtree( double rLogSize; /* Logarithm of the number of rows in the table */ pNew = pBuilder->pNew; - pSrc = pBuilder->pTabList->a + pNew->iTab; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; assert( !IsVirtual(pSrc->pTab) ); if( pSrc->pIndex ){ @@ -4295,8 +4294,8 @@ static int whereLoopAddBtree( /* Automatic indexes */ if( !pBuilder->pBest - && pBuilder->pTabList->nSrc>1 - && (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 + && pTabList->nSrc>1 + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->viaCoroutine && !pSrc->notIndexed && !pSrc->isCorrelated @@ -4356,9 +4355,9 @@ static int whereLoopAddBtree( /* Full scan via index */ if( (m==0 || b) && pProbe->bUnordered==0 - && (pBuilder->pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis - && OptimizationEnabled(pBuilder->pParse->db, SQLITE_CoverIdxScan) + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; @@ -4384,6 +4383,7 @@ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ Bitmask mExtra /* Extra prerequesites for using this table */ ){ + WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ WhereClause *pWC; /* The WHERE clause */ struct SrcList_item *pSrc; /* The FROM clause term to search */ @@ -4401,11 +4401,12 @@ static int whereLoopAddVirtual( WhereLoop *pNew; int rc = SQLITE_OK; - pParse = pBuilder->pParse; + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; db = pParse->db; pWC = pBuilder->pWC; pNew = pBuilder->pNew; - pSrc = &pBuilder->pTabList->a[pNew->iTab]; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; pTab = pSrc->pTab; assert( IsVirtual(pTab) ); pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); @@ -4532,6 +4533,7 @@ whereLoopAddVtab_exit: ** btrees or virtual tables. */ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereInfo *pWInfo = pBuilder->pWInfo; WhereClause *pWC; WhereLoop *pNew; WhereTerm *pTerm, *pWCEnd; @@ -4542,9 +4544,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ WhereLoop sBest; struct SrcList_item *pItem; - pWC = pBuilder->pWC; - if( pWC->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; + if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; @@ -4559,7 +4560,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ double nRow = 0; Bitmask prereq = mExtra; - pItem = pBuilder->pTabList->a + pNew->iTab; + pItem = pWInfo->pTabList->a + pNew->iTab; iCur = pItem->iCursor; sSubBuild = *pBuilder; sSubBuild.pOrderBy = 0; @@ -4569,11 +4570,9 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ if( (pOrTerm->eOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; }else if( pOrTerm->leftCursor==iCur ){ - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; + tempWC.pWInfo = pWC->pWInfo; tempWC.pOuter = pWC; tempWC.op = TK_AND; - tempWC.wctrlFlags = 0; tempWC.nTerm = 1; tempWC.a = pOrTerm; sSubBuild.pWC = &tempWC; @@ -4610,14 +4609,15 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ ** Add all WhereLoop objects for all tables */ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; Bitmask mExtra = 0; Bitmask mPrior = 0; int iTab; - SrcList *pTabList = pBuilder->pTabList; + SrcList *pTabList = pWInfo->pTabList; struct SrcList_item *pItem; WhereClause *pWC = pBuilder->pWC; - sqlite3 *db = pBuilder->db; - int nTabList = pBuilder->pWInfo->nLevel; + sqlite3 *db = pWInfo->pParse->db; + int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; WhereLoop *pNew; @@ -4637,7 +4637,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ } for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; - pNew->maskSelf = getMask(pWC->pMaskSet, pItem->iCursor); + pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ mExtra = mPrior; } @@ -4698,7 +4698,6 @@ static int wherePathSatisfiesOrderBy( Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ Bitmask obDone; /* Mask of all ORDER BY terms */ Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ - WhereMaskSet *pMaskSet; /* WhereMaskSet object for this where clause */ /* @@ -4739,7 +4738,6 @@ static int wherePathSatisfiesOrderBy( isOrderDistinct = 1; obDone = MASKBIT(nOrderBy)-1; orderDistinctMask = 0; - pMaskSet = pWInfo->pWC->pMaskSet; for(iLoop=0; isOrderDistinct && obSataLoop[iLoop] : pLast; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); @@ -4870,7 +4868,7 @@ static int wherePathSatisfiesOrderBy( Expr *p; if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; - if( (exprTableUsage(pMaskSet, p)&~orderDistinctMask)==0 ){ + if( (exprTableUsage(&pWInfo->sMaskSet, p)&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } } @@ -5239,9 +5237,6 @@ WhereInfo *sqlite3WhereBegin( /* Variable initialization */ memset(&sWLB, 0, sizeof(sWLB)); - sWLB.pParse = pParse; - sWLB.db = pParse->db; - sWLB.pTabList = pTabList; sWLB.pOrderBy = pOrderBy; /* The number of tables in the FROM clause is limited by the number of @@ -5269,11 +5264,7 @@ WhereInfo *sqlite3WhereBegin( */ db = pParse->db; nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, - nByteWInfo + - sizeof(WhereClause) + - sizeof(WhereMaskSet) - ); + pWInfo = sqlite3DbMallocZero(db, nByteWInfo); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); pWInfo = 0; @@ -5285,12 +5276,11 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pOrderBy = pOrderBy; pWInfo->pDistinct = pDistinct; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWInfo->pWC[1]; + pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; - sWLB.pWC = pWInfo->pWC; + sWLB.pWC = &pWInfo->sWC; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -5300,9 +5290,9 @@ WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWInfo->pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(&pWInfo->sWC, pWInfo); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWInfo->pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(&pWInfo->sWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -5347,7 +5337,7 @@ WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, pWInfo->pWC); + exprAnalyzeAll(pTabList, &pWInfo->sWC); if( db->mallocFailed ){ goto whereBeginError; } @@ -5356,7 +5346,7 @@ WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse,pTabList,pWInfo->pWC,pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse,pTabList,&pWInfo->sWC,pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5467,7 +5457,7 @@ WhereInfo *sqlite3WhereBegin( } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLoop->wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWInfo->pWC, pTabItem, notReady, pLevel); + constructAutomaticIndex(pParse, &pWInfo->sWC, pTabItem, notReady, pLevel); }else #endif if( pLoop->wsFlags & WHERE_INDEXED ){ @@ -5482,7 +5472,7 @@ WhereInfo *sqlite3WhereBegin( VdbeComment((v, "%s", pIx->zName)); } sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWInfo->pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -5495,7 +5485,7 @@ WhereInfo *sqlite3WhereBegin( for(ii=0; iia[ii]; explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, ii, wctrlFlags, notReady); + notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; } From e3be82cf188d5e3582c4d1c605870ba119d31b83 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Jun 2013 19:25:42 +0000 Subject: [PATCH 069/109] Remove some commented-out code that was mistakenly left in the previous check-in. FossilOrigin-Name: b4a5dbad36bceabc5c5350e7676af6ad42de04eb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9f0a3553bc..7068f05a1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sand\sreorganize\sthe\selements\sof\sthe\svarious\sobjects\sin\sthe\nanalysis\stree\sfor\sthe\sNGQP. -D 2013-06-06T19:16:33.565 +C Remove\ssome\scommented-out\scode\sthat\swas\smistakenly\sleft\sin\sthe\sprevious\ncheck-in. +D 2013-06-06T19:25:42.472 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e3b7d7ef7aebae993b5ee37b7fc104f1dcea4bc0 +F src/where.c 0b652e425f7a958a66591b082731c4e8379d8954 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1574653b9b4522b489d4c62d9cf70166bb3bddfd -R 5e3fae80c355d69a47b06cb86e66bb8c +P bfc76ae1e94fc5b7fd2c9484a36a8dfe6655d165 +R 6ac62cf90895a44c1fcede6b2ff8eacf U drh -Z 38b156b13260a994e987fde6c73313cc +Z b5c1236cedf9282cba7e1987b8db3ea9 diff --git a/manifest.uuid b/manifest.uuid index 5ddbbffbb4..0f8825c921 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bfc76ae1e94fc5b7fd2c9484a36a8dfe6655d165 \ No newline at end of file +b4a5dbad36bceabc5c5350e7676af6ad42de04eb \ No newline at end of file diff --git a/src/where.c b/src/where.c index d86e3c6464..21277f1d25 100644 --- a/src/where.c +++ b/src/where.c @@ -314,9 +314,6 @@ struct WhereMaskSet { */ struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ -// sqlite3 *db; /* Database connection */ -// Parse *pParse; /* Parsing context */ -// SrcList *pTabList; /* FROM clause */ WhereClause *pWC; /* WHERE clause terms */ ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ From 4efc9298fb4fde4f5001b8d0e3155bbde5cffcea Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Jun 2013 23:02:03 +0000 Subject: [PATCH 070/109] Improved management of the space to hold WhereLoop.aLTerm[]. FossilOrigin-Name: d4141ecbea3abbe83525910684fbd89eb74eeb34 --- manifest | 12 +- manifest.uuid | 2 +- src/where.c | 338 +++++++++++++++++++++++++++++--------------------- 3 files changed, 207 insertions(+), 145 deletions(-) diff --git a/manifest b/manifest index 7068f05a1e..b68a0110a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\scommented-out\scode\sthat\swas\smistakenly\sleft\sin\sthe\sprevious\ncheck-in. -D 2013-06-06T19:25:42.472 +C Improved\smanagement\sof\sthe\sspace\sto\shold\sWhereLoop.aLTerm[]. +D 2013-06-06T23:02:03.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0b652e425f7a958a66591b082731c4e8379d8954 +F src/where.c e029cd3fe9f2fcc857cf19d05b352b2359638271 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bfc76ae1e94fc5b7fd2c9484a36a8dfe6655d165 -R 6ac62cf90895a44c1fcede6b2ff8eacf +P b4a5dbad36bceabc5c5350e7676af6ad42de04eb +R c565385814d78fee98948398a72cac6f U drh -Z b5c1236cedf9282cba7e1987b8db3ea9 +Z 2b77e3c42c0b772f55707fe4d25f7e48 diff --git a/manifest.uuid b/manifest.uuid index 0f8825c921..bc7a4d415f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4a5dbad36bceabc5c5350e7676af6ad42de04eb \ No newline at end of file +d4141ecbea3abbe83525910684fbd89eb74eeb34 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 21277f1d25..08569fc60b 100644 --- a/src/where.c +++ b/src/where.c @@ -45,6 +45,7 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; +typedef float WhereCost; /* ** For each nested loop in a WHERE clause implementation, the WhereInfo @@ -98,11 +99,12 @@ struct WhereLoop { #endif u8 iTab; /* Position in FROM clause of table for this loop */ u8 iSortIdx; /* Sorting index number. 0==None */ - u16 nTerm; /* Number of entries in aTerm[] */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ u32 wsFlags; /* WHERE_* flags describing the plan */ - double rSetup; /* One-time setup cost (ex: create transient index) */ - double rRun; /* Cost of running each loop */ - double nOut; /* Estimated number of output rows */ + WhereCost rSetup; /* One-time setup cost (ex: create transient index) */ + WhereCost rRun; /* Cost of running each loop */ + WhereCost nOut; /* Estimated number of output rows */ union { struct { /* Information for internal btree tables */ int nEq; /* Number of equality constraints */ @@ -116,10 +118,14 @@ struct WhereLoop { char *idxStr; /* Index identifier string */ } vtab; } u; - WhereTerm **aTerm; /* WhereTerms used */ + WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ }; +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + /* ** Each instance of this object holds a sequence of WhereLoop objects ** that implement some or all of the entire query plan. @@ -127,8 +133,8 @@ struct WhereLoop { struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ - double nRow; /* Estimated number of rows generated by this path */ - double rCost; /* Total cost of this path */ + WhereCost nRow; /* Estimated number of rows generated by this path */ + WhereCost rCost; /* Total cost of this path */ u8 isOrdered; /* True if this path satisfies ORDER BY */ u8 isOrderedValid; /* True if the isOrdered field is valid */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ @@ -318,7 +324,6 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereLoop *pBest; /* If non-NULL, store single best loop here */ - int mxTerm; /* Maximum number of aTerm[] entries on pNew */ }; /* @@ -346,8 +351,8 @@ struct WhereInfo { WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - double nRowOut; /* Estimated number of output rows */ + WhereCost savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + WhereCost nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; @@ -399,7 +404,7 @@ struct WhereInfo { ** Return the estimated number of output rows from a WHERE clause */ double sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return pWInfo->nRowOut; + return (double)pWInfo->nRowOut; } /* @@ -1823,9 +1828,9 @@ static int isDistinctRedundant( ** complexity. Because N is just a guess, it is no great tragedy if ** logN is a little off. */ -static double estLog(double N){ - double logN = 1; - double x = 10; +static WhereCost estLog(WhereCost N){ + WhereCost logN = 1; + WhereCost x = 10; while( N>x ){ logN += 1; x *= 10; @@ -1946,23 +1951,21 @@ static void constructAutomaticIndex( pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; - pLoop->aTerm = sqlite3DbRealloc(pParse->db, pLoop->aTerm, - mxConstraint*sizeof(pLoop->aTerm[0])); - if( pLoop->aTerm==0 ) return; - for(pTerm=pWC->a; pTermnTerma; pTermnLTermu.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS ); testcase( iCol==BMS-1 ); if( (idxCols & cMask)==0 ){ - pLoop->aTerm[nColumn++] = pTerm; + if( whereLoopResize(pParse->db, pLoop, nColumn+1) ) return; + pLoop->aLTerm[nColumn++] = pTerm; idxCols |= cMask; } } } assert( nColumn>0 ); - pLoop->u.btree.nEq = pLoop->nTerm = nColumn; + pLoop->u.btree.nEq = pLoop->nLTerm = nColumn; pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED | WHERE_TEMP_INDEX; @@ -2118,7 +2121,6 @@ static sqlite3_index_info *allocateIndexInfo( + sizeof(*pIdxOrderBy)*nOrderBy ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return 0; } @@ -2241,7 +2243,7 @@ static int whereKeyStats( int i, eType; int isEq = 0; i64 v; - double r, rS; + WhereCost r, rS; assert( roundUp==0 || roundUp==1 ); assert( pIdx->nSample>0 ); @@ -2459,7 +2461,7 @@ static int whereRangeScanEst( int nEq, /* index into p->aCol[] of the range-compared column */ WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - double *pRangeDiv /* OUT: Reduce search space by this divisor */ + WhereCost *pRangeDiv /* OUT: Reduce search space by this divisor */ ){ int rc = SQLITE_OK; @@ -2498,9 +2500,9 @@ static int whereRangeScanEst( } if( rc==SQLITE_OK ){ if( iUpper<=iLower ){ - *pRangeDiv = (double)p->aiRowEst[0]; + *pRangeDiv = (WhereCost)p->aiRowEst[0]; }else{ - *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower); + *pRangeDiv = (WhereCost)p->aiRowEst[0]/(WhereCost)(iUpper - iLower); } /*WHERETRACE(("range scan regions: %u..%u div=%g\n", (u32)iLower, (u32)iUpper, *pRangeDiv));*/ @@ -2513,9 +2515,9 @@ static int whereRangeScanEst( UNUSED_PARAMETER(nEq); #endif assert( pLower || pUpper ); - *pRangeDiv = (double)1; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (double)4; - if( pUpper ) *pRangeDiv *= (double)4; + *pRangeDiv = (WhereCost)1; + if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (WhereCost)4; + if( pUpper ) *pRangeDiv *= (WhereCost)4; return rc; } @@ -2541,7 +2543,7 @@ static int whereEqualScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - double *pnRow /* Write the revised row estimate here */ + WhereCost *pnRow /* Write the revised row estimate here */ ){ sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ u8 aff; /* Column affinity */ @@ -2590,11 +2592,11 @@ static int whereInScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - double *pnRow /* Write the revised row estimate here */ + WhereCost *pnRow /* Write the revised row estimate here */ ){ int rc = SQLITE_OK; /* Subfunction return code */ - double nEst; /* Number of rows for a single term */ - double nRowEst = (double)0; /* New estimate of the number of rows */ + WhereCost nEst; /* Number of rows for a single term */ + WhereCost nRowEst = (WhereCost)0; /* New estimate of the number of rows */ int i; /* Loop counter */ assert( p->aSample!=0 ); @@ -2858,7 +2860,7 @@ static int codeAllEqualityTerms( assert( pIdx->nColumn>=nEq ); for(j=0; jaTerm[j]; + pTerm = pLoop->aLTerm[j]; assert( pTerm!=0 ); /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ @@ -3132,14 +3134,14 @@ static Bitmask codeOneLoopStart( */ int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; - int nConstraint = pLoop->nTerm; + int nConstraint = pLoop->nLTerm; sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; jaTerm[j]; + pTerm = pLoop->aLTerm[j]; if( pTerm->eOperator & WO_IN ){ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); addrNotFound = pLevel->addrNxt; @@ -3155,7 +3157,7 @@ static Bitmask codeOneLoopStart( pLoop->u.vtab.needFree = 0; for(j=0; ju.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pLoop->aTerm[j]); + disableTerm(pLevel, pLoop->aLTerm[j]); } } pLevel->op = OP_VNext; @@ -3176,7 +3178,7 @@ static Bitmask codeOneLoopStart( */ assert( pLoop->u.btree.nEq==1 ); iReleaseReg = sqlite3GetTempReg(pParse); - pTerm = pLoop->aTerm[0]; + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); assert( omitTable==0 ); @@ -3202,8 +3204,8 @@ static Bitmask codeOneLoopStart( assert( omitTable==0 ); j = 0; pStart = pEnd = 0; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aTerm[j++]; - if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aTerm[j++]; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; if( bRev ){ pTerm = pStart; pStart = pEnd; @@ -3361,11 +3363,11 @@ static Bitmask codeOneLoopStart( */ j = nEq; if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = pLoop->aTerm[j++]; + pRangeStart = pLoop->aLTerm[j++]; nExtraReg = 1; } if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = pLoop->aTerm[j++]; + pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; } @@ -3574,7 +3576,7 @@ static Bitmask codeOneLoopStart( int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - pTerm = pLoop->aTerm[0]; + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); @@ -3860,19 +3862,27 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %05x N %d", p->wsFlags, p->nTerm); + sqlite3DebugPrintf(" fg %05x N %d", p->wsFlags, p->nLTerm); sqlite3DebugPrintf(" cost %.2g,%.2g,%.2g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } #endif /* -** Deallocate internal memory used by a WhereLoop object +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. */ -static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - sqlite3DbFree(db, p->aTerm); - p->aTerm = 0; - p->nTerm = 0; +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ if( p->u.vtab.needFree ) sqlite3_free(p->u.vtab.idxStr); p->u.vtab.needFree = 0; @@ -3884,6 +3894,61 @@ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ } } + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + if( whereLoopResize(db, pTo, pFrom->nLTerm) ) return SQLITE_NOMEM; + whereLoopClearUnion(db, pTo); + pTo->prereq = pFrom->prereq; + pTo->maskSelf = pFrom->maskSelf; + pTo->iTab = pFrom->iTab; + pTo->iSortIdx = pFrom->iSortIdx; + pTo->nLTerm = pFrom->nLTerm; + pTo->rSetup = pFrom->rSetup; + pTo->rRun = pFrom->rRun; + pTo->nOut = pFrom->nOut; + if( pTo->nLTerm ){ + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + } + pTo->wsFlags = pFrom->wsFlags; + pTo->u = pFrom->u; + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_TEMP_INDEX)!=0 && pFrom->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, pFrom->u.btree.pIndex->zColAff); + sqlite3DbFree(db, pFrom->u.btree.pIndex); + pFrom->u.btree.pIndex = 0; + } + return SQLITE_OK; +} + /* ** Delete a WhereLoop object */ @@ -3934,8 +3999,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** dependencies */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; - WhereTerm **paTerm = 0; + WhereLoop **ppPrev, *p, *pNext = 0; WhereInfo *pWInfo = pBuilder->pWInfo; sqlite3 *db = pWInfo->pParse->db; @@ -3946,17 +4010,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ */ if( (p = pBuilder->pBest)!=0 ){ if( p->maskSelf!=0 ){ - if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){ + WhereCost rCost = p->rRun + p->rSetup; + WhereCost rTemplate = pTemplate->rRun + pTemplate->rSetup; + if( rCost < rTemplate ){ goto whereLoopInsert_noop; } - if( p->rRun+p->rSetup == pTemplate->rRun+pTemplate->rSetup - && p->prereq <= pTemplate->prereq ){ + if( rCost == rTemplate && p->prereq <= pTemplate->prereq ){ goto whereLoopInsert_noop; } } - *p = *pTemplate; - p->aTerm = 0; - p->u.vtab.needFree = 0; + whereLoopXfer(db, p, pTemplate); #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf("ins-best: "); @@ -3976,7 +4039,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ && p->rRun<=pTemplate->rRun ){ /* p is equal or better than pTemplate */ - if( p->nTermnTerm + if( p->nLTermnLTerm && (p->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && p->u.btree.pIndex==pTemplate->u.btree.pIndex @@ -4019,30 +4082,18 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } #endif if( p==0 ){ - p = pToFree = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; + whereLoopInit(p); } - if( pTemplate->nTerm ){ - paTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); - if( paTerm==0 ){ - sqlite3DbFree(db, pToFree); - return SQLITE_NOMEM; - } - } - *p = *pTemplate; + whereLoopXfer(db, p, pTemplate); p->pNextLoop = pNext; *ppPrev = p; - p->aTerm = paTerm; - if( p->nTerm ){ - memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0])); - } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } - }else{ - pTemplate->u.vtab.needFree = 0; } return SQLITE_OK; @@ -4077,11 +4128,15 @@ static int whereLoopAddBtreeIndex( WhereTerm *pTerm; /* A WhereTerm under consideration */ int opMask; /* Valid operators for constraints */ WhereScan scan; /* Iterator for WHERE terms */ - WhereLoop savedLoop; /* Saved original content of pNew[] */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + int saved_nEq; /* Original value of pNew->u.btree.nEq */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + WhereCost saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ tRowcnt iRowEst; /* Estimated index selectivity */ - double rLogSize; /* Logarithm of table size */ + WhereCost rLogSize; /* Logarithm of table size */ WhereTerm *pTop, *pBtm; /* Top and bottom range constraints */ pNew = pBuilder->pNew; @@ -4108,18 +4163,22 @@ static int whereLoopAddBtreeIndex( } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); - savedLoop = *pNew; - pNew->rSetup = (double)0; + saved_nEq = pNew->u.btree.nEq; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pNew->rSetup = (WhereCost)0; rLogSize = estLog(pProbe->aiRowEst[0]); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 1; if( pTerm->prereqRight & pNew->maskSelf ) continue; - pNew->wsFlags = savedLoop.wsFlags; - pNew->u.btree.nEq = savedLoop.u.btree.nEq; - pNew->nTerm = savedLoop.nTerm; - if( pNew->nTerm>=pBuilder->mxTerm ) break; /* Repeated column in index */ - pNew->aTerm[pNew->nTerm++] = pTerm; - pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; pNew->rRun = rLogSize; if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; @@ -4133,7 +4192,7 @@ static int whereLoopAddBtreeIndex( } pNew->rRun *= nIn; pNew->u.btree.nEq++; - pNew->nOut = (double)iRowEst * nInMul * nIn; + pNew->nOut = (WhereCost)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_EQ) ){ assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 || nInMul==1 ); @@ -4146,12 +4205,12 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_ONEROW; } pNew->u.btree.nEq++; - pNew->nOut = (double)iRowEst * nInMul; + pNew->nOut = (WhereCost)iRowEst * nInMul; }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; nIn = 2; /* Assume IS NULL matches two rows */ - pNew->nOut = (double)iRowEst * nInMul * nIn; + pNew->nOut = (WhereCost)iRowEst * nInMul * nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; @@ -4160,14 +4219,14 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aTerm[pNew->nTerm-2] : 0; + pNew->aLTerm[pNew->nLTerm-2] : 0; } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ - double rDiv; + WhereCost rDiv; whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, pBtm, pTop, &rDiv); - pNew->nOut = savedLoop.nOut/rDiv; + pNew->nOut = saved_nOut/rDiv; } #ifdef SQLITE_ENABLE_STAT3 if( pNew->u.btree.nEq==1 && pProbe->nSample ){ @@ -4198,7 +4257,11 @@ static int whereLoopAddBtreeIndex( whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); } } - *pNew = savedLoop; + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; return rc; } @@ -4252,8 +4315,8 @@ static int whereLoopAddBtree( int rc = SQLITE_OK; /* Return code */ int iSortIdx = 1; /* Index number */ int b; /* A boolean value */ - double rSize; /* number of rows in the table */ - double rLogSize; /* Logarithm of the number of rows in the table */ + WhereCost rSize; /* number of rows in the table */ + WhereCost rLogSize; /* Logarithm of the number of rows in the table */ pNew = pBuilder->pNew; pWInfo = pBuilder->pWInfo; @@ -4286,7 +4349,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = (double)pSrc->pTab->nRowEst; + rSize = (WhereCost)pSrc->pTab->nRowEst; rLogSize = estLog(rSize); /* Automatic indexes */ @@ -4306,10 +4369,10 @@ static int whereLoopAddBtree( if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; pNew->u.btree.pIndex = 0; - pNew->nTerm = 1; - pNew->aTerm[0] = pTerm; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; pNew->rSetup = 20*rLogSize*pSrc->pTab->nRowEst; - pNew->nOut = (double)10; + pNew->nOut = (WhereCost)10; pNew->rRun = rLogSize + pNew->nOut; pNew->wsFlags = WHERE_TEMP_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; @@ -4322,9 +4385,9 @@ static int whereLoopAddBtree( */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ pNew->u.btree.nEq = 0; - pNew->nTerm = 0; + pNew->nLTerm = 0; pNew->iSortIdx = 0; - pNew->rSetup = (double)0; + pNew->rSetup = (WhereCost)0; pNew->prereq = mExtra; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); @@ -4392,6 +4455,7 @@ static int whereLoopAddVirtual( WhereTerm *pTerm; int i, j; int iTerm, mxTerm; + int nConstraint; int seenIn = 0; /* True if an IN operator is seen */ int seenVar = 0; /* True if a non-constant constraint is seen */ int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ @@ -4411,9 +4475,11 @@ static int whereLoopAddVirtual( pNew->prereq = 0; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; - pNew->nTerm = 0; + pNew->nLTerm = 0; pNew->u.vtab.needFree = 0; pUsage = pIdxInfo->aConstraintUsage; + nConstraint = pIdxInfo->nConstraint; + if( whereLoopResize(db, pNew, nConstraint) ) return SQLITE_NOMEM; for(iPhase=0; iPhase<=3; iPhase++){ if( !seenIn && (iPhase&1)!=0 ){ @@ -4456,23 +4522,23 @@ static int whereLoopAddVirtual( pIdxInfo->idxNum = 0; pIdxInfo->needToFreeIdxStr = 0; pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + /* ((WhereCost)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((WhereCost)2); rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; pNew->prereq = 0; mxTerm = -1; - for(i=0; imxTerm; i++) pNew->aTerm[i] = 0; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; pNew->u.vtab.omitMask = 0; - for(i=0; inConstraint; i++, pIdxCons++){ + for(i=0; i=0 ){ - if( iTerm>=pBuilder->mxTerm ) break; j = pIdxCons->iTermOffset; - if( iTerm>=pIdxInfo->nConstraint + if( iTerm>=nConstraint || j<0 || j>=pWC->nTerm - || pNew->aTerm[iTerm]!=0 + || pNew->aLTerm[iTerm]!=0 ){ rc = SQLITE_ERROR; sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); @@ -4480,7 +4546,8 @@ static int whereLoopAddVirtual( } pTerm = &pWC->a[j]; pNew->prereq |= pTerm->prereqRight; - pNew->aTerm[iTerm] = pTerm; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; if( iTerm>mxTerm ) mxTerm = iTerm; if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ @@ -4500,17 +4567,18 @@ static int whereLoopAddVirtual( } } } - if( i>=pIdxInfo->nConstraint ){ - pNew->nTerm = mxTerm+1; + if( i>=nConstraint ){ + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); pNew->u.vtab.idxNum = pIdxInfo->idxNum; pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) && pIdxInfo->orderByConsumed); - pNew->rSetup = (double)0; + pNew->rSetup = (WhereCost)0; pNew->rRun = pIdxInfo->estimatedCost; - pNew->nOut = (double)25; + pNew->nOut = (WhereCost)25; whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -4545,6 +4613,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; + whereLoopInit(&sBest); for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 @@ -4553,8 +4622,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; WhereTerm *pOrTerm; - double rTotal = 0; - double nRow = 0; + WhereCost rTotal = 0; + WhereCost nRow = 0; Bitmask prereq = mExtra; pItem = pWInfo->pTabList->a + pNew->iTab; @@ -4577,21 +4646,24 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ continue; } sBest.maskSelf = 0; + sBest.rSetup = 0; + sBest.rRun = 0; if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mExtra); }else{ rc = whereLoopAddBtree(&sSubBuild, mExtra); } if( sBest.maskSelf==0 ) break; - assert( sBest.rSetup==(double)0 ); + assert( sBest.rSetup==(WhereCost)0 ); rTotal += sBest.rRun; nRow += sBest.nOut; prereq |= sBest.prereq; } - pNew->nTerm = 1; - pNew->aTerm[0] = pTerm; + assert( pNew->nLSlot>=1 ); + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; pNew->wsFlags = WHERE_MULTI_OR; - pNew->rSetup = (double)0; + pNew->rSetup = (WhereCost)0; pNew->rRun = rTotal; pNew->nOut = nRow; pNew->prereq = prereq; @@ -4599,6 +4671,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ rc = whereLoopInsert(pBuilder, pNew); } } + whereLoopClear(pWInfo->pParse->db, &sBest); return rc; } @@ -4612,7 +4685,6 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ int iTab; SrcList *pTabList = pWInfo->pTabList; struct SrcList_item *pItem; - WhereClause *pWC = pBuilder->pWC; sqlite3 *db = pWInfo->pParse->db; int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; @@ -4621,17 +4693,8 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pBuilder->pNew = pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); if( pNew==0 ) return SQLITE_NOMEM; - pBuilder->mxTerm = pWC->nTerm+1; - while( pWC->pOuter ){ - pWC = pWC->pOuter; - pBuilder->mxTerm += pWC->nTerm; - } - pWC = pBuilder->pWC; - pNew->aTerm = sqlite3DbMallocZero(db,pBuilder->mxTerm*sizeof(pNew->aTerm[0])); - if( pNew->aTerm==0 ){ - rc = SQLITE_NOMEM; - goto whereLoopAddAll_end; - } + pNew->aLTerm = pNew->aLTermSpace; + pNew->nLSlot = ArraySize(pNew->aLTermSpace); for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); @@ -4649,7 +4712,6 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } -whereLoopAddAll_end: whereLoopDelete(db, pBuilder->pNew); pBuilder->pNew = 0; return rc; @@ -4755,7 +4817,7 @@ static int wherePathSatisfiesOrderBy( ** in the ORDER BY clause. */ for(i=0; iu.btree.nEq; i++){ - pTerm = pLoop->aTerm[i]; + pTerm = pLoop->aLTerm[i]; if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))==0 ) continue; iColumn = pTerm->u.leftColumn; for(j=0; ju.btree.nEq - && ((i = pLoop->aTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ) isOrderDistinct = 0; continue; @@ -4898,15 +4960,15 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation ** error occurs. */ -static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ +static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ - double rCost; /* Cost of a path */ - double mxCost; /* Maximum cost of a set of paths */ - double rSortCost; /* Cost to do a sort */ + WhereCost rCost; /* Cost of a path */ + WhereCost mxCost; /* Maximum cost of a set of paths */ + WhereCost rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -4937,12 +4999,12 @@ static int wherePathSolver(WhereInfo *pWInfo, double nRowEst){ } /* Seed the search with a single WherePath containing zero WhereLoops */ - aFrom[0].nRow = (double)1; + aFrom[0].nRow = (WhereCost)1; nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ - rSortCost = (double)0; + rSortCost = (WhereCost)0; if( pWInfo->pOrderBy==0 || nRowEst<=0.0 ){ aFrom[0].isOrderedValid = 1; }else{ @@ -5412,7 +5474,7 @@ WhereInfo *sqlite3WhereBegin( */ sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; - pWInfo->nRowOut = (double)1; + pWInfo->nRowOut = (WhereCost)1; for(ii=0, pLevel=pWInfo->a; ii Date: Thu, 6 Jun 2013 23:44:25 +0000 Subject: [PATCH 071/109] Performance improvements. FossilOrigin-Name: 9f8e84ab9874a8af826763b1669adb57abd493ea --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 +++++++++++++---------------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index b68a0110a6..b0f5052613 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\smanagement\sof\sthe\sspace\sto\shold\sWhereLoop.aLTerm[]. -D 2013-06-06T23:02:03.759 +C Performance\simprovements. +D 2013-06-06T23:44:25.383 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e029cd3fe9f2fcc857cf19d05b352b2359638271 +F src/where.c fb4bee9bfeb4c1c2019a2ed36cb49d1f72e8dda7 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b4a5dbad36bceabc5c5350e7676af6ad42de04eb -R c565385814d78fee98948398a72cac6f +P d4141ecbea3abbe83525910684fbd89eb74eeb34 +R cd5ddc1e68b8c783ab773654e8fe4f69 U drh -Z 2b77e3c42c0b772f55707fe4d25f7e48 +Z c534f818416a6634a12552ce3fcffbb2 diff --git a/manifest.uuid b/manifest.uuid index bc7a4d415f..b12d1e6402 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4141ecbea3abbe83525910684fbd89eb74eeb34 \ No newline at end of file +9f8e84ab9874a8af826763b1669adb57abd493ea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 08569fc60b..994df30244 100644 --- a/src/where.c +++ b/src/where.c @@ -3883,18 +3883,19 @@ static void whereLoopInit(WhereLoop *p){ ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. */ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ - if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - if( p->u.vtab.needFree ) sqlite3_free(p->u.vtab.idxStr); - p->u.vtab.needFree = 0; - p->u.vtab.idxStr = 0; - }else if( (p->wsFlags & WHERE_TEMP_INDEX)!=0 && p->u.btree.pIndex!=0 ){ - sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3DbFree(db, p->u.btree.pIndex); - p->u.btree.pIndex = 0; + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_TEMP_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_TEMP_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } } } - /* ** Deallocate internal memory used by a WhereLoop object */ @@ -3941,9 +3942,7 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ pTo->u = pFrom->u; if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ pFrom->u.vtab.needFree = 0; - }else if( (pFrom->wsFlags & WHERE_TEMP_INDEX)!=0 && pFrom->u.btree.pIndex!=0 ){ - sqlite3DbFree(db, pFrom->u.btree.pIndex->zColAff); - sqlite3DbFree(db, pFrom->u.btree.pIndex); + }else if( (pFrom->wsFlags & WHERE_TEMP_INDEX)!=0 ){ pFrom->u.btree.pIndex = 0; } return SQLITE_OK; @@ -4048,7 +4047,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ /* Overwrite an existing WhereLoop with an similar one that uses ** more terms of the index */ pNext = p->pNextLoop; - whereLoopClear(db, p); break; }else{ /* pTemplate is not helpful. @@ -4062,7 +4060,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with a better one */ pNext = p->pNextLoop; - whereLoopClear(db, p); break; } } @@ -4613,7 +4610,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; - whereLoopInit(&sBest); for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 @@ -4626,6 +4622,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ WhereCost nRow = 0; Bitmask prereq = mExtra; + whereLoopInit(&sBest); pItem = pWInfo->pTabList->a + pNew->iTab; iCur = pItem->iCursor; sSubBuild = *pBuilder; @@ -4669,9 +4666,9 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->prereq = prereq; memset(&pNew->u, 0, sizeof(pNew->u)); rc = whereLoopInsert(pBuilder, pNew); + whereLoopClear(pWInfo->pParse->db, &sBest); } } - whereLoopClear(pWInfo->pParse->db, &sBest); return rc; } From a20141593a5b49e14bbc82c64b5e11cafbdeba57 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Jun 2013 00:29:23 +0000 Subject: [PATCH 072/109] Further prepare-time performance improvements. FossilOrigin-Name: 02741d177bfc2fca23bc99974cd899eba13cb59a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 33 +++++++++++---------------------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index b0f5052613..6c3bd8a20c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovements. -D 2013-06-06T23:44:25.383 +C Further\sprepare-time\sperformance\simprovements. +D 2013-06-07T00:29:23.782 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c fb4bee9bfeb4c1c2019a2ed36cb49d1f72e8dda7 +F src/where.c bbd8aad0e54692a107d0f68111a289149814b10c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d4141ecbea3abbe83525910684fbd89eb74eeb34 -R cd5ddc1e68b8c783ab773654e8fe4f69 +P 9f8e84ab9874a8af826763b1669adb57abd493ea +R 8930578735fbc55c72ec3d27796f6bd0 U drh -Z c534f818416a6634a12552ce3fcffbb2 +Z c23263f2bef48916be7bbfa11913b59d diff --git a/manifest.uuid b/manifest.uuid index b12d1e6402..aaaa9ad535 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f8e84ab9874a8af826763b1669adb57abd493ea \ No newline at end of file +02741d177bfc2fca23bc99974cd899eba13cb59a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 994df30244..3b825214f5 100644 --- a/src/where.c +++ b/src/where.c @@ -99,9 +99,6 @@ struct WhereLoop { #endif u8 iTab; /* Position in FROM clause of table for this loop */ u8 iSortIdx; /* Sorting index number. 0==None */ - u16 nLTerm; /* Number of entries in aLTerm[] */ - u16 nLSlot; /* Number of slots allocated for aLTerm[] */ - u32 wsFlags; /* WHERE_* flags describing the plan */ WhereCost rSetup; /* One-time setup cost (ex: create transient index) */ WhereCost rRun; /* Cost of running each loop */ WhereCost nOut; /* Estimated number of output rows */ @@ -118,6 +115,11 @@ struct WhereLoop { char *idxStr; /* Index identifier string */ } vtab; } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ @@ -3927,19 +3929,8 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ if( whereLoopResize(db, pTo, pFrom->nLTerm) ) return SQLITE_NOMEM; whereLoopClearUnion(db, pTo); - pTo->prereq = pFrom->prereq; - pTo->maskSelf = pFrom->maskSelf; - pTo->iTab = pFrom->iTab; - pTo->iSortIdx = pFrom->iSortIdx; - pTo->nLTerm = pFrom->nLTerm; - pTo->rSetup = pFrom->rSetup; - pTo->rRun = pFrom->rRun; - pTo->nOut = pFrom->nOut; - if( pTo->nLTerm ){ - memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); - } - pTo->wsFlags = pFrom->wsFlags; - pTo->u = pFrom->u; + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ pFrom->u.vtab.needFree = 0; }else if( (pFrom->wsFlags & WHERE_TEMP_INDEX)!=0 ){ @@ -4685,13 +4676,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; - WhereLoop *pNew; + WhereLoop *pNew, sNew; /* Loop over the tables in the join, from left to right */ - pBuilder->pNew = pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->aLTerm = pNew->aLTermSpace; - pNew->nLSlot = ArraySize(pNew->aLTermSpace); + pBuilder->pNew = pNew = &sNew; + whereLoopInit(pNew); for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); @@ -4709,7 +4698,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } - whereLoopDelete(db, pBuilder->pNew); + whereLoopClear(db, pNew); pBuilder->pNew = 0; return rc; } From 23fec451a6bdb4d0b598592c66567ef7f6485330 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Jun 2013 02:04:19 +0000 Subject: [PATCH 073/109] Must faster computation of estimated logarithm. FossilOrigin-Name: dfbca3acaeb862f24f25e7810a16ff2066111ff4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 ++++++------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c3bd8a20c..d0734bd638 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sprepare-time\sperformance\simprovements. -D 2013-06-07T00:29:23.782 +C Must\sfaster\scomputation\sof\sestimated\slogarithm. +D 2013-06-07T02:04:19.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c bbd8aad0e54692a107d0f68111a289149814b10c +F src/where.c 70e1ebd1c3f828f4062fc2b5ada2461419020dae F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9f8e84ab9874a8af826763b1669adb57abd493ea -R 8930578735fbc55c72ec3d27796f6bd0 +P 02741d177bfc2fca23bc99974cd899eba13cb59a +R 8ee17766ef9ac6851fd1329c34593963 U drh -Z c23263f2bef48916be7bbfa11913b59d +Z adf16c6d5fc0c897d258a12782d83ad0 diff --git a/manifest.uuid b/manifest.uuid index aaaa9ad535..8ae2ff38a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02741d177bfc2fca23bc99974cd899eba13cb59a \ No newline at end of file +dfbca3acaeb862f24f25e7810a16ff2066111ff4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3b825214f5..c1fc4301d7 100644 --- a/src/where.c +++ b/src/where.c @@ -1831,13 +1831,11 @@ static int isDistinctRedundant( ** logN is a little off. */ static WhereCost estLog(WhereCost N){ - WhereCost logN = 1; - WhereCost x = 10; - while( N>x ){ - logN += 1; - x *= 10; - } - return logN; + u32 a; + assert( sizeof(WhereCost)==4 ); /* 32-bit float input */ + if( N<=0.0 ) return 0.0; + memcpy(&a, &N, 4); + return ((a >>= 23)-127)*0.3; } /* @@ -5398,6 +5396,7 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); + /* TBD: if( nTablist==1 ) whereCommonCase(&sWLB); */ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; From 60c96cd7890abc4cf8526afc46b62c971e89b8e2 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 9 Jun 2013 17:21:25 +0000 Subject: [PATCH 074/109] High-speed version of NGQP. Still has some minor problems. FossilOrigin-Name: db2415fa677b84cd0f6dd424283c94e98d246e3b --- manifest | 15 ++++--- manifest.uuid | 2 +- src/where.c | 117 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 105 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index d0734bd638..39d6b51064 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Must\sfaster\scomputation\sof\sestimated\slogarithm. -D 2013-06-07T02:04:19.774 +C High-speed\sversion\sof\sNGQP.\s\sStill\shas\ssome\sminor\sproblems. +D 2013-06-09T17:21:25.979 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 70e1ebd1c3f828f4062fc2b5ada2461419020dae +F src/where.c 2dae18fe0b79aa90f213c5f2ff385a7ac56d36d5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 02741d177bfc2fca23bc99974cd899eba13cb59a -R 8ee17766ef9ac6851fd1329c34593963 +P dfbca3acaeb862f24f25e7810a16ff2066111ff4 +R e3ea6bb6c8cd62c47ea6f6e83f99515a +T *branch * nextgen-query-plan-fast +T *sym-nextgen-query-plan-fast * +T -sym-nextgen-query-plan-exp * U drh -Z adf16c6d5fc0c897d258a12782d83ad0 +Z 13597db4c2da76541af04a4f735ffed4 diff --git a/manifest.uuid b/manifest.uuid index 8ae2ff38a7..3a88e72860 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dfbca3acaeb862f24f25e7810a16ff2066111ff4 \ No newline at end of file +db2415fa677b84cd0f6dd424283c94e98d246e3b \ No newline at end of file diff --git a/src/where.c b/src/where.c index c1fc4301d7..135633e1cf 100644 --- a/src/where.c +++ b/src/where.c @@ -5168,6 +5168,75 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ return SQLITE_OK; } +/* +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite3_prepare() +** times for the common case. +** +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. +*/ +static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + struct SrcList_item *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int i, j; + int nOrderBy; + Table *pTab; + Index *pIdx; + + pWInfo = pBuilder->pWInfo; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->zIndex ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pWInfo->a[0].pWLoop = pLoop; + pLoop->wsFlags = 0; + nOrderBy = pWInfo->pOrderBy ? pWInfo->pOrderBy->nExpr : 0; + pTerm = findTerm(pWC, iCur, -1, 1, WO_EQ, 0); + if( pTerm ){ + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + pLoop->rRun = (WhereCost)10; + pLoop->nOut = (WhereCost)1; + pWInfo->nRowOut = 1; + pWInfo->nOBSat = nOrderBy; + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(j=0; jnColumn; j++){ + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 1, WO_EQ, pIdx); + if( pTerm==0 ) break; + whereLoopResize(pWInfo->pParse->db, pLoop, j); + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nColumn ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW; + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + pLoop->rRun = (WhereCost)15; + pLoop->nOut = (WhereCost)1; + pWInfo->nRowOut = 1; + pWInfo->nOBSat = nOrderBy; + break; + } + } + return pLoop->wsFlags!=0; +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -5307,7 +5376,7 @@ WhereInfo *sqlite3WhereBegin( */ db = pParse->db; nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, nByteWInfo); + pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); pWInfo = 0; @@ -5324,6 +5393,8 @@ WhereInfo *sqlite3WhereBegin( pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)&pWInfo->a[nTabList]; + whereLoopInit(sWLB.pNew); /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -5396,30 +5467,32 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); - /* TBD: if( nTablist==1 ) whereCommonCase(&sWLB); */ - rc = whereLoopAddAll(&sWLB); - if( rc ) goto whereBeginError; - - /* Display all of the WhereLoop objects if wheretrace is enabled */ + if( nTabList!=1 || whereSimpleFastCase(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + + /* Display all of the WhereLoop objects if wheretrace is enabled */ #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ - WhereLoop *p; - int i = 0; - static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; - for(p=pWInfo->pLoops; p; p=p->pNextLoop){ - p->cId = zLabel[(i++)%sizeof(zLabel)]; - whereLoopPrint(p, pTabList); + if( sqlite3WhereTrace ){ + WhereLoop *p; + int i = 0; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops; p; p=p->pNextLoop){ + p->cId = zLabel[(i++)%sizeof(zLabel)]; + whereLoopPrint(p, pTabList); + } + } +#endif + + wherePathSolver(pWInfo, -1); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut); + if( db->mallocFailed ) goto whereBeginError; } } -#endif - - wherePathSolver(pWInfo, -1); - if( db->mallocFailed ) goto whereBeginError; - if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut); - if( db->mallocFailed ) goto whereBeginError; - }else if( db->flags & SQLITE_ReverseOrder ){ + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ pWInfo->revMask = (Bitmask)(-1); } if( pParse->nErr || db->mallocFailed ){ From 92a121f46e27cdef9d79a8f192fc62617b8b3218 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 12:15:47 +0000 Subject: [PATCH 075/109] Minor problems in the high-speed NGQP fixed. FossilOrigin-Name: 20eeccf1f24dd762f1bee9c3fe628c70ff01627f --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 36 ++++++++++++++++++++++++++---------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 39d6b51064..7682ff8ad2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C High-speed\sversion\sof\sNGQP.\s\sStill\shas\ssome\sminor\sproblems. -D 2013-06-09T17:21:25.979 +C Minor\sproblems\sin\sthe\shigh-speed\sNGQP\sfixed. +D 2013-06-10T12:15:47.024 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2dae18fe0b79aa90f213c5f2ff385a7ac56d36d5 +F src/where.c 9fff9a5f3f227857aa0590026837a78696d3bba7 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,10 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dfbca3acaeb862f24f25e7810a16ff2066111ff4 -R e3ea6bb6c8cd62c47ea6f6e83f99515a -T *branch * nextgen-query-plan-fast -T *sym-nextgen-query-plan-fast * -T -sym-nextgen-query-plan-exp * +P db2415fa677b84cd0f6dd424283c94e98d246e3b +R 1c99bcc201a3fe7c000749ac05c30a32 U drh -Z 13597db4c2da76541af04a4f735ffed4 +Z f1d76f2fd1cf61ecde305cf79590397b diff --git a/manifest.uuid b/manifest.uuid index 3a88e72860..6bd8b33dc7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db2415fa677b84cd0f6dd424283c94e98d246e3b \ No newline at end of file +20eeccf1f24dd762f1bee9c3fe628c70ff01627f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 135633e1cf..e6b3c3a534 100644 --- a/src/where.c +++ b/src/where.c @@ -4281,6 +4281,21 @@ static int indexMightHelpWithOrderBy( return 0; } +/* +** Return a bitmask where 1s indicate that the corresponding column of +** the table is used by an index. Only the first 63 columns are considered. +*/ +static Bitmask columnsUsedByIndex(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( xpNew->iTab. That table is guaranteed to be @@ -4388,14 +4403,7 @@ static int whereLoopAddBtree( rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; }else{ - Bitmask m = pSrc->colUsed; - int j; - for(j=pProbe->nColumn-1; j>=0; j--){ - int x = pProbe->aiColumn[j]; - if( xcolUsed & ~columnsUsedByIndex(pProbe); pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; /* Full scan via index */ @@ -5186,7 +5194,7 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ WhereTerm *pTerm; WhereLoop *pLoop; int iCur; - int i, j; + int j; int nOrderBy; Table *pTab; Index *pIdx; @@ -5202,6 +5210,11 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ pLoop = pBuilder->pNew; pWInfo->a[0].pWLoop = pLoop; pLoop->wsFlags = 0; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif nOrderBy = pWInfo->pOrderBy ? pWInfo->pOrderBy->nExpr : 0; pTerm = findTerm(pWC, iCur, -1, 1, WO_EQ, 0); if( pTerm ){ @@ -5223,7 +5236,10 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nColumn ) continue; - pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( (pItem->colUsed & ~columnsUsedByIndex(pIdx))==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } pLoop->nLTerm = j; pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; From 43b85ef5c6c76ba34acbcb10fdd64bf53582695c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 12:34:45 +0000 Subject: [PATCH 076/109] Performance improvements for whereScan methods. FossilOrigin-Name: aae14350a37ad50e4607953ab496cba006032873 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 +++++-------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 914fd84078..a55fe673ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\shigh-speed\sbypass\sfor\sthe\sNGQP\sfor\sthe\scommon\scase\sof\sa\ssimply\squery\nwith\squality\sconstraints\sthat\soutputs\sa\ssingle\srow. -D 2013-06-10T12:17:03.683 +C Performance\simprovements\sfor\swhereScan\smethods. +D 2013-06-10T12:34:45.199 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 9fff9a5f3f227857aa0590026837a78696d3bba7 +F src/where.c eddcadac3d6d9ec6e969e2d2c9a07c18ee8326c0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dfbca3acaeb862f24f25e7810a16ff2066111ff4 20eeccf1f24dd762f1bee9c3fe628c70ff01627f -R 1c99bcc201a3fe7c000749ac05c30a32 +P 8d1ba309211e8afa8be5520bbc1eecf6deacae07 +R eb89dcd4fc9448ac2059e0ee34e53951 U drh -Z f19cc889e5aa39da8409c75eaebc0bb3 +Z 8ab3224e95cf812319014fdeea79f2dd diff --git a/manifest.uuid b/manifest.uuid index 493af68286..00d4b0108f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d1ba309211e8afa8be5520bbc1eecf6deacae07 \ No newline at end of file +aae14350a37ad50e4607953ab496cba006032873 \ No newline at end of file diff --git a/src/where.c b/src/where.c index e6b3c3a534..37d1b151c8 100644 --- a/src/where.c +++ b/src/where.c @@ -231,7 +231,6 @@ struct WhereTerm { ** terms in the WHERE clause that are useful to the query planner. */ struct WhereScan { - WhereTerm *pCurrent; /* Most recent match */ WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ char *zCollName; /* Required collating sequence, if not NULL */ @@ -779,12 +778,13 @@ WhereTerm *whereScanNext(WhereScan *pScan){ Expr *pX; /* An expression being tested */ WhereClause *pWC; /* Shorthand for pScan->pWC */ WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ while( pScan->iEquiv<=pScan->nEquiv ){ iCur = pScan->aEquiv[pScan->iEquiv-2]; iColumn = pScan->aEquiv[pScan->iEquiv-1]; while( (pWC = pScan->pWC)!=0 ){ - for(pTerm=pWC->a+pScan->k; pScan->knTerm; pScan->k++, pTerm++){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaEquiv) @@ -828,20 +828,18 @@ WhereTerm *whereScanNext(WhereScan *pScan){ ){ continue; } - pScan->pCurrent = pTerm; - pScan->k++; + pScan->k = k+1; return pTerm; } } } pWC = pScan->pWC = pScan->pWC->pOuter; - pScan->k = 0; + k = 0; } pScan->pWC = pScan->pOrigWC; - pScan->k = 0; + k = 0; pScan->iEquiv += 2; } - pScan->pCurrent = 0; return 0; } @@ -867,7 +865,6 @@ WhereTerm *whereScanInit( int j; /* memset(pScan, 0, sizeof(*pScan)); */ - pScan->pCurrent = 0; pScan->pOrigWC = pWC; pScan->pWC = pWC; if( pIdx && iColumn>=0 ){ From 3b75ffaacab82d95b469ce2099df7b60b3b4a718 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 14:56:25 +0000 Subject: [PATCH 077/109] Simplification and performance tweak to the high-speed NGQP bypass. FossilOrigin-Name: 0f8a38ee54208d6a477aa2482cd277b4808450f0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 32 +++++++++++++++----------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index a55fe673ef..d8a42f44d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovements\sfor\swhereScan\smethods. -D 2013-06-10T12:34:45.199 +C Simplification\sand\sperformance\stweak\sto\sthe\shigh-speed\sNGQP\sbypass. +D 2013-06-10T14:56:25.426 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c eddcadac3d6d9ec6e969e2d2c9a07c18ee8326c0 +F src/where.c 2e75418eb48dbaa675c3e0a112cbd697bd66588c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8d1ba309211e8afa8be5520bbc1eecf6deacae07 -R eb89dcd4fc9448ac2059e0ee34e53951 +P aae14350a37ad50e4607953ab496cba006032873 +R c9c6efd382ece49dba030de75ba0bdde U drh -Z 8ab3224e95cf812319014fdeea79f2dd +Z 90f206413d760722d5039cb0107594a6 diff --git a/manifest.uuid b/manifest.uuid index 00d4b0108f..36d5135459 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aae14350a37ad50e4607953ab496cba006032873 \ No newline at end of file +0f8a38ee54208d6a477aa2482cd277b4808450f0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 37d1b151c8..8f25d1d968 100644 --- a/src/where.c +++ b/src/where.c @@ -5192,7 +5192,6 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ WhereLoop *pLoop; int iCur; int j; - int nOrderBy; Table *pTab; Index *pIdx; @@ -5205,29 +5204,19 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; - pWInfo->a[0].pWLoop = pLoop; pLoop->wsFlags = 0; - pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); - pWInfo->a[0].iTabCur = iCur; -#ifdef SQLITE_DEBUG - pLoop->cId = '0'; -#endif - nOrderBy = pWInfo->pOrderBy ? pWInfo->pOrderBy->nExpr : 0; - pTerm = findTerm(pWC, iCur, -1, 1, WO_EQ, 0); + pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; pLoop->u.btree.nEq = 1; pLoop->rRun = (WhereCost)10; - pLoop->nOut = (WhereCost)1; - pWInfo->nRowOut = 1; - pWInfo->nOBSat = nOrderBy; }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; for(j=0; jnColumn; j++){ - pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 1, WO_EQ, pIdx); + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); if( pTerm==0 ) break; whereLoopResize(pWInfo->pParse->db, pLoop, j); pLoop->aLTerm[j] = pTerm; @@ -5241,13 +5230,22 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; pLoop->rRun = (WhereCost)15; - pLoop->nOut = (WhereCost)1; - pWInfo->nRowOut = 1; - pWInfo->nOBSat = nOrderBy; break; } } - return pLoop->wsFlags!=0; + if( pLoop->wsFlags ){ + pLoop->nOut = (WhereCost)1; + pWInfo->a[0].pWLoop = pLoop; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + pWInfo->nOBSat = pWInfo->pOrderBy ? pWInfo->pOrderBy->nExpr : 0; +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif + return 1; + } + return 0; } /* From b8a8e8a5d2f45c74ab624f57a333afa96fe252a1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 19:12:39 +0000 Subject: [PATCH 078/109] First attempt to store costs and row counts as a logarithm. FossilOrigin-Name: 9e8109673c3a87e379f5a5a97a8b0d5a1afe853d --- manifest | 15 ++-- manifest.uuid | 2 +- src/where.c | 234 +++++++++++++++++++++++++++++--------------------- 3 files changed, 147 insertions(+), 104 deletions(-) diff --git a/manifest b/manifest index d8a42f44d6..7323ba878c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sand\sperformance\stweak\sto\sthe\shigh-speed\sNGQP\sbypass. -D 2013-06-10T14:56:25.426 +C First\sattempt\sto\sstore\scosts\sand\srow\scounts\sas\sa\slogarithm. +D 2013-06-10T19:12:39.512 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2e75418eb48dbaa675c3e0a112cbd697bd66588c +F src/where.c ae52899cfb8710b5f63c01ac64030b20f284dd5e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,7 +1094,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aae14350a37ad50e4607953ab496cba006032873 -R c9c6efd382ece49dba030de75ba0bdde +P 0f8a38ee54208d6a477aa2482cd277b4808450f0 +R 17f9a2533926060b9b736553f681baac +T *branch * nextgen-query-plan-logcost +T *sym-nextgen-query-plan-logcost * +T -sym-nextgen-query-plan-exp * U drh -Z 90f206413d760722d5039cb0107594a6 +Z a107d2813c152067f49ec61599513e6e diff --git a/manifest.uuid b/manifest.uuid index 36d5135459..78304d046a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f8a38ee54208d6a477aa2482cd277b4808450f0 \ No newline at end of file +9e8109673c3a87e379f5a5a97a8b0d5a1afe853d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8f25d1d968..7d2f1f73bb 100644 --- a/src/where.c +++ b/src/where.c @@ -45,7 +45,7 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; -typedef float WhereCost; +typedef unsigned short int WhereCost; /* 10 times log2() of run-time */ /* ** For each nested loop in a WHERE clause implementation, the WhereInfo @@ -1820,6 +1820,48 @@ static int isDistinctRedundant( return 0; } +/* +** The sum of two WhereCosts +*/ +static WhereCost whereCostAdd(WhereCost a, WhereCost b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a>=b ){ + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; + }else{ + if( b>a+49 ) return b; + if( b>a+31 ) return b+1; + return b+x[b-a]; + } +} + +/* +** Convert an integer into a WhereCost +*/ +static WhereCost whereCostFromInt(tRowcnt x){ + static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + WhereCost y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} + /* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating @@ -1828,11 +1870,7 @@ static int isDistinctRedundant( ** logN is a little off. */ static WhereCost estLog(WhereCost N){ - u32 a; - assert( sizeof(WhereCost)==4 ); /* 32-bit float input */ - if( N<=0.0 ) return 0.0; - memcpy(&a, &N, 4); - return ((a >>= 23)-127)*0.3; + return whereCostFromInt(N) - 33; } /* @@ -2240,7 +2278,7 @@ static int whereKeyStats( int i, eType; int isEq = 0; i64 v; - WhereCost r, rS; + double r, rS; assert( roundUp==0 || roundUp==1 ); assert( pIdx->nSample>0 ); @@ -2496,11 +2534,11 @@ static int whereRangeScanEst( sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK ){ - if( iUpper<=iLower ){ - *pRangeDiv = (WhereCost)p->aiRowEst[0]; - }else{ - *pRangeDiv = (WhereCost)p->aiRowEst[0]/(WhereCost)(iUpper - iLower); + WhereCost iBase = whereCostFromInt(p->aiRowEst[0]); + if( iUpper>iLower ){ + iBase -= whereCostFromInt(iUpper - iLower); } + *pRangeDiv = iBase; /*WHERETRACE(("range scan regions: %u..%u div=%g\n", (u32)iLower, (u32)iUpper, *pRangeDiv));*/ return SQLITE_OK; @@ -2512,9 +2550,13 @@ static int whereRangeScanEst( UNUSED_PARAMETER(nEq); #endif assert( pLower || pUpper ); - *pRangeDiv = (WhereCost)1; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (WhereCost)4; - if( pUpper ) *pRangeDiv *= (WhereCost)4; + *pRangeDiv = 0; + if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ + *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + } + if( pUpper ){ + *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + } return rc; } @@ -2540,7 +2582,7 @@ static int whereEqualScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - WhereCost *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ u8 aff; /* Column affinity */ @@ -2589,12 +2631,12 @@ static int whereInScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - WhereCost *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - int rc = SQLITE_OK; /* Subfunction return code */ - WhereCost nEst; /* Number of rows for a single term */ - WhereCost nRowEst = (WhereCost)0; /* New estimate of the number of rows */ - int i; /* Loop counter */ + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ @@ -2982,7 +3024,6 @@ static void explainOneScan( Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ char *zMsg; /* Text to add to EQP output */ - sqlite3_int64 nRow; /* Expected number of rows visited by scan */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ @@ -3037,13 +3078,7 @@ static void explainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ - testcase( wctrlFlags & WHERE_ORDERBY_MIN ); - nRow = 1; - }else{ - nRow = (sqlite3_int64)pLoop->nOut; - } - zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); + zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg); sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); } } @@ -3828,7 +3863,7 @@ static Bitmask codeOneLoopStart( ** Print a WhereLoop object for debugging purposes */ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ - int nb = 2*((pTabList->nSrc+15)/16); + int nb = 1+(pTabList->nSrc+7)/8; struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; sqlite3DebugPrintf("%c %2d.%0*llx.%0*llx", p->cId, @@ -3860,8 +3895,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3_free(z); } sqlite3DebugPrintf(" fg %05x N %d", p->wsFlags, p->nLTerm); - sqlite3DebugPrintf(" cost %.2g,%.2g,%.2g\n", - p->prereq, p->rSetup, p->rRun, p->nOut); + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); } #endif @@ -3995,8 +4029,8 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ */ if( (p = pBuilder->pBest)!=0 ){ if( p->maskSelf!=0 ){ - WhereCost rCost = p->rRun + p->rSetup; - WhereCost rTemplate = pTemplate->rRun + pTemplate->rSetup; + WhereCost rCost = whereCostAdd(p->rRun,p->rSetup); + WhereCost rTemplate = whereCostAdd(pTemplate->rRun,pTemplate->rSetup); if( rCost < rTemplate ){ goto whereLoopInsert_noop; } @@ -4102,7 +4136,7 @@ static int whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ struct SrcList_item *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ - int nInMul /* Number of iterations due to IN */ + WhereCost nInMul /* log(Number of iterations due to IN) */ ){ WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ Parse *pParse = pWInfo->pParse; /* Parsing context */ @@ -4118,7 +4152,7 @@ static int whereLoopAddBtreeIndex( WhereCost saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - tRowcnt iRowEst; /* Estimated index selectivity */ + WhereCost nRowEst; /* Estimated index selectivity */ WhereCost rLogSize; /* Logarithm of table size */ WhereTerm *pTop, *pBtm; /* Top and bottom range constraints */ @@ -4139,10 +4173,10 @@ static int whereLoopAddBtreeIndex( if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - iRowEst = pProbe->aiRowEst[pNew->u.btree.nEq+1]; + nRowEst = whereCostFromInt(pProbe->aiRowEst[pNew->u.btree.nEq+1]); }else{ iCol = -1; - iRowEst = 1; + nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4151,10 +4185,10 @@ static int whereLoopAddBtreeIndex( saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; - pNew->rSetup = (WhereCost)0; - rLogSize = estLog(pProbe->aiRowEst[0]); + pNew->rSetup = 0; + rLogSize = estLog(whereCostFromInt(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ - int nIn = 1; + int nIn = 0; if( pTerm->prereqRight & pNew->maskSelf ) continue; pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; @@ -4168,32 +4202,32 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 25; + nIn = 46; /* whereCostFromInt(25) */ }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ - nIn = pExpr->x.pList->nExpr; + nIn = whereCostFromInt(pExpr->x.pList->nExpr); } - pNew->rRun *= nIn; + pNew->rRun += nIn; pNew->u.btree.nEq++; - pNew->nOut = (WhereCost)iRowEst * nInMul * nIn; + pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_EQ) ){ assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 - || nInMul==1 ); + || nInMul==0 ); pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 - || (pProbe->onError!=OE_None && nInMul==1 + || (pProbe->onError!=OE_None && nInMul==0 && pNew->u.btree.nEq==pProbe->nColumn-1) ){ testcase( pNew->wsFlags & WHERE_COLUMN_IN ); pNew->wsFlags |= WHERE_ONEROW; } pNew->u.btree.nEq++; - pNew->nOut = (WhereCost)iRowEst * nInMul; + pNew->nOut = nRowEst + nInMul; }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; - nIn = 2; /* Assume IS NULL matches two rows */ - pNew->nOut = (WhereCost)iRowEst * nInMul * nIn; + nIn = 10; /* Assume IS NULL matches two rows */ + pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; @@ -4209,27 +4243,29 @@ static int whereLoopAddBtreeIndex( WhereCost rDiv; whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, pBtm, pTop, &rDiv); - pNew->nOut = saved_nOut/rDiv; + pNew->nOut = saved_nOut - rDiv; } #ifdef SQLITE_ENABLE_STAT3 if( pNew->u.btree.nEq==1 && pProbe->nSample ){ + tRowcnt nOut = 0; if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, - &pNew->nOut); + rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &nOut); }else if( (pTerm->eOperator & WO_IN) && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, - &pNew->nOut); - + rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); } + pNew->nOut = whereCostFromInt(nOut); } #endif if( pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK) ){ - pNew->rRun += pNew->nOut; /* Unit step cost to reach each row */ + /* Step cost for each output row */ + pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); }else{ /* Each row involves a step of the index, then a binary search of ** the main table */ - pNew->rRun += pNew->nOut*(1 + rLogSize); + pNew->rRun = rLogSize>90 ? + whereCostAdd(pNew->rRun, pNew->nOut+rLogSize-90) : + pNew->rRun; } /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); @@ -4237,7 +4273,7 @@ static int whereLoopAddBtreeIndex( && pNew->u.btree.nEq<=pProbe->nColumn && pProbe->zName!=0 ){ - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } } pNew->prereq = saved_prereq; @@ -4313,8 +4349,8 @@ static int whereLoopAddBtree( int rc = SQLITE_OK; /* Return code */ int iSortIdx = 1; /* Index number */ int b; /* A boolean value */ - WhereCost rSize; /* number of rows in the table */ - WhereCost rLogSize; /* Logarithm of the number of rows in the table */ + WhereCost rSize; /* number of rows in the table */ + WhereCost rLogSize; /* Logarithm of the number of rows in the table */ pNew = pBuilder->pNew; pWInfo = pBuilder->pWInfo; @@ -4347,7 +4383,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = (WhereCost)pSrc->pTab->nRowEst; + rSize = whereCostFromInt(pSrc->pTab->nRowEst); rLogSize = estLog(rSize); /* Automatic indexes */ @@ -4369,9 +4405,10 @@ static int whereLoopAddBtree( pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; - pNew->rSetup = 20*rLogSize*pSrc->pTab->nRowEst; - pNew->nOut = (WhereCost)10; - pNew->rRun = rLogSize + pNew->nOut; + assert( 43==whereCostFromInt(20) ); + pNew->rSetup = 43 + rLogSize + rSize; + pNew->nOut = 33; assert( 33==whereCostFromInt(10) ); + pNew->rRun = whereCostAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_TEMP_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; rc = whereLoopInsert(pBuilder, pNew); @@ -4385,7 +4422,7 @@ static int whereLoopAddBtree( pNew->u.btree.nEq = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; - pNew->rSetup = (WhereCost)0; + pNew->rSetup = 0; pNew->prereq = mExtra; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); @@ -4396,7 +4433,7 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = (rSize + rLogSize)*(3+b); /* 4x penalty for a full-scan */ + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 + b*4; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; }else{ @@ -4412,12 +4449,12 @@ static int whereLoopAddBtree( ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize); + pNew->rRun = whereCostAdd(rSize,rLogSize) + ((m==0 && b) ? 10 : 0); rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; } } - rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ @@ -4567,9 +4604,9 @@ static int whereLoopAddVirtual( pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) && pIdxInfo->orderByConsumed); - pNew->rSetup = (WhereCost)0; - pNew->rRun = pIdxInfo->estimatedCost; - pNew->nOut = (WhereCost)25; + pNew->rSetup = 0; + pNew->rRun = whereCostFromInt((tRowcnt)pIdxInfo->estimatedCost); + pNew->nOut = 46; assert( 46 == whereCostFromInt(25) ); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -4645,16 +4682,16 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ rc = whereLoopAddBtree(&sSubBuild, mExtra); } if( sBest.maskSelf==0 ) break; - assert( sBest.rSetup==(WhereCost)0 ); - rTotal += sBest.rRun; - nRow += sBest.nOut; + assert( sBest.rSetup==0 ); + rTotal = whereCostAdd(rTotal, sBest.rRun); + nRow = whereCostAdd(nRow, sBest.nOut); prereq |= sBest.prereq; } assert( pNew->nLSlot>=1 ); pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; pNew->wsFlags = WHERE_MULTI_OR; - pNew->rSetup = (WhereCost)0; + pNew->rSetup = 0; pNew->rRun = rTotal; pNew->nOut = nRow; pNew->prereq = prereq; @@ -4679,10 +4716,10 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; - WhereLoop *pNew, sNew; + WhereLoop *pNew; /* Loop over the tables in the join, from left to right */ - pBuilder->pNew = pNew = &sNew; + pNew = pBuilder->pNew; whereLoopInit(pNew); for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; @@ -4702,7 +4739,6 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ if( rc || db->mallocFailed ) break; } whereLoopClear(db, pNew); - pBuilder->pNew = 0; return rc; } @@ -4988,20 +5024,20 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ } /* Seed the search with a single WherePath containing zero WhereLoops */ - aFrom[0].nRow = (WhereCost)1; + aFrom[0].nRow = 0; nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ - rSortCost = (WhereCost)0; - if( pWInfo->pOrderBy==0 || nRowEst<=0.0 ){ + rSortCost = 0; + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrderedValid = 1; }else{ /* Compute an estimate on the cost to sort the entire result set */ - rSortCost = nRowEst*estLog(nRowEst); + rSortCost = nRowEst + estLog(nRowEst); #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace>=2 ){ - sqlite3DebugPrintf("---- sort cost=%-7.2g\n", rSortCost); + sqlite3DebugPrintf("---- sort cost=%-3d\n", rSortCost); } #endif } @@ -5021,7 +5057,8 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; + rCost = whereCostAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rCost = whereCostAdd(rCost, pFrom->rCost); maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( !isOrderedValid ){ switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, iLoop==nLoop-1, @@ -5033,7 +5070,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ case 0: /* No. pFrom+pWLoop will require a separate sort */ isOrdered = 0; isOrderedValid = 1; - rCost += rSortCost; + rCost = whereCostAdd(rCost, rSortCost); break; default: /* Cannot tell yet. Try again on the next iteration */ break; @@ -5051,7 +5088,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( nTo>=mxChoice && rCost>=mxCost ){ #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf("Skip %s cost=%-7.2g order=%c\n", + sqlite3DebugPrintf("Skip %s cost=%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); } @@ -5069,7 +5106,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pTo = &aTo[jj]; #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf("New %s cost=%-7.2g order=%c\n", + sqlite3DebugPrintf("New %s cost=%-3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); } @@ -5079,10 +5116,10 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( - "Skip %s cost=%-7.2g order=%c", + "Skip %s cost=%-3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); - sqlite3DebugPrintf(" vs %s cost=%-7.2g order=%c\n", + sqlite3DebugPrintf(" vs %s cost=%-3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); } @@ -5093,10 +5130,10 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( - "Update %s cost=%-7.2g order=%c", + "Update %s cost=%-3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); - sqlite3DebugPrintf(" was %s cost=%-7.2g order=%c\n", + sqlite3DebugPrintf(" was %s cost=%-3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); } @@ -5105,7 +5142,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ /* pWLoop is a winner. Add it to the set of best so far */ pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; pTo->revLoop = revMask; - pTo->nRow = pFrom->nRow * pWLoop->nOut; + pTo->nRow = pFrom->nRow + pWLoop->nOut; pTo->rCost = rCost; pTo->isOrderedValid = isOrderedValid; pTo->isOrdered = isOrdered; @@ -5124,7 +5161,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( sqlite3WhereTrace>=2 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iirCost, pTo->nRow, pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); if( pTo->isOrderedValid && pTo->isOrdered ){ @@ -5184,7 +5221,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ ** no-frills query planner. Return zero if this query needs the ** general-purpose query planner. */ -static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ +static int whereShortCut(WhereLoopBuilder *pBuilder){ WhereInfo *pWInfo; struct SrcList_item *pItem; WhereClause *pWC; @@ -5211,7 +5248,7 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; pLoop->u.btree.nEq = 1; - pLoop->rRun = (WhereCost)10; + pLoop->rRun = 33; /* 33 == whereCostFromInt(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; @@ -5229,7 +5266,7 @@ static int whereSimpleFastCase(WhereLoopBuilder *pBuilder){ pLoop->nLTerm = j; pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; - pLoop->rRun = (WhereCost)15; + pLoop->rRun = 39; /* 39 == whereCostFromInt(15) */ break; } } @@ -5406,6 +5443,9 @@ WhereInfo *sqlite3WhereBegin( sWLB.pWC = &pWInfo->sWC; sWLB.pNew = (WhereLoop*)&pWInfo->a[nTabList]; whereLoopInit(sWLB.pNew); +#ifdef SQLITE_DEBUG + sWLB.pNew->cId = '*'; +#endif /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -5478,7 +5518,7 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); - if( nTabList!=1 || whereSimpleFastCase(&sWLB)==0 ){ + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; From c63367ef6848896d8fc602c7aba12977089c72e0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 20:46:50 +0000 Subject: [PATCH 079/109] Fix some minor issues with logarithmic cost in NGQP. FossilOrigin-Name: 69cf877283d362915edddf1822fbf7a9f86278b3 --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/select.c | 12 ++++++------ src/sqliteInt.h | 6 +++--- src/where.c | 30 ++++++++++++++++++++++++++---- 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 7323ba878c..739f017f0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sattempt\sto\sstore\scosts\sand\srow\scounts\sas\sa\slogarithm. -D 2013-06-10T19:12:39.512 +C Fix\ssome\sminor\sissues\swith\slogarithmic\scost\sin\sNGQP. +D 2013-06-10T20:46:50.026 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,12 +215,12 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 5e8fe15b1c5b2cc592880d3f039c6a6da55bf253 +F src/select.c ddcf0c795f690968de66ab21051e764ca1c13f09 F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 7b85b09d746cde295c7a0f60711421a28f5c4905 +F src/sqliteInt.h 4284f3a915d6ee86656aa0d47649855e8df8081e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c ae52899cfb8710b5f63c01ac64030b20f284dd5e +F src/where.c 402d3f74aa8f72bd0d77df6d414cd0ad538411e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1094,10 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0f8a38ee54208d6a477aa2482cd277b4808450f0 -R 17f9a2533926060b9b736553f681baac -T *branch * nextgen-query-plan-logcost -T *sym-nextgen-query-plan-logcost * -T -sym-nextgen-query-plan-exp * +P 9e8109673c3a87e379f5a5a97a8b0d5a1afe853d +R 6b6adbb839b36bd753eb8d10af218a35 U drh -Z a107d2813c152067f49ec61599513e6e +Z 0c1be235257af3633fe6569a03c10208 diff --git a/manifest.uuid b/manifest.uuid index 78304d046a..98fac7263e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e8109673c3a87e379f5a5a97a8b0d5a1afe853d \ No newline at end of file +69cf877283d362915edddf1822fbf7a9f86278b3 \ No newline at end of file diff --git a/src/select.c b/src/select.c index d160513bd3..ee994a8c20 100644 --- a/src/select.c +++ b/src/select.c @@ -1539,7 +1539,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); }else{ - if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; + if( p->nSelectRow > n ) p->nSelectRow = n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -1733,9 +1733,9 @@ static int multiSelect( p->nSelectRow += pPrior->nSelectRow; if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && p->nSelectRow > (double)nLimit + && p->nSelectRow > nLimit ){ - p->nSelectRow = (double)nLimit; + p->nSelectRow = nLimit; } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -4239,7 +4239,7 @@ int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = (double)LARGEST_INT64; + p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && addrSortIndex>=0 ){ sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; @@ -4320,9 +4320,9 @@ int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->iAlias = 0; } - if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; + if( p->nSelectRow>100 ) p->nSelectRow = 100; }else{ - p->nSelectRow = (double)1; + p->nSelectRow = 1; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 76835385fb..aa1ea51011 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2042,7 +2042,7 @@ struct Select { u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ - double nSelectRow; /* Estimated number of result rows */ + u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -2226,7 +2226,7 @@ struct Parse { /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ - double nQueryLoop; /* Estimated number of iterations of a query */ + u32 nQueryLoop; /* Estimated number of iterations of a query */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ @@ -2796,7 +2796,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); -double sqlite3WhereOutputRowCount(WhereInfo*); +u64 sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); int sqlite3WhereContinueLabel(WhereInfo*); diff --git a/src/where.c b/src/where.c index 7d2f1f73bb..4ddb370124 100644 --- a/src/where.c +++ b/src/where.c @@ -45,7 +45,13 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; -typedef unsigned short int WhereCost; /* 10 times log2() of run-time */ + +/* +** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The +** maximum cost is 64*(2**63) which becomes 6900. So all costs can be +** be stored in a 16-bit unsigned integer without risk of overflow. +*/ +typedef unsigned short int WhereCost; /* ** For each nested loop in a WHERE clause implementation, the WhereInfo @@ -401,11 +407,25 @@ struct WhereInfo { #define WHERE_TEMP_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_COVER_SCAN 0x00008000 /* Full scan of a covering index */ + +/* Convert a WhereCost value (10 times log2(X)) into its integer value X. +*/ +static u64 whereCostToInt(WhereCost x){ + u64 n; + if( x<=10 ) return 1; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>=3 ) return (n+8)<<(x-3); + return (n+8)>>(3-x); +} + /* ** Return the estimated number of output rows from a WHERE clause */ -double sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return (double)pWInfo->nRowOut; +u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return whereCostToInt(pWInfo->nRowOut); } /* @@ -4716,6 +4736,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; + u8 priorJoinType = 0; WhereLoop *pNew; /* Loop over the tables in the join, from left to right */ @@ -4724,9 +4745,10 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); - if( (pItem->jointype & (JT_LEFT|JT_CROSS))!=0 ){ + if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ mExtra = mPrior; } + priorJoinType = pItem->jointype; if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(pBuilder, mExtra); }else{ From 5822d6feb28aa9023abe7133b32d40e91eb3e943 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Jun 2013 23:30:09 +0000 Subject: [PATCH 080/109] Fix test cases for the new EXPLAIN QUERY PLAN format. Add the wherecosttest tool. Other fixes to logarithm cost. FossilOrigin-Name: aa580e368e3c398b8377b80342dfdd906324c248 --- ext/rtree/rtree6.test | 20 +-- ext/rtree/rtree8.test | 1 - manifest | 177 +++++++++++++------------- manifest.uuid | 2 +- src/where.c | 12 +- test/all.test | 2 - test/analyze3.test | 18 +-- test/analyze4.test | 2 +- test/analyze6.test | 22 ++-- test/analyze7.test | 32 ++--- test/analyze8.test | 22 ++-- test/async5.test | 1 - test/autoindex1.test | 24 ++-- test/backup4.test | 1 - test/btreefault.test | 1 - test/capi3e.test | 2 +- test/close.test | 1 - test/corruptF.test | 1 - test/e_createtable.test | 6 +- test/e_fkey.test | 18 +-- test/eqp.test | 256 +++++++++++++++++++------------------- test/exclusive.test | 1 - test/fallocate.test | 1 - test/filefmt.test | 1 - test/fts3aa.test | 1 - test/fts3ao.test | 1 - test/fts3atoken.test | 2 - test/fts3auto.test | 1 - test/fts3aux1.test | 34 ++--- test/fts3corrupt.test | 1 - test/fts3defer2.test | 1 - test/fts3expr3.test | 4 - test/fts3malloc.test | 1 - test/fts3matchinfo.test | 1 - test/fts3prefix2.test | 1 - test/fts3query.test | 16 +-- test/fts3shared.test | 1 - test/fts3snippet.test | 8 +- test/fts3sort.test | 1 - test/fts3tok1.test | 2 - test/fts3tok_err.test | 2 - test/fts4content.test | 1 - test/incrblob3.test | 1 - test/incrblob4.test | 1 - test/incrblobfault.test | 1 - test/incrvacuum3.test | 1 - test/indexedby.test | 54 ++++---- test/io.test | 1 - test/ioerr6.test | 1 - test/lock7.test | 1 - test/misc7.test | 6 +- test/notify3.test | 1 - test/pager1.test | 1 - test/pagerfault.test | 1 - test/pagerfault2.test | 1 - test/pagerfault3.test | 1 - test/securedel2.test | 1 - test/shared8.test | 1 - test/sharedlock.test | 1 - test/tkt-385a5b56b9.test | 8 +- test/tkt-3a77c9714e.test | 1 - test/tkt-3fe897352e.test | 6 +- test/tkt-78e04e52ea.test | 6 +- test/tkt-7a31705a7e6.test | 1 - test/tkt-7bbfb7d442.test | 2 - test/tkt-c48d99d690.test | 1 - test/tkt-d11f09d36e.test | 1 - test/tkt-f3e5abed55.test | 1 - test/tkt-f973c7ac31.test | 1 - test/tkt3442.test | 6 +- test/tkt3918.test | 1 - test/tkt3929.test | 1 - test/unordered.test | 26 ++-- test/veryquick.test | 1 - test/wal8.test | 1 - test/walcksum.test | 1 - test/walcrash.test | 1 - test/walcrash2.test | 1 - test/walcrash3.test | 1 - test/walro.test | 2 - test/walshared.test | 1 - test/where3.test | 42 +++---- test/where7.test | 6 +- test/where9.test | 20 +-- test/whereC.test | 1 - tool/wherecosttest.c | 111 +++++++++++++++++ 86 files changed, 537 insertions(+), 492 deletions(-) create mode 100644 tool/wherecosttest.c diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index 320b993368..17985ee525 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -74,36 +74,36 @@ do_test rtree6-1.5 { do_eqp_test rtree6.2.1 { SELECT * FROM t1,t2 WHERE k=+ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~25 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.2 { SELECT * FROM t1,t2 WHERE k=ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~25 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.3 { SELECT * FROM t1,t2 WHERE k=ii } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~25 rows)} - 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.4 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~25 rows)} - 0 1 1 {SCAN TABLE t2 (~100000 rows)} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} + 0 1 1 {SCAN TABLE t2} } do_eqp_test rtree6.2.5 { SELECT * FROM t1,t2 WHERE k=ii AND x1maskSelf==0 ? "ins-init: " : "ins-best: "); whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif + whereLoopXfer(db, p, pTemplate); return SQLITE_OK; } @@ -4138,7 +4138,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ whereLoopInsert_noop: #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); + sqlite3DebugPrintf(pBuilder->pBest ? "ins-skip: " : "ins-noop: "); whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif @@ -4283,9 +4283,8 @@ static int whereLoopAddBtreeIndex( }else{ /* Each row involves a step of the index, then a binary search of ** the main table */ - pNew->rRun = rLogSize>90 ? - whereCostAdd(pNew->rRun, pNew->nOut+rLogSize-90) : - pNew->rRun; + WhereCost rStepAndSearch = rLogSize>80 ? rLogSize-80 : 1; + pNew->rRun = whereCostAdd(pNew->rRun, rStepAndSearch); } /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); @@ -5255,6 +5254,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ Index *pIdx; pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; pTab = pItem->pTab; diff --git a/test/all.test b/test/all.test index cd2a430390..f6e722f2df 100644 --- a/test/all.test +++ b/test/all.test @@ -48,5 +48,3 @@ if {$::tcl_platform(platform)=="unix"} { } finish_test - - diff --git a/test/analyze3.test b/test/analyze3.test index f705bc6661..c25d04422e 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -97,10 +97,10 @@ do_test analyze3-1.1.1 { do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x? AND x200 AND x<300 } @@ -146,10 +146,10 @@ do_test analyze3-1.2.1 { } {} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x? AND x12 AND x<20 } } {161 0 4760} @@ -193,10 +193,10 @@ do_test analyze3-1.3.1 { } {} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x? AND x200 AND x<300 } @@ -248,10 +248,10 @@ do_test analyze3-2.1 { } {} do_eqp_test analyze3-2.2 { SELECT count(a) FROM t1 WHERE b LIKE 'a%' -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (b>? AND b? AND b? AND b? AND b? AND b? AND b? AND c? AND c? AND c? AND c? AND owner_change_date? AND owner_change_date= 0} {incr i -1} { } finish_test - diff --git a/test/e_createtable.test b/test/e_createtable.test index 351a0f7220..249b5b0e77 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -1368,13 +1368,13 @@ do_execsql_test 4.10.0 { } do_createtable_tests 4.10 { 1 "EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b = 5" - {0 0 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (b=?) (~1 rows)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (b=?)}} 2 "EXPLAIN QUERY PLAN SELECT * FROM t2 ORDER BY b, c" - {0 0 0 {SCAN TABLE t2 USING INDEX sqlite_autoindex_t2_1 (~1000000 rows)}} + {0 0 0 {SCAN TABLE t2 USING INDEX sqlite_autoindex_t2_1}} 3 "EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE b=10 AND c>10" - {0 0 0 {SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (b=? AND c>?) (~2 rows)}} + {0 0 0 {SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (b=? AND c>?)}} } # EVIDENCE-OF: R-45493-35653 A CHECK constraint may be attached to a diff --git a/test/e_fkey.test b/test/e_fkey.test index 001ba6c386..976ed16325 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -974,15 +974,15 @@ do_execsql_test e_fkey-25.2 { EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; EXPLAIN QUERY PLAN SELECT rowid FROM track WHERE trackartist = ?; } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SCAN TABLE track (~100000 rows)} + 0 0 0 {SCAN TABLE artist} + 0 0 0 {SCAN TABLE track} } do_execsql_test e_fkey-25.3 { PRAGMA foreign_keys = ON; EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SCAN TABLE track (~100000 rows)} + 0 0 0 {SCAN TABLE artist} + 0 0 0 {SCAN TABLE track} } do_test e_fkey-25.4 { execsql { @@ -1099,15 +1099,15 @@ do_test e_fkey-27.2 { do_execsql_test e_fkey-27.3 { EXPLAIN QUERY PLAN UPDATE artist SET artistid = ?, artistname = ? } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?) (~10 rows)} - 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?) (~10 rows)} + 0 0 0 {SCAN TABLE artist} + 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?)} + 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?)} } do_execsql_test e_fkey-27.4 { EXPLAIN QUERY PLAN DELETE FROM artist } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?) (~10 rows)} + 0 0 0 {SCAN TABLE artist} + 0 0 0 {SEARCH TABLE track USING COVERING INDEX trackindex (trackartist=?)} } diff --git a/test/eqp.test b/test/eqp.test index 454f2afbdd..f137920309 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -43,37 +43,37 @@ do_execsql_test 1.1 { do_eqp_test 1.2 { SELECT * FROM t2, t1 WHERE t1.a=1 OR t1.b=2; } { - 0 0 1 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows)} - 0 0 1 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~10 rows)} - 0 1 0 {SCAN TABLE t2 (~1000000 rows)} + 0 0 1 {SEARCH TABLE t1 USING INDEX i1 (a=?)} + 0 0 1 {SEARCH TABLE t1 USING INDEX i2 (b=?)} + 0 1 0 {SCAN TABLE t2} } do_eqp_test 1.3 { SELECT * FROM t2 CROSS JOIN t1 WHERE t1.a=1 OR t1.b=2; } { - 0 0 0 {SCAN TABLE t2 (~1000000 rows)} - 0 1 1 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows)} - 0 1 1 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~10 rows)} + 0 0 0 {SCAN TABLE t2} + 0 1 1 {SEARCH TABLE t1 USING INDEX i1 (a=?)} + 0 1 1 {SEARCH TABLE t1 USING INDEX i2 (b=?)} } do_eqp_test 1.3 { SELECT a FROM t1 ORDER BY a } { - 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1} } do_eqp_test 1.4 { SELECT a FROM t1 ORDER BY +a } { - 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_eqp_test 1.5 { SELECT a FROM t1 WHERE a=4 } { - 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)} } do_eqp_test 1.6 { SELECT DISTINCT count(*) FROM t3 GROUP BY a; } { - 0 0 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 0 {SCAN TABLE t3} 0 0 0 {USE TEMP B-TREE FOR GROUP BY} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } @@ -81,40 +81,40 @@ do_eqp_test 1.6 { do_eqp_test 1.7 { SELECT * FROM t3 JOIN (SELECT 1) } { - 0 0 1 {SCAN SUBQUERY 1 (~1 rows)} - 0 1 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 1 {SCAN SUBQUERY 1} + 0 1 0 {SCAN TABLE t3} } do_eqp_test 1.8 { SELECT * FROM t3 JOIN (SELECT 1 UNION SELECT 2) } { 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (UNION)} - 0 0 1 {SCAN SUBQUERY 1 (~2 rows)} - 0 1 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 1 {SCAN SUBQUERY 1} + 0 1 0 {SCAN TABLE t3} } do_eqp_test 1.9 { SELECT * FROM t3 JOIN (SELECT 1 EXCEPT SELECT a FROM t3 LIMIT 17) } { - 3 0 0 {SCAN TABLE t3 (~1000000 rows)} + 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (EXCEPT)} - 0 0 1 {SCAN SUBQUERY 1 (~17 rows)} - 0 1 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 1 {SCAN SUBQUERY 1} + 0 1 0 {SCAN TABLE t3} } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) } { - 3 0 0 {SCAN TABLE t3 (~1000000 rows)} + 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (INTERSECT)} - 0 0 1 {SCAN SUBQUERY 1 (~1 rows)} - 0 1 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 1 {SCAN SUBQUERY 1} + 0 1 0 {SCAN TABLE t3} } do_eqp_test 1.11 { SELECT * FROM t3 JOIN (SELECT 1 UNION ALL SELECT a FROM t3 LIMIT 17) } { - 3 0 0 {SCAN TABLE t3 (~1000000 rows)} + 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 (UNION ALL)} - 0 0 1 {SCAN SUBQUERY 1 (~17 rows)} - 0 1 0 {SCAN TABLE t3 (~1000000 rows)} + 0 0 1 {SCAN SUBQUERY 1} + 0 1 0 {SCAN TABLE t3} } #------------------------------------------------------------------------- @@ -129,48 +129,48 @@ do_execsql_test 2.1 { } det 2.2.1 "SELECT DISTINCT min(x), max(x) FROM t1 GROUP BY x ORDER BY 1" { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR GROUP BY} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } det 2.2.2 "SELECT DISTINCT min(x), max(x) FROM t2 GROUP BY x ORDER BY 1" { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } det 2.2.3 "SELECT DISTINCT * FROM t1" { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } det 2.2.4 "SELECT DISTINCT * FROM t1, t2" { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} - 0 1 1 {SCAN TABLE t2 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE t2} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } det 2.2.5 "SELECT DISTINCT * FROM t1, t2 ORDER BY t1.x" { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} - 0 1 1 {SCAN TABLE t2 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE t2} 0 0 0 {USE TEMP B-TREE FOR DISTINCT} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } det 2.2.6 "SELECT DISTINCT t2.x FROM t1, t2 ORDER BY t2.x" { - 0 0 1 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} - 0 1 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 1 {SCAN TABLE t2 USING COVERING INDEX t2i1} + 0 1 0 {SCAN TABLE t1} } det 2.3.1 "SELECT max(x) FROM t2" { - 0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1 (~1 rows)} + 0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1} } det 2.3.2 "SELECT min(x) FROM t2" { - 0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1 (~1 rows)} + 0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1} } det 2.3.3 "SELECT min(x), max(x) FROM t2" { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} } det 2.4.1 "SELECT * FROM t1 WHERE rowid=?" { - 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)} } @@ -181,39 +181,39 @@ det 2.4.1 "SELECT * FROM t1 WHERE rowid=?" { do_eqp_test 3.1.1 { SELECT (SELECT x FROM t1 AS sub) FROM t1; } { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} - 1 0 0 {SCAN TABLE t1 AS sub (~1000000 rows)} + 1 0 0 {SCAN TABLE t1 AS sub} } do_eqp_test 3.1.2 { SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub); } { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} - 1 0 0 {SCAN TABLE t1 AS sub (~1000000 rows)} + 1 0 0 {SCAN TABLE t1 AS sub} } do_eqp_test 3.1.3 { SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub ORDER BY y); } { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} - 1 0 0 {SCAN TABLE t1 AS sub (~1000000 rows)} + 1 0 0 {SCAN TABLE t1 AS sub} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_eqp_test 3.1.4 { SELECT * FROM t1 WHERE (SELECT x FROM t2 ORDER BY x); } { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} - 1 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} } det 3.2.1 { SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) ORDER BY y LIMIT 5 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 0 0 0 {SCAN SUBQUERY 1 (~10 rows)} + 0 0 0 {SCAN SUBQUERY 1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } det 3.2.2 { @@ -222,34 +222,34 @@ det 3.2.2 { (SELECT * FROM t2 ORDER BY x LIMIT 10) AS x2 ORDER BY x2.y LIMIT 5 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 USING INDEX t2i1 (~1000000 rows)} - 0 0 0 {SCAN SUBQUERY 1 AS x1 (~10 rows)} - 0 1 1 {SCAN SUBQUERY 2 AS x2 (~10 rows)} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 0 0 0 {SCAN SUBQUERY 1 AS x1} + 0 1 1 {SCAN SUBQUERY 2 AS x2} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } det 3.3.1 { SELECT * FROM t1 WHERE y IN (SELECT y FROM t2) } { - 0 0 0 {SCAN TABLE t1 (~100000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE LIST SUBQUERY 1} - 1 0 0 {SCAN TABLE t2 (~1000000 rows)} + 1 0 0 {SCAN TABLE t2} } det 3.3.2 { SELECT * FROM t1 WHERE y IN (SELECT y FROM t2 WHERE t1.x!=t2.x) } { - 0 0 0 {SCAN TABLE t1 (~500000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE CORRELATED LIST SUBQUERY 1} - 1 0 0 {SCAN TABLE t2 (~500000 rows)} + 1 0 0 {SCAN TABLE t2} } det 3.3.3 { SELECT * FROM t1 WHERE EXISTS (SELECT y FROM t2 WHERE t1.x!=t2.x) } { - 0 0 0 {SCAN TABLE t1 (~500000 rows)} + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 1} - 1 0 0 {SCAN TABLE t2 (~500000 rows)} + 1 0 0 {SCAN TABLE t2} } #------------------------------------------------------------------------- @@ -258,43 +258,43 @@ det 3.3.3 { do_eqp_test 4.1.1 { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} + 2 0 0 {SCAN TABLE t2} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} } do_eqp_test 4.1.2 { SELECT * FROM t1 UNION ALL SELECT * FROM t2 ORDER BY 2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} } do_eqp_test 4.1.3 { SELECT * FROM t1 UNION SELECT * FROM t2 ORDER BY 2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.1.4 { SELECT * FROM t1 INTERSECT SELECT * FROM t2 ORDER BY 2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.1.5 { SELECT * FROM t1 EXCEPT SELECT * FROM t2 ORDER BY 2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } @@ -302,35 +302,35 @@ do_eqp_test 4.1.5 { do_eqp_test 4.2.2 { SELECT * FROM t1 UNION ALL SELECT * FROM t2 ORDER BY 1 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 USING INDEX t2i1 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} } do_eqp_test 4.2.3 { SELECT * FROM t1 UNION SELECT * FROM t2 ORDER BY 1 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.2.4 { SELECT * FROM t1 INTERSECT SELECT * FROM t2 ORDER BY 1 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.2.5 { SELECT * FROM t1 EXCEPT SELECT * FROM t2 ORDER BY 1 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } @@ -338,28 +338,28 @@ do_eqp_test 4.2.5 { do_eqp_test 4.3.1 { SELECT x FROM t1 UNION SELECT x FROM t2 } { - 1 0 0 {SCAN TABLE t1 (~1000000 rows)} - 2 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1} + 2 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 USING TEMP B-TREE (UNION)} } do_eqp_test 4.3.2 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 } { - 2 0 0 {SCAN TABLE t1 (~1000000 rows)} - 3 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 2 0 0 {SCAN TABLE t1} + 3 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (UNION)} - 4 0 0 {SCAN TABLE t1 (~1000000 rows)} + 4 0 0 {SCAN TABLE t1} 0 0 0 {COMPOUND SUBQUERIES 1 AND 4 USING TEMP B-TREE (UNION)} } do_eqp_test 4.3.3 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 ORDER BY 1 } { - 2 0 0 {SCAN TABLE t1 (~1000000 rows)} + 2 0 0 {SCAN TABLE t1} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} - 3 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1 (~1000000 rows)} + 3 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 (UNION)} - 4 0 0 {SCAN TABLE t1 (~1000000 rows)} + 4 0 0 {SCAN TABLE t1} 4 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 4 (UNION)} } @@ -371,127 +371,127 @@ do_eqp_test 4.3.3 { drop_all_tables # EVIDENCE-OF: R-64208-08323 sqlite> EXPLAIN QUERY PLAN SELECT a, b -# FROM t1 WHERE a=1; 0|0|0|SCAN TABLE t1 (~100000 rows) +# FROM t1 WHERE a=1; 0|0|0|SCAN TABLE t1 do_execsql_test 5.1.0 { CREATE TABLE t1(a, b) } det 5.1.1 "SELECT a, b FROM t1 WHERE a=1" { - 0 0 0 {SCAN TABLE t1 (~100000 rows)} + 0 0 0 {SCAN TABLE t1} } # EVIDENCE-OF: R-09022-44606 sqlite> CREATE INDEX i1 ON t1(a); # sqlite> EXPLAIN QUERY PLAN SELECT a, b FROM t1 WHERE a=1; -# 0|0|0|SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows) +# 0|0|0|SEARCH TABLE t1 USING INDEX i1 (a=?) do_execsql_test 5.2.0 { CREATE INDEX i1 ON t1(a) } det 5.2.1 "SELECT a, b FROM t1 WHERE a=1" { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } # EVIDENCE-OF: R-62228-34103 sqlite> CREATE INDEX i2 ON t1(a, b); # sqlite> EXPLAIN QUERY PLAN SELECT a, b FROM t1 WHERE a=1; -# 0|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows) +# 0|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) do_execsql_test 5.3.0 { CREATE INDEX i2 ON t1(a, b) } det 5.3.1 "SELECT a, b FROM t1 WHERE a=1" { - 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?)} } # EVIDENCE-OF: R-22253-05302 sqlite> EXPLAIN QUERY PLAN SELECT t1.*, # t2.* FROM t1, t2 WHERE t1.a=1 AND t1.b>2; 0|0|0|SEARCH TABLE t1 -# USING COVERING INDEX i2 (a=? AND b>?) (~3 rows) 0|1|1|SCAN TABLE t2 -# (~1000000 rows) +# USING COVERING INDEX i2 (a=? AND b>?) 0|1|1|SCAN TABLE t2 +# do_execsql_test 5.4.0 {CREATE TABLE t2(c, d)} det 5.4.1 "SELECT t1.*, t2.* FROM t1, t2 WHERE t1.a=1 AND t1.b>2" { - 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?) (~2 rows)} - 0 1 1 {SCAN TABLE t2 (~1000000 rows)} + 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?)} + 0 1 1 {SCAN TABLE t2} } # EVIDENCE-OF: R-21040-07025 sqlite> EXPLAIN QUERY PLAN SELECT t1.*, # t2.* FROM t2, t1 WHERE t1.a=1 AND t1.b>2; 0|0|1|SEARCH TABLE t1 -# USING COVERING INDEX i2 (a=? AND b>?) (~3 rows) 0|1|0|SCAN TABLE t2 -# (~1000000 rows) +# USING COVERING INDEX i2 (a=? AND b>?) 0|1|0|SCAN TABLE t2 +# det 5.5 "SELECT t1.*, t2.* FROM t2, t1 WHERE t1.a=1 AND t1.b>2" { - 0 0 1 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?) (~2 rows)} - 0 1 0 {SCAN TABLE t2 (~1000000 rows)} + 0 0 1 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?)} + 0 1 0 {SCAN TABLE t2} } # EVIDENCE-OF: R-39007-61103 sqlite> CREATE INDEX i3 ON t1(b); # sqlite> EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=1 OR b=2; -# 0|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows) -# 0|0|0|SEARCH TABLE t1 USING INDEX i3 (b=?) (~10 rows) +# 0|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) +# 0|0|0|SEARCH TABLE t1 USING INDEX i3 (b=?) do_execsql_test 5.5.0 {CREATE INDEX i3 ON t1(b)} det 5.6.1 "SELECT * FROM t1 WHERE a=1 OR b=2" { - 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows)} - 0 0 0 {SEARCH TABLE t1 USING INDEX i3 (b=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?)} + 0 0 0 {SEARCH TABLE t1 USING INDEX i3 (b=?)} } # EVIDENCE-OF: R-33025-54904 sqlite> EXPLAIN QUERY PLAN SELECT c, d -# FROM t2 ORDER BY c; 0|0|0|SCAN TABLE t2 (~1000000 rows) 0|0|0|USE TEMP +# FROM t2 ORDER BY c; 0|0|0|SCAN TABLE t2 0|0|0|USE TEMP # B-TREE FOR ORDER BY det 5.7 "SELECT c, d FROM t2 ORDER BY c" { - 0 0 0 {SCAN TABLE t2 (~1000000 rows)} + 0 0 0 {SCAN TABLE t2} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } # EVIDENCE-OF: R-38854-22809 sqlite> CREATE INDEX i4 ON t2(c); # sqlite> EXPLAIN QUERY PLAN SELECT c, d FROM t2 ORDER BY c; -# 0|0|0|SCAN TABLE t2 USING INDEX i4 (~1000000 rows) +# 0|0|0|SCAN TABLE t2 USING INDEX i4 do_execsql_test 5.8.0 {CREATE INDEX i4 ON t2(c)} det 5.8.1 "SELECT c, d FROM t2 ORDER BY c" { - 0 0 0 {SCAN TABLE t2 USING INDEX i4 (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING INDEX i4} } # EVIDENCE-OF: R-29884-43993 sqlite> EXPLAIN QUERY PLAN SELECT # (SELECT b FROM t1 WHERE a=0), (SELECT a FROM t1 WHERE b=t2.c) FROM t2; -# 0|0|0|SCAN TABLE t2 (~1000000 rows) 0|0|0|EXECUTE SCALAR SUBQUERY 1 -# 1|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows) +# 0|0|0|SCAN TABLE t2 0|0|0|EXECUTE SCALAR SUBQUERY 1 +# 1|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) # 0|0|0|EXECUTE CORRELATED SCALAR SUBQUERY 2 2|0|0|SEARCH TABLE t1 USING -# INDEX i3 (b=?) (~10 rows) +# INDEX i3 (b=?) det 5.9 { SELECT (SELECT b FROM t1 WHERE a=0), (SELECT a FROM t1 WHERE b=t2.c) FROM t2 } { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i4 (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i4} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} - 1 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?) (~10 rows)} + 1 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?)} 0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} - 2 0 0 {SEARCH TABLE t1 USING INDEX i3 (b=?) (~10 rows)} + 2 0 0 {SEARCH TABLE t1 USING INDEX i3 (b=?)} } # EVIDENCE-OF: R-17911-16445 sqlite> EXPLAIN QUERY PLAN SELECT # count(*) FROM (SELECT max(b) AS x FROM t1 GROUP BY a) GROUP BY x; -# 1|0|0|SCAN TABLE t1 USING COVERING INDEX i2 (~1000000 rows) 0|0|0|SCAN -# SUBQUERY 1 (~1000000 rows) 0|0|0|USE TEMP B-TREE FOR GROUP BY +# 1|0|0|SCAN TABLE t1 USING COVERING INDEX i2 0|0|0|SCAN +# SUBQUERY 1 0|0|0|USE TEMP B-TREE FOR GROUP BY det 5.10 { SELECT count(*) FROM (SELECT max(b) AS x FROM t1 GROUP BY a) GROUP BY x } { - 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i2 (~1000000 rows)} - 0 0 0 {SCAN SUBQUERY 1 (~100 rows)} + 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i2} + 0 0 0 {SCAN SUBQUERY 1} 0 0 0 {USE TEMP B-TREE FOR GROUP BY} } # EVIDENCE-OF: R-18544-33103 sqlite> EXPLAIN QUERY PLAN SELECT * FROM # (SELECT * FROM t2 WHERE c=1), t1; 0|0|0|SEARCH TABLE t2 USING INDEX i4 -# (c=?) (~10 rows) 0|1|1|SCAN TABLE t1 (~1000000 rows) +# (c=?) 0|1|1|SCAN TABLE t1 det 5.11 "SELECT * FROM (SELECT * FROM t2 WHERE c=1), t1" { - 0 0 0 {SEARCH TABLE t2 USING INDEX i4 (c=?) (~10 rows)} - 0 1 1 {SCAN TABLE t1 USING COVERING INDEX i2 (~1000000 rows)} + 0 0 0 {SEARCH TABLE t2 USING INDEX i4 (c=?)} + 0 1 1 {SCAN TABLE t1 USING COVERING INDEX i2} } # EVIDENCE-OF: R-40701-42164 sqlite> EXPLAIN QUERY PLAN SELECT a FROM -# t1 UNION SELECT c FROM t2; 1|0|0|SCAN TABLE t1 (~1000000 rows) -# 2|0|0|SCAN TABLE t2 (~1000000 rows) 0|0|0|COMPOUND SUBQUERIES 1 AND 2 +# t1 UNION SELECT c FROM t2; 1|0|0|SCAN TABLE t1 +# 2|0|0|SCAN TABLE t2 0|0|0|COMPOUND SUBQUERIES 1 AND 2 # USING TEMP B-TREE (UNION) det 5.12 "SELECT a FROM t1 UNION SELECT c FROM t2" { - 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 2 0 0 {SCAN TABLE t2 USING COVERING INDEX i4 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i1} + 2 0 0 {SCAN TABLE t2 USING COVERING INDEX i4} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 USING TEMP B-TREE (UNION)} } # EVIDENCE-OF: R-61538-24748 sqlite> EXPLAIN QUERY PLAN SELECT a FROM # t1 EXCEPT SELECT d FROM t2 ORDER BY 1; 1|0|0|SCAN TABLE t1 USING -# COVERING INDEX i2 (~1000000 rows) 2|0|0|SCAN TABLE t2 (~1000000 rows) +# COVERING INDEX i2 2|0|0|SCAN TABLE t2 # 2|0|0|USE TEMP B-TREE FOR ORDER BY 0|0|0|COMPOUND SUBQUERIES 1 AND 2 # (EXCEPT) det 5.13 "SELECT a FROM t1 EXCEPT SELECT d FROM t2 ORDER BY 1" { - 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i2 (~1000000 rows)} - 2 0 0 {SCAN TABLE t2 (~1000000 rows)} + 1 0 0 {SCAN TABLE t1 USING COVERING INDEX i2} + 2 0 0 {SCAN TABLE t2} 2 0 0 {USE TEMP B-TREE FOR ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } @@ -531,8 +531,8 @@ proc do_peqp_test {tn sql res} { do_peqp_test 6.1 { SELECT a FROM t1 EXCEPT SELECT d FROM t2 ORDER BY 1 } [string trimleft { -1 0 0 SCAN TABLE t1 USING COVERING INDEX i2 (~1000000 rows) -2 0 0 SCAN TABLE t2 (~1000000 rows) +1 0 0 SCAN TABLE t1 USING COVERING INDEX i2 +2 0 0 SCAN TABLE t2 2 0 0 USE TEMP B-TREE FOR ORDER BY 0 0 0 COMPOUND SUBQUERIES 1 AND 2 (EXCEPT) }] @@ -550,7 +550,7 @@ do_execsql_test 7.0 { } det 7.1 "SELECT count(*) FROM t1" { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 0 {SCAN TABLE t1} } det 7.2 "SELECT count(*) FROM t2" { @@ -572,7 +572,7 @@ db close sqlite3 db test.db det 7.4 "SELECT count(*) FROM t1" { - 0 0 0 {SCAN TABLE t1 (~2 rows)} + 0 0 0 {SCAN TABLE t1} } det 7.5 "SELECT count(*) FROM t2" { diff --git a/test/exclusive.test b/test/exclusive.test index ffde891533..c000dfefa4 100644 --- a/test/exclusive.test +++ b/test/exclusive.test @@ -506,4 +506,3 @@ do_execsql_test exclusive-6.5 { } {exclusive} finish_test - diff --git a/test/fallocate.test b/test/fallocate.test index 8a5fa32360..f523c2cc4a 100644 --- a/test/fallocate.test +++ b/test/fallocate.test @@ -143,4 +143,3 @@ if {!$skipwaltests} { finish_test - diff --git a/test/filefmt.test b/test/filefmt.test index bc1af18e8b..2df1424436 100644 --- a/test/filefmt.test +++ b/test/filefmt.test @@ -248,4 +248,3 @@ do_test filefmt-4.4 { db2 close finish_test - diff --git a/test/fts3aa.test b/test/fts3aa.test index 5d79e9318f..bde38af553 100644 --- a/test/fts3aa.test +++ b/test/fts3aa.test @@ -224,4 +224,3 @@ do_catchsql_test fts3aa-7.5 { finish_test - diff --git a/test/fts3ao.test b/test/fts3ao.test index 786667a7f3..91693094db 100644 --- a/test/fts3ao.test +++ b/test/fts3ao.test @@ -220,4 +220,3 @@ do_execsql_test 5.2 { } {0 6} finish_test - diff --git a/test/fts3atoken.test b/test/fts3atoken.test index 9277bfb8e6..b7722c7d04 100644 --- a/test/fts3atoken.test +++ b/test/fts3atoken.test @@ -193,5 +193,3 @@ do_test fts3token-internal { finish_test - - diff --git a/test/fts3auto.test b/test/fts3auto.test index 20b2812dcc..20640d29ac 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -707,4 +707,3 @@ foreach {tn create} { set sqlite_fts3_enable_parentheses $sfep finish_test - diff --git a/test/fts3aux1.test b/test/fts3aux1.test index f421b7f6ef..d17ac85df5 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -105,10 +105,10 @@ db func rec rec # do_execsql_test 2.1.1.1 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term='braid' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1:} } do_execsql_test 2.1.1.2 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term='braid' -} {0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)}} +} {0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:}} # Now show that using "term='braid'" means the virtual table returns # only 1 row to SQLite, but "+term='braid'" means all 19 are returned. @@ -154,24 +154,24 @@ do_execsql_test 2.1.5 { SELECT * FROM terms WHERE term=NULL } {} do_execsql_test 2.2.1.1 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term>'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 2: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 2:} } do_execsql_test 2.2.1.2 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term>'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:} } do_execsql_test 2.2.1.3 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term<'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 4: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 4:} } do_execsql_test 2.2.1.4 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term<'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:} } do_execsql_test 2.2.1.5 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE term BETWEEN 'brags' AND 'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 6: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 6:} } do_execsql_test 2.2.1.6 { EXPLAIN QUERY PLAN SELECT * FROM terms WHERE +term BETWEEN 'brags' AND 'brain' -} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} } +} { 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:} } do_test 2.2.2.1 { set cnt 0 @@ -335,7 +335,7 @@ foreach {tn sort orderby} { 9 1 "ORDER BY occurrences DESC" } { - set res [list 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)}] + set res [list 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:}] if {$sort} { lappend res 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } set sql "SELECT * FROM terms $orderby" @@ -410,32 +410,32 @@ proc do_plansql_test {tn sql r} { do_plansql_test 4.2 { SELECT y FROM x2, terms WHERE y = term AND col = '*' } { - 0 0 0 {SCAN TABLE x2 (~1000000 rows)} - 0 1 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} + 0 0 0 {SCAN TABLE x2} + 0 1 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 1:} a b c d e f g h i j k l } do_plansql_test 4.3 { SELECT y FROM terms, x2 WHERE y = term AND col = '*' } { - 0 0 1 {SCAN TABLE x2 (~1000000 rows)} - 0 1 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1: (~25 rows)} + 0 0 1 {SCAN TABLE x2} + 0 1 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 1:} a b c d e f g h i j k l } do_plansql_test 4.4 { SELECT y FROM x3, terms WHERE y = term AND col = '*' } { - 0 0 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} - 0 1 0 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?) (~10 rows)} + 0 0 1 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:} + 0 1 0 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?)} a b c d e f g h i j k l } do_plansql_test 4.5 { SELECT y FROM terms, x3 WHERE y = term AND occurrences>1 AND col = '*' } { - 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0: (~25 rows)} - 0 1 1 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?) (~10 rows)} + 0 0 0 {SCAN TABLE terms VIRTUAL TABLE INDEX 0:} + 0 1 1 {SEARCH TABLE x3 USING COVERING INDEX i1 (y=?)} a k l } diff --git a/test/fts3corrupt.test b/test/fts3corrupt.test index ea4c9a9d3b..cb50e3e46a 100644 --- a/test/fts3corrupt.test +++ b/test/fts3corrupt.test @@ -166,4 +166,3 @@ do_test 5.3.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB finish_test - diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 337359af6e..608371f15d 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -153,4 +153,3 @@ foreach {tn sql} { finish_test - diff --git a/test/fts3expr3.test b/test/fts3expr3.test index e3cc2619c6..a8d7319266 100644 --- a/test/fts3expr3.test +++ b/test/fts3expr3.test @@ -204,7 +204,3 @@ do_faultsim_test fts3expr3-fault-1 -faults oom-* -body { set sqlite_fts3_enable_parentheses 0 finish_test - - - - diff --git a/test/fts3malloc.test b/test/fts3malloc.test index 7eeee7fe0d..efd1e062f6 100644 --- a/test/fts3malloc.test +++ b/test/fts3malloc.test @@ -301,4 +301,3 @@ do_write_test fts3_malloc-5.3 ft_content { finish_test - diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 3998c9a40c..3da3a77427 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -427,4 +427,3 @@ do_execsql_test 8.3 { } finish_test - diff --git a/test/fts3prefix2.test b/test/fts3prefix2.test index e3da3b7593..e033fb6bcb 100644 --- a/test/fts3prefix2.test +++ b/test/fts3prefix2.test @@ -59,4 +59,3 @@ do_execsql_test 1.3 { } finish_test - diff --git a/test/fts3query.test b/test/fts3query.test index 51b84be906..2d12351ae9 100644 --- a/test/fts3query.test +++ b/test/fts3query.test @@ -118,26 +118,26 @@ do_test fts3query-4.1 { do_eqp_test fts3query-4.2 { SELECT t1.number FROM t1, ft WHERE t1.number=ft.rowid ORDER BY t1.date } { - 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 1 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~25 rows)} + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1} + 0 1 1 {SCAN TABLE ft VIRTUAL TABLE INDEX 1:} } do_eqp_test fts3query-4.3 { SELECT t1.number FROM ft, t1 WHERE t1.number=ft.rowid ORDER BY t1.date } { - 0 0 1 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 0 {SCAN TABLE ft VIRTUAL TABLE INDEX 1: (~25 rows)} + 0 0 1 {SCAN TABLE t1 USING COVERING INDEX i1} + 0 1 0 {SCAN TABLE ft VIRTUAL TABLE INDEX 1:} } do_eqp_test fts3query-4.4 { SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date } { - 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 1 {SEARCH TABLE bt USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1} + 0 1 1 {SEARCH TABLE bt USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test fts3query-4.5 { SELECT t1.number FROM bt, t1 WHERE t1.number=bt.rowid ORDER BY t1.date } { - 0 0 1 {SCAN TABLE t1 USING COVERING INDEX i1 (~1000000 rows)} - 0 1 0 {SEARCH TABLE bt USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 1 {SCAN TABLE t1 USING COVERING INDEX i1} + 0 1 0 {SEARCH TABLE bt USING INTEGER PRIMARY KEY (rowid=?)} } diff --git a/test/fts3shared.test b/test/fts3shared.test index a587a437bf..0943c0510f 100644 --- a/test/fts3shared.test +++ b/test/fts3shared.test @@ -174,4 +174,3 @@ dbW close dbR close sqlite3_enable_shared_cache $::enable_shared_cache finish_test - diff --git a/test/fts3snippet.test b/test/fts3snippet.test index b8646cdfaa..f19db925ba 100644 --- a/test/fts3snippet.test +++ b/test/fts3snippet.test @@ -432,10 +432,10 @@ foreach {DO_MALLOC_TEST enc} { {2 2 1 3 3 3 6 3 0 0 0 2 3 2} }] - # EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the - # "query by rowid" or "linear scan" strategies, then the snippet and - # offsets both return an empty string, and the matchinfo function - # returns a blob value zero bytes in size. + # EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the + # "query by rowid" or "linear scan" strategies, then the snippet and + # offsets both return an empty string, and the matchinfo function + # returns a blob value zero bytes in size. # set r 1000000 ;# A rowid that exists in table ft do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r diff --git a/test/fts3sort.test b/test/fts3sort.test index be7560432b..218bf3e827 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -182,4 +182,3 @@ do_execsql_test 3.2 { finish_test - diff --git a/test/fts3tok1.test b/test/fts3tok1.test index 98e55a06c1..4faed34d12 100644 --- a/test/fts3tok1.test +++ b/test/fts3tok1.test @@ -113,5 +113,3 @@ do_catchsql_test 2.1 { finish_test - - diff --git a/test/fts3tok_err.test b/test/fts3tok_err.test index aaa7272b7b..df0d4beebd 100644 --- a/test/fts3tok_err.test +++ b/test/fts3tok_err.test @@ -45,5 +45,3 @@ do_faultsim_test fts3tok_err-2 -faults oom* -prep { finish_test - - diff --git a/test/fts4content.test b/test/fts4content.test index 302f14e510..6b2cd3cc8e 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -623,4 +623,3 @@ do_execsql_test 10.7 { } finish_test - diff --git a/test/incrblob3.test b/test/incrblob3.test index 4c49f1563c..5f2e860d08 100644 --- a/test/incrblob3.test +++ b/test/incrblob3.test @@ -269,4 +269,3 @@ db close tvfs delete finish_test - diff --git a/test/incrblob4.test b/test/incrblob4.test index a96356b06f..a92e373536 100644 --- a/test/incrblob4.test +++ b/test/incrblob4.test @@ -87,4 +87,3 @@ do_test 3.3 { } {} finish_test - diff --git a/test/incrblobfault.test b/test/incrblobfault.test index d125471c4b..0c1d93d460 100644 --- a/test/incrblobfault.test +++ b/test/incrblobfault.test @@ -67,4 +67,3 @@ do_faultsim_test 3 -prep { } finish_test - diff --git a/test/incrvacuum3.test b/test/incrvacuum3.test index f01dc100b8..e901bfe1f5 100644 --- a/test/incrvacuum3.test +++ b/test/incrvacuum3.test @@ -151,4 +151,3 @@ foreach {T jrnl_mode} { } finish_test - diff --git a/test/indexedby.test b/test/indexedby.test index 7ccc4de24a..955d185619 100644 --- a/test/indexedby.test +++ b/test/indexedby.test @@ -42,15 +42,15 @@ proc EQP {sql} { # do_execsql_test indexedby-1.2 { EXPLAIN QUERY PLAN select * from t1 WHERE a = 10; -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-1.3 { EXPLAIN QUERY PLAN select * from t1 ; -} {0 0 0 {SCAN TABLE t1 (~1000000 rows)}} +} {0 0 0 {SCAN TABLE t1}} do_execsql_test indexedby-1.4 { EXPLAIN QUERY PLAN select * from t1, t2 WHERE c = 10; } { - 0 0 1 {SEARCH TABLE t2 USING INDEX i3 (c=?) (~10 rows)} - 0 1 0 {SCAN TABLE t1 (~1000000 rows)} + 0 0 1 {SEARCH TABLE t2 USING INDEX i3 (c=?)} + 0 1 0 {SCAN TABLE t1} } # Parser tests. Test that an INDEXED BY or NOT INDEX clause can be @@ -85,15 +85,15 @@ do_test indexedby-2.7 { # do_execsql_test indexedby-3.1 { EXPLAIN QUERY PLAN SELECT * FROM t1 NOT INDEXED WHERE a = 'one' AND b = 'two' -} {0 0 0 {SCAN TABLE t1 (~10000 rows)}} +} {0 0 0 {SCAN TABLE t1}} do_execsql_test indexedby-3.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 INDEXED BY i1 WHERE a = 'one' AND b = 'two' -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-3.3 { EXPLAIN QUERY PLAN SELECT * FROM t1 INDEXED BY i2 WHERE a = 'one' AND b = 'two' -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?)}} do_test indexedby-3.4 { catchsql { SELECT * FROM t1 INDEXED BY i2 WHERE a = 'one' } } {1 {cannot use index: i2}} @@ -110,11 +110,11 @@ do_test indexedby-3.7 { do_execsql_test indexedby-3.8 { EXPLAIN QUERY PLAN SELECT * FROM t3 INDEXED BY sqlite_autoindex_t3_1 ORDER BY e -} {0 0 0 {SCAN TABLE t3 USING INDEX sqlite_autoindex_t3_1 (~1000000 rows)}} +} {0 0 0 {SCAN TABLE t3 USING INDEX sqlite_autoindex_t3_1}} do_execsql_test indexedby-3.9 { EXPLAIN QUERY PLAN SELECT * FROM t3 INDEXED BY sqlite_autoindex_t3_1 WHERE e = 10 -} {0 0 0 {SEARCH TABLE t3 USING INDEX sqlite_autoindex_t3_1 (e=?) (~1 rows)}} +} {0 0 0 {SEARCH TABLE t3 USING INDEX sqlite_autoindex_t3_1 (e=?)}} do_test indexedby-3.10 { catchsql { SELECT * FROM t3 INDEXED BY sqlite_autoindex_t3_1 WHERE f = 10 } } {1 {cannot use index: sqlite_autoindex_t3_1}} @@ -127,14 +127,14 @@ do_test indexedby-3.11 { do_execsql_test indexedby-4.1 { EXPLAIN QUERY PLAN SELECT * FROM t1, t2 WHERE a = c } { - 0 0 0 {SCAN TABLE t1 (~1000000 rows)} - 0 1 1 {SEARCH TABLE t2 USING INDEX i3 (c=?) (~10 rows)} + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SEARCH TABLE t2 USING INDEX i3 (c=?)} } do_execsql_test indexedby-4.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 INDEXED BY i1, t2 WHERE a = c } { - 0 0 1 {SCAN TABLE t2 (~1000000 rows)} - 0 1 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~10 rows)} + 0 0 1 {SCAN TABLE t2} + 0 1 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } do_test indexedby-4.3 { catchsql { @@ -154,10 +154,10 @@ do_test indexedby-4.4 { do_execsql_test indexedby-5.1 { CREATE VIEW v2 AS SELECT * FROM t1 INDEXED BY i1 WHERE a > 5; EXPLAIN QUERY PLAN SELECT * FROM v2 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?) (~250000 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} do_execsql_test indexedby-5.2 { EXPLAIN QUERY PLAN SELECT * FROM v2 WHERE b = 10 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?) (~25000 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} do_test indexedby-5.3 { execsql { DROP INDEX i1 } catchsql { SELECT * FROM v2 } @@ -178,28 +178,28 @@ do_test indexedby-5.5 { # do_execsql_test indexedby-6.1 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b = 10 ORDER BY rowid -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?)}} do_execsql_test indexedby-6.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 NOT INDEXED WHERE b = 10 ORDER BY rowid -} {0 0 0 {SCAN TABLE t1 USING INTEGER PRIMARY KEY (~100000 rows)}} +} {0 0 0 {SCAN TABLE t1 USING INTEGER PRIMARY KEY}} # Test that "INDEXED BY" can be used in a DELETE statement. # do_execsql_test indexedby-7.1 { EXPLAIN QUERY PLAN DELETE FROM t1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} do_execsql_test indexedby-7.2 { EXPLAIN QUERY PLAN DELETE FROM t1 NOT INDEXED WHERE a = 5 -} {0 0 0 {SCAN TABLE t1 (~100000 rows)}} +} {0 0 0 {SCAN TABLE t1}} do_execsql_test indexedby-7.3 { EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} do_execsql_test indexedby-7.4 { EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 AND b = 10 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-7.5 { EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i2 WHERE a = 5 AND b = 10 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?)}} do_test indexedby-7.6 { catchsql { DELETE FROM t1 INDEXED BY i2 WHERE a = 5} } {1 {cannot use index: i2}} @@ -208,21 +208,21 @@ do_test indexedby-7.6 { # do_execsql_test indexedby-8.1 { EXPLAIN QUERY PLAN UPDATE t1 SET rowid=rowid+1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} do_execsql_test indexedby-8.2 { EXPLAIN QUERY PLAN UPDATE t1 NOT INDEXED SET rowid=rowid+1 WHERE a = 5 -} {0 0 0 {SCAN TABLE t1 (~100000 rows)}} +} {0 0 0 {SCAN TABLE t1}} do_execsql_test indexedby-8.3 { EXPLAIN QUERY PLAN UPDATE t1 INDEXED BY i1 SET rowid=rowid+1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} do_execsql_test indexedby-8.4 { EXPLAIN QUERY PLAN UPDATE t1 INDEXED BY i1 SET rowid=rowid+1 WHERE a = 5 AND b = 10 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-8.5 { EXPLAIN QUERY PLAN UPDATE t1 INDEXED BY i2 SET rowid=rowid+1 WHERE a = 5 AND b = 10 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?) (~2 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b=?)}} do_test indexedby-8.6 { catchsql { UPDATE t1 INDEXED BY i2 SET rowid=rowid+1 WHERE a = 5} } {1 {cannot use index: i2}} diff --git a/test/io.test b/test/io.test index 11f9cc8422..c5086c10ec 100644 --- a/test/io.test +++ b/test/io.test @@ -641,4 +641,3 @@ foreach {tn sql} { sqlite3_simulate_device -char {} -sectorsize 0 finish_test - diff --git a/test/ioerr6.test b/test/ioerr6.test index 66c48ad2e8..d1847b2877 100644 --- a/test/ioerr6.test +++ b/test/ioerr6.test @@ -89,4 +89,3 @@ do_faultsim_test 3 -faults full* -prep { } finish_test - diff --git a/test/lock7.test b/test/lock7.test index 49bc147a22..02f2c6c630 100644 --- a/test/lock7.test +++ b/test/lock7.test @@ -58,4 +58,3 @@ db1 close db2 close finish_test - diff --git a/test/misc7.test b/test/misc7.test index 72c1cd64ff..96062a93b9 100644 --- a/test/misc7.test +++ b/test/misc7.test @@ -269,17 +269,17 @@ ifcapable explain { CREATE TABLE abc(a PRIMARY KEY, b, c); EXPLAIN QUERY PLAN SELECT * FROM abc AS t2 WHERE rowid = 1; } { - 0 0 0 {SEARCH TABLE abc AS t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SEARCH TABLE abc AS t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_execsql_test misc7-14.2 { EXPLAIN QUERY PLAN SELECT * FROM abc AS t2 WHERE a = 1; } {0 0 0 - {SEARCH TABLE abc AS t2 USING INDEX sqlite_autoindex_abc_1 (a=?) (~1 rows)} + {SEARCH TABLE abc AS t2 USING INDEX sqlite_autoindex_abc_1 (a=?)} } do_execsql_test misc7-14.3 { EXPLAIN QUERY PLAN SELECT * FROM abc AS t2 ORDER BY a; } {0 0 0 - {SCAN TABLE abc AS t2 USING INDEX sqlite_autoindex_abc_1 (~1000000 rows)} + {SCAN TABLE abc AS t2 USING INDEX sqlite_autoindex_abc_1} } } diff --git a/test/notify3.test b/test/notify3.test index 446f010780..4b5e8016b4 100644 --- a/test/notify3.test +++ b/test/notify3.test @@ -150,4 +150,3 @@ catch { db2 close } sqlite3_enable_shared_cache $esc finish_test - diff --git a/test/pager1.test b/test/pager1.test index 72e47805a4..4a3b6fa2d8 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2815,4 +2815,3 @@ do_test 43.3 { } {0 1 0} finish_test - diff --git a/test/pagerfault.test b/test/pagerfault.test index 23754fa9de..796f531c37 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1546,4 +1546,3 @@ sqlite3_shutdown sqlite3_config_uri 0 finish_test - diff --git a/test/pagerfault2.test b/test/pagerfault2.test index 6cdb99a3ed..7f6c995efd 100644 --- a/test/pagerfault2.test +++ b/test/pagerfault2.test @@ -96,4 +96,3 @@ do_faultsim_test pagerfault2-2 -faults oom-transient -prep { sqlite3_memdebug_vfs_oom_test 1 finish_test - diff --git a/test/pagerfault3.test b/test/pagerfault3.test index 3d192e6d43..a4301cd17d 100644 --- a/test/pagerfault3.test +++ b/test/pagerfault3.test @@ -61,4 +61,3 @@ do_faultsim_test pagerfault3-1 -faults ioerr-transient -prep { } finish_test - diff --git a/test/securedel2.test b/test/securedel2.test index b20f4f9212..9badc560d4 100644 --- a/test/securedel2.test +++ b/test/securedel2.test @@ -92,4 +92,3 @@ do_test 1.6.2 { } {0} finish_test - diff --git a/test/shared8.test b/test/shared8.test index 600e02bf7e..73f0d47627 100644 --- a/test/shared8.test +++ b/test/shared8.test @@ -110,4 +110,3 @@ do_test 1.8 { foreach db {db1 db2 db3 db4} { catch { $db close } } sqlite3_enable_shared_cache $::enable_shared_cache finish_test - diff --git a/test/sharedlock.test b/test/sharedlock.test index 1e78eeafdf..9dd7fe2bff 100644 --- a/test/sharedlock.test +++ b/test/sharedlock.test @@ -52,4 +52,3 @@ db2 close sqlite3_enable_shared_cache $::enable_shared_cache finish_test - diff --git a/test/tkt-385a5b56b9.test b/test/tkt-385a5b56b9.test index 8184864865..1338435ed6 100644 --- a/test/tkt-385a5b56b9.test +++ b/test/tkt-385a5b56b9.test @@ -35,19 +35,19 @@ do_execsql_test 2.0 { } do_eqp_test 2.1 { SELECT DISTINCT x FROM t2 } { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2x (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2x} } do_eqp_test 2.2 { SELECT DISTINCT y FROM t2 } { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2y (~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2y} } do_eqp_test 2.3 { SELECT DISTINCT x, y FROM t2 WHERE y=10 } { - 0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?) (~1 rows)} + 0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?)} } do_eqp_test 2.4 { SELECT DISTINCT x, y FROM t2 WHERE x=10 } { - 0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x=?) (~1 rows)} + 0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x=?)} } finish_test diff --git a/test/tkt-3a77c9714e.test b/test/tkt-3a77c9714e.test index 6eaec16cdd..b7d366f36e 100644 --- a/test/tkt-3a77c9714e.test +++ b/test/tkt-3a77c9714e.test @@ -70,4 +70,3 @@ do_execsql_test 2.2 { finish_test - diff --git a/test/tkt-3fe897352e.test b/test/tkt-3fe897352e.test index deafe48e0e..bc2b2033e7 100644 --- a/test/tkt-3fe897352e.test +++ b/test/tkt-3fe897352e.test @@ -16,9 +16,9 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl - -# The following tests use hex_to_utf16be() and hex_to_utf16le() which -# which are only available if SQLite is built with UTF16 support. + +# The following tests use hex_to_utf16be() and hex_to_utf16le() which +# which are only available if SQLite is built with UTF16 support. ifcapable {!utf16} { finish_test return diff --git a/test/tkt-78e04e52ea.test b/test/tkt-78e04e52ea.test index a664ceb9e6..e906d3b7f9 100644 --- a/test/tkt-78e04e52ea.test +++ b/test/tkt-78e04e52ea.test @@ -44,7 +44,7 @@ do_test tkt-78e04-1.4 { execsql { EXPLAIN QUERY PLAN SELECT * FROM "" WHERE "" LIKE 'abc%'; } -} {0 0 0 {SCAN TABLE USING COVERING INDEX i1 (~500000 rows)}} +} {0 0 0 {SCAN TABLE USING COVERING INDEX i1}} do_test tkt-78e04-1.5 { execsql { DROP TABLE ""; @@ -57,12 +57,12 @@ do_test tkt-78e04-2.1 { CREATE INDEX "" ON t2(x); EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE x=5; } -} {0 0 0 {SEARCH TABLE t2 USING COVERING INDEX (x=?) (~10 rows)}} +} {0 0 0 {SEARCH TABLE t2 USING COVERING INDEX (x=?)}} do_test tkt-78e04-2.2 { execsql { DROP INDEX ""; EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE x=2; } -} {0 0 0 {SCAN TABLE t2 (~100000 rows)}} +} {0 0 0 {SCAN TABLE t2}} finish_test diff --git a/test/tkt-7a31705a7e6.test b/test/tkt-7a31705a7e6.test index 64701220a8..e3e402917d 100644 --- a/test/tkt-7a31705a7e6.test +++ b/test/tkt-7a31705a7e6.test @@ -23,4 +23,3 @@ do_execsql_test tkt-7a31705a7e6-1.1 { CREATE TABLE t2x (b INTEGER PRIMARY KEY); SELECT t1.a FROM ((t1 JOIN t2 ON t1.a=t2.a) AS x JOIN t2x ON x.b=t2x.b) as y; } {} - diff --git a/test/tkt-7bbfb7d442.test b/test/tkt-7bbfb7d442.test index dcb9b16058..56d4caeb3e 100644 --- a/test/tkt-7bbfb7d442.test +++ b/test/tkt-7bbfb7d442.test @@ -152,5 +152,3 @@ do_execsql_test 2.3 { finish_test - - diff --git a/test/tkt-c48d99d690.test b/test/tkt-c48d99d690.test index 9b40579917..6d1b9dbe61 100644 --- a/test/tkt-c48d99d690.test +++ b/test/tkt-c48d99d690.test @@ -23,4 +23,3 @@ do_test 1.1 { do_test 1.2 { execsql VACUUM } {} finish_test - diff --git a/test/tkt-d11f09d36e.test b/test/tkt-d11f09d36e.test index 70657706c4..ffd3d21928 100644 --- a/test/tkt-d11f09d36e.test +++ b/test/tkt-d11f09d36e.test @@ -59,4 +59,3 @@ do_test tkt-d11f09d36e.5 { } {ok} finish_test - diff --git a/test/tkt-f3e5abed55.test b/test/tkt-f3e5abed55.test index b3f5d5656c..3c793d4f4d 100644 --- a/test/tkt-f3e5abed55.test +++ b/test/tkt-f3e5abed55.test @@ -114,4 +114,3 @@ if {[permutation]!="inmemory_journal"} { finish_test - diff --git a/test/tkt-f973c7ac31.test b/test/tkt-f973c7ac31.test index 882e86a0a0..454309057f 100644 --- a/test/tkt-f973c7ac31.test +++ b/test/tkt-f973c7ac31.test @@ -84,4 +84,3 @@ foreach {tn sql} { finish_test - diff --git a/test/tkt3442.test b/test/tkt3442.test index ae03949d54..ee9169ce2c 100644 --- a/test/tkt3442.test +++ b/test/tkt3442.test @@ -49,10 +49,10 @@ proc EQP {sql} { ifcapable explain { do_test tkt3442-1.2 { EQP { SELECT node FROM listhash WHERE id='5000' LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?)}} do_test tkt3442-1.3 { EQP { SELECT node FROM listhash WHERE id="5000" LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?)}} } @@ -61,7 +61,7 @@ ifcapable explain { ifcapable explain { do_test tkt3442-1.4 { EQP { SELECT node FROM listhash WHERE id=5000 LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?)}} } do_test tkt3442-1.5 { catchsql { diff --git a/test/tkt3918.test b/test/tkt3918.test index c46ad8f6c1..e20ee152fc 100644 --- a/test/tkt3918.test +++ b/test/tkt3918.test @@ -57,4 +57,3 @@ do_test tkt3918.5 { } {} finish_test - diff --git a/test/tkt3929.test b/test/tkt3929.test index 3ed4d288c4..db02bb841e 100644 --- a/test/tkt3929.test +++ b/test/tkt3929.test @@ -50,4 +50,3 @@ do_test tkt3930-1.2 { integrity_check tkt3930-1.3 finish_test - diff --git a/test/unordered.test b/test/unordered.test index 4aa8310f89..78183b4923 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -40,28 +40,28 @@ foreach idxmode {ordered unordered} { sqlite3 db test.db foreach {tn sql r(ordered) r(unordered)} { 1 "SELECT * FROM t1 ORDER BY a" - {0 0 0 {SCAN TABLE t1 USING INDEX i1 (~128 rows)}} - {0 0 0 {SCAN TABLE t1 (~128 rows)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} + {0 0 0 {SCAN TABLE t1 USING INDEX i1}} + {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} 2 "SELECT * FROM t1 WHERE a >?" - {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?) (~32 rows)}} - {0 0 0 {SCAN TABLE t1 (~42 rows)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} + {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" - {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)}} - {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)} + {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} 4 "SELECT max(a) FROM t1" - {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (~1 rows)}} - {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (~1 rows)}} + {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1}} + {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1}} 5 "SELECT group_concat(b) FROM t1 GROUP BY a" - {0 0 0 {SCAN TABLE t1 USING INDEX i1 (~128 rows)}} - {0 0 0 {SCAN TABLE t1 (~128 rows)} 0 0 0 {USE TEMP B-TREE FOR GROUP BY}} + {0 0 0 {SCAN TABLE t1 USING INDEX i1}} + {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR GROUP BY}} 6 "SELECT * FROM t1 WHERE a = ?" - {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)}} - {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} 7 "SELECT count(*) FROM t1" {0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1(~128 rows)}} - {0 0 0 {SCAN TABLE t1 (~128 rows)}} + {0 0 0 {SCAN TABLE t1}} } { do_eqp_test 1.$idxmode.$tn $sql $r($idxmode) } diff --git a/test/veryquick.test b/test/veryquick.test index ca82b22b92..c8d6ce8f8b 100644 --- a/test/veryquick.test +++ b/test/veryquick.test @@ -16,4 +16,3 @@ source $testdir/permutations.test run_test_suite veryquick finish_test - diff --git a/test/wal8.test b/test/wal8.test index 339953895c..0682fce35b 100644 --- a/test/wal8.test +++ b/test/wal8.test @@ -88,4 +88,3 @@ do_execsql_test 3.1 { } {t1} finish_test - diff --git a/test/walcksum.test b/test/walcksum.test index 08278dd054..3005a758ac 100644 --- a/test/walcksum.test +++ b/test/walcksum.test @@ -390,4 +390,3 @@ foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { } finish_test - diff --git a/test/walcrash.test b/test/walcrash.test index adc4841dd1..2a647188b0 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -293,4 +293,3 @@ for {set i 1} {$i < $REPEATS} {incr i} { } finish_test - diff --git a/test/walcrash2.test b/test/walcrash2.test index 7116281c2d..9c93bcdef3 100644 --- a/test/walcrash2.test +++ b/test/walcrash2.test @@ -96,4 +96,3 @@ do_test walcrash2-1.3 { catch { db2 close } finish_test - diff --git a/test/walcrash3.test b/test/walcrash3.test index c2c9a6d518..71dfb4555a 100644 --- a/test/walcrash3.test +++ b/test/walcrash3.test @@ -126,4 +126,3 @@ for {set i 2} {$i<10000 && [set_test_counter errors]==$nInitialErr} {incr i} { } finish_test - diff --git a/test/walro.test b/test/walro.test index 465ce838c2..f1ed275a1d 100644 --- a/test/walro.test +++ b/test/walro.test @@ -291,5 +291,3 @@ do_multiclient_test tn { } finish_test - - diff --git a/test/walshared.test b/test/walshared.test index 73a3fb8bc1..fbbdeb4de3 100644 --- a/test/walshared.test +++ b/test/walshared.test @@ -60,4 +60,3 @@ do_test walshared-1.4 { sqlite3_enable_shared_cache $::enable_shared_cache finish_test - diff --git a/test/where3.test b/test/where3.test index 842f683749..cee74ba434 100644 --- a/test/where3.test +++ b/test/where3.test @@ -236,15 +236,15 @@ do_execsql_test where3-3.0 { ANALYZE; explain query plan SELECT * FROM t302, t301 WHERE t302.x=5 AND t301.a=t302.y; } { - 0 0 0 {SCAN TABLE t302 (~1 rows)} - 0 1 1 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SCAN TABLE t302} + 0 1 1 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)} } do_execsql_test where3-3.1 { explain query plan SELECT * FROM t301, t302 WHERE t302.x=5 AND t301.a=t302.y; } { - 0 0 1 {SCAN TABLE t302 (~1 rows)} - 0 1 0 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 1 {SCAN TABLE t302} + 0 1 0 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)} } # Verify that when there are multiple tables in a join which must be @@ -258,25 +258,25 @@ do_execsql_test where3-4.0 { EXPLAIN QUERY PLAN SELECT * FROM t400, t401, t402 WHERE t402.z GLOB 'abc*'; } { - 0 0 2 {SCAN TABLE t402 (~500000 rows)} - 0 1 0 {SCAN TABLE t400 (~1000000 rows)} - 0 2 1 {SCAN TABLE t401 (~1000000 rows)} + 0 0 2 {SCAN TABLE t402} + 0 1 0 {SCAN TABLE t400} + 0 2 1 {SCAN TABLE t401} } do_execsql_test where3-4.1 { EXPLAIN QUERY PLAN SELECT * FROM t400, t401, t402 WHERE t401.r GLOB 'abc*'; } { - 0 0 1 {SCAN TABLE t401 (~500000 rows)} - 0 1 0 {SCAN TABLE t400 (~1000000 rows)} - 0 2 2 {SCAN TABLE t402 (~1000000 rows)} + 0 0 1 {SCAN TABLE t401} + 0 1 0 {SCAN TABLE t400} + 0 2 2 {SCAN TABLE t402} } do_execsql_test where3-4.2 { EXPLAIN QUERY PLAN SELECT * FROM t400, t401, t402 WHERE t400.c GLOB 'abc*'; } { - 0 0 0 {SCAN TABLE t400 (~500000 rows)} - 0 1 1 {SCAN TABLE t401 (~1000000 rows)} - 0 2 2 {SCAN TABLE t402 (~1000000 rows)} + 0 0 0 {SCAN TABLE t400} + 0 1 1 {SCAN TABLE t401} + 0 2 2 {SCAN TABLE t402} } # Verify that a performance regression encountered by firefox @@ -308,8 +308,8 @@ do_execsql_test where3-5.0 { AND bbb.parent = 4 ORDER BY bbb.title COLLATE NOCASE ASC; } { - 0 0 0 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?) (~10 rows)} - 0 1 1 {SEARCH TABLE bbb USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?)} + 0 1 1 {SEARCH TABLE bbb USING INTEGER PRIMARY KEY (rowid=?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_execsql_test where3-5.1 { @@ -321,8 +321,8 @@ do_execsql_test where3-5.1 { AND bbb.parent = 4 ORDER BY bbb.title COLLATE NOCASE ASC; } { - 0 0 0 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?) (~10 rows)} - 0 1 1 {SEARCH TABLE aaa AS bbb USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 0 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?)} + 0 1 1 {SEARCH TABLE aaa AS bbb USING INTEGER PRIMARY KEY (rowid=?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_execsql_test where3-5.2 { @@ -334,8 +334,8 @@ do_execsql_test where3-5.2 { AND bbb.parent = 4 ORDER BY bbb.title COLLATE NOCASE ASC; } { - 0 0 1 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?) (~10 rows)} - 0 1 0 {SEARCH TABLE bbb USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 1 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?)} + 0 1 0 {SEARCH TABLE bbb USING INTEGER PRIMARY KEY (rowid=?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_execsql_test where3-5.3 { @@ -347,8 +347,8 @@ do_execsql_test where3-5.3 { AND bbb.parent = 4 ORDER BY bbb.title COLLATE NOCASE ASC; } { - 0 0 1 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?) (~10 rows)} - 0 1 0 {SEARCH TABLE aaa AS bbb USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} + 0 0 1 {SEARCH TABLE aaa USING INDEX aaa_333 (fk=?)} + 0 1 0 {SEARCH TABLE aaa AS bbb USING INTEGER PRIMARY KEY (rowid=?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } diff --git a/test/where7.test b/test/where7.test index b6cd7ccbbc..3cb8781200 100644 --- a/test/where7.test +++ b/test/where7.test @@ -23339,9 +23339,9 @@ do_execsql_test where7-3.1 { OR t301.c8 = 1407424651264000) ORDER BY t302.c5 LIMIT 200; } { - 0 0 1 {SEARCH TABLE t301 USING COVERING INDEX t301_c4 (c4=?) (~10 rows)} - 0 0 1 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} - 0 1 0 {SEARCH TABLE t302 USING INDEX t302_c8_c3 (c8=? AND c3>?) (~2 rows)} + 0 0 1 {SEARCH TABLE t301 USING COVERING INDEX t301_c4 (c4=?)} + 0 0 1 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)} + 0 1 0 {SEARCH TABLE t302 USING INDEX t302_c8_c3 (c8=? AND c3>?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } diff --git a/test/where9.test b/test/where9.test index 1e94fdff78..d6a8c597b4 100644 --- a/test/where9.test +++ b/test/where9.test @@ -362,9 +362,9 @@ ifcapable explain { SELECT t2.a FROM t1, t2 WHERE t1.a=80 AND ((t1.c=t2.c AND t1.d=t2.d) OR t1.f=t2.f) } { - 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} - 0 1 1 {SEARCH TABLE t2 USING INDEX t2d (d=?) (~2 rows)} - 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2f (f=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)} + 0 1 1 {SEARCH TABLE t2 USING INDEX t2d (d=?)} + 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2f (f=?)} } do_execsql_test where9-3.2 { EXPLAIN QUERY PLAN @@ -372,9 +372,9 @@ ifcapable explain { FROM t1 LEFT JOIN t2 ON (t1.c+1=t2.c AND t1.d=t2.d) OR (t1.f||'x')=t2.f WHERE t1.a=80 } { - 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)} - 0 1 1 {SEARCH TABLE t2 USING INDEX t2d (d=?) (~2 rows)} - 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2f (f=?) (~10 rows)} + 0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)} + 0 1 1 {SEARCH TABLE t2 USING INDEX t2d (d=?)} + 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2f (f=?)} } } @@ -453,8 +453,8 @@ ifcapable explain { do_execsql_test where9-5.1 { EXPLAIN QUERY PLAN SELECT a FROM t1 WHERE b>1000 AND (c=31031 OR d IS NULL) } { - 0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c=?) (~3 rows)} - 0 0 0 {SEARCH TABLE t1 USING INDEX t1d (d=?) (~3 rows)} + 0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c=?)} + 0 0 0 {SEARCH TABLE t1 USING INDEX t1d (d=?)} } # In contrast, b=1000 is preferred over any OR-clause. @@ -462,7 +462,7 @@ ifcapable explain { do_execsql_test where9-5.2 { EXPLAIN QUERY PLAN SELECT a FROM t1 WHERE b=1000 AND (c=31031 OR d IS NULL) } { - 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?) (~5 rows)} + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)} } # Likewise, inequalities in an AND are preferred over inequalities in @@ -471,7 +471,7 @@ ifcapable explain { do_execsql_test where9-5.3 { EXPLAIN QUERY PLAN SELECT a FROM t1 WHERE b>1000 AND (c>=31031 OR d IS NULL) } { - 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>?) (~125000 rows)} + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>?)} } } diff --git a/test/whereC.test b/test/whereC.test index 9fa1bbaf95..bd7fe9ad82 100644 --- a/test/whereC.test +++ b/test/whereC.test @@ -67,4 +67,3 @@ foreach {tn sql res} { finish_test - diff --git a/tool/wherecosttest.c b/tool/wherecosttest.c new file mode 100644 index 0000000000..3055358fc2 --- /dev/null +++ b/tool/wherecosttest.c @@ -0,0 +1,111 @@ +/* +** 2013-06-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a simple command-line utility for converting from +** integers and WhereCost values and back again and for doing simple +** arithmetic operations (multiple and add) on WhereCost values. +** +** Usage: +** +** ./wherecosttest ARGS +** +** Arguments: +** +** 'x' Multiple the top two elements of the stack +** '+' Add the top two elements of the stack +** NUM Convert NUM from integer to WhereCost and push onto the stack +** ^NUM Interpret NUM as a WhereCost and push onto stack. +** +** Examples: +** +** To convert 123 from WhereCost to integer: +** +** ./wherecosttest ^123 +** +** To convert 123456 from integer to WhereCost: +** +** ./wherecosttest 123456 +** +*/ +#include +#include +#include + +typedef unsigned short int WhereCost; /* 10 times log2() */ + +WhereCost whereCostMultiply(WhereCost a, WhereCost b){ return a+b; } +WhereCost whereCostAdd(WhereCost a, WhereCost b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( ab+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; +} +WhereCost whereCostFromInteger(int x){ + static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + WhereCost y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} +static unsigned long int whereCostToInt(WhereCost x){ + unsigned long int n; + if( x<=10 ) return 1; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>=3 ) return (n+8)<<(x-3); + return (n+8)>>(3-x); +} + +int main(int argc, char **argv){ + int i; + int n = 0; + WhereCost a[100]; + for(i=1; i=2 ){ + a[n-2] = whereCostAdd(a[n-2],a[n-1]); + n--; + } + }else if( z[0]=='x' ){ + if( n>=2 ){ + a[n-2] = whereCostMultiply(a[n-2],a[n-1]); + n--; + } + }else if( z[0]=='^' ){ + a[n++] = atoi(z+1); + }else{ + a[n++] = whereCostFromInteger(atoi(z)); + } + } + for(i=n-1; i>=0; i--){ + printf("%d (%lu)\n", a[i], whereCostToInt(a[i])); + } + return 0; +} From 8636e9c55cb46f8923ebbb8a1014010d9c813472 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Jun 2013 01:50:08 +0000 Subject: [PATCH 081/109] Handle virtual tables correctly when using logarithmic costs. Fixes to test cases. FossilOrigin-Name: e612664aa2e24ed5e222be2c7fe16e210ac9bded --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/where.c | 39 ++++++++++++++++++++++++++++++++++----- test/analyze5.test | 32 ++++++++++++++++---------------- test/between.test | 4 ++-- test/eqp.test | 4 ++-- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index c92fe224c3..cda6c0adff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scases\sfor\sthe\snew\sEXPLAIN\sQUERY\sPLAN\sformat.\s\sAdd\sthe\nwherecosttest\stool.\s\sOther\sfixes\sto\slogarithm\scost. -D 2013-06-10T23:30:09.637 +C Handle\svirtual\stables\scorrectly\swhen\susing\slogarithmic\scosts.\s\sFixes\nto\stest\scases. +D 2013-06-11T01:50:08.263 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 672b76db7422b7bbc94fe1be42183c6356e56e96 +F src/where.c 72f9aa6c35dadf0d432c2d80fd117d8887f70473 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -303,7 +303,7 @@ F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/analyze.test f8ab7d15858b4093b06caf5e57e2a5ff7104bdae F test/analyze3.test 69863b446539f8849a996c2aa0b50461c9cecea4 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 -F test/analyze5.test 713354664c5ff1853ab2cbcb740f0cf5cb7c802e +F test/analyze5.test 3e57f192307be931f9ab2f6ff456f9063358ac77 F test/analyze6.test cdbf9887d40ab41301f570fe85c6e1525dd3b1c9 F test/analyze7.test 7de3ab66e1981303e783102a012d62cb876bf1d5 F test/analyze8.test ea4972c76820ac8d6a0754e6f5b851292f0f5a61 @@ -335,7 +335,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa -F test/between.test e6896728a636e3e610bcb2f87c9305120c809bc5 +F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 7c79f1ef0c6c2c2bc1e7bd895596fab32bfb4796 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 @@ -440,7 +440,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test ac506be979f611719a2a2de8476fe608fc9d66a4 +F test/eqp.test 68dd66c8fa05551d056ebd7fc8ae3b2e1cb5836e F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 69cf877283d362915edddf1822fbf7a9f86278b3 -R 6bac639abb86e2ec49ce16ea155e2dc3 +P aa580e368e3c398b8377b80342dfdd906324c248 +R dc216beb84f13406fad3ccd046a844fe U drh -Z f81162c822de40652566c68321ce2c25 +Z a1ab90c9180f43596814678177a2e938 diff --git a/manifest.uuid b/manifest.uuid index 072a7b4e84..b4b006445a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa580e368e3c398b8377b80342dfdd906324c248 \ No newline at end of file +e612664aa2e24ed5e222be2c7fe16e210ac9bded \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2b0049b9c3..4a27b9e382 100644 --- a/src/where.c +++ b/src/where.c @@ -1882,6 +1882,23 @@ static WhereCost whereCostFromInt(tRowcnt x){ return a[x&7] + y - 10; } +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Convert a double (as received from xBestIndex of a virtual table) +** into a WhereCost +*/ +static WhereCost whereCostFromDouble(double x){ + u64 a; + WhereCost e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=1 ) return 0; + if( x<=2000000000 ) return whereCostFromInt((tRowcnt)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + /* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating @@ -4088,6 +4105,14 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** more terms of the index */ pNext = p->pNextLoop; break; + }else if( p->nOut>pTemplate->nOut + && p->rSetup==pTemplate->rSetup + && p->rRun==pTemplate->rRun + ){ + /* Overwrite an existing WhereLoop with the same cost but more + ** outputs */ + pNext = p->pNextLoop; + break; }else{ /* pTemplate is not helpful. ** Return without changing or adding anything */ @@ -4263,7 +4288,7 @@ static int whereLoopAddBtreeIndex( WhereCost rDiv; whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, pBtm, pTop, &rDiv); - pNew->nOut = saved_nOut - rDiv; + pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10; } #ifdef SQLITE_ENABLE_STAT3 if( pNew->u.btree.nEq==1 && pProbe->nSample ){ @@ -4482,6 +4507,7 @@ static int whereLoopAddBtree( return rc; } +#ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. @@ -4569,8 +4595,7 @@ static int whereLoopAddVirtual( pIdxInfo->idxNum = 0; pIdxInfo->needToFreeIdxStr = 0; pIdxInfo->orderByConsumed = 0; - /* ((WhereCost)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((WhereCost)2); + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; @@ -4624,7 +4649,7 @@ static int whereLoopAddVirtual( pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) && pIdxInfo->orderByConsumed); pNew->rSetup = 0; - pNew->rRun = whereCostFromInt((tRowcnt)pIdxInfo->estimatedCost); + pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost); pNew->nOut = 46; assert( 46 == whereCostFromInt(25) ); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ @@ -4639,6 +4664,7 @@ whereLoopAddVtab_exit: sqlite3DbFree(db, pIdxInfo); return rc; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ /* ** Add WhereLoop entries to handle OR terms. This works for either @@ -4695,9 +4721,12 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sBest.maskSelf = 0; sBest.rSetup = 0; sBest.rRun = 0; +#ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mExtra); - }else{ + }else +#endif + { rc = whereLoopAddBtree(&sSubBuild, mExtra); } if( sBest.maskSelf==0 ) break; diff --git a/test/analyze5.test b/test/analyze5.test index 1041d709d2..c400780fff 100644 --- a/test/analyze5.test +++ b/test/analyze5.test @@ -156,13 +156,14 @@ foreach {testid where index rows} { } { # Verify that the expected index is used with the expected row count - do_test analyze5-1.${testid}a { - set x [lindex [eqp "SELECT * FROM t1 WHERE $where"] 3] - set idx {} - regexp {INDEX (t1.) } $x all idx - regexp {~([0-9]+) rows} $x all nrow - list $idx $nrow - } [list $index $rows] + # No longer valid due to an EXPLAIN QUERY PLAN output format change + # do_test analyze5-1.${testid}a { + # set x [lindex [eqp "SELECT * FROM t1 WHERE $where"] 3] + # set idx {} + # regexp {INDEX (t1.) } $x all idx + # regexp {~([0-9]+) rows} $x all nrow + # list $idx $nrow + # } [list $index $rows] # Verify that the same result is achieved regardless of whether or not # the index is used @@ -202,15 +203,14 @@ foreach {testid where index rows} { } { # Verify that the expected index is used with the expected row count -if {$testid==50299} {breakpoint; set sqlite_where_trace 1} - do_test analyze5-1.${testid}a { - set x [lindex [eqp "SELECT * FROM t1 WHERE $where"] 3] - set idx {} - regexp {INDEX (t1.) } $x all idx - regexp {~([0-9]+) rows} $x all nrow - list $idx $nrow - } [list $index $rows] -if {$testid==50299} exit + # No longer valid due to an EXPLAIN QUERY PLAN format change + # do_test analyze5-1.${testid}a { + # set x [lindex [eqp "SELECT * FROM t1 WHERE $where"] 3] + # set idx {} + # regexp {INDEX (t1.) } $x all idx + # regexp {~([0-9]+) rows} $x all nrow + # list $idx $nrow + # } [list $index $rows] # Verify that the same result is achieved regardless of whether or not # the index is used diff --git a/test/between.test b/test/between.test index f089cd95d3..df4c67995c 100644 --- a/test/between.test +++ b/test/between.test @@ -58,10 +58,10 @@ proc queryplan {sql} { set eqp [execsql "EXPLAIN QUERY PLAN $sql"] # puts eqp=$eqp foreach {a b c x} $eqp { - if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\y} \ $x all as tab idx]} { lappend data $tab $idx - } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\y} $x all as tab]} { lappend data $tab * } } diff --git a/test/eqp.test b/test/eqp.test index f137920309..ad7d4edad2 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -554,7 +554,7 @@ det 7.1 "SELECT count(*) FROM t1" { } det 7.2 "SELECT count(*) FROM t2" { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i1(~1000000 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i1} } do_execsql_test 7.3 { @@ -576,7 +576,7 @@ det 7.4 "SELECT count(*) FROM t1" { } det 7.5 "SELECT count(*) FROM t2" { - 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i1(~3 rows)} + 0 0 0 {SCAN TABLE t2 USING COVERING INDEX i1} } From 8a4380d76156ba0ffbcdb64b5e37f82dee6385c6 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Jun 2013 02:32:50 +0000 Subject: [PATCH 082/109] Fixes to EXPLAIN QUERY PLAN output. Change weights back to something closer to what they are in legacy. More test case fixes. FossilOrigin-Name: 36373b85f9a97840aa06e24ae31c12fcfbae084e --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/select.c | 7 +++---- src/where.c | 2 +- test/like.test | 6 +++--- test/where.test | 16 ++++++++-------- test/where2.test | 4 ++-- test/where3.test | 4 ++-- test/whereE.test | 8 ++++---- 9 files changed, 36 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index cda6c0adff..6c5cbae82b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\svirtual\stables\scorrectly\swhen\susing\slogarithmic\scosts.\s\sFixes\nto\stest\scases. -D 2013-06-11T01:50:08.263 +C Fixes\sto\sEXPLAIN\sQUERY\sPLAN\soutput.\s\sChange\sweights\sback\sto\ssomething\scloser\nto\swhat\sthey\sare\sin\slegacy.\s\sMore\stest\scase\sfixes. +D 2013-06-11T02:32:50.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c ddcf0c795f690968de66ab21051e764ca1c13f09 +F src/select.c 588ae13fc3d10dc812a0832fd143374583beb847 F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 72f9aa6c35dadf0d432c2d80fd117d8887f70473 +F src/where.c 1241512fecb60142384231ba902e3b68a550c0b8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -620,7 +620,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 F test/lastinsert.test 474d519c68cb79d07ecae56a763aa7f322c72f51 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test 52e650adfa208325f928b847571d77b647af07c3 +F test/like.test 935fb4f608e3ea126891496a6e99b9468372bf5c F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 @@ -1029,9 +1029,9 @@ F test/walro.test 6cc247a0cc9b36aeea2057dd28a922a1cdfbd630 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 054a6b6f7933c5a5f50d0bcd650b5eccb450cc81 -F test/where2.test 116fb0d6e98a423d12eb9a65906218ce09936674 -F test/where3.test 1f2a9c997243dc39c67d38fe6d73254ab2f880b8 +F test/where.test cd13d4804b8f2986b176fb45c0a43fbbba64215a +F test/where2.test 58cc3f85c082015144bc34c7ffdbd64134f650ee +F test/where3.test f2a7027e1971d583069947edba0a31563e86284f F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1043,7 +1043,7 @@ F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 -F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d +F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aa580e368e3c398b8377b80342dfdd906324c248 -R dc216beb84f13406fad3ccd046a844fe +P e612664aa2e24ed5e222be2c7fe16e210ac9bded +R 22fdd5c799c9a709871f8decba5b6ac4 U drh -Z a1ab90c9180f43596814678177a2e938 +Z 5d13a5c0b3748ceb3b9bdd15d82353e6 diff --git a/manifest.uuid b/manifest.uuid index b4b006445a..5783c7e87f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e612664aa2e24ed5e222be2c7fe16e210ac9bded \ No newline at end of file +36373b85f9a97840aa06e24ae31c12fcfbae084e \ No newline at end of file diff --git a/src/select.c b/src/select.c index ee994a8c20..8972faacc6 100644 --- a/src/select.c +++ b/src/select.c @@ -3884,11 +3884,10 @@ static void explainSimpleCount( Index *pIdx /* Index used to optimize scan, or NULL */ ){ if( pParse->explain==2 ){ - char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s %s%s(~%d rows)", + char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s", pTab->zName, - pIdx ? "USING COVERING INDEX " : "", - pIdx ? pIdx->zName : "", - pTab->nRowEst + pIdx ? " USING COVERING INDEX " : "", + pIdx ? pIdx->zName : "" ); sqlite3VdbeAddOp4( pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC diff --git a/src/where.c b/src/where.c index 4a27b9e382..330faf771c 100644 --- a/src/where.c +++ b/src/where.c @@ -4308,7 +4308,7 @@ static int whereLoopAddBtreeIndex( }else{ /* Each row involves a step of the index, then a binary search of ** the main table */ - WhereCost rStepAndSearch = rLogSize>80 ? rLogSize-80 : 1; + WhereCost rStepAndSearch = whereCostAdd(10, rLogSize>17 ? rLogSize-17 : 1); pNew->rRun = whereCostAdd(pNew->rRun, rStepAndSearch); } /* TBD: Adjust nOut for additional constraints */ diff --git a/test/like.test b/test/like.test index f68a93537b..230dc74fed 100644 --- a/test/like.test +++ b/test/like.test @@ -166,13 +166,13 @@ proc queryplan {sql} { set eqp [execsql "EXPLAIN QUERY PLAN $sql"] # puts eqp=$eqp foreach {a b c x} $eqp { - if {[regexp { TABLE (\w+ AS )?(\w+) USING COVERING INDEX (\w+)\W} \ + if {[regexp { TABLE (\w+ AS )?(\w+) USING COVERING INDEX (\w+)\y} \ $x all as tab idx]} { lappend data {} $idx - } elseif {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + } elseif {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\y} \ $x all as tab idx]} { lappend data $tab $idx - } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\y} $x all as tab]} { lappend data $tab * } } diff --git a/test/where.test b/test/where.test index 8610fa6f3d..39324bf1a6 100644 --- a/test/where.test +++ b/test/where.test @@ -67,7 +67,7 @@ do_test where-1.1.1 { } {3 121 10 3} do_eqp_test where-1.1.2 { SELECT x, y, w FROM t1 WHERE w=10 -} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.1.3 { db status step } {0} @@ -79,13 +79,13 @@ do_test where-1.1.5 { } {99} do_eqp_test where-1.1.6 { SELECT x, y, w FROM t1 WHERE +w=10 -} {*SCAN TABLE t1 *} +} {*SCAN TABLE t1*} do_test where-1.1.7 { count {SELECT x, y, w AS abc FROM t1 WHERE abc=10} } {3 121 10 3} do_eqp_test where-1.1.8 { SELECT x, y, w AS abc FROM t1 WHERE abc=10 -} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.1.9 { db status step } {0} @@ -106,19 +106,19 @@ do_test where-1.4.1 { } {11 3 144 3} do_eqp_test where-1.4.2 { SELECT w, x, y FROM t1 WHERE 11=w AND x>2 -} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.4.3 { count {SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2} } {11 3 144 3} do_eqp_test where-1.4.4 { SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2 -} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.5 { count {SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2} } {3 144 3} do_eqp_test where-1.5.2 { SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2 -} {*SEARCH TABLE t1 USING INDEX i1w (w=?) *} +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.6 { count {SELECT x, y FROM t1 WHERE y<200 AND x>2 AND w=11} } {3 144 3} @@ -130,10 +130,10 @@ do_test where-1.8 { } {3 144 3} do_eqp_test where-1.8.2 { SELECT x, y FROM t1 WHERE w>10 AND y=144 AND x=3 -} {*SEARCH TABLE t1 USING INDEX i1xy (x=? AND y=?) *} +} {*SEARCH TABLE t1 USING INDEX i1xy (x=? AND y=?)*} do_eqp_test where-1.8.3 { SELECT x, y FROM t1 WHERE y=144 AND x=3 -} {*SEARCH TABLE t1 USING COVERING INDEX i1xy (x=? AND y=?) *} +} {*SEARCH TABLE t1 USING COVERING INDEX i1xy (x=? AND y=?)*} do_test where-1.9 { count {SELECT x, y FROM t1 WHERE y=144 AND w>10 AND x=3} } {3 144 3} diff --git a/test/where2.test b/test/where2.test index d7fb514a87..c55d4a12a3 100644 --- a/test/where2.test +++ b/test/where2.test @@ -76,10 +76,10 @@ proc queryplan {sql} { set eqp [execsql "EXPLAIN QUERY PLAN $sql"] # puts eqp=$eqp foreach {a b c x} $eqp { - if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\y} \ $x all as tab idx]} { lappend data $tab $idx - } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\y} $x all as tab]} { lappend data $tab * } } diff --git a/test/where3.test b/test/where3.test index cee74ba434..e6acafed07 100644 --- a/test/where3.test +++ b/test/where3.test @@ -111,10 +111,10 @@ proc queryplan {sql} { set eqp [execsql "EXPLAIN QUERY PLAN $sql"] # puts eqp=$eqp foreach {a b c x} $eqp { - if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\W} \ + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\y} \ $x all as tab idx]} { lappend data $tab $idx - } elseif {[regexp { TABLE (\w+ AS )?(\w+)\W} $x all as tab]} { + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\y} $x all as tab]} { lappend data $tab * } } diff --git a/test/whereE.test b/test/whereE.test index e686a4628e..a6b8f481b2 100644 --- a/test/whereE.test +++ b/test/whereE.test @@ -47,16 +47,16 @@ do_execsql_test 1.1 { CREATE UNIQUE INDEX t2zx ON t2(z,x); EXPLAIN QUERY PLAN SELECT x FROM t1, t2 WHERE a=z AND c=x; -} {/.*SCAN TABLE t1 .*SEARCH TABLE t2 .*/} +} {/.*SCAN TABLE t1.*SEARCH TABLE t2.*/} do_execsql_test 1.2 { EXPLAIN QUERY PLAN SELECT x FROM t2, t1 WHERE a=z AND c=x; -} {/.*SCAN TABLE t1 .*SEARCH TABLE t2 .*/} +} {/.*SCAN TABLE t1.*SEARCH TABLE t2.*/} do_execsql_test 1.3 { ANALYZE; EXPLAIN QUERY PLAN SELECT x FROM t1, t2 WHERE a=z AND c=x; -} {/.*SCAN TABLE t1 .*SEARCH TABLE t2 .*/} +} {/.*SCAN TABLE t1.*SEARCH TABLE t2.*/} do_execsql_test 1.4 { EXPLAIN QUERY PLAN SELECT x FROM t2, t1 WHERE a=z AND c=x; -} {/.*SCAN TABLE t1 .*SEARCH TABLE t2 .*/} +} {/.*SCAN TABLE t1.*SEARCH TABLE t2.*/} finish_test From 8e23daf372c57084e7921e48c1b7968b631bb94d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Jun 2013 13:30:04 +0000 Subject: [PATCH 083/109] Fix the Parse.nQueryLoop state variable to work with NGQP. FossilOrigin-Name: f1cac24f06b9c71cfa472fdcf2da4cd8689a7cc3 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 6 +++--- src/prepare.c | 4 ++-- src/sqliteInt.h | 2 +- src/where.c | 9 +++++---- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index f2ea8783ae..9e3a53b7f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sa\slogarithmic\srather\sthan\slinear\scost\sand\srow-count\smeasures.\nDo\snot\sreport\srow\scount\sestimates\sin\sEQP\soutput. -D 2013-06-11T02:36:41.347 +C Fix\sthe\sParse.nQueryLoop\sstate\svariable\sto\swork\swith\sNGQP. +D 2013-06-11T13:30:04.431 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c ac9d259eea3123faa05fabe6dd8717696aca72b1 +F src/expr.c 9cc9e4c85197b567e5f97f3296700faf7747810f F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 5c50c1ea31fd864b0fe921fe1a8d4c55acd609ef @@ -210,7 +210,7 @@ F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c d23d07716de96c7c0c2503ec5051a4384c3fb938 F src/pragma.c 67a611bd4be0754f27ee13eb87932c3b14415862 -F src/prepare.c 743e484233c51109666d402f470523553b41797c +F src/prepare.c 213de4aae5e29cfbb32202d1c09f4fc4b5a51b09 F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 4284f3a915d6ee86656aa0d47649855e8df8081e +F src/sqliteInt.h f2e51444e15f7dd35c1ee139f95f43b0bccf3fb5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 1241512fecb60142384231ba902e3b68a550c0b8 +F src/where.c 572357246bee5389751427c670cb0222fe36be78 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0f8a38ee54208d6a477aa2482cd277b4808450f0 36373b85f9a97840aa06e24ae31c12fcfbae084e -R 22fdd5c799c9a709871f8decba5b6ac4 +P b777b1097dcf9dfeb1b86c71e1b5f6918d68c975 +R 1aa8d10751da8bc78a75655bc8318da4 U drh -Z 00ce9a3b8547724477b5f04e4eb4057d +Z 0a42be2e93f4ddc55a00a5d0e3551610 diff --git a/manifest.uuid b/manifest.uuid index 06a0d20b6c..3db045bf33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b777b1097dcf9dfeb1b86c71e1b5f6918d68c975 \ No newline at end of file +f1cac24f06b9c71cfa472fdcf2da4cd8689a7cc3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c1a27ebf84..4e834ffe8e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1596,15 +1596,15 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ /* Could not found an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. */ - double savedNQueryLoop = pParse->nQueryLoop; + u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; eType = IN_INDEX_EPH; if( prNotFound ){ *prNotFound = rMayHaveNull = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ - testcase( pParse->nQueryLoop>(double)1 ); - pParse->nQueryLoop = (double)1; + testcase( pParse->nQueryLoop>1 ); + pParse->nQueryLoop = 1; if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } diff --git a/src/prepare.c b/src/prepare.c index d78d83cbd8..319bb5012e 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -592,7 +592,7 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); pParse->db = db; - pParse->nQueryLoop = (double)1; + pParse->nQueryLoop = 1; if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -614,7 +614,7 @@ static int sqlite3Prepare( }else{ sqlite3RunParser(pParse, zSql, &zErrMsg); } - assert( 1==(int)pParse->nQueryLoop ); + assert( 1==pParse->nQueryLoop ); if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aa1ea51011..d3d6fcf70a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2226,7 +2226,7 @@ struct Parse { /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ - u32 nQueryLoop; /* Estimated number of iterations of a query */ + u32 grep nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ diff --git a/src/where.c b/src/where.c index 330faf771c..d493dba46f 100644 --- a/src/where.c +++ b/src/where.c @@ -358,7 +358,7 @@ struct WhereInfo { WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - WhereCost savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ WhereCost nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; @@ -3099,7 +3099,7 @@ static void explainOneScan( }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); - if( flags&WHERE_COLUMN_EQ ){ + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowidpBest - && pTabList->nSrc>1 +// && pTabList->nSrc>1 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->viaCoroutine && !pSrc->notIndexed @@ -5074,7 +5074,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ } /* Seed the search with a single WherePath containing zero WhereLoops */ - aFrom[0].nRow = 0; + aFrom[0].nRow = pWInfo->pParse->nQueryLoop; nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller @@ -5615,6 +5615,7 @@ WhereInfo *sqlite3WhereBegin( } #endif WHERETRACE(("*** Optimizer Finished ***\n")); + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; #if 0 /* FIXME: Add this back in? */ /* If the caller is an UPDATE or DELETE statement that is requesting From 4f402f26b15902d22f13df5e77cfdc4d8a5622cb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Jun 2013 18:59:38 +0000 Subject: [PATCH 084/109] Improved processing of DISTINCT. FossilOrigin-Name: ba897100fed291d2025f68d09334f9985312298b --- ext/misc/fuzzer.c | 6 +-- manifest | 16 +++--- manifest.uuid | 2 +- src/sqliteInt.h | 6 ++- src/where.c | 126 ++++++++++++++++++++++++++++++---------------- 5 files changed, 100 insertions(+), 56 deletions(-) diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 023bdb1d09..fe41cda8c2 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -1079,7 +1079,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; int seenMatch = 0; const struct sqlite3_index_constraint *pConstraint; - double rCost = 100000; + double rCost = 1e12; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ @@ -1095,7 +1095,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 1; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; - rCost /= 1000000.0; + rCost /= 1e6; } if( (iPlan & 2)==0 && pConstraint->iColumn==1 @@ -1132,7 +1132,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30; + if( seenMatch && (iPlan&1)==0 ) rCost = 1e99; pIdxInfo->estimatedCost = rCost; return SQLITE_OK; diff --git a/manifest b/manifest index 9e3a53b7f0..bdcdfbf749 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sParse.nQueryLoop\sstate\svariable\sto\swork\swith\sNGQP. -D 2013-06-11T13:30:04.431 +C Improved\sprocessing\sof\sDISTINCT. +D 2013-06-11T18:59:38.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -108,7 +108,7 @@ F ext/icu/icu.c 7538f98eab2854cf17fa5f7797bffa6c76e3863b F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c eae8454cd9dcb287b2a3ec2e65a865a4ac5f0d06 F ext/misc/closure.c 997c20ddf35f85ab399f4a02a557a9baa822ec32 -F ext/misc/fuzzer.c 79ac20b68794dfe03626f3a31681e6c240583052 +F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/percentile.c 4fb5e46c4312b0be74e8e497ac18f805f0e3e6c5 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h f2e51444e15f7dd35c1ee139f95f43b0bccf3fb5 +F src/sqliteInt.h 026a52f3e24c15603fddb78025e786b96c4c0d2c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 572357246bee5389751427c670cb0222fe36be78 +F src/where.c 2afa8f1a71dd088b137a7772f5c5dfee70c8fbe8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b777b1097dcf9dfeb1b86c71e1b5f6918d68c975 -R 1aa8d10751da8bc78a75655bc8318da4 +P f1cac24f06b9c71cfa472fdcf2da4cd8689a7cc3 +R f41c25fe4af487f857bb0cfd93c46d44 U drh -Z 0a42be2e93f4ddc55a00a5d0e3551610 +Z 15fc6bc86393280f1819e7074b4ff66c diff --git a/manifest.uuid b/manifest.uuid index 3db045bf33..64bfde3e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1cac24f06b9c71cfa472fdcf2da4cd8689a7cc3 \ No newline at end of file +ba897100fed291d2025f68d09334f9985312298b \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d3d6fcf70a..b22689462c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1967,8 +1967,10 @@ struct SrcList { #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ -/* Allowed values for WhereInfo.eDistinct and DistinctCtx.eTnctType */ +/* Allowed return values from sqlite3WhereIsDistinct() +*/ #define WHERE_DISTINCT_NOOP 0 /* DISTINCT keyword not used */ #define WHERE_DISTINCT_UNIQUE 1 /* No duplicates */ #define WHERE_DISTINCT_ORDERED 2 /* All duplicates are adjacent */ @@ -2226,7 +2228,7 @@ struct Parse { /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ - u32 grep nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ + u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ diff --git a/src/where.c b/src/where.c index d493dba46f..c21f3fc494 100644 --- a/src/where.c +++ b/src/where.c @@ -345,9 +345,11 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pDistinct; /* DISTINCT ON values, or NULL */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + WhereCost nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 bOBSat; /* ORDER BY satisfied by indices */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ @@ -355,11 +357,9 @@ struct WhereInfo { int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int nLevel; /* Number of nested loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereClause sWC; /* Decomposition of the WHERE clause */ - WhereLoop *pLoops; /* List of all WhereLoop objects */ - int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - WhereCost nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; @@ -441,7 +441,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** Return FALSE if the output needs to be sorted. */ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat>0; + return pWInfo->bOBSat!=0; } /* @@ -1776,15 +1776,16 @@ static int findIndexCol( /* ** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. A DISTINCT list is redundant if the database contains a -** UNIQUE index that guarantees that the result of the query will be distinct -** anyway. +** is redundant. +** +** A DISTINCT list is redundant if the database contains some set of +** columns that are unique and non-null. */ static int isDistinctRedundant( - Parse *pParse, - SrcList *pTabList, - WhereClause *pWC, - ExprList *pDistinct + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ ){ Table *pTab; Index *pIdx; @@ -3418,7 +3419,7 @@ static Bitmask codeOneLoopStart( ** this requires some special handling. */ if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pWInfo->nOBSat>0) + && (pWInfo->bOBSat!=0) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ @@ -4493,7 +4494,9 @@ static int whereLoopAddBtree( ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize) + ((m==0 && b) ? 10 : 0); + pNew->rRun = whereCostAdd(rSize,rLogSize); + if( m!=0 ) pNew->rRun += rLogSize; + if( b ) pNew->rRun--; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; } @@ -4804,11 +4807,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ */ static int wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ - int nLoop, /* Number of entries in pPath->aLoop[] */ - int isLastLoop, /* True if pLast is the inner-most loop */ + u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + u8 isLastLoop, /* True if pLast is the inner-most loop */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ - Bitmask *pRevMask /* Mask of WhereLoops to run in reverse order */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ ){ u8 revSet; /* True if rev is known */ u8 rev; /* Composite sort order */ @@ -4823,7 +4828,6 @@ static int wherePathSatisfiesOrderBy( int iCur; /* Cursor number for current WhereLoop */ int iColumn; /* A column number within table iCur */ WhereLoop *pLoop; /* Current WhereLoop being processed. */ - ExprList *pOrderBy = pWInfo->pOrderBy; /* the ORDER BY clause */ WhereTerm *pTerm; /* A single term of the WHERE clause */ Expr *pOBExpr; /* An expression from the ORDER BY clause */ CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ @@ -4960,7 +4964,7 @@ static int wherePathSatisfiesOrderBy( for(i=0; bOnce && ia[i].pExpr); - if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ) bOnce = 0; + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; @@ -5111,8 +5115,9 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ rCost = whereCostAdd(rCost, pFrom->rCost); maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, iLoop==nLoop-1, - pWLoop, &revMask) ){ + switch( wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, iLoop==nLoop-1, pWLoop, &revMask) ){ case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ isOrdered = 1; isOrderedValid = 1; @@ -5249,9 +5254,22 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pLevel->iFrom = pWLoop->iTab; /* FIXME: Omit the iFrom field */ pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; } + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->pDistinct + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinct, pFrom, + WHERE_DISTINCTBY, nLoop-1, 1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } if( pFrom->isOrdered ){ - pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; - pWInfo->revMask = pFrom->revLoop; + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + }else{ + pWInfo->bOBSat = 1; + pWInfo->revMask = pFrom->revLoop; + } } pWInfo->nRowOut = pFrom->nRow; @@ -5327,7 +5345,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; - pWInfo->nOBSat = pWInfo->pOrderBy ? pWInfo->pOrderBy->nExpr : 0; + if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; + if( pWInfo->pDistinct ) pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; #ifdef SQLITE_DEBUG pLoop->cId = '0'; #endif @@ -5414,15 +5433,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ** pOrderBy is a pointer to the ORDER BY clause of a SELECT statement, ** if there is one. If there is no ORDER BY clause or if this routine ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. -** -** If an index can be used so that the natural output order of the table -** scan is correct for the ORDER BY clause, then that index is used and -** the returned WhereInfo.nOBSat field is set to pOrderBy->nExpr. This -** is an optimization that prevents an unnecessary sort of the result set -** if an index appropriate for the ORDER BY clause already exists. -** -** If the where clause loops cannot be arranged to provide the correct -** output order, then WhereInfo.nOBSat is 0. */ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ @@ -5558,13 +5568,32 @@ WhereInfo *sqlite3WhereBegin( goto whereBeginError; } + /* If the ORDER BY (or GROUP BY) clause contains references to general + ** expressions, then we won't be able to satisfy it using indices, so + ** go ahead and disable it now. + */ + if( pOrderBy ){ + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite3ExprSkipCollate(pOrderBy->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ){ + pWInfo->pOrderBy = pOrderBy = 0; + break; + } + } + } + /* Check if the DISTINCT qualifier, if there is one, is redundant. ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse,pTabList,&pWInfo->sWC,pDistinct) ){ - pDistinct = 0; - pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + if( pDistinct ){ + if( isDistinctRedundant(pParse,pTabList,&pWInfo->sWC,pDistinct) ){ + pDistinct = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pDistinct; + } } /* Construct the WhereLoop objects */ @@ -5587,7 +5616,7 @@ WhereInfo *sqlite3WhereBegin( } #endif - wherePathSolver(pWInfo, -1); + wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ wherePathSolver(pWInfo, pWInfo->nRowOut); @@ -5603,12 +5632,25 @@ WhereInfo *sqlite3WhereBegin( #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ int ii; - sqlite3DebugPrintf("---- Solution"); - if( pWInfo->nOBSat ){ - sqlite3DebugPrintf(" ORDER BY omitted rev=0x%llx\n", pWInfo->revMask); - }else{ - sqlite3DebugPrintf("\n"); + sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->bOBSat ){ + sqlite3DebugPrintf(" ORDERBY=0x%llx", pWInfo->revMask); } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite3DebugPrintf(" DISTINCT=unique"); + break; + } + case WHERE_DISTINCT_ORDERED: { + sqlite3DebugPrintf(" DISTINCT=ordered"); + break; + } + case WHERE_DISTINCT_UNORDERED: { + sqlite3DebugPrintf(" DISTINCT=unordered"); + break; + } + } + sqlite3DebugPrintf("\n"); for(ii=0; iia[ii].pWLoop, pTabList); } From e217efc82218d232320402bdc13c05058058d9b2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Jun 2013 03:48:41 +0000 Subject: [PATCH 085/109] Continue refining the NGQP FossilOrigin-Name: 40567fddd468d00295275af8df09a7a1785e684a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index bdcdfbf749..0f0b79ba59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sprocessing\sof\sDISTINCT. -D 2013-06-11T18:59:38.240 +C Continue\srefining\sthe\sNGQP +D 2013-06-12T03:48:41.127 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2afa8f1a71dd088b137a7772f5c5dfee70c8fbe8 +F src/where.c 3f38f769ef1365d7c8ff5b0ae9341b32f49bf46e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f1cac24f06b9c71cfa472fdcf2da4cd8689a7cc3 -R f41c25fe4af487f857bb0cfd93c46d44 +P ba897100fed291d2025f68d09334f9985312298b +R 5ed376a4396f55c99b307b2ce75a8d86 U drh -Z 15fc6bc86393280f1819e7074b4ff66c +Z 47a7a33cc642ca844caf03a37d1ebaba diff --git a/manifest.uuid b/manifest.uuid index 64bfde3e8b..d84595d4b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba897100fed291d2025f68d09334f9985312298b \ No newline at end of file +40567fddd468d00295275af8df09a7a1785e684a \ No newline at end of file diff --git a/src/where.c b/src/where.c index c21f3fc494..059c2c1015 100644 --- a/src/where.c +++ b/src/where.c @@ -74,7 +74,7 @@ struct WhereLevel { int addrNxt; /* Jump here to start the next IN combination */ int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ - u8 iFrom; /* FIXME: Which entry in the FROM clause */ + u8 iFrom; /* Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ union { /* Information that depends on plan.wsFlags */ @@ -4248,7 +4248,7 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 46; /* whereCostFromInt(25) */ + nIn = 46; assert( 46==whereCostFromInt(25) ); }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ nIn = whereCostFromInt(pExpr->x.pList->nExpr); @@ -4272,7 +4272,7 @@ static int whereLoopAddBtreeIndex( }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; - nIn = 10; /* Assume IS NULL matches two rows */ + nIn = 10; assert( 10==whereCostFromInt(2) ); pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; @@ -4303,15 +4303,13 @@ static int whereLoopAddBtreeIndex( pNew->nOut = whereCostFromInt(nOut); } #endif - if( pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK) ){ - /* Step cost for each output row */ - pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); - }else{ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ /* Each row involves a step of the index, then a binary search of ** the main table */ - WhereCost rStepAndSearch = whereCostAdd(10, rLogSize>17 ? rLogSize-17 : 1); - pNew->rRun = whereCostAdd(pNew->rRun, rStepAndSearch); + pNew->rRun = whereCostAdd(pNew->rRun, rLogSize>27 ? rLogSize-17 : 10); } + /* Step cost for each output row */ + pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -5251,7 +5249,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ for(iLoop=0; iLoopa + iLoop; pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; - pLevel->iFrom = pWLoop->iTab; /* FIXME: Omit the iFrom field */ + pLevel->iFrom = pWLoop->iTab; pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; } if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 @@ -5572,12 +5570,14 @@ WhereInfo *sqlite3WhereBegin( ** expressions, then we won't be able to satisfy it using indices, so ** go ahead and disable it now. */ - if( pOrderBy ){ + if( pOrderBy && pDistinct ){ for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOrderBy->a[ii].pExpr); if( pExpr->op!=TK_COLUMN ){ pWInfo->pOrderBy = pOrderBy = 0; break; + }else if( pExpr->iColumn<0 ){ + break; } } } From fd5874d23dce6e96aba056485b328e3801b69270 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Jun 2013 14:52:39 +0000 Subject: [PATCH 086/109] Add the "queryplanner" test permutation. Continuing refinements to NGQP. FossilOrigin-Name: 25e2cde105a19293bdb9c001b48624e5d7f8c4e5 --- manifest | 20 +++++------ manifest.uuid | 2 +- src/where.c | 34 +++++++++---------- test/permutations.test | 77 ++++++++++++++++++++++++++++++++++++++++++ test/where8.test | 2 +- test/where9.test | 36 ++++++++++++++------ test/whereF.test | 8 ++--- 7 files changed, 136 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 0f0b79ba59..2445afe8f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continue\srefining\sthe\sNGQP -D 2013-06-12T03:48:41.127 +C Add\sthe\s"queryplanner"\stest\spermutation.\s\sContinuing\srefinements\sto\sNGQP. +D 2013-06-12T14:52:39.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 3f38f769ef1365d7c8ff5b0ae9341b32f49bf46e +F src/where.c 67d70ce776aae6e81eddc182357bd5542dc9e023 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -707,7 +707,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test 4614301e38398df7fdd5f28f4ed8f272b328251b -F test/permutations.test d997a947ab8aabb15f763d50a030b3c11e8ef1b6 +F test/permutations.test 2ab3937fdd34e18712be8f4159ac354c4d27ce1d F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1036,15 +1036,15 @@ F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test e0741d85b6069effe53c6ed7682e18aa078f4b11 -F test/where8.test d6a283eb7348a8967d44e2a753f117ab0d21d4f3 +F test/where8.test f6b9559723564a042927ee0f22003ac9bed71b21 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 4094299aea1c5a1ebe172ad832c2467cba23133a +F test/where9.test f5752185fc19a1ee826a2386a249f05dbc25979b F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f -F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 +F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ba897100fed291d2025f68d09334f9985312298b -R 5ed376a4396f55c99b307b2ce75a8d86 +P 40567fddd468d00295275af8df09a7a1785e684a +R 8ad423cc8976bc762dc67ac33e49e948 U drh -Z 47a7a33cc642ca844caf03a37d1ebaba +Z 7a74b3f46b1e20d0df5f0a7a123b8d69 diff --git a/manifest.uuid b/manifest.uuid index d84595d4b9..943b9de7f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40567fddd468d00295275af8df09a7a1785e684a \ No newline at end of file +25e2cde105a19293bdb9c001b48624e5d7f8c4e5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 059c2c1015..06470f6fa4 100644 --- a/src/where.c +++ b/src/where.c @@ -604,7 +604,7 @@ static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ /* ** Initialize an expression mask set (a WhereMaskSet object) */ -#define initMaskSet(P) memset(P, 0, sizeof(*P)) +#define initMaskSet(P) (P)->n=0 /* ** Return the bitmask for the given cursor number. Return 0 if @@ -4361,7 +4361,7 @@ static int indexMightHelpWithOrderBy( ** Return a bitmask where 1s indicate that the corresponding column of ** the table is used by an index. Only the first 63 columns are considered. */ -static Bitmask columnsUsedByIndex(Index *pIdx){ +static Bitmask columnsInIndex(Index *pIdx){ Bitmask m = 0; int j; for(j=pIdx->nColumn-1; j>=0; j--){ @@ -4431,7 +4431,6 @@ static int whereLoopAddBtree( /* Automatic indexes */ if( !pBuilder->pBest -// && pTabList->nSrc>1 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->viaCoroutine && !pSrc->notIndexed @@ -4476,11 +4475,11 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 + b*4; + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 - b; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; }else{ - Bitmask m = pSrc->colUsed & ~columnsUsedByIndex(pProbe); + Bitmask m = pSrc->colUsed & ~columnsInIndex(pProbe); pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; /* Full scan via index */ @@ -4493,8 +4492,7 @@ static int whereLoopAddBtree( pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; pNew->rRun = whereCostAdd(rSize,rLogSize); - if( m!=0 ) pNew->rRun += rLogSize; - if( b ) pNew->rRun--; + pNew->rRun += ((m!=0) ? rLogSize : 10) - b; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; } @@ -4737,15 +4735,17 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ prereq |= sBest.prereq; } assert( pNew->nLSlot>=1 ); - pNew->nLTerm = 1; - pNew->aLTerm[0] = pTerm; - pNew->wsFlags = WHERE_MULTI_OR; - pNew->rSetup = 0; - pNew->rRun = rTotal; - pNew->nOut = nRow; - pNew->prereq = prereq; - memset(&pNew->u, 0, sizeof(pNew->u)); - rc = whereLoopInsert(pBuilder, pNew); + if( sBest.maskSelf ){ + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->rRun = rTotal; + pNew->nOut = nRow; + pNew->prereq = prereq; + memset(&pNew->u, 0, sizeof(pNew->u)); + rc = whereLoopInsert(pBuilder, pNew); + } whereLoopClear(pWInfo->pParse->db, &sBest); } } @@ -5327,7 +5327,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ } if( j!=pIdx->nColumn ) continue; pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( (pItem->colUsed & ~columnsUsedByIndex(pIdx))==0 ){ + if( (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ pLoop->wsFlags |= WHERE_IDX_ONLY; } pLoop->nLTerm = j; diff --git a/test/permutations.test b/test/permutations.test index bc3ceb8085..7b633019ae 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -213,6 +213,83 @@ test_suite "nofaultsim" -prefix "" -description { unset -nocomplain ::G(valgrind) } +test_suite "queryplanner" -prefix "" -description { + Tests of the query planner and query optimizer +} -files { + alter2.test alter3.test alter4.test alter.test analyze3.test + analyze4.test analyze5.test analyze6.test analyze7.test analyze8.test + analyze.test attach2.test attach3.test attach4.test attachmalloc.test + attach.test autoinc.test autoindex1.test between.test cast.test + check.test closure01.test coalesce.test collate1.test collate2.test + collate3.test collate4.test collate5.test collate6.test collate7.test + collate8.test collate9.test collateA.test colmeta.test colname.test + conflict.test count.test coveridxscan.test createtab.test cse.test + date.test dbstatus2.test dbstatus.test default.test delete2.test + delete3.test delete.test descidx1.test descidx2.test descidx3.test + distinctagg.test distinct.test e_createtable.test e_delete.test + e_droptrigger.test e_dropview.test e_expr.test e_fkey.test e_insert.test + eqp.test e_reindex.test e_resolve.test e_select2.test e_select.test + e_update.test exists.test expr.test fkey1.test fkey2.test fkey3.test + fkey4.test fkey5.test fkey_malloc.test format4.test func2.test + func3.test func.test fuzz2.test fuzz3.test fuzzer1.test fuzz-oss1.test + fuzz.test in2.test in3.test in4.test in5.test index2.test index3.test + index4.test index5.test indexedby.test indexfault.test index.test + insert2.test insert3.test insert4.test insert5.test insert.test + instr.test in.test intpkey.test join2.test join3.test join4.test + join5.test join6.test join.test like2.test like.test limit.test + minmax2.test minmax3.test minmax4.test minmax.test misc1.test misc2.test + misc3.test misc4.test misc5.test misc6.test misc7.test orderby1.test + orderby2.test orderby3.test orderby4.test randexpr1.test regexp1.test + reindex.test rowhash.test rowid.test schema2.test schema3.test + schema4.test schema5.test schema.test securedel2.test securedel.test + select1.test select2.test select3.test select4.test select5.test + select6.test select7.test select8.test select9.test selectA.test + selectB.test selectC.test selectD.test selectE.test sidedelete.test + sort.test spellfix.test subquery2.test subquery.test subselect.test + substr.test tkt-02a8e81d44.test tkt1435.test tkt1443.test tkt1444.test + tkt1449.test tkt1473.test tkt1501.test tkt1512.test tkt1514.test + tkt1536.test tkt1537.test tkt1567.test tkt1644.test tkt1667.test + tkt1873.test tkt2141.test tkt2192.test tkt2213.test tkt2251.test + tkt2285.test tkt2332.test tkt2339.test tkt2391.test tkt2409.test + tkt2450.test tkt2565.test tkt2640.test tkt2643.test tkt2686.test + tkt-26ff0c2d1e.test tkt2767.test tkt2817.test tkt2820.test tkt2822.test + tkt2832.test tkt2854.test tkt2920.test tkt2927.test tkt2942.test + tkt-2a5629202f.test tkt-2d1a5c67d.test tkt-2ea2425d34.test tkt3080.test + tkt3093.test tkt3121.test tkt-31338dca7e.test tkt-313723c356.test + tkt3201.test tkt3292.test tkt3298.test tkt3334.test tkt3346.test + tkt3357.test tkt3419.test tkt3424.test tkt3442.test tkt3457.test + tkt3461.test tkt3493.test tkt3508.test tkt3522.test tkt3527.test + tkt3541.test tkt3554.test tkt3581.test tkt35xx.test tkt3630.test + tkt3718.test tkt3731.test tkt3757.test tkt3761.test tkt3762.test + tkt3773.test tkt3791.test tkt3793.test tkt3810.test tkt3824.test + tkt3832.test tkt3838.test tkt3841.test tkt-385a5b56b9.test tkt3871.test + tkt3879.test tkt-38cb5df375.test tkt3911.test tkt3918.test tkt3922.test + tkt3929.test tkt3935.test tkt3992.test tkt3997.test tkt-3998683a16.test + tkt-3a77c9714e.test tkt-3fe897352e.test tkt4018.test tkt-4a03edc4c8.test + tkt-4dd95f6943.test tkt-54844eea3f.test tkt-5d863f876e.test + tkt-5e10420e8d.test tkt-5ee23731f.test tkt-6bfb98dfc0.test + tkt-752e1646fc.test tkt-78e04e52ea.test tkt-7a31705a7e6.test + tkt-7bbfb7d442.test tkt-80ba201079.test tkt-80e031a00f.test + tkt-8454a207b9.test tkt-91e2e8ba6f.test tkt-94c04eaadb.test + tkt-9d68c883.test tkt-a7b7803e.test tkt-b1d3a2e531.test + tkt-b351d95f9.test tkt-b72787b1.test tkt-bd484a090c.test + tkt-bdc6bbbb38.test tkt-c48d99d690.test tkt-cbd054fa6b.test + tkt-d11f09d36e.test tkt-d635236375.test tkt-d82e3f3721.test + tkt-f3e5abed55.test tkt-f777251dc7a.test tkt-f7b4edec.test + tkt-f973c7ac31.test tkt-fa7bf5ec.test tkt-fc62af4523.test + tkt-fc7bd6358f.test trigger1.test trigger2.test trigger3.test + trigger4.test trigger5.test trigger6.test trigger7.test trigger8.test + trigger9.test triggerA.test triggerB.test triggerC.test triggerD.test + types2.test types3.test types.test unique.test unordered.test + update.test view.test vtab1.test vtab2.test vtab3.test vtab4.test + vtab5.test vtab6.test vtab7.test vtab8.test vtab9.test vtab_alter.test + vtabA.test vtabB.test vtabC.test vtabD.test vtab_err.test vtabE.test + vtabF.test where2.test where3.test where4.test where5.test where6.test + where7.test where8m.test where8.test where9.test whereA.test whereB.test + whereC.test whereD.test whereE.test whereF.test wherelimit.test + where.test +} + lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: diff --git a/test/where8.test b/test/where8.test index 9b6014e708..6890e3ac59 100644 --- a/test/where8.test +++ b/test/where8.test @@ -268,7 +268,7 @@ do_test where8-3.12 { execsql_status { SELECT a, d FROM t1, t2 WHERE (a=d OR b=e) AND +a<5 ORDER BY a } -} {1 1 2 2 3 3 4 2 4 4 0 0} +} {1 1 2 2 3 3 4 2 4 4 9 0} do_test where8-3.13 { execsql_status { SELECT a, d FROM t1, t2 WHERE (a=d OR b=e) AND +a<5 diff --git a/test/where9.test b/test/where9.test index d6a8c597b4..4ffcad1470 100644 --- a/test/where9.test +++ b/test/where9.test @@ -420,7 +420,7 @@ do_test where9-4.5 { AND (c=31031 OR d IS NULL) ORDER BY +a } -} {1 {cannot use index: t1b}} +} {1 {no query solution}} do_test where9-4.6 { count_steps { SELECT a FROM t1 NOT INDEXED @@ -436,7 +436,7 @@ do_test where9-4.7 { AND (c=31031 OR d IS NULL) ORDER BY +a } -} {1 {cannot use index: t1c}} +} {1 {no query solution}} do_test where9-4.8 { catchsql { SELECT a FROM t1 INDEXED BY t1d @@ -444,7 +444,7 @@ do_test where9-4.8 { AND (c=31031 OR d IS NULL) ORDER BY +a } -} {1 {cannot use index: t1d}} +} {1 {no query solution}} ifcapable explain { # The (c=31031 OR d IS NULL) clause is preferred over b>1000 because @@ -598,7 +598,7 @@ do_test where9-6.3.5 { OR (b NOT NULL AND c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 98 sort 0} ;# DELETEs rows 90 91 92 97 +} {scan 0 sort 0} ;# DELETEs rows 90 91 92 97 do_test where9-6.3.6 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -615,7 +615,7 @@ do_test where9-6.3.7 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 98 sort 0} ;# Add 100 to rowids 90 91 92 97 +} {scan 0 sort 0} ;# Add 100 to rowids 90 91 92 97 do_test where9-6.3.8 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -705,7 +705,7 @@ do_test where9-6.6.1 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 98 sort 0} ;# DELETEs rows 90 91 92 97 +} {scan 0 sort 0} ;# DELETEs rows 90 91 92 97 do_test where9-6.6.2 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -722,7 +722,7 @@ do_test where9-6.6.3 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 98 sort 0} ;# Add 100 to rowids 90 91 92 97 +} {scan 0 sort 0} ;# Add 100 to rowids 90 91 92 97 do_test where9-6.6.4 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -768,19 +768,35 @@ do_test where9-6.7.4 { do_test where9-6.8.1 { catchsql { DELETE FROM t1 INDEXED BY t1b - WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) OR (b NOT NULL AND c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {1 {cannot use index: t1b}} +} {1 {no query solution}} do_test where9-6.8.2 { + catchsql { + UPDATE t1 INDEXED BY t1b SET a=a+100 + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } +} {1 {no query solution}} +do_test where9-6.8.3 { catchsql { UPDATE t1 INDEXED BY t1b SET a=a+100 WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) OR (b NOT NULL AND c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {1 {cannot use index: t1b}} +} {0 {}} +do_test where9-6.8.4 { + catchsql { + DELETE FROM t1 INDEXED BY t1b + WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } +} {0 {}} ############################################################################ # Test cases where terms inside an OR series are combined with AND terms diff --git a/test/whereF.test b/test/whereF.test index 57bdbee058..c98312a262 100644 --- a/test/whereF.test +++ b/test/whereF.test @@ -46,7 +46,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix x +set testprefix whereF do_execsql_test 1.0 { PRAGMA automatic_index = 0; @@ -63,7 +63,7 @@ foreach {tn sql} { } { do_test 1.$tn { db eval "EXPLAIN QUERY PLAN $sql" - } {/.*SCAN TABLE t2 .*SEARCH TABLE t1 .*/} + } {/.*SCAN TABLE t2\y.*SEARCH TABLE t1\y.*/} } do_execsql_test 2.0 { @@ -84,7 +84,7 @@ foreach {tn sql} { } { do_test 2.$tn { db eval "EXPLAIN QUERY PLAN $sql" - } {/.*SCAN TABLE t2 .*SEARCH TABLE t1 .*/} + } {/.*SCAN TABLE t2\y.*SEARCH TABLE t1\y.*/} } do_execsql_test 3.0 { @@ -109,7 +109,7 @@ foreach {tn sql} { } { do_test 3.$tn { db eval "EXPLAIN QUERY PLAN $sql" - } {/.*SCAN TABLE t2 .*SEARCH TABLE t1 .*/} + } {/.*SCAN TABLE t2\y.*SEARCH TABLE t1\y.*/} } finish_test From 4fe425ad798b7d11ba0adc4078df34cca5d4c848 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Jun 2013 17:08:06 +0000 Subject: [PATCH 087/109] "make test" now passing. FossilOrigin-Name: addd7f466d6ff55f82d907286650c26b06e9397b --- manifest | 40 ++++++++++++++++++++-------------------- manifest.uuid | 2 +- src/where.c | 13 +++++++++++-- test/autoindex1.test | 2 +- test/distinct.test | 2 +- test/eqp.test | 2 +- test/indexedby.test | 22 +++++++++++----------- test/orderby1.test | 10 ++++------ test/permutations.test | 15 +++++++-------- test/tkt-2a5629202f.test | 2 +- test/unordered.test | 4 ++-- test/vtab1.test | 25 +++++++++++++------------ test/vtab6.test | 4 ++-- test/where.test | 2 +- test/where2.test | 36 ++++++++++++++++++------------------ test/where3.test | 2 ++ test/where7.test | 4 ++-- 17 files changed, 98 insertions(+), 89 deletions(-) diff --git a/manifest b/manifest index 2445afe8f0..8b92ace178 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"queryplanner"\stest\spermutation.\s\sContinuing\srefinements\sto\sNGQP. -D 2013-06-12T14:52:39.194 +C "make\stest"\snow\spassing. +D 2013-06-12T17:08:06.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 67d70ce776aae6e81eddc182357bd5542dc9e023 +F src/where.c 58522e12fcb559ac138d1cf51348b6861345968e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -322,7 +322,7 @@ F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf -F test/autoindex1.test fadbdf682948e0be840a3958d1df5501cfca9d6e +F test/autoindex1.test fac4207316e4e954abd086eafdd571ef093c5f5d F test/autovacuum.test 9f22a7733f39c56ef6a5665d10145ac25d8cb574 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -419,7 +419,7 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 84da1414b2e6887fffd5ed571311b344c5b082ce +F test/distinct.test 44028aaf161a5e80a2f229622b3a174d3b352810 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test ddf3b2e4506e0813f46b69ccf55757c5570cc181 F test/e_delete.test 89aa84d3d1bd284a0689ede04bce10226a5aeaa5 @@ -440,7 +440,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 68dd66c8fa05551d056ebd7fc8ae3b2e1cb5836e +F test/eqp.test 8ef4d89dc44d8988b3ea6e60902905d739508db0 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -586,7 +586,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/indexedby.test e06e1bf5a933b53f7dbeedf45749d0d37c7067ff +F test/indexedby.test 0e959308707c808515c3a51363f7a9835027108c F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 489aa12a027c83d291f5034a83c8c32e6be1dca2 @@ -691,7 +691,7 @@ F test/notnull.test 2afad748d18fd66d01f66463de73b3e2501fb226 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 3c5ff32c23364fb3d567ad53b945837736e701ce +F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -707,7 +707,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test 4614301e38398df7fdd5f28f4ed8f272b328251b -F test/permutations.test 2ab3937fdd34e18712be8f4159ac354c4d27ce1d +F test/permutations.test 02a3c0174c943734d710ac01ca465198bff05819 F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -821,7 +821,7 @@ F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 -F test/tkt-2a5629202f.test befaa77b90426a5ca8025724279e0bc26336610e +F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 @@ -975,7 +975,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test cadb172bbd5a2e83cd644d186ccd602085e54edc F test/unixexcl.test a9870e46cc6f8390a494513d4f2bf55b5a8b3e46 -F test/unordered.test 08b4ea1c273891ebc10d5e5c3a352ac5c62a6fbf +F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c F test/update.test 8bc86fd7ef1a00014f76dc6a6a7c974df4aef172 F test/uri.test 63e03df051620a18f794b4f4adcdefb3c23b6751 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -986,12 +986,12 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 4057630287bfa5955628fe90a13d4c225d1c7352 -F test/vtab1.test 4403f987860ebddef1ce2de6db7216421035339d +F test/vtab1.test 1550abf90bc2b996f8c34e0be3fdb251af54fa41 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 F test/vtab5.test 889f444970393c73f1e077e2bdc5d845e157a391 -F test/vtab6.test c7f290d172609d636fbfc58166eadcb55d5c117c +F test/vtab6.test 5f5380c425e52993560ab4763db4f826d2ba7b09 F test/vtab7.test ae560ebea870ed04e9aa4177cc302f910faaabb5 F test/vtab8.test e19fa4a538fcd1bb66c22825fa8f71618fb13583 F test/vtab9.test ea58d2b95d61955f87226381716b2d0b1d4e4f9b @@ -1029,13 +1029,13 @@ F test/walro.test 6cc247a0cc9b36aeea2057dd28a922a1cdfbd630 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test cd13d4804b8f2986b176fb45c0a43fbbba64215a -F test/where2.test 58cc3f85c082015144bc34c7ffdbd64134f650ee -F test/where3.test f2a7027e1971d583069947edba0a31563e86284f +F test/where.test da54153a4c1571ea1b95659e5bec8119edf786aa +F test/where2.test dcf0ffafe0de55051c1373835a5a57aee6b50094 +F test/where3.test 157071521ceabc06bfd4d37106e4270a8956364d F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b -F test/where7.test e0741d85b6069effe53c6ed7682e18aa078f4b11 +F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test f6b9559723564a042927ee0f22003ac9bed71b21 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test f5752185fc19a1ee826a2386a249f05dbc25979b @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 40567fddd468d00295275af8df09a7a1785e684a -R 8ad423cc8976bc762dc67ac33e49e948 +P 25e2cde105a19293bdb9c001b48624e5d7f8c4e5 +R 74299348f4ebeefec70a5406f9e251d7 U drh -Z 7a74b3f46b1e20d0df5f0a7a123b8d69 +Z ad22135d0c0bd81f6331e3b3327b23e1 diff --git a/manifest.uuid b/manifest.uuid index 943b9de7f4..16f0323d65 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25e2cde105a19293bdb9c001b48624e5d7f8c4e5 \ No newline at end of file +addd7f466d6ff55f82d907286650c26b06e9397b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 06470f6fa4..9886d4e214 100644 --- a/src/where.c +++ b/src/where.c @@ -1908,7 +1908,8 @@ static WhereCost whereCostFromDouble(double x){ ** logN is a little off. */ static WhereCost estLog(WhereCost N){ - return whereCostFromInt(N) - 33; + WhereCost x = whereCostFromInt(N); + return x>33 ? x - 33 : 0; } /* @@ -4431,7 +4432,8 @@ static int whereLoopAddBtree( /* Automatic indexes */ if( !pBuilder->pBest - && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && pSrc->pIndex==0 && !pSrc->viaCoroutine && !pSrc->notIndexed && !pSrc->isCorrelated @@ -5526,6 +5528,13 @@ WhereInfo *sqlite3WhereBegin( pWhere = 0; } + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->bOBSat = 1; + if( pDistinct ) pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + /* Assign a bit from the bitmask to every term in the FROM clause. ** ** When assigning bitmask values to FROM clause cursors, it must be diff --git a/test/autoindex1.test b/test/autoindex1.test index f8ab0c9d57..9d67def05f 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -245,7 +245,7 @@ do_execsql_test autoindex1-600 { 1 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} 2 0 0 {SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date0 ORDER BY c,a LIMIT 3 } -} {1 100 4 sort} +} {1 100 4 nosort} do_test where-6.9.8 { cksort { SELECT * FROM t3 WHERE a=1 AND c>0 ORDER BY a DESC, c ASC LIMIT 3 diff --git a/test/where2.test b/test/where2.test index c55d4a12a3..e57192d85c 100644 --- a/test/where2.test +++ b/test/where2.test @@ -283,12 +283,12 @@ do_test where2-6.3 { queryplan { SELECT * FROM t1 WHERE w=99 OR w=100 OR 6=+w ORDER BY +w } -} {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 {}} +} {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 *} do_test where2-6.4 { queryplan { SELECT * FROM t1 WHERE w=99 OR +w=100 OR 6=w ORDER BY +w } -} {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 {}} +} {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 *} set ::idx {} ifcapable subquery {set ::idx i1zyx} @@ -324,7 +324,7 @@ do_test where2-6.7 { -- SELECT * FROM t2249b CROSS JOIN t2249a WHERE a=b; } -} {123 0123 nosort t2249b {} t2249a {}} +} {123 0123 nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.9 { queryplan { -- The + operator removes affinity from the rhs. No conversions @@ -332,13 +332,13 @@ do_test where2-6.9 { -- SELECT * FROM t2249b CROSS JOIN t2249a WHERE a=+b; } -} {nosort t2249b {} {} sqlite_autoindex_t2249a_1} +} {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.9.2 { # The same thing but with the expression flipped around. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE +b=a } -} {nosort t2249b {} {} sqlite_autoindex_t2249a_1} +} {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.10 { queryplan { -- Use + on both sides of the comparison to disable indices @@ -346,32 +346,32 @@ do_test where2-6.10 { -- SELECT * FROM t2249b CROSS JOIN t2249a WHERE +a=+b; } -} {nosort t2249b {} t2249a {}} +} {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.11 { # This will not attempt the OR optimization because of the a=b # comparison. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE a=b OR a='hello'; } -} {123 0123 nosort t2249b {} t2249a {}} +} {123 0123 nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.11.2 { # Permutations of the expression terms. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE b=a OR a='hello'; } -} {123 0123 nosort t2249b {} t2249a {}} +} {123 0123 nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.11.3 { # Permutations of the expression terms. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE 'hello'=a OR b=a; } -} {123 0123 nosort t2249b {} t2249a {}} +} {123 0123 nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.11.4 { # Permutations of the expression terms. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE a='hello' OR b=a; } -} {123 0123 nosort t2249b {} t2249a {}} +} {123 0123 nosort t2249b * t2249a sqlite_autoindex_t2249a_1} ifcapable explain&&subquery { # These tests are not run if subquery support is not included in the # build. This is because these tests test the "a = 1 OR a = 2" to @@ -385,7 +385,7 @@ ifcapable explain&&subquery { queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE a=+b OR a='hello'; } - } {nosort t2249b {} {} sqlite_autoindex_t2249a_1} + } {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.12.2 { # In this case, the +b disables the affinity conflict and allows # the OR optimization to be used again. The result is now an empty @@ -393,7 +393,7 @@ ifcapable explain&&subquery { queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE a='hello' OR +b=a; } - } {nosort t2249b {} {} sqlite_autoindex_t2249a_1} + } {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.12.3 { # In this case, the +b disables the affinity conflict and allows # the OR optimization to be used again. The result is now an empty @@ -401,14 +401,14 @@ ifcapable explain&&subquery { queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE +b=a OR a='hello'; } - } {nosort t2249b {} {} sqlite_autoindex_t2249a_1} + } {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} do_test where2-6.13 { # The addition of +a on the second term disabled the OR optimization. # But we should still get the same empty-set result as in where2-6.9. queryplan { SELECT * FROM t2249b CROSS JOIN t2249a WHERE a=+b OR +a='hello'; } - } {nosort t2249b {} t2249a {}} + } {nosort t2249b * t2249a sqlite_autoindex_t2249a_1} } # Variations on the order of terms in a WHERE clause in order @@ -417,7 +417,7 @@ do_test where2-6.20 { queryplan { SELECT * FROM t2249a x CROSS JOIN t2249a y WHERE x.a=y.a } -} {0123 0123 nosort x {} {} sqlite_autoindex_t2249a_1} +} {0123 0123 nosort x sqlite_autoindex_t2249a_1 y sqlite_autoindex_t2249a_1} ifcapable explain&&subquery { # These tests are not run if subquery support is not included in the # build. This is because these tests test the "a = 1 OR a = 2" to @@ -428,17 +428,17 @@ ifcapable explain&&subquery { queryplan { SELECT * FROM t2249a x CROSS JOIN t2249a y WHERE x.a=y.a OR y.a='hello' } - } {0123 0123 nosort x {} {} sqlite_autoindex_t2249a_1} + } {0123 0123 nosort x sqlite_autoindex_t2249a_1 y sqlite_autoindex_t2249a_1} do_test where2-6.22 { queryplan { SELECT * FROM t2249a x CROSS JOIN t2249a y WHERE y.a=x.a OR y.a='hello' } - } {0123 0123 nosort x {} {} sqlite_autoindex_t2249a_1} + } {0123 0123 nosort x sqlite_autoindex_t2249a_1 y sqlite_autoindex_t2249a_1} do_test where2-6.23 { queryplan { SELECT * FROM t2249a x CROSS JOIN t2249a y WHERE y.a='hello' OR x.a=y.a } - } {0123 0123 nosort x {} {} sqlite_autoindex_t2249a_1} + } {0123 0123 nosort x sqlite_autoindex_t2249a_1 y sqlite_autoindex_t2249a_1} } # Unique queries (queries that are guaranteed to return only a single diff --git a/test/where3.test b/test/where3.test index e6acafed07..a5137173f2 100644 --- a/test/where3.test +++ b/test/where3.test @@ -247,6 +247,7 @@ do_execsql_test where3-3.1 { 0 1 0 {SEARCH TABLE t301 USING INTEGER PRIMARY KEY (rowid=?)} } +if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be # full table scans that the query planner attempts put the table with # the fewest number of output rows as the outer loop. @@ -278,6 +279,7 @@ do_execsql_test where3-4.2 { 0 1 1 {SCAN TABLE t401} 0 2 2 {SCAN TABLE t402} } +} ;# endif # Verify that a performance regression encountered by firefox # has been fixed. diff --git a/test/where7.test b/test/where7.test index 3cb8781200..5032c698b2 100644 --- a/test/where7.test +++ b/test/where7.test @@ -23303,7 +23303,7 @@ do_test where7-2.1001.2 { # # The test case that follows is code from an actual # application with identifiers change and unused columns -# remove. +# removed. # do_execsql_test where7-3.1 { CREATE TABLE t301 ( @@ -23332,7 +23332,7 @@ do_execsql_test where7-3.1 { EXPLAIN QUERY PLAN SELECT t302.c1 - FROM t302 JOIN t301 ON t302.c8 = t301.c8 + FROM t302 JOIN t301 ON t302.c8 = +t301.c8 WHERE t302.c2 = 19571 AND t302.c3 > 1287603136 AND (t301.c4 = 1407449685622784 From 95ed68df55bce1b194dc6dd5b94dc8ed59506854 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Jun 2013 17:55:50 +0000 Subject: [PATCH 088/109] Bug fixes in the handling of virtual tables. FossilOrigin-Name: 25c0f7292a20c0db6ef176966d9987f29c7d73e5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4944aabb1c..86301aff88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\schanges\sfrom\strunk. -D 2013-06-12T17:17:45.907 +C Bug\sfixes\sin\sthe\shandling\sof\svirtual\stables. +D 2013-06-12T17:55:50.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 97f5d9c3a9dfa802d91f7723cd1d782a3ce78e7a +F src/where.c 568aef7548e2cad24f03b6099f07aa66b2fa269c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P addd7f466d6ff55f82d907286650c26b06e9397b 24fc9d4438a5615dd20af5419456166df83a72ea -R 400236daf5743c1455f090bd73116cba +P f2e15b1974e55373b5819e3b2326653f890f409f +R 6fa5c9c088caf080e0b4a5ec6cf7a456 U drh -Z eafccaa2bb28a08ba2c2409a2846f8df +Z 738850606e13a26064a82bd0faf55587 diff --git a/manifest.uuid b/manifest.uuid index 9a98bfb402..86fd19a1a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2e15b1974e55373b5819e3b2326653f890f409f \ No newline at end of file +25c0f7292a20c0db6ef176966d9987f29c7d73e5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index abf4ebf56e..5b699a8663 100644 --- a/src/where.c +++ b/src/where.c @@ -3211,6 +3211,7 @@ static Bitmask codeOneLoopStart( for(j=0; jaLTerm[j]; + if( pTerm==0 ) continue; if( pTerm->eOperator & WO_IN ){ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); addrNotFound = pLevel->addrNxt; @@ -4704,7 +4705,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sSubBuild.pOrderBy = 0; sSubBuild.pBest = &sBest; - for(pOrTerm=pOrWC->a; pOrTerma; rc==SQLITE_OK && pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; }else if( pOrTerm->leftCursor==iCur ){ From 3b48e8c91a631d35c2479087d1db944d19d72f77 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Jun 2013 20:18:16 +0000 Subject: [PATCH 089/109] Activate the one-pass optimization. Update comments, especially the descriptions of the various objects. FossilOrigin-Name: e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d --- manifest | 12 ++-- manifest.uuid | 2 +- src/where.c | 178 +++++++++++++++++++++++++++----------------------- 3 files changed, 105 insertions(+), 87 deletions(-) diff --git a/manifest b/manifest index 86301aff88..b3d0bc075d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfixes\sin\sthe\shandling\sof\svirtual\stables. -D 2013-06-12T17:55:50.456 +C Activate\sthe\sone-pass\soptimization.\s\sUpdate\scomments,\sespecially\sthe\ndescriptions\sof\sthe\svarious\sobjects. +D 2013-06-12T20:18:16.225 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 568aef7548e2cad24f03b6099f07aa66b2fa269c +F src/where.c 4209851115ec883e004f0e0ec496215c9df10e33 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f2e15b1974e55373b5819e3b2326653f890f409f -R 6fa5c9c088caf080e0b4a5ec6cf7a456 +P 25c0f7292a20c0db6ef176966d9987f29c7d73e5 +R abdb63c5ef041b9c2f729e7a4c7abdae U drh -Z 738850606e13a26064a82bd0faf55587 +Z 9e2bd249132b4bdcba34d19dbfd465f2 diff --git a/manifest.uuid b/manifest.uuid index 86fd19a1a4..f1e38ab6d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25c0f7292a20c0db6ef176966d9987f29c7d73e5 \ No newline at end of file +e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5b699a8663..62cef439ba 100644 --- a/src/where.c +++ b/src/where.c @@ -27,10 +27,10 @@ #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) -# define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X +# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X # define WHERETRACE_ENABLED 1 #else -# define WHERETRACE(X) +# define WHERETRACE(K,X) #endif /* Forward reference @@ -48,23 +48,32 @@ typedef struct WhereScan WhereScan; /* ** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The -** maximum cost is 64*(2**63) which becomes 6900. So all costs can be -** be stored in a 16-bit unsigned integer without risk of overflow. +** maximum cost for ordinary tables is 64*(2**63) which becomes 6900. +** (Virtual tables can return a larger cost, but let's assume they do not.) +** So all costs can be stored in a 16-bit unsigned integer without risk +** of overflow. +** +** Costs are estimates, so don't go to the computational trouble to compute +** 10*log2(X) exactly. Instead, a close estimate is used. Any value of +** X<=1 is stored as 0. X=2 is 10. X=3 is 16. X=1000 is 99. etc. +** */ typedef unsigned short int WhereCost; /* -** For each nested loop in a WHERE clause implementation, the WhereInfo -** structure contains a single instance of this structure. This structure -** is intended to be private to the where.c module and should not be -** access or modified by other modules. +** This object contains information needed to implement a single nestd +** loop in WHERE clause. ** -** The pIdxInfo field is used to help pick the best index on a -** virtual table. The pIdxInfo pointer contains indexing -** information for the i-th table in the FROM clause before reordering. -** All the pIdxInfo pointers are freed by whereInfoFree() in where.c. -** All other information in the i-th WhereLevel object for the i-th table -** after FROM clause ordering. +** Contrast this object with WhereLoop. This object describes the +** implementation of the loop. WhereLoop describes the algorithm. +** This object contains a pointer to the WhereLoop algorithm as one of +** its elements. +** +** The WhereInfo object contains a single instance of this object for +** each term in the FROM clause (which is to say, for each of the +** nested loops as implemented). The order of WhereLevel objects determines +** the loop nested order, with WhereInfo.a[0] being the outer loop and +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. */ struct WhereLevel { int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ @@ -92,10 +101,18 @@ struct WhereLevel { }; /* -** Each instance of this object represents a way of evaluating one -** term of a join. The WhereClause object holds a table of these -** objects using (maskSelf,prereq,) as the primary key. Note that the -** same join term might have multiple associated WhereLoop objects. +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. */ struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ @@ -136,7 +153,21 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); /* ** Each instance of this object holds a sequence of WhereLoop objects -** that implement some or all of the entire query plan. +** that implement some or all of a query plan. +** +** Think of each WhereLoop objects as a node in a graph, which arcs +** showing dependences and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependences are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the choosen query plan. */ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ @@ -323,7 +354,8 @@ struct WhereMaskSet { }; /* -** This object is a factory for WhereLoop objects for a particular query. +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. */ struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ @@ -339,6 +371,9 @@ struct WhereLoopBuilder { ** half does the tail of the WHERE loop. An instance of ** this structure is returned by the first half and passed ** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. */ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ @@ -364,9 +399,10 @@ struct WhereInfo { }; /* -** Bitmasks for the operators that indices are able to exploit. An +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for -** terms in the where clause. +** particular WhereTerms within a WhereClause. */ #define WO_IN 0x001 #define WO_EQ 0x002 @@ -385,9 +421,9 @@ struct WhereInfo { #define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ /* -** Value for wsFlags returned by bestIndex() and stored in -** WhereLevel.wsFlags. These flags determine which search -** strategies are appropriate. +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. */ #define WHERE_COLUMN_EQ 0x00000001 /* x=EXPR or x IN (...) or x IS NULL */ #define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ @@ -409,6 +445,7 @@ struct WhereInfo { /* Convert a WhereCost value (10 times log2(X)) into its integer value X. +** A rough approximation is used. The value returned is not exact. */ static u64 whereCostToInt(WhereCost x){ u64 n; @@ -602,7 +639,7 @@ static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ } /* -** Initialize an expression mask set (a WhereMaskSet object) +** Initialize a WhereMaskSet object */ #define initMaskSet(P) (P)->n=0 @@ -635,18 +672,9 @@ static void createMask(WhereMaskSet *pMaskSet, int iCursor){ } /* -** This routine walks (recursively) an expression tree and generates +** These routine walk (recursively) an expression tree and generates ** a bitmask indicating which tables are used in that expression ** tree. -** -** In order for this routine to work, the calling function must have -** previously invoked sqlite3ResolveExprNames() on the expression. See -** the header comment on that routine for additional information. -** The sqlite3ResolveExprNames() routines looks for column names and -** sets their opcodes to TK_COLUMN and their Expr.iTable fields to -** the VDBE cursor number of the table. This routine just has to -** translate the cursor numbers into bitmask values and OR all -** the bitmasks together. */ static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); @@ -700,7 +728,7 @@ static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", and "IN". +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" ** ** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be ** of one of the following forms: column = expression column > expression @@ -727,10 +755,9 @@ static int allowedOp(int op){ ** are converted into "Y op X". ** ** If left/right precedence rules come into play when determining the -** collating -** side of the comparison, it remains associated with the same side after -** the commutation. So "Y collate NOCASE op X" becomes -** "X op Y". This is because any collation sequence on +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on ** the left hand side of a comparison overrides any collation sequence ** attached to the right. For the same reason the EP_Collate flag ** is not commuted. @@ -871,6 +898,11 @@ WhereTerm *whereScanNext(WhereScan *pScan){ ** for terms of the form "X " where X is column iColumn of table ** iCur. The must be one of the operators described by opMask. ** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** ** If X is not the INTEGER PRIMARY KEY then X must be compatible with ** index pIdx. */ @@ -959,8 +991,6 @@ static void exprAnalyze(SrcList*, WhereClause*, int); /* ** Call exprAnalyze on all terms in a WHERE clause. -** -** */ static void exprAnalyzeAll( SrcList *pTabList, /* the FROM clause */ @@ -1739,11 +1769,8 @@ static void exprAnalyze( } /* -** This function searches the expression list passed as the second argument -** for an expression of type TK_COLUMN that refers to the same column and -** uses the same collation sequence as the iCol'th column of index pIdx. -** Argument iBase is the cursor number used for the table that pIdx refers -** to. +** This function searches pList for a entry that matches the iCol-th column +** of index pIdx. ** ** If such an expression is found, its index in pList->a[] is returned. If ** no expression is found, -1 is returned. @@ -1778,7 +1805,7 @@ static int findIndexCol( ** Return true if the DISTINCT expression-list passed as the third argument ** is redundant. ** -** A DISTINCT list is redundant if the database contains some set of +** A DISTINCT list is redundant if the database contains some subset of ** columns that are unique and non-null. */ static int isDistinctRedundant( @@ -1842,7 +1869,10 @@ static int isDistinctRedundant( } /* -** The sum of two WhereCosts +** The (an approximate) sum of two WhereCosts. This computation is +** not a simple "+" operator because WhereCost is stored as a logarithmic +** value. +** */ static WhereCost whereCostAdd(WhereCost a, WhereCost b){ static const unsigned char x[] = { @@ -1868,7 +1898,8 @@ static WhereCost whereCostAdd(WhereCost a, WhereCost b){ } /* -** Convert an integer into a WhereCost +** Convert an integer into a WhereCost. In other words, compute a +** good approximatation for 10*log2(x). */ static WhereCost whereCostFromInt(tRowcnt x){ static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; @@ -1886,7 +1917,8 @@ static WhereCost whereCostFromInt(tRowcnt x){ #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Convert a double (as received from xBestIndex of a virtual table) -** into a WhereCost +** into a WhereCost. In other words, compute an approximation for +** 10*log2(x). */ static WhereCost whereCostFromDouble(double x){ u64 a; @@ -1901,11 +1933,7 @@ static WhereCost whereCostFromDouble(double x){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ /* -** Prepare a crude estimate of the logarithm of the input value. -** The results need not be exact. This is only used for estimating -** the total cost of performing operations with O(logN) or O(NlogN) -** complexity. Because N is just a guess, it is no great tragedy if -** logN is a little off. +** Estimate the logarithm of the input value to base 2. */ static WhereCost estLog(WhereCost N){ WhereCost x = whereCostFromInt(N); @@ -2158,8 +2186,6 @@ static sqlite3_index_info *allocateIndexInfo( int nOrderBy; sqlite3_index_info *pIdxInfo; - /*WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName));*/ - /* Count the number of possible WHERE clause constraints referring ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ @@ -2250,8 +2276,8 @@ static sqlite3_index_info *allocateIndexInfo( /* ** The table object reference passed as the second argument to this function ** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite3_index_info pointer passed -** as the argument. +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. ** ** If an error occurs, pParse is populated with an error message and a ** non-zero value is returned. Otherwise, 0 is returned and the output @@ -2266,7 +2292,6 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ int i; int rc; - /*WHERETRACE(("xBestIndex for %s\n", pTab->zName));*/ TRACE_IDX_INPUTS(p); rc = pVtab->pModule->xBestIndex(pVtab, p); TRACE_IDX_OUTPUTS(p); @@ -2578,8 +2603,8 @@ static int whereRangeScanEst( iBase -= whereCostFromInt(iUpper - iLower); } *pRangeDiv = iBase; - /*WHERETRACE(("range scan regions: %u..%u div=%g\n", - (u32)iLower, (u32)iUpper, *pRangeDiv));*/ + WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", + (u32)iLower, (u32)iUpper, *pRangeDiv)); return SQLITE_OK; } } @@ -2640,7 +2665,7 @@ static int whereEqualScanEst( if( pRhs==0 ) return SQLITE_NOTFOUND; rc = whereKeyStats(pParse, p, pRhs, 0, a); if( rc==SQLITE_OK ){ - /*WHERETRACE(("equality scan regions: %d\n", (int)a[1]));*/ + WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); *pnRow = a[1]; } whereEqualScanEst_cancel: @@ -2686,7 +2711,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; *pnRow = nRowEst; - /*WHERETRACE(("IN row estimate: est=%g\n", nRowEst));*/ + WHERETRACE(0x100,("IN row estimate: est=%g\n", nRowEst)); } return rc; } @@ -5060,9 +5085,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ nLoop = pWInfo->nLevel; mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace>=2 ) sqlite3DebugPrintf("---- begin solver\n"); -#endif + WHERETRACE(0x002, ("---- begin solver\n")); /* Allocate and initialize space for aTo and aFrom */ ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; @@ -5088,11 +5111,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ }else{ /* Compute an estimate on the cost to sort the entire result set */ rSortCost = nRowEst + estLog(nRowEst); -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace>=2 ){ - sqlite3DebugPrintf("---- sort cost=%-3d\n", rSortCost); - } -#endif + WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } /* Compute successively longer WherePaths using the previous generation @@ -5605,7 +5624,7 @@ WhereInfo *sqlite3WhereBegin( } /* Construct the WhereLoop objects */ - WHERETRACE(("*** Optimizer Start ***\n")); + WHERETRACE(0xffff,("*** Optimizer Start ***\n")); if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; @@ -5664,21 +5683,20 @@ WhereInfo *sqlite3WhereBegin( } } #endif - WHERETRACE(("*** Optimizer Finished ***\n")); + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; -#if 0 /* FIXME: Add this back in? */ /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. ** The one-pass algorithm only works if the WHERE clause constraints ** the statement to update a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_ONEROW)!=0 ){ + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ pWInfo->okOnePass = 1; - pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; } -#endif /* Open all tables in the pTabList and any indices selected for ** searching those tables. From 12ffbc7e690fb2b6f001493c16efcdaa24f9be8b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Jun 2013 14:51:53 +0000 Subject: [PATCH 090/109] Fix an off-by-one error in the WhereCost to integer conversion. FossilOrigin-Name: b5ca80d924f8c6d31f036247ba6e20d234f4482e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- tool/wherecosttest.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b3d0bc075d..91e4990458 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Activate\sthe\sone-pass\soptimization.\s\sUpdate\scomments,\sespecially\sthe\ndescriptions\sof\sthe\svarious\sobjects. -D 2013-06-12T20:18:16.225 +C Fix\san\soff-by-one\serror\sin\sthe\sWhereCost\sto\sinteger\sconversion. +D 2013-06-13T14:51:53.839 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 4209851115ec883e004f0e0ec496215c9df10e33 +F src/where.c c6944d98be4dfd0f080c2e205909ca6c25c21224 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,9 +1093,9 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b +F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 25c0f7292a20c0db6ef176966d9987f29c7d73e5 -R abdb63c5ef041b9c2f729e7a4c7abdae +P e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d +R 70bdc5e3664356410e482d1133202557 U drh -Z 9e2bd249132b4bdcba34d19dbfd465f2 +Z 22748fc276d8e873b764764d1effa482 diff --git a/manifest.uuid b/manifest.uuid index f1e38ab6d5..4172d0480a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d \ No newline at end of file +b5ca80d924f8c6d31f036247ba6e20d234f4482e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 62cef439ba..8593dc4d94 100644 --- a/src/where.c +++ b/src/where.c @@ -449,7 +449,7 @@ struct WhereInfo { */ static u64 whereCostToInt(WhereCost x){ u64 n; - if( x<=10 ) return 1; + if( x<10 ) return 1; n = x%10; x /= 10; if( n>=5 ) n -= 2; diff --git a/tool/wherecosttest.c b/tool/wherecosttest.c index 3055358fc2..39d4a7a368 100644 --- a/tool/wherecosttest.c +++ b/tool/wherecosttest.c @@ -73,7 +73,7 @@ WhereCost whereCostFromInteger(int x){ } static unsigned long int whereCostToInt(WhereCost x){ unsigned long int n; - if( x<=10 ) return 1; + if( x<10 ) return 1; n = x%10; x /= 10; if( n>=5 ) n -= 2; From e1e2e9acd6a66c7a3b6cf133dcf76711271d7049 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Jun 2013 15:16:53 +0000 Subject: [PATCH 091/109] Make the MIN() and MAX() macros available in sqliteInt.h. Add TUNING comments to the NGQP and adjust costs slightly. FossilOrigin-Name: 3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 --- manifest | 22 +++++----- manifest.uuid | 2 +- src/backup.c | 6 --- src/memjournal.c | 6 --- src/os_win.c | 7 --- src/sqliteInt.h | 6 +++ src/where.c | 102 ++++++++++++++++++++++++++++++------------- test/autoindex1.test | 6 +++ 8 files changed, 96 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 91e4990458..d0d1a72243 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\sWhereCost\sto\sinteger\sconversion. -D 2013-06-13T14:51:53.839 +C Make\sthe\sMIN()\sand\sMAX()\smacros\savailable\sin\ssqliteInt.h.\s\sAdd\sTUNING\ncomments\sto\sthe\sNGQP\sand\sadjust\scosts\sslightly. +D 2013-06-13T15:16:53.047 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -159,7 +159,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c 27e541b9b5b48b41eb899b22a49ff42384899151 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 +F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0 @@ -191,7 +191,7 @@ F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f -F src/memjournal.c 41a598445c8f249bd9b26ecae700c60dcf34cbf3 +F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 @@ -202,7 +202,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42c9b8b7c61c9fa3561258f523be5749e52ed0e0 -F src/os_win.c 5f018dbd4cec25c5b47e11432b946a7d2ccee60b +F src/os_win.c 074cb2b9bca6a1c2bd72acf04666cdc554bfaa9b F src/pager.c 79df56da9dd49aceaa4cac207484a9a82cba40ae F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 9acfcc83ddbf0cf82f0ed9582ccf0ad6c366ff37 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 026a52f3e24c15603fddb78025e786b96c4c0d2c +F src/sqliteInt.h 82a0f3e7c3410cc748c80268e0831dba5ea98cb4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c6944d98be4dfd0f080c2e205909ca6c25c21224 +F src/where.c 2e8e0a18dcef3a56a5874a33ba5ec4339f6911ce F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -322,7 +322,7 @@ F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf -F test/autoindex1.test fac4207316e4e954abd086eafdd571ef093c5f5d +F test/autoindex1.test 6a22a566b2dcbc968a62f8282af81574055d1bbf F test/autovacuum.test 9f22a7733f39c56ef6a5665d10145ac25d8cb574 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d -R 70bdc5e3664356410e482d1133202557 +P b5ca80d924f8c6d31f036247ba6e20d234f4482e +R 224bd6bce7ee1e3846617c8c2c148fe1 U drh -Z 22748fc276d8e873b764764d1effa482 +Z df43f94251ce9f1e894782fd34764619 diff --git a/manifest.uuid b/manifest.uuid index 4172d0480a..b4af3a8785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5ca80d924f8c6d31f036247ba6e20d234f4482e \ No newline at end of file +3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 252f61cfca..6ef9374ec6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -15,12 +15,6 @@ #include "sqliteInt.h" #include "btreeInt.h" -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** Structure allocated for each backup operation. */ diff --git a/src/memjournal.c b/src/memjournal.c index 05725948f6..65ed378b38 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -31,12 +31,6 @@ typedef struct FileChunk FileChunk; */ #define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** The rollback journal is composed of a linked list of these structures. */ diff --git a/src/os_win.c b/src/os_win.c index 07b74cbe22..bdf025aa4c 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -84,13 +84,6 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); #endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ -/* -** Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** Some Microsoft compilers lack this definition. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b22689462c..b2cab7a7ad 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -395,6 +395,12 @@ #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) #endif +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) diff --git a/src/where.c b/src/where.c index 8593dc4d94..000df0e107 100644 --- a/src/where.c +++ b/src/where.c @@ -57,6 +57,10 @@ typedef struct WhereScan WhereScan; ** 10*log2(X) exactly. Instead, a close estimate is used. Any value of ** X<=1 is stored as 0. X=2 is 10. X=3 is 16. X=1000 is 99. etc. ** +** The tool/wherecosttest.c source file implements a command-line program +** that will convert between WhereCost to integers and do addition and +** multiplication on WhereCost values. That command-line program is a +** useful utility to have around when working with this module. */ typedef unsigned short int WhereCost; @@ -1901,7 +1905,7 @@ static WhereCost whereCostAdd(WhereCost a, WhereCost b){ ** Convert an integer into a WhereCost. In other words, compute a ** good approximatation for 10*log2(x). */ -static WhereCost whereCostFromInt(tRowcnt x){ +static WhereCost whereCost(tRowcnt x){ static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; WhereCost y = 40; if( x<8 ){ @@ -1925,7 +1929,7 @@ static WhereCost whereCostFromDouble(double x){ WhereCost e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=1 ) return 0; - if( x<=2000000000 ) return whereCostFromInt((tRowcnt)x); + if( x<=2000000000 ) return whereCost((tRowcnt)x); memcpy(&a, &x, 8); e = (a>>52) - 1022; return e*10; @@ -1936,7 +1940,7 @@ static WhereCost whereCostFromDouble(double x){ ** Estimate the logarithm of the input value to base 2. */ static WhereCost estLog(WhereCost N){ - WhereCost x = whereCostFromInt(N); + WhereCost x = whereCost(N); return x>33 ? x - 33 : 0; } @@ -2598,9 +2602,9 @@ static int whereRangeScanEst( sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK ){ - WhereCost iBase = whereCostFromInt(p->aiRowEst[0]); + WhereCost iBase = whereCost(p->aiRowEst[0]); if( iUpper>iLower ){ - iBase -= whereCostFromInt(iUpper - iLower); + iBase -= whereCost(iUpper - iLower); } *pRangeDiv = iBase; WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", @@ -2615,11 +2619,13 @@ static int whereRangeScanEst( #endif assert( pLower || pUpper ); *pRangeDiv = 0; + /* TUNING: Each inequality constraint reduces the search space 4-fold. + ** A BETWEEN operator, therefore, reduces the search space 16-fold */ if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + *pRangeDiv += 20; assert( 20==whereCost(4) ); } if( pUpper ){ - *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + *pRangeDiv += 20; assert( 20==whereCost(4) ); } return rc; } @@ -4244,7 +4250,7 @@ static int whereLoopAddBtreeIndex( if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = whereCostFromInt(pProbe->aiRowEst[pNew->u.btree.nEq+1]); + nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); }else{ iCol = -1; nRowEst = 0; @@ -4257,7 +4263,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(whereCostFromInt(pProbe->aiRowEst[0])); + rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; if( pTerm->prereqRight & pNew->maskSelf ) continue; @@ -4267,16 +4273,16 @@ static int whereLoopAddBtreeIndex( if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; + pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 46; assert( 46==whereCostFromInt(25) ); + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==whereCost(25) ); }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ - nIn = whereCostFromInt(pExpr->x.pList->nExpr); + nIn = whereCost(pExpr->x.pList->nExpr); } pNew->rRun += nIn; pNew->u.btree.nEq++; @@ -4297,7 +4303,8 @@ static int whereLoopAddBtreeIndex( }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; - nIn = 10; assert( 10==whereCostFromInt(2) ); + /* TUNING: IS NULL selects 2 rows */ + nIn = 10; assert( 10==whereCost(2) ); pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; @@ -4325,7 +4332,7 @@ static int whereLoopAddBtreeIndex( && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); } - pNew->nOut = whereCostFromInt(nOut); + pNew->nOut = whereCost(nOut); } #endif if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ @@ -4451,7 +4458,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = whereCostFromInt(pSrc->pTab->nRowEst); + rSize = whereCost(pSrc->pTab->nRowEst); rLogSize = estLog(rSize); /* Automatic indexes */ @@ -4473,9 +4480,12 @@ static int whereLoopAddBtree( pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; - assert( 43==whereCostFromInt(20) ); - pNew->rSetup = 43 + rLogSize + rSize; - pNew->nOut = 33; assert( 33==whereCostFromInt(10) ); + /* TUNING: One-time cost for computing the automatic index is + ** approximately 6*N*log2(N) where N is the number of rows in + ** the table being indexed. */ + pNew->rSetup = rLogSize + rSize + 26; assert( 26==whereCost(6) ); + /* TUNING: Each index lookup yields 10 rows in the table */ + pNew->nOut = 33; assert( 33==whereCost(10) ); pNew->rRun = whereCostAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_TEMP_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; @@ -4501,7 +4511,12 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 - b; + /* TUNING: Cost of full table scan is 3*(N + log2(N)). + ** + The extra 3 factor is to encourage the use of indexed lookups + ** over full scans. A smaller constant 2 is used for covering + ** index scans so that a covering index scan will be favored over + ** a table scan. */ + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; }else{ @@ -4517,8 +4532,22 @@ static int whereLoopAddBtree( ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize); - pNew->rRun += ((m!=0) ? rLogSize : 10) - b; + if( m==0 ){ + /* TUNING: Cost of a covering index scan is 2*(N + log2(N)). + ** + The extra 2 factor is to encourage the use of indexed lookups + ** over index scans. A table scan uses a factor of 3 so that + ** index scans are favored over table scans. + ** + If this covering index might also help satisfy the ORDER BY + ** clause, then the cost is fudged down slightly so that this + ** index is favored above other indices that have no hope of + ** helping with the ORDER BY. */ + pNew->rRun = 10 + whereCostAdd(rSize,rLogSize) - b; + }else{ + assert( b!=0 ); + /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) + ** which we will simplify to just N*log2(N) */ + pNew->rRun = rSize + rLogSize; + } rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; } @@ -4675,7 +4704,8 @@ static int whereLoopAddVirtual( && pIdxInfo->orderByConsumed); pNew->rSetup = 0; pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost); - pNew->nOut = 46; assert( 46 == whereCostFromInt(25) ); + /* TUNING: Every virtual table query returns 25 rows */ + pNew->nOut = 46; assert( 46==whereCost(25) ); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -5066,6 +5096,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ @@ -5081,8 +5112,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ WhereLoop **pX; /* Used to divy up the pSpace memory */ char *pSpace; /* Temporary memory used by this routine */ - db = pWInfo->pParse->db; + pParse = pWInfo->pParse; + db = pParse->db; nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver\n")); @@ -5099,8 +5134,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pFrom->aLoop = pX; } - /* Seed the search with a single WherePath containing zero WhereLoops */ - aFrom[0].nRow = pWInfo->pParse->nQueryLoop; + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 25. If the cost + ** of computing an automatic index is not paid back within the first 25 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==whereCost(25) ); nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller @@ -5109,7 +5148,8 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrderedValid = 1; }else{ - /* Compute an estimate on the cost to sort the entire result set */ + /* TUNING: Estimated cost of sorting is N*log2(N) where N is the + ** number of output rows. */ rSortCost = nRowEst + estLog(nRowEst); WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } @@ -5254,7 +5294,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ } if( nFrom==0 ){ - sqlite3ErrorMsg(pWInfo->pParse, "no query solution"); + sqlite3ErrorMsg(pParse, "no query solution"); sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } @@ -5335,7 +5375,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; pLoop->u.btree.nEq = 1; - pLoop->rRun = 33; /* 33 == whereCostFromInt(10) */ + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==whereCost(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; @@ -5353,7 +5394,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->nLTerm = j; pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; - pLoop->rRun = 39; /* 39 == whereCostFromInt(15) */ + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==whereCost(15) */ break; } } diff --git a/test/autoindex1.test b/test/autoindex1.test index 9d67def05f..f7b0977fbc 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -78,6 +78,9 @@ do_test autoindex1-202 { do_test autoindex1-210 { db eval { PRAGMA automatic_index=ON; + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } } {11 911 22 922 33 933 44 944 55 955 66 966 77 977 88 988} @@ -143,6 +146,9 @@ do_test autoindex1-401 { do_execsql_test autoindex1-500 { CREATE TABLE t501(a INTEGER PRIMARY KEY, b); CREATE TABLE t502(x INTEGER PRIMARY KEY, y); + INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t501',null,'1000000'); + INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t502',null,'1000'); + ANALYZE sqlite_master; EXPLAIN QUERY PLAN SELECT b FROM t501 WHERE t501.a IN (SELECT x FROM t502 WHERE y=?); From bbe8b24b8d96b8e0c79c68869358e7501ad007a8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Jun 2013 15:50:59 +0000 Subject: [PATCH 092/109] Restore the ability to do a BETWEEN query on the rowid. Also fix a nearby comment. FossilOrigin-Name: 459a7b9068310b36fed950940d251c4b798ffc58 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d0d1a72243..1780bda7cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sMIN()\sand\sMAX()\smacros\savailable\sin\ssqliteInt.h.\s\sAdd\sTUNING\ncomments\sto\sthe\sNGQP\sand\sadjust\scosts\sslightly. -D 2013-06-13T15:16:53.047 +C Restore\sthe\sability\sto\sdo\sa\sBETWEEN\squery\son\sthe\srowid.\s\sAlso\sfix\sa\nnearby\scomment. +D 2013-06-13T15:50:59.429 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2e8e0a18dcef3a56a5874a33ba5ec4339f6911ce +F src/where.c 82b519b79e4efc45461f6ce3990292c01b5adfdd F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b5ca80d924f8c6d31f036247ba6e20d234f4482e -R 224bd6bce7ee1e3846617c8c2c148fe1 +P 3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 +R 1257e951bd26adefbb24e4a4d17e2b37 U drh -Z df43f94251ce9f1e894782fd34764619 +Z e1ac17bd2fbb6e506f40edf419e837ee diff --git a/manifest.uuid b/manifest.uuid index b4af3a8785..fe46658266 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 \ No newline at end of file +459a7b9068310b36fed950940d251c4b798ffc58 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 000df0e107..14bbf88a5b 100644 --- a/src/where.c +++ b/src/where.c @@ -4345,8 +4345,7 @@ static int whereLoopAddBtreeIndex( /* TBD: Adjust nOut for additional constraints */ rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEq<=pProbe->nColumn - && pProbe->zName!=0 + && pNew->u.btree.nEq<(pProbe->nColumn + (pProbe->zName!=0)) ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } From 53cfbe92d1ca41e6873e7c897023187d656f41c1 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Jun 2013 17:28:22 +0000 Subject: [PATCH 093/109] Make sure that disabling the covering index scan optimization does not prevent a covering index from being used to satisfy an ORDER BY clause. FossilOrigin-Name: e8b7ea8202c443bfc8a978588c7d2cfaa14a8fea --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 19 +++++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 1780bda7cd..97ae5f18b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restore\sthe\sability\sto\sdo\sa\sBETWEEN\squery\son\sthe\srowid.\s\sAlso\sfix\sa\nnearby\scomment. -D 2013-06-13T15:50:59.429 +C Make\ssure\sthat\sdisabling\sthe\scovering\sindex\sscan\soptimization\sdoes\snot\nprevent\sa\scovering\sindex\sfrom\sbeing\sused\sto\ssatisfy\san\sORDER\sBY\sclause. +D 2013-06-13T17:28:22.026 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 82b519b79e4efc45461f6ce3990292c01b5adfdd +F src/where.c 700085a8d8e0edf13db11e110577320ec7348edc F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 -R 1257e951bd26adefbb24e4a4d17e2b37 +P 459a7b9068310b36fed950940d251c4b798ffc58 +R 242dd2338fa025aa7191faa8afaf3ae6 U drh -Z e1ac17bd2fbb6e506f40edf419e837ee +Z 7fc4ebaffcaa0d84212cec77e769d428 diff --git a/manifest.uuid b/manifest.uuid index fe46658266..10384ea54c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -459a7b9068310b36fed950940d251c4b798ffc58 \ No newline at end of file +e8b7ea8202c443bfc8a978588c7d2cfaa14a8fea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 14bbf88a5b..46f741d159 100644 --- a/src/where.c +++ b/src/where.c @@ -445,7 +445,6 @@ struct WhereInfo { #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x00004000 /* Uses an ephemeral index */ -#define WHERE_COVER_SCAN 0x00008000 /* Full scan of a covering index */ /* Convert a WhereCost value (10 times log2(X)) into its integer value X. @@ -3614,8 +3613,7 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - if( (pLoop->wsFlags & (WHERE_COLUMN_EQ | WHERE_COLUMN_RANGE | - WHERE_COLUMN_NULL | WHERE_COLUMN_IN))==0 ){ + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ assert( pLevel->p5==0 ); @@ -4375,6 +4373,7 @@ static int indexMightHelpWithOrderBy( int iCol; int ii; + if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; iCol = pIndex->aiColumn[0]; for(ii=0; iinExpr; ii++){ @@ -4503,6 +4502,8 @@ static int whereLoopAddBtree( pNew->prereq = mExtra; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); if( pProbe->tnum<=0 ){ /* Integer primary key index */ pNew->wsFlags = WHERE_IPK; @@ -4523,11 +4524,13 @@ static int whereLoopAddBtree( pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; /* Full scan via index */ - if( (m==0 || b) - && pProbe->bUnordered==0 - && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 - && sqlite3GlobalConfig.bUseCis - && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + if( b + || ( m==0 + && pProbe->bUnordered==0 + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + ) ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; From 6d38147c191e9c7a34b505f2c3a8926bbddec9a6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Jun 2013 17:58:08 +0000 Subject: [PATCH 094/109] An index might be useful for ORDER BY if any indexed column is in the ORDER BY clause, not just the first indexed column. FossilOrigin-Name: ade473b5ae3fe2162b0ec29731d8e864a9301e07 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 9 ++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 97ae5f18b7..a5722835d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sdisabling\sthe\scovering\sindex\sscan\soptimization\sdoes\snot\nprevent\sa\scovering\sindex\sfrom\sbeing\sused\sto\ssatisfy\san\sORDER\sBY\sclause. -D 2013-06-13T17:28:22.026 +C An\sindex\smight\sbe\suseful\sfor\sORDER\sBY\sif\sany\sindexed\scolumn\sis\sin\sthe\nORDER\sBY\sclause,\snot\sjust\sthe\sfirst\sindexed\scolumn. +D 2013-06-13T17:58:08.642 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 700085a8d8e0edf13db11e110577320ec7348edc +F src/where.c ba5f6766cfc7fcee945d45120cbb6e39a881b8e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 459a7b9068310b36fed950940d251c4b798ffc58 -R 242dd2338fa025aa7191faa8afaf3ae6 +P e8b7ea8202c443bfc8a978588c7d2cfaa14a8fea +R 20475a52b4ff0eaa6499f9e0ae4a1e9c U drh -Z 7fc4ebaffcaa0d84212cec77e769d428 +Z f8930bf04903145489a941260cbef3e2 diff --git a/manifest.uuid b/manifest.uuid index 10384ea54c..1431942f12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8b7ea8202c443bfc8a978588c7d2cfaa14a8fea \ No newline at end of file +ade473b5ae3fe2162b0ec29731d8e864a9301e07 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 46f741d159..d702668aab 100644 --- a/src/where.c +++ b/src/where.c @@ -4370,18 +4370,17 @@ static int indexMightHelpWithOrderBy( int iCursor ){ ExprList *pOB; - int iCol; - int ii; + int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; - iCol = pIndex->aiColumn[0]; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); if( pExpr->op!=TK_COLUMN ) return 0; if( pExpr->iTable==iCursor ){ - if( pExpr->iColumn==iCol ) return 1; - return 0; + for(jj=0; jjnColumn; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } } } return 0; From b8916be945bebb123a03e0193a567f4934c787ba Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Jun 2013 02:51:48 +0000 Subject: [PATCH 095/109] Add a new ORDER BY optimization that bypasses ORDER BY terms that are constrained by == and IS NULL terms of the WHERE clause. FossilOrigin-Name: b920bb70bb009b7c54e7667544c9810c5ee25e19 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/where.c | 33 +++++++++++++++- test/orderby5.test | 97 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 test/orderby5.test diff --git a/manifest b/manifest index a5722835d2..ed857f034a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\sindex\smight\sbe\suseful\sfor\sORDER\sBY\sif\sany\sindexed\scolumn\sis\sin\sthe\nORDER\sBY\sclause,\snot\sjust\sthe\sfirst\sindexed\scolumn. -D 2013-06-13T17:58:08.642 +C Add\sa\snew\sORDER\sBY\soptimization\sthat\sbypasses\sORDER\sBY\sterms\sthat\sare\nconstrained\sby\s==\sand\sIS\sNULL\sterms\sof\sthe\sWHERE\sclause. +D 2013-06-14T02:51:48.285 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c ba5f6766cfc7fcee945d45120cbb6e39a881b8e0 +F src/where.c 7d406cca2f60afbbfb57470f0d4981798bccf6e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -695,6 +695,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 +F test/orderby5.test 02eca502a0f97c77ce383b0dfa17e99c6a107b8d F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 16b649c8f0b38d446acbcff8a7bf055a9be43276 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1095,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e8b7ea8202c443bfc8a978588c7d2cfaa14a8fea -R 20475a52b4ff0eaa6499f9e0ae4a1e9c +P ade473b5ae3fe2162b0ec29731d8e864a9301e07 +R eae0305c76446978c9d6694aa033807b U drh -Z f8930bf04903145489a941260cbef3e2 +Z 288ef6a3a068d55d460d1b487a44477a diff --git a/manifest.uuid b/manifest.uuid index 1431942f12..ade080fb52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ade473b5ae3fe2162b0ec29731d8e864a9301e07 \ No newline at end of file +b920bb70bb009b7c54e7667544c9810c5ee25e19 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d702668aab..2163ee7a56 100644 --- a/src/where.c +++ b/src/where.c @@ -4891,7 +4891,7 @@ static int wherePathSatisfiesOrderBy( Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ Bitmask obDone; /* Mask of all ORDER BY terms */ Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ - + Bitmask ready; /* Mask of inner loops */ /* ** We say the WhereLoop is "one-row" if it generates no more than one @@ -4931,10 +4931,39 @@ static int wherePathSatisfiesOrderBy( isOrderDistinct = 1; obDone = MASKBIT(nOrderBy)-1; orderDistinctMask = 0; + ready = 0; for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; pLoop = iLoopaLoop[iLoop] : pLast; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL, 0); + if( pTerm==0 ) continue; + if( pOBExpr->iColumn>=0 ){ + const char *z1, *z2; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + z1 = pColl->zName; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); + if( !pColl ) pColl = db->pDfltColl; + z2 = pColl->zName; + if( sqlite3StrICmp(z1, z2)!=0 ) continue; + } + obSat |= MASKBIT(i); + } + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ if( pLoop->wsFlags & WHERE_IPK ){ pIndex = 0; @@ -5066,7 +5095,7 @@ static int wherePathSatisfiesOrderBy( } } } - } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ if( obSat==obDone ) return 1; if( !isOrderDistinct ) return 0; if( isLastLoop ) return 1; diff --git a/test/orderby5.test b/test/orderby5.test new file mode 100644 index 0000000000..5bae5db717 --- /dev/null +++ b/test/orderby5.test @@ -0,0 +1,97 @@ +# 2013-06-14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the optimizations that disable +# ORDER BY clauses work correctly +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby5 + +# Generate test data for a join. Verify that the join gets the +# correct answer. +# +do_execsql_test 1.1 { + CREATE TABLE t1(a,b,c); + CREATE INDEX t1bc ON t1(b,c); + + EXPLAIN QUERY PLAN + SELECT DISTINCT a, b, c FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.2.1 { + EXPLAIN QUERY PLAN + SELECT DISTINCT a, c, b FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.2.2 { + EXPLAIN QUERY PLAN + SELECT DISTINCT a, c, b FROM t1 WHERE a='xyz' COLLATE nocase; +} {/B-TREE/} +do_execsql_test 1.2.3 { + EXPLAIN QUERY PLAN + SELECT DISTINCT a COLLATE nocase, c, b FROM t1 WHERE a='xyz'; +} {/B-TREE/} +do_execsql_test 1.2.4 { + EXPLAIN QUERY PLAN + SELECT DISTINCT a COLLATE nocase, c, b FROM t1 WHERE a='xyz' COLLATE nocase; +} {~/B-TREE/} +do_execsql_test 1.3 { + EXPLAIN QUERY PLAN + SELECT DISTINCT b, a, c FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.4 { + EXPLAIN QUERY PLAN + SELECT DISTINCT b, c, a FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.5 { + EXPLAIN QUERY PLAN + SELECT DISTINCT c, a, b FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.6 { + EXPLAIN QUERY PLAN + SELECT DISTINCT c, b, a FROM t1 WHERE a=0; +} {~/B-TREE/} +do_execsql_test 1.7 { + EXPLAIN QUERY PLAN + SELECT DISTINCT c, b, a FROM t1 WHERE +a=0; +} {/B-TREE/} +do_execsql_test 2.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; +} {~/B-TREE/} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; +} {/B-TREE/} +do_execsql_test 2.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY b, a, c; +} {~/B-TREE/} +do_execsql_test 2.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY b, c, a; +} {~/B-TREE/} +do_execsql_test 2.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY a, c, b; +} {/B-TREE/} +do_execsql_test 2.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY c, a, b; +} {/B-TREE/} +do_execsql_test 2.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a=0 ORDER BY c, b, a; +} {/B-TREE/} + + +finish_test From f003076032281c2f2dfc884f2e59934f811af6fe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Jun 2013 13:27:01 +0000 Subject: [PATCH 096/109] Comment tweaks in where.c. No changes to code. FossilOrigin-Name: cecc5fdd5d8fbad7d9e8c275b9ba9ade3dbee8ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 12 +++++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ed857f034a..299b950889 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snew\sORDER\sBY\soptimization\sthat\sbypasses\sORDER\sBY\sterms\sthat\sare\nconstrained\sby\s==\sand\sIS\sNULL\sterms\sof\sthe\sWHERE\sclause. -D 2013-06-14T02:51:48.285 +C Comment\stweaks\sin\swhere.c.\s\sNo\schanges\sto\scode. +D 2013-06-14T13:27:01.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 7d406cca2f60afbbfb57470f0d4981798bccf6e8 +F src/where.c cd2ee239c40ef7ffc952e176bfb98ff13a638b20 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ade473b5ae3fe2162b0ec29731d8e864a9301e07 -R eae0305c76446978c9d6694aa033807b +P b920bb70bb009b7c54e7667544c9810c5ee25e19 +R 382f7260a0330db641dd26f80cde0534 U drh -Z 288ef6a3a068d55d460d1b487a44477a +Z 568b215d3c8e178d6fdb12bf215bc7ae diff --git a/manifest.uuid b/manifest.uuid index ade080fb52..6a7fb20d1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b920bb70bb009b7c54e7667544c9810c5ee25e19 \ No newline at end of file +cecc5fdd5d8fbad7d9e8c275b9ba9ade3dbee8ef \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2163ee7a56..33370c383f 100644 --- a/src/where.c +++ b/src/where.c @@ -65,7 +65,7 @@ typedef struct WhereScan WhereScan; typedef unsigned short int WhereCost; /* -** This object contains information needed to implement a single nestd +** This object contains information needed to implement a single nested ** loop in WHERE clause. ** ** Contrast this object with WhereLoop. This object describes the @@ -5944,12 +5944,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } } - /* If this scan uses an index, make code substitutions to read data - ** from the index in preference to the table. Sometimes, this means - ** the table need never be read from. This is a performance boost, - ** as the vdbe level waits until the table is read before actually - ** seeking the table cursor to the record corresponding to the current - ** position in the index. + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. ** ** Calls to the code generator in between sqlite3WhereBegin and ** sqlite3WhereEnd will have created code that references the table From 613ba1ea9c851b35cbf131e5da52244b0d1081f1 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Jun 2013 15:11:45 +0000 Subject: [PATCH 097/109] Fix compiler warnings. FossilOrigin-Name: 3e8ac46918c68723bd199dbec8b0901457d524a9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 6 +++--- src/where.c | 13 ++++--------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 299b950889..c6e1871a02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\sin\swhere.c.\s\sNo\schanges\sto\scode. -D 2013-06-14T13:27:01.630 +C Fix\scompiler\swarnings. +D 2013-06-15T15:11:45.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 588ae13fc3d10dc812a0832fd143374583beb847 +F src/select.c 94a755b3d3788cf171c98064f2fa0ce6350fd6ca F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c cd2ee239c40ef7ffc952e176bfb98ff13a638b20 +F src/where.c df0d274ff134fab313f576f232e586756339e7a1 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b920bb70bb009b7c54e7667544c9810c5ee25e19 -R 382f7260a0330db641dd26f80cde0534 +P cecc5fdd5d8fbad7d9e8c275b9ba9ade3dbee8ef +R ef11f7373ad17a9fe10dde15795cd3f5 U drh -Z 568b215d3c8e178d6fdb12bf215bc7ae +Z 56f2db275e8c6ae6e91e7c2a5db315eb diff --git a/manifest.uuid b/manifest.uuid index 6a7fb20d1e..f8be393895 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cecc5fdd5d8fbad7d9e8c275b9ba9ade3dbee8ef \ No newline at end of file +3e8ac46918c68723bd199dbec8b0901457d524a9 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 8972faacc6..0c706bdbc2 100644 --- a/src/select.c +++ b/src/select.c @@ -1538,8 +1538,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); - }else{ - if( p->nSelectRow > n ) p->nSelectRow = n; + }else if( n>=0 && p->nSelectRow>(u64)n ){ + p->nSelectRow = n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -1733,7 +1733,7 @@ static int multiSelect( p->nSelectRow += pPrior->nSelectRow; if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && p->nSelectRow > nLimit + && nLimit>0 && p->nSelectRow > (u64)nLimit ){ p->nSelectRow = nLimit; } diff --git a/src/where.c b/src/where.c index 33370c383f..7777d7e878 100644 --- a/src/where.c +++ b/src/where.c @@ -2927,8 +2927,6 @@ static int codeEqualityTerm( static int codeAllEqualityTerms( Parse *pParse, /* Parsing context */ WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - WhereClause *pWC, /* The WHERE clause */ - Bitmask notReady, /* Which parts of FROM have not yet been coded */ int bRev, /* Reverse the order of IN operators */ int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ @@ -3475,9 +3473,7 @@ static Bitmask codeOneLoopStart( ** and store the values of those terms in an array of registers ** starting at regBase. */ - regBase = codeAllEqualityTerms( - pParse, pLevel, pWC, notReady, bRev, nExtraReg, &zStartAff - ); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); zEndAff = sqlite3DbStrDup(pParse->db, zStartAff); addrNxt = pLevel->addrNxt; @@ -4568,8 +4564,7 @@ static int whereLoopAddBtree( ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. */ static int whereLoopAddVirtual( - WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra /* Extra prerequesites for using this table */ + WhereLoopBuilder *pBuilder /* WHERE clause information */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ @@ -4779,7 +4774,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sBest.rRun = 0; #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mExtra); + rc = whereLoopAddVirtual(&sSubBuild); }else #endif { @@ -4836,7 +4831,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ } priorJoinType = pItem->jointype; if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(pBuilder, mExtra); + rc = whereLoopAddVirtual(pBuilder); }else{ rc = whereLoopAddBtree(pBuilder, mExtra); } From 75bf6b984ac35bdb16dbb42a06e4596cde365127 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Jun 2013 14:18:21 +0000 Subject: [PATCH 098/109] Use automatic indices on subqueries of the FROM clause when appropriate. FossilOrigin-Name: e8f124284ee0d0e373dc4431668630f1e17015c1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 - test/autoindex1.test | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c6e1871a02..c2a22a6d6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings. -D 2013-06-15T15:11:45.875 +C Use\sautomatic\sindices\son\ssubqueries\sof\sthe\sFROM\sclause\swhen\sappropriate. +D 2013-06-17T14:18:21.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c df0d274ff134fab313f576f232e586756339e7a1 +F src/where.c 74ecd744a5a9d18cc8e66f4c234bdac11c472eee F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -322,7 +322,7 @@ F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf -F test/autoindex1.test 6a22a566b2dcbc968a62f8282af81574055d1bbf +F test/autoindex1.test 314f12f87667861ac965c41587f9df82c42fff65 F test/autovacuum.test 9f22a7733f39c56ef6a5665d10145ac25d8cb574 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P cecc5fdd5d8fbad7d9e8c275b9ba9ade3dbee8ef -R ef11f7373ad17a9fe10dde15795cd3f5 +P 3e8ac46918c68723bd199dbec8b0901457d524a9 +R cb395ca086e8d43cd185b18cda8198f7 U drh -Z 56f2db275e8c6ae6e91e7c2a5db315eb +Z 602d1da2f8b38f776a1f4f76f4cb8197 diff --git a/manifest.uuid b/manifest.uuid index f8be393895..b46fbe6550 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e8ac46918c68723bd199dbec8b0901457d524a9 \ No newline at end of file +e8f124284ee0d0e373dc4431668630f1e17015c1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7777d7e878..0977dbea8d 100644 --- a/src/where.c +++ b/src/where.c @@ -5770,7 +5770,6 @@ WhereInfo *sqlite3WhereBegin( */ sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; - pWInfo->nRowOut = (WhereCost)1; for(ii=0, pLevel=pWInfo->a; ii? AND owner_change_date Date: Mon, 17 Jun 2013 18:20:48 +0000 Subject: [PATCH 099/109] Simplifications to the NGQP. Add the queryplantest makefile target. Add testcase() macros in the NGQP. FossilOrigin-Name: eaf1f1b405ec2c498092527fae00e5dbe9e176c1 --- main.mk | 3 +++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 3 ++- src/where.c | 39 +++++++++++---------------------------- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/main.mk b/main.mk index 845df4beaa..d8c4c29168 100644 --- a/main.mk +++ b/main.mk @@ -593,6 +593,9 @@ soaktest: testfixture$(EXE) sqlite3$(EXE) fulltestonly: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/full.test +queryplantest: testfixture$(EXE) sqlite3$(EXE) + ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner + test: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/veryquick.test diff --git a/manifest b/manifest index c2a22a6d6e..9a336de116 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sautomatic\sindices\son\ssubqueries\sof\sthe\sFROM\sclause\swhen\sappropriate. -D 2013-06-17T14:18:21.734 +C Simplifications\sto\sthe\sNGQP.\s\sAdd\sthe\squeryplantest\smakefile\starget.\s\sAdd\ntestcase()\smacros\sin\sthe\sNGQP. +D 2013-06-17T18:20:48.993 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -138,7 +138,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f2b23a6bde8f1c6e86b957e4d94eab0add520b0d -F main.mk e536751ac719806209c51f5dc63022a5dd40c631 +F main.mk c4335dbdb004d37e00e23d6ff226e55df0db5b21 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -215,7 +215,7 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 94a755b3d3788cf171c98064f2fa0ce6350fd6ca +F src/select.c d5a1b9bc3fb451e68ce907df253c6ac17e7310f7 F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 74ecd744a5a9d18cc8e66f4c234bdac11c472eee +F src/where.c 74627cb9415ad421004de574cfafe6ed0359554b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3e8ac46918c68723bd199dbec8b0901457d524a9 -R cb395ca086e8d43cd185b18cda8198f7 +P e8f124284ee0d0e373dc4431668630f1e17015c1 +R 5f6127cf4cee34c1f05fd60f463f5e2a U drh -Z 602d1da2f8b38f776a1f4f76f4cb8197 +Z 08d0c98ccbb424b49a424bc18bdd25d5 diff --git a/manifest.uuid b/manifest.uuid index b46fbe6550..2d08ee9b92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8f124284ee0d0e373dc4431668630f1e17015c1 \ No newline at end of file +eaf1f1b405ec2c498092527fae00e5dbe9e176c1 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0c706bdbc2..40b7ec86f0 100644 --- a/src/select.c +++ b/src/select.c @@ -4401,7 +4401,8 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + WHERE_GROUPBY, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereIsOrdered(pWInfo) ){ /* The optimizer is able to deliver rows in group by order so diff --git a/src/where.c b/src/where.c index 0977dbea8d..cf55858f7e 100644 --- a/src/where.c +++ b/src/where.c @@ -4321,6 +4321,8 @@ static int whereLoopAddBtreeIndex( if( pNew->u.btree.nEq==1 && pProbe->nSample ){ tRowcnt nOut = 0; if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( pTerm->eOperator & WO_EQ ); + testcase( pTerm->eOperator & WO_ISNULL ); rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &nOut); }else if( (pTerm->eOperator & WO_IN) && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ @@ -4861,7 +4863,6 @@ static int wherePathSatisfiesOrderBy( WherePath *pPath, /* The WherePath to check */ u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ u16 nLoop, /* Number of entries in pPath->aLoop[] */ - u8 isLastLoop, /* True if pLast is the inner-most loop */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ ){ @@ -4946,6 +4947,8 @@ static int wherePathSatisfiesOrderBy( pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, ~ready, WO_EQ|WO_ISNULL, 0); if( pTerm==0 ) continue; + testcase( pTerm->eOperator & WO_EQ ); + testcase( pTerm->eOperator & WO_ISNULL ); if( pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); @@ -4970,30 +4973,6 @@ static int wherePathSatisfiesOrderBy( isOrderDistinct = pIndex->onError!=OE_None; } - /* For every term of the index that is constrained by == or IS NULL, - ** mark off corresponding ORDER BY terms wherever they occur - ** in the ORDER BY clause. - */ - for(i=0; iu.btree.nEq; i++){ - pTerm = pLoop->aLTerm[i]; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))==0 ) continue; - iColumn = pTerm->u.leftColumn; - for(j=0; ja[j].pExpr); - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; - if( iColumn>=0 ){ - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[j].pExpr); - if( !pColl ) pColl = db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pIndex->azColl[i])!=0 ) continue; - } - obSat |= MASKBIT(j); - } - if( obSat==obDone ) return 1; - } - /* Loop through all columns of the index and deal with the ones ** that are not constrained by == or IN. */ @@ -5043,6 +5022,8 @@ static int wherePathSatisfiesOrderBy( for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; @@ -5093,7 +5074,6 @@ static int wherePathSatisfiesOrderBy( } /* End the loop over all WhereLoops from outer-most down to inner-most */ if( obSat==obDone ) return 1; if( !isOrderDistinct ) return 0; - if( isLastLoop ) return 1; return -1; } @@ -5200,7 +5180,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( !isOrderedValid ){ switch( wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, - iLoop, iLoop==nLoop-1, pWLoop, &revMask) ){ + iLoop, pWLoop, &revMask) ){ case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ isOrdered = 1; isOrderedValid = 1; @@ -5326,9 +5306,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ /* Find the lowest cost path. pFrom will be left pointing to that path */ pFrom = aFrom; + assert( nFrom==1 ); +#if 0 /* The following is needed if nFrom is ever more than 1 */ for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; } +#endif assert( pWInfo->nLevel==nLoop ); /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLooppDistinct, pFrom, - WHERE_DISTINCTBY, nLoop-1, 1, pFrom->aLoop[nLoop-1], ¬Used); + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } if( pFrom->isOrdered ){ From 7963b0e85321b7af85a632cab1c2e0b410adaf6a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Jun 2013 21:37:40 +0000 Subject: [PATCH 100/109] Add more testcase() macros. Fix a memory leak following OOM in the virtual table analysis logic. FossilOrigin-Name: b61402af690ac08b68974f3c807096b0cffd9bc0 --- manifest | 12 +++---- manifest.uuid | 2 +- src/where.c | 86 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 9a336de116..8f9dc7b011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplifications\sto\sthe\sNGQP.\s\sAdd\sthe\squeryplantest\smakefile\starget.\s\sAdd\ntestcase()\smacros\sin\sthe\sNGQP. -D 2013-06-17T18:20:48.993 +C Add\smore\stestcase()\smacros.\s\sFix\sa\smemory\sleak\sfollowing\sOOM\sin\sthe\nvirtual\stable\sanalysis\slogic. +D 2013-06-17T21:37:40.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 74627cb9415ad421004de574cfafe6ed0359554b +F src/where.c b911a484d36d5dca41448e493974368d8147a2e1 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e8f124284ee0d0e373dc4431668630f1e17015c1 -R 5f6127cf4cee34c1f05fd60f463f5e2a +P eaf1f1b405ec2c498092527fae00e5dbe9e176c1 +R 7a1603a9005d87a42c0df561c944734f U drh -Z 08d0c98ccbb424b49a424bc18bdd25d5 +Z 30a954d83fb3f9269803abab8ea42340 diff --git a/manifest.uuid b/manifest.uuid index 2d08ee9b92..a6139b6aec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eaf1f1b405ec2c498092527fae00e5dbe9e176c1 \ No newline at end of file +b61402af690ac08b68974f3c807096b0cffd9bc0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index cf55858f7e..e3ab83c592 100644 --- a/src/where.c +++ b/src/where.c @@ -2114,6 +2114,8 @@ static void constructAutomaticIndex( if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); if( (idxCols & cMask)==0 ){ Expr *pX = pTerm->pExpr; idxCols |= cMask; @@ -3046,7 +3048,6 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ int *aiColumn = pIndex->aiColumn; StrAccum txt; - if( pIndex==0 ) return 0; if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ return 0; } @@ -3113,7 +3114,7 @@ static void explainOneScan( zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); } if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 - && pLoop->u.btree.pIndex!=0 + && ALWAYS(pLoop->u.btree.pIndex!=0) ){ char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, @@ -3133,7 +3134,7 @@ static void explainOneScan( zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); - }else if( flags&WHERE_TOP_LIMIT ){ + }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowideOperator & WO_LE ); - testcase( pRangeStart && pRangeStart->eOperator & WO_GE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); start_constraints = pRangeStart || nEq>0; @@ -3889,6 +3890,8 @@ static Bitmask codeOneLoopStart( pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); if( pAlt==0 ) continue; if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IN ); VdbeNoopComment((v, "begin transitive constraint")); sEq = *pAlt->pExpr; sEq.pLeft = pE->pLeft; @@ -4095,6 +4098,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereCost rCost = whereCostAdd(p->rRun,p->rSetup); WhereCost rTemplate = whereCostAdd(pTemplate->rRun,pTemplate->rSetup); if( rCost < rTemplate ){ + testcase( rCost==rTemplate-1 ); goto whereLoopInsert_noop; } if( rCost == rTemplate && p->prereq <= pTemplate->prereq ){ @@ -4120,6 +4124,8 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ + testcase( p->rSetup==pTemplate->rSetup ); + testcase( p->rRun==pTemplate->rRun ); /* p is equal or better than pTemplate */ if( p->nLTermnLTerm && (p->wsFlags & WHERE_INDEXED)!=0 @@ -4145,14 +4151,26 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ goto whereLoopInsert_noop; } } + testcase( (p->prereq & pTemplate->prereq)==p->prereq + && p->rSetup==pTemplate->rSetup+1 ); + testcase( (p->prereq & pTemplate->prereq)==p->prereq + && p->rSetup<=pTemplate->rSetup + && p->rRun==pTemplate->rRun+1 ); if( (p->prereq & pTemplate->prereq)==pTemplate->prereq && p->rSetup>=pTemplate->rSetup && p->rRun>=pTemplate->rRun ){ /* Overwrite an existing WhereLoop with a better one */ + testcase( p->rSetup==pTemplate->rSetup ); + testcase( p->rRun==pTemplate->rRun ); pNext = p->pNextLoop; break; } + testcase( (p->prereq & pTemplate->prereq)==pTemplate->prereq + && p->rSetup==pTemplate->rSetup-1 ); + testcase( (p->prereq & pTemplate->prereq)==pTemplate->prereq + && p->rSetup>=pTemplate->rSetup-1 + && p->rRun==pTemplate->rRun-1 ); } /* If we reach this point it means that either p[] should be overwritten @@ -4231,7 +4249,6 @@ static int whereLoopAddBtreeIndex( if( db->mallocFailed ) return SQLITE_NOMEM; assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); - assert( pNew->u.btree.nEq<=pProbe->nColumn ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; @@ -4242,6 +4259,7 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + assert( pNew->u.btree.nEq<=pProbe->nColumn ); if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); @@ -4301,10 +4319,15 @@ static int whereLoopAddBtreeIndex( nIn = 10; assert( 10==whereCost(2) ); pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ + testcase( pTerm->eOperator & WO_GT ); + testcase( pTerm->eOperator & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ + }else{ + assert( pTerm->eOperator & (WO_LT|WO_LE) ); + testcase( pTerm->eOperator & WO_LT ); + testcase( pTerm->eOperator & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? @@ -4393,6 +4416,8 @@ static Bitmask columnsInIndex(Index *pIdx){ int j; for(j=pIdx->nColumn-1; j>=0; j--){ int x = pIdx->aiColumn[j]; + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); if( xu.vtab.needFree = 0; pUsage = pIdxInfo->aConstraintUsage; nConstraint = pIdxInfo->nConstraint; - if( whereLoopResize(db, pNew, nConstraint) ) return SQLITE_NOMEM; + if( whereLoopResize(db, pNew, nConstraint) ){ + sqlite3DbFree(db, pIdxInfo); + return SQLITE_NOMEM; + } for(iPhase=0; iPhase<=3; iPhase++){ if( !seenIn && (iPhase&1)!=0 ){ @@ -4621,9 +4649,10 @@ static int whereLoopAddVirtual( pIdxCons->usable = 0; if( (pTerm->eOperator & WO_IN)!=0 ){ seenIn = 1; - }else if( pTerm->prereqRight!=0 ){ + } + if( pTerm->prereqRight!=0 ){ seenVar = 1; - }else{ + }else if( (pTerm->eOperator & WO_IN)==0 ){ pIdxCons->usable = 1; } break; @@ -4668,11 +4697,16 @@ static int whereLoopAddVirtual( sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); goto whereLoopAddVtab_exit; } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); pTerm = &pWC->a[j]; pNew->prereq |= pTerm->prereqRight; assert( iTermnLSlot ); pNew->aLTerm[iTerm] = pTerm; if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ if( pUsage[i].omit==0 ){ @@ -4923,6 +4957,7 @@ static int wherePathSatisfiesOrderBy( if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ isOrderDistinct = 1; obDone = MASKBIT(nOrderBy)-1; @@ -4947,9 +4982,7 @@ static int wherePathSatisfiesOrderBy( pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, ~ready, WO_EQ|WO_ISNULL, 0); if( pTerm==0 ) continue; - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - if( pOBExpr->iColumn>=0 ){ + if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -4985,7 +5018,10 @@ static int wherePathSatisfiesOrderBy( if( ju.btree.nEq && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ - if( i & WO_ISNULL ) isOrderDistinct = 0; + if( i & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } continue; } @@ -4999,6 +5035,7 @@ static int wherePathSatisfiesOrderBy( if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ /* The ROWID column at the end */ + assert( j==nColumn ); iColumn = -1; revIdx = 0; } @@ -5037,7 +5074,10 @@ static int wherePathSatisfiesOrderBy( break; } if( isMatch ){ - if( iColumn<0 ) distinctColumns = 1; + if( iColumn<0 ){ + testcase( distinctColumns==0 ); + distinctColumns = 1; + } obSat |= MASKBIT(i); if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ /* Make sure the sort order is compatible in an ORDER BY clause. @@ -5052,7 +5092,10 @@ static int wherePathSatisfiesOrderBy( } }else{ /* No match found */ - if( j==0 || jmaskLoop==maskNew && pTo->isOrderedValid==isOrderedValid ){ + testcase( jj==nTo-1 ); break; } } @@ -5242,8 +5286,10 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); } #endif + testcase( pTo->rCost==rCost ); continue; } + testcase( pTo->rCost==rCost+1 ); /* A new and better score for a previously created equivalent path */ #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace&0x4 ){ @@ -5779,8 +5825,8 @@ WhereInfo *sqlite3WhereBegin( && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); - testcase( pTab->nCol==BMS-1 ); - testcase( pTab->nCol==BMS ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); if( !pWInfo->okOnePass && pTab->nColcolUsed; int n = 0; From 81186b4392bd2710d4e8761f0fa097da68c69430 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Jun 2013 01:52:41 +0000 Subject: [PATCH 101/109] Remove some redundant and unreachable code. FossilOrigin-Name: 4c6d58d75d51e1ce829aec214617c3a89e784a2d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 19 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 8f9dc7b011..56a8a23589 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\stestcase()\smacros.\s\sFix\sa\smemory\sleak\sfollowing\sOOM\sin\sthe\nvirtual\stable\sanalysis\slogic. -D 2013-06-17T21:37:40.990 +C Remove\ssome\sredundant\sand\sunreachable\scode. +D 2013-06-18T01:52:41.578 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c b911a484d36d5dca41448e493974368d8147a2e1 +F src/where.c 86cc9705ef7de9ef712619485d3bff3910d58b12 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P eaf1f1b405ec2c498092527fae00e5dbe9e176c1 -R 7a1603a9005d87a42c0df561c944734f +P b61402af690ac08b68974f3c807096b0cffd9bc0 +R ddde8ef3fd559d48220a86d476077d7b U drh -Z 30a954d83fb3f9269803abab8ea42340 +Z a276f283a2c7c1a652996ed6754c6f02 diff --git a/manifest.uuid b/manifest.uuid index a6139b6aec..f73027ac71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b61402af690ac08b68974f3c807096b0cffd9bc0 \ No newline at end of file +4c6d58d75d51e1ce829aec214617c3a89e784a2d \ No newline at end of file diff --git a/src/where.c b/src/where.c index e3ab83c592..800194c6ef 100644 --- a/src/where.c +++ b/src/where.c @@ -2041,7 +2041,6 @@ static void constructAutomaticIndex( WhereLoop *pLoop; /* The Loop object */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - const int mxConstraint = 10; /* Maximum number of constraints */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -2056,7 +2055,7 @@ static void constructAutomaticIndex( pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; - for(pTerm=pWC->a; pTermnLTerma; pTermu.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -3305,6 +3304,7 @@ static Bitmask codeOneLoopStart( pStart = pEnd = 0; if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); if( bRev ){ pTerm = pStart; pStart = pEnd; @@ -3359,11 +3359,7 @@ static Bitmask codeOneLoopStart( pLevel->op = bRev ? OP_Prev : OP_Next; pLevel->p1 = iCur; pLevel->p2 = start; - if( pStart==0 && pEnd==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); - } + assert( pLevel->p5==0 ); if( testOp!=OP_Noop ){ iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); @@ -3798,8 +3794,8 @@ static Bitmask codeOneLoopStart( ** be available. */ pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_TEMP_INDEX)==0 ); if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 - && (pSubLoop->wsFlags & WHERE_TEMP_INDEX)==0 && (ii==0 || pSubLoop->u.btree.pIndex==pCov) ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); @@ -5099,7 +5095,10 @@ static int wherePathSatisfiesOrderBy( break; } } /* end Loop over all index columns */ - if( distinctColumns ) isOrderDistinct = 1; + if( distinctColumns ){ + testcase( isOrderDistinct==0 ); + isOrderDistinct = 1; + } } /* end-if not one-row */ /* Mark off any other ORDER BY terms that reference pLoop */ @@ -5749,7 +5748,7 @@ WhereInfo *sqlite3WhereBegin( if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ pWInfo->revMask = (Bitmask)(-1); } - if( pParse->nErr || db->mallocFailed ){ + if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; } #ifdef WHERETRACE_ENABLED From 4a5acf8e5ce52face22d05831453e7d8fb78574e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Jun 2013 20:06:23 +0000 Subject: [PATCH 102/109] Adjustments to testcase() macros for improved testability. FossilOrigin-Name: 4fbb0c4d26c54aaefbe5397cde2a0b9d2ce3885f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/where.c | 18 ++++++++---------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 56a8a23589..3817eca13b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sredundant\sand\sunreachable\scode. -D 2013-06-18T01:52:41.578 +C Adjustments\sto\stestcase()\smacros\sfor\simproved\stestability. +D 2013-06-18T20:06:23.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c 9cc9e4c85197b567e5f97f3296700faf7747810f +F src/expr.c 3286e703adde34865d3a2cb62f9f4f31e3d2f2ed F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 5c50c1ea31fd864b0fe921fe1a8d4c55acd609ef @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 86cc9705ef7de9ef712619485d3bff3910d58b12 +F src/where.c 85b7b80a350951444c8e3e45bf2f7e6cc6d34683 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b61402af690ac08b68974f3c807096b0cffd9bc0 -R ddde8ef3fd559d48220a86d476077d7b +P 4c6d58d75d51e1ce829aec214617c3a89e784a2d +R f77d52adfdd29eb20a93333ac0fbc3fa U drh -Z a276f283a2c7c1a652996ed6754c6f02 +Z e0df54c8e9d824f28c0e34a1e5fe6134 diff --git a/manifest.uuid b/manifest.uuid index f73027ac71..cf33cbf052 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c6d58d75d51e1ce829aec214617c3a89e784a2d \ No newline at end of file +4fbb0c4d26c54aaefbe5397cde2a0b9d2ce3885f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 4e834ffe8e..d066e9b8bc 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1603,8 +1603,8 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ *prNotFound = rMayHaveNull = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ - testcase( pParse->nQueryLoop>1 ); - pParse->nQueryLoop = 1; + testcase( pParse->nQueryLoop>0 ); + pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } diff --git a/src/where.c b/src/where.c index 800194c6ef..dc4a515b90 100644 --- a/src/where.c +++ b/src/where.c @@ -4097,7 +4097,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ testcase( rCost==rTemplate-1 ); goto whereLoopInsert_noop; } - if( rCost == rTemplate && p->prereq <= pTemplate->prereq ){ + if( rCost==rTemplate && (p->prereq & pTemplate->prereq)==p->prereq ){ goto whereLoopInsert_noop; } } @@ -4120,9 +4120,9 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ - testcase( p->rSetup==pTemplate->rSetup ); + /* This branch taken when p is equal or better than pTemplate in + ** all of (1) dependences (2) setup-cost, and (3) run-cost. */ testcase( p->rRun==pTemplate->rRun ); - /* p is equal or better than pTemplate */ if( p->nLTermnLTerm && (p->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 @@ -4147,8 +4147,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ goto whereLoopInsert_noop; } } - testcase( (p->prereq & pTemplate->prereq)==p->prereq - && p->rSetup==pTemplate->rSetup+1 ); testcase( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun==pTemplate->rRun+1 ); @@ -4156,16 +4154,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ && p->rSetup>=pTemplate->rSetup && p->rRun>=pTemplate->rRun ){ - /* Overwrite an existing WhereLoop with a better one */ + /* Overwrite an existing WhereLoop with a better one: one that is + ** better at one of (1) dependences, (2) setup-cost, or (3) run-cost + ** and is no worse in any of those categories. */ testcase( p->rSetup==pTemplate->rSetup ); testcase( p->rRun==pTemplate->rRun ); pNext = p->pNextLoop; break; } testcase( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rSetup==pTemplate->rSetup-1 ); - testcase( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rSetup>=pTemplate->rSetup-1 + && p->rSetup>=pTemplate->rSetup && p->rRun==pTemplate->rRun-1 ); } @@ -4303,7 +4301,7 @@ static int whereLoopAddBtreeIndex( || (pProbe->onError!=OE_None && nInMul==0 && pNew->u.btree.nEq==pProbe->nColumn-1) ){ - testcase( pNew->wsFlags & WHERE_COLUMN_IN ); + assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); pNew->wsFlags |= WHERE_ONEROW; } pNew->u.btree.nEq++; From c7f0d229a7c9f56f0f7f79eee0d69d3f1b54fc78 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Jun 2013 03:27:12 +0000 Subject: [PATCH 103/109] Fix compiler warnings. Fix a harmless off-by-one error in the solver. FossilOrigin-Name: 10021941d0258951b916e788881df140113f0597 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/prepare.c | 4 ++-- src/where.c | 15 +++++++++++---- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3817eca13b..d232347684 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustments\sto\stestcase()\smacros\sfor\simproved\stestability. -D 2013-06-18T20:06:23.048 +C Fix\scompiler\swarnings.\s\sFix\sa\sharmless\soff-by-one\serror\sin\sthe\ssolver. +D 2013-06-19T03:27:12.614 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -210,7 +210,7 @@ F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c d23d07716de96c7c0c2503ec5051a4384c3fb938 F src/pragma.c 67a611bd4be0754f27ee13eb87932c3b14415862 -F src/prepare.c 213de4aae5e29cfbb32202d1c09f4fc4b5a51b09 +F src/prepare.c 2306be166bbeddf454e18bf8b21dba8388d05328 F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 85b7b80a350951444c8e3e45bf2f7e6cc6d34683 +F src/where.c 5cc059bd283fe9f99800270c797fabdcbef8009a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4c6d58d75d51e1ce829aec214617c3a89e784a2d -R f77d52adfdd29eb20a93333ac0fbc3fa +P 4fbb0c4d26c54aaefbe5397cde2a0b9d2ce3885f +R dae7f42197a8d862e7c14dd3769614f8 U drh -Z e0df54c8e9d824f28c0e34a1e5fe6134 +Z e26ab5beb4ba36d99e52a8a301bdec10 diff --git a/manifest.uuid b/manifest.uuid index cf33cbf052..011a8dd71f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fbb0c4d26c54aaefbe5397cde2a0b9d2ce3885f \ No newline at end of file +10021941d0258951b916e788881df140113f0597 \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index 319bb5012e..28145aa4e7 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -592,7 +592,7 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); pParse->db = db; - pParse->nQueryLoop = 1; + pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -614,7 +614,7 @@ static int sqlite3Prepare( }else{ sqlite3RunParser(pParse, zSql, &zErrMsg); } - assert( 1==pParse->nQueryLoop ); + assert( 0==pParse->nQueryLoop ); if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM; diff --git a/src/where.c b/src/where.c index dc4a515b90..73fdcaa431 100644 --- a/src/where.c +++ b/src/where.c @@ -4237,7 +4237,7 @@ static int whereLoopAddBtreeIndex( int rc = SQLITE_OK; /* Return code */ WhereCost nRowEst; /* Estimated index selectivity */ WhereCost rLogSize; /* Logarithm of table size */ - WhereTerm *pTop, *pBtm; /* Top and bottom range constraints */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4257,6 +4257,7 @@ static int whereLoopAddBtreeIndex( if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); + if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; }else{ iCol = -1; nRowEst = 0; @@ -4786,7 +4787,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sSubBuild.pOrderBy = 0; sSubBuild.pBest = &sBest; - for(pOrTerm=pOrWC->a; rc==SQLITE_OK && pOrTerma; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; }else if( pOrTerm->leftCursor==iCur ){ @@ -4810,6 +4811,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ { rc = whereLoopAddBtree(&sSubBuild, mExtra); } + /* sBest.maskSelf is always zero if an error occurs */ + assert( rc==SQLITE_OK || sBest.maskSelf==0 ); if( sBest.maskSelf==0 ) break; assert( sBest.rSetup==0 ); rTotal = whereCostAdd(rTotal, sBest.rRun); @@ -5135,6 +5138,10 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** attempts to find the lowest cost path that visits each WhereLoop ** once. This path is then loaded into the pWInfo->a[].pWLoop fields. ** +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation ** error occurs. */ @@ -5146,7 +5153,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ WhereCost rCost; /* Cost of a path */ - WhereCost mxCost; /* Maximum cost of a set of paths */ + WhereCost mxCost = 0; /* Maximum cost of a set of paths */ WhereCost rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ @@ -5739,7 +5746,7 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut); + wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } } From dbb80234dabefcb518fb09fa073b6b09bd30f3af Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Jun 2013 12:34:13 +0000 Subject: [PATCH 104/109] Simplify and add invariants to the WhereLoop merging logic inside of whereLoopInsert(). FossilOrigin-Name: 8f27f35f288434b9e7bc503c608f1e2b590ade4d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 38 +++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index d232347684..37bf8cca81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings.\s\sFix\sa\sharmless\soff-by-one\serror\sin\sthe\ssolver. -D 2013-06-19T03:27:12.614 +C Simplify\sand\sadd\sinvariants\sto\sthe\sWhereLoop\smerging\slogic\sinside\sof\nwhereLoopInsert(). +D 2013-06-19T12:34:13.186 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 5cc059bd283fe9f99800270c797fabdcbef8009a +F src/where.c e5ecc3d12c5b9d66d13f786e80641614530d1957 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4fbb0c4d26c54aaefbe5397cde2a0b9d2ce3885f -R dae7f42197a8d862e7c14dd3769614f8 +P 10021941d0258951b916e788881df140113f0597 +R 38e2e388820d13ceac39bea07ba5a7cc U drh -Z e26ab5beb4ba36d99e52a8a301bdec10 +Z 087e132035ea5a36d1e5d81e4344a21a diff --git a/manifest.uuid b/manifest.uuid index 011a8dd71f..8aac6362f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10021941d0258951b916e788881df140113f0597 \ No newline at end of file +8f27f35f288434b9e7bc503c608f1e2b590ade4d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 73fdcaa431..2626306c24 100644 --- a/src/where.c +++ b/src/where.c @@ -4115,14 +4115,30 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** priority over pTemplate. */ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue; + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + if( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ /* This branch taken when p is equal or better than pTemplate in ** all of (1) dependences (2) setup-cost, and (3) run-cost. */ - testcase( p->rRun==pTemplate->rRun ); + assert( p->rSetup==pTemplate->rSetup ); if( p->nLTermnLTerm && (p->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 @@ -4133,38 +4149,22 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** more terms of the index */ pNext = p->pNextLoop; break; - }else if( p->nOut>pTemplate->nOut - && p->rSetup==pTemplate->rSetup - && p->rRun==pTemplate->rRun - ){ - /* Overwrite an existing WhereLoop with the same cost but more - ** outputs */ - pNext = p->pNextLoop; - break; }else{ /* pTemplate is not helpful. ** Return without changing or adding anything */ goto whereLoopInsert_noop; } } - testcase( (p->prereq & pTemplate->prereq)==p->prereq - && p->rSetup<=pTemplate->rSetup - && p->rRun==pTemplate->rRun+1 ); if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rSetup>=pTemplate->rSetup && p->rRun>=pTemplate->rRun + && ALWAYS(p->rSetup>=pTemplate->rSetup) /* See SETUP-INVARIANT above */ ){ /* Overwrite an existing WhereLoop with a better one: one that is ** better at one of (1) dependences, (2) setup-cost, or (3) run-cost ** and is no worse in any of those categories. */ - testcase( p->rSetup==pTemplate->rSetup ); - testcase( p->rRun==pTemplate->rRun ); pNext = p->pNextLoop; break; } - testcase( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rSetup>=pTemplate->rSetup - && p->rRun==pTemplate->rRun-1 ); } /* If we reach this point it means that either p[] should be overwritten From e8ae583ebd689f6c95cf29cfe4f474366a7613dd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Jun 2013 13:32:46 +0000 Subject: [PATCH 105/109] Fix a harmless uninitialized variable warning. FossilOrigin-Name: 9d3ef3bd2c6281784a537a22a87279f420649935 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 37bf8cca81..cb3eff1fbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sand\sadd\sinvariants\sto\sthe\sWhereLoop\smerging\slogic\sinside\sof\nwhereLoopInsert(). -D 2013-06-19T12:34:13.186 +C Fix\sa\sharmless\suninitialized\svariable\swarning. +D 2013-06-19T13:32:46.775 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e5ecc3d12c5b9d66d13f786e80641614530d1957 +F src/where.c 0d12d709b410bc19e6a38dd48565d01aa2157a48 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 10021941d0258951b916e788881df140113f0597 -R 38e2e388820d13ceac39bea07ba5a7cc +P 8f27f35f288434b9e7bc503c608f1e2b590ade4d +R 5141bf3c3afc19733578e654fa12a0bb U drh -Z 087e132035ea5a36d1e5d81e4344a21a +Z f8bc370f664ab6cce8282fa162fbfbcd diff --git a/manifest.uuid b/manifest.uuid index 8aac6362f2..c36b341294 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f27f35f288434b9e7bc503c608f1e2b590ade4d \ No newline at end of file +9d3ef3bd2c6281784a537a22a87279f420649935 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2626306c24..7b7b071b41 100644 --- a/src/where.c +++ b/src/where.c @@ -4909,7 +4909,7 @@ static int wherePathSatisfiesOrderBy( int i, j; /* Loop counters */ int iCur; /* Cursor number for current WhereLoop */ int iColumn; /* A column number within table iCur */ - WhereLoop *pLoop; /* Current WhereLoop being processed. */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ WhereTerm *pTerm; /* A single term of the WHERE clause */ Expr *pOBExpr; /* An expression from the ORDER BY clause */ CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ From ad01d892a702ba66b578f512efa660680bc0d6bd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Jun 2013 13:59:49 +0000 Subject: [PATCH 106/109] Additional compiler warning fixes. FossilOrigin-Name: 8d2ae8e2f343f82f6fba6d0e89cee7f15b444aa3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 4 ++-- src/where.c | 7 +++---- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index cb3eff1fbd..a1804c7bf0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\suninitialized\svariable\swarning. -D 2013-06-19T13:32:46.775 +C Additional\scompiler\swarning\sfixes. +D 2013-06-19T13:59:49.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0 F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 -F src/build.c eadda12ec0cae14dcdc13e97fd9a3fd2f02e9e8f +F src/build.c 42239cfd95533e4aacf4d58b4724c8f858de5ced F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0d12d709b410bc19e6a38dd48565d01aa2157a48 +F src/where.c 88884f3a0355ece00fa0d1a3566c4ed6fb5a8bed F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8f27f35f288434b9e7bc503c608f1e2b590ade4d -R 5141bf3c3afc19733578e654fa12a0bb +P 9d3ef3bd2c6281784a537a22a87279f420649935 +R 576d65f25fc5d448aeaee421b4a0d4ef U drh -Z f8bc370f664ab6cce8282fa162fbfbcd +Z 2fbaa2eda000406e9f5d447a3a6c804c diff --git a/manifest.uuid b/manifest.uuid index c36b341294..350fdf4563 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d3ef3bd2c6281784a537a22a87279f420649935 \ No newline at end of file +8d2ae8e2f343f82f6fba6d0e89cee7f15b444aa3 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 94fafff9fd..0a3922c952 100644 --- a/src/build.c +++ b/src/build.c @@ -3187,7 +3187,7 @@ SrcList *sqlite3SrcListEnlarge( } pSrc = pNew; nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = (u16)nGot; + pSrc->nAlloc = (u8)nGot; } /* Move existing slots that come after the newly inserted slots @@ -3195,7 +3195,7 @@ SrcList *sqlite3SrcListEnlarge( for(i=pSrc->nSrc-1; i>=iStart; i--){ pSrc->a[i+nExtra] = pSrc->a[i]; } - pSrc->nSrc += (i16)nExtra; + pSrc->nSrc += (i8)nExtra; /* Zero the newly allocated slots */ memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); diff --git a/src/where.c b/src/where.c index 7b7b071b41..f1cee90de1 100644 --- a/src/where.c +++ b/src/where.c @@ -883,7 +883,7 @@ WhereTerm *whereScanNext(WhereScan *pScan){ } } } - pWC = pScan->pWC = pScan->pWC->pOuter; + pScan->pWC = pScan->pWC->pOuter; k = 0; } pScan->pWC = pScan->pOrigWC; @@ -3433,7 +3433,6 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - k = (nEq==pIdx->nColumn ? -1 : pIdx->aiColumn[nEq]); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3687,7 +3686,7 @@ static Bitmask codeOneLoopStart( pOrTab = sqlite3StackAllocRaw(pParse->db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (i16)(nNotReady + 1); + pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); origSrc = pWInfo->pTabList->a; @@ -4346,7 +4345,7 @@ static int whereLoopAddBtreeIndex( && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); } - pNew->nOut = whereCost(nOut); + if( rc==SQLITE_OK ) pNew->nOut = whereCost(nOut); } #endif if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ From 74f91d446213475d5880ecbf6428291796f42d26 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Jun 2013 18:01:44 +0000 Subject: [PATCH 107/109] Add in the cost of doing a table lookup on OR searches. Make test case changes to deal with difference in STAT3 behavior. FossilOrigin-Name: d97898e8e3990ae8c1882c9102b57692d8810730 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/where.c | 10 ++++---- test/where9.test | 63 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index a1804c7bf0..b27c436553 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Additional\scompiler\swarning\sfixes. -D 2013-06-19T13:59:49.313 +C Add\sin\sthe\scost\sof\sdoing\sa\stable\slookup\son\sOR\ssearches.\s\sMake\stest\scase\nchanges\sto\sdeal\swith\sdifference\sin\sSTAT3\sbehavior. +D 2013-06-19T18:01:44.089 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 88884f3a0355ece00fa0d1a3566c4ed6fb5a8bed +F src/where.c ac9f8c4ee8ece1e8a4c93802792dbb0d629a7fa0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1039,7 +1039,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test f6b9559723564a042927ee0f22003ac9bed71b21 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test f5752185fc19a1ee826a2386a249f05dbc25979b +F test/where9.test 9a7fda4a4512abc26a855e8b2b6572b200f6019b F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d3ef3bd2c6281784a537a22a87279f420649935 -R 576d65f25fc5d448aeaee421b4a0d4ef +P 8d2ae8e2f343f82f6fba6d0e89cee7f15b444aa3 +R f44808387531e19eca9d858ab5ea40e0 U drh -Z 2fbaa2eda000406e9f5d447a3a6c804c +Z 1152d5230daa64158b83c69b1e514ae0 diff --git a/manifest.uuid b/manifest.uuid index 350fdf4563..6cc8090fec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d2ae8e2f343f82f6fba6d0e89cee7f15b444aa3 \ No newline at end of file +d97898e8e3990ae8c1882c9102b57692d8810730 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f1cee90de1..b7dab2105d 100644 --- a/src/where.c +++ b/src/where.c @@ -630,8 +630,8 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ ** the WhereClause.a[] array. The slot[] array grows as needed to contain ** all terms of the WHERE clause. */ -static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ - pWC->op = (u8)op; +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + pWC->op = op; if( pExpr==0 ) return; if( pExpr->op!=op ){ whereClauseInsert(pWC, pExpr, 0); @@ -4516,6 +4516,7 @@ static int whereLoopAddBtree( pNew->iSortIdx = 0; pNew->rSetup = 0; pNew->prereq = mExtra; + pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ @@ -4526,7 +4527,6 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - pNew->nOut = rSize; /* TUNING: Cost of full table scan is 3*(N + log2(N)). ** + The extra 3 factor is to encourage the use of indexed lookups ** over full scans. A smaller constant 2 is used for covering @@ -4549,7 +4549,6 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - pNew->nOut = rSize; if( m==0 ){ /* TUNING: Cost of a covering index scan is 2*(N + log2(N)). ** + The extra 2 factor is to encourage the use of indexed lookups @@ -4824,7 +4823,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->aLTerm[0] = pTerm; pNew->wsFlags = WHERE_MULTI_OR; pNew->rSetup = 0; - pNew->rRun = rTotal; + /* TUNING: Multiple by 3.5 for the secondary table lookup */ + pNew->rRun = rTotal + 18; assert( 18==whereCost(7)-whereCost(2) ); pNew->nOut = nRow; pNew->prereq = prereq; memset(&pNew->u, 0, sizeof(pNew->u)); diff --git a/test/where9.test b/test/where9.test index 4ffcad1470..5b000d967e 100644 --- a/test/where9.test +++ b/test/where9.test @@ -598,7 +598,7 @@ do_test where9-6.3.5 { OR (b NOT NULL AND c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 0 sort 0} ;# DELETEs rows 90 91 92 97 +} {scan 98 sort 0} ;# DELETEs rows 90 91 92 97 do_test where9-6.3.6 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -615,7 +615,7 @@ do_test where9-6.3.7 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 0 sort 0} ;# Add 100 to rowids 90 91 92 97 +} {scan 98 sort 0} ;# Add 100 to rowids 90 91 92 97 do_test where9-6.3.8 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -705,7 +705,7 @@ do_test where9-6.6.1 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 0 sort 0} ;# DELETEs rows 90 91 92 97 +} {scan 98 sort 0} ;# DELETEs rows 90 91 92 97 do_test where9-6.6.2 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -722,7 +722,7 @@ do_test where9-6.6.3 { OR (b NOT NULL AND +c IS NULL AND d NOT NULL) OR (b NOT NULL AND c NOT NULL AND d IS NULL) } -} {scan 0 sort 0} ;# Add 100 to rowids 90 91 92 97 +} {scan 98 sort 0} ;# Add 100 to rowids 90 91 92 97 do_test where9-6.6.4 { db eval { SELECT count(*) FROM t1 UNION ALL @@ -781,23 +781,44 @@ do_test where9-6.8.2 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {1 {no query solution}} -do_test where9-6.8.3 { - catchsql { - UPDATE t1 INDEXED BY t1b SET a=a+100 - WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) - OR (b NOT NULL AND c IS NULL AND d NOT NULL) - OR (b NOT NULL AND c NOT NULL AND d IS NULL) - } -} {0 {}} -do_test where9-6.8.4 { - catchsql { - DELETE FROM t1 INDEXED BY t1b - WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) - OR (b NOT NULL AND c IS NULL AND d NOT NULL) - OR (b NOT NULL AND c NOT NULL AND d IS NULL) - } -} {0 {}} - +ifcapable stat3 { + # When STAT3 is enabled, the "b NOT NULL" terms get translated + # into b>NULL, which can be satified by the index t1b. It is a very + # expensive way to do the query, but it works, and so a solution is possible. + do_test where9-6.8.3-stat3 { + catchsql { + UPDATE t1 INDEXED BY t1b SET a=a+100 + WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } + } {0 {}} + do_test where9-6.8.4-stat3 { + catchsql { + DELETE FROM t1 INDEXED BY t1b + WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } + } {0 {}} +} else { + do_test where9-6.8.3 { + catchsql { + UPDATE t1 INDEXED BY t1b SET a=a+100 + WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } + } {1 {no query solution}} + do_test where9-6.8.4 { + catchsql { + DELETE FROM t1 INDEXED BY t1b + WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) + } + } {1 {no query solution}} +} ############################################################################ # Test cases where terms inside an OR series are combined with AND terms # external to the OR clause. In other words, cases where From 472eae8a4d4d69895e85234aff425fdc835957ff Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Jun 2013 17:32:28 +0000 Subject: [PATCH 108/109] Add a NEVER() macro and an explanation comment around an unreachable branch in the STAT3 logic. FossilOrigin-Name: 604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7c057f18d9..e7e71b5de4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pull\sin\sthe\sposix_fallocate()\schange\sfrom\strunk. -D 2013-06-20T14:17:39.289 +C Add\sa\sNEVER()\smacro\sand\san\sexplanation\scomment\saround\san\sunreachable\sbranch\nin\sthe\sSTAT3\slogic. +D 2013-06-20T17:32:28.451 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c ac9f8c4ee8ece1e8a4c93802792dbb0d629a7fa0 +F src/where.c 8d6c07d9641bf107e03a2b613550d90b8c7f4a82 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bf5764067ab848e19e5971cbdf892c633495e325 b674462243138fcee192ef05d434665e30c681c4 -R 91cfb3838b57db0708acca963a7bb8c5 +P d94db3fd921890ab1d6414ab629410ae50779686 +R 4877ecf4e1d53d3b49ba9f1fadfb6d66 U drh -Z 767fbda1b22b9d1069574da955ceb02c +Z c939c499a0659dd37def1fcbcc79c922 diff --git a/manifest.uuid b/manifest.uuid index fc397231db..bda9422793 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d94db3fd921890ab1d6414ab629410ae50779686 \ No newline at end of file +604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index b7dab2105d..cdc0e16839 100644 --- a/src/where.c +++ b/src/where.c @@ -2409,9 +2409,10 @@ static int whereKeyStats( assert( pColl->enc==SQLITE_UTF8 ); }else{ pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl); - if( pColl==0 ){ - return SQLITE_ERROR; - } + /* If the collating sequence was unavailable, we should have failed + ** long ago and never reached this point. But we'll check just to + ** be doubly sure. */ + if( NEVER(pColl==0) ) return SQLITE_ERROR; z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); if( !z ){ return SQLITE_NOMEM; From 2c036cff3d2fa5e993448ef1aeaf9e1eab388749 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Jun 2013 00:34:13 +0000 Subject: [PATCH 109/109] Fix an uninitialized variable detected by valgrind. Unclear whether or not this should apply to trunk. FossilOrigin-Name: 19ab4811d542ba781aeb6a4eb3c74642677225e1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e7e71b5de4..a758c3108d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sNEVER()\smacro\sand\san\sexplanation\scomment\saround\san\sunreachable\sbranch\nin\sthe\sSTAT3\slogic. -D 2013-06-20T17:32:28.451 +C Fix\san\suninitialized\svariable\sdetected\sby\svalgrind.\s\sUnclear\swhether\sor\snot\nthis\sshould\sapply\sto\strunk. +D 2013-06-26T00:34:13.504 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c 3286e703adde34865d3a2cb62f9f4f31e3d2f2ed +F src/expr.c 2b47ae9da6c9f34eff6736962ea2e102c6c4a755 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 5c50c1ea31fd864b0fe921fe1a8d4c55acd609ef @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d94db3fd921890ab1d6414ab629410ae50779686 -R 4877ecf4e1d53d3b49ba9f1fadfb6d66 +P 604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 +R 2ed2dcc6fe1201d795c5181c6afd8a91 U drh -Z c939c499a0659dd37def1fcbcc79c922 +Z 9f76be0e05737d7f0f0ba2111ccc0b78 diff --git a/manifest.uuid b/manifest.uuid index bda9422793..4852f22866 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 \ No newline at end of file +19ab4811d542ba781aeb6a4eb3c74642677225e1 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d066e9b8bc..2c0419aa2e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -920,6 +920,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); pItem->sortOrder = pOldItem->sortOrder; pItem->done = 0; + pItem->bSpanIsTab = pOldItem->bSpanIsTab; pItem->iOrderByCol = pOldItem->iOrderByCol; pItem->iAlias = pOldItem->iAlias; }