diff --git a/manifest b/manifest index 91e4990458..d0d1a72243 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\sWhereCost\sto\sinteger\sconversion. -D 2013-06-13T14:51:53.839 +C Make\sthe\sMIN()\sand\sMAX()\smacros\savailable\sin\ssqliteInt.h.\s\sAdd\sTUNING\ncomments\sto\sthe\sNGQP\sand\sadjust\scosts\sslightly. +D 2013-06-13T15:16:53.047 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -159,7 +159,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c 27e541b9b5b48b41eb899b22a49ff42384899151 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 +F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0 @@ -191,7 +191,7 @@ F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f -F src/memjournal.c 41a598445c8f249bd9b26ecae700c60dcf34cbf3 +F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 @@ -202,7 +202,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42c9b8b7c61c9fa3561258f523be5749e52ed0e0 -F src/os_win.c 5f018dbd4cec25c5b47e11432b946a7d2ccee60b +F src/os_win.c 074cb2b9bca6a1c2bd72acf04666cdc554bfaa9b F src/pager.c 79df56da9dd49aceaa4cac207484a9a82cba40ae F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 9acfcc83ddbf0cf82f0ed9582ccf0ad6c366ff37 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 026a52f3e24c15603fddb78025e786b96c4c0d2c +F src/sqliteInt.h 82a0f3e7c3410cc748c80268e0831dba5ea98cb4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c6944d98be4dfd0f080c2e205909ca6c25c21224 +F src/where.c 2e8e0a18dcef3a56a5874a33ba5ec4339f6911ce F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -322,7 +322,7 @@ F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf -F test/autoindex1.test fac4207316e4e954abd086eafdd571ef093c5f5d +F test/autoindex1.test 6a22a566b2dcbc968a62f8282af81574055d1bbf F test/autovacuum.test 9f22a7733f39c56ef6a5665d10145ac25d8cb574 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d -R 70bdc5e3664356410e482d1133202557 +P b5ca80d924f8c6d31f036247ba6e20d234f4482e +R 224bd6bce7ee1e3846617c8c2c148fe1 U drh -Z 22748fc276d8e873b764764d1effa482 +Z df43f94251ce9f1e894782fd34764619 diff --git a/manifest.uuid b/manifest.uuid index 4172d0480a..b4af3a8785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5ca80d924f8c6d31f036247ba6e20d234f4482e \ No newline at end of file +3a72af2a95b04b8e195ef17cb3e9d9021a4f0915 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 252f61cfca..6ef9374ec6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -15,12 +15,6 @@ #include "sqliteInt.h" #include "btreeInt.h" -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** Structure allocated for each backup operation. */ diff --git a/src/memjournal.c b/src/memjournal.c index 05725948f6..65ed378b38 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -31,12 +31,6 @@ typedef struct FileChunk FileChunk; */ #define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** The rollback journal is composed of a linked list of these structures. */ diff --git a/src/os_win.c b/src/os_win.c index 07b74cbe22..bdf025aa4c 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -84,13 +84,6 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); #endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ -/* -** Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif - /* ** Some Microsoft compilers lack this definition. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b22689462c..b2cab7a7ad 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -395,6 +395,12 @@ #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) #endif +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) diff --git a/src/where.c b/src/where.c index 8593dc4d94..000df0e107 100644 --- a/src/where.c +++ b/src/where.c @@ -57,6 +57,10 @@ typedef struct WhereScan WhereScan; ** 10*log2(X) exactly. Instead, a close estimate is used. Any value of ** X<=1 is stored as 0. X=2 is 10. X=3 is 16. X=1000 is 99. etc. ** +** The tool/wherecosttest.c source file implements a command-line program +** that will convert between WhereCost to integers and do addition and +** multiplication on WhereCost values. That command-line program is a +** useful utility to have around when working with this module. */ typedef unsigned short int WhereCost; @@ -1901,7 +1905,7 @@ static WhereCost whereCostAdd(WhereCost a, WhereCost b){ ** Convert an integer into a WhereCost. In other words, compute a ** good approximatation for 10*log2(x). */ -static WhereCost whereCostFromInt(tRowcnt x){ +static WhereCost whereCost(tRowcnt x){ static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; WhereCost y = 40; if( x<8 ){ @@ -1925,7 +1929,7 @@ static WhereCost whereCostFromDouble(double x){ WhereCost e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=1 ) return 0; - if( x<=2000000000 ) return whereCostFromInt((tRowcnt)x); + if( x<=2000000000 ) return whereCost((tRowcnt)x); memcpy(&a, &x, 8); e = (a>>52) - 1022; return e*10; @@ -1936,7 +1940,7 @@ static WhereCost whereCostFromDouble(double x){ ** Estimate the logarithm of the input value to base 2. */ static WhereCost estLog(WhereCost N){ - WhereCost x = whereCostFromInt(N); + WhereCost x = whereCost(N); return x>33 ? x - 33 : 0; } @@ -2598,9 +2602,9 @@ static int whereRangeScanEst( sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK ){ - WhereCost iBase = whereCostFromInt(p->aiRowEst[0]); + WhereCost iBase = whereCost(p->aiRowEst[0]); if( iUpper>iLower ){ - iBase -= whereCostFromInt(iUpper - iLower); + iBase -= whereCost(iUpper - iLower); } *pRangeDiv = iBase; WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", @@ -2615,11 +2619,13 @@ static int whereRangeScanEst( #endif assert( pLower || pUpper ); *pRangeDiv = 0; + /* TUNING: Each inequality constraint reduces the search space 4-fold. + ** A BETWEEN operator, therefore, reduces the search space 16-fold */ if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + *pRangeDiv += 20; assert( 20==whereCost(4) ); } if( pUpper ){ - *pRangeDiv += 20; assert( 20==whereCostFromInt(4) ); + *pRangeDiv += 20; assert( 20==whereCost(4) ); } return rc; } @@ -4244,7 +4250,7 @@ static int whereLoopAddBtreeIndex( if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = whereCostFromInt(pProbe->aiRowEst[pNew->u.btree.nEq+1]); + nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); }else{ iCol = -1; nRowEst = 0; @@ -4257,7 +4263,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(whereCostFromInt(pProbe->aiRowEst[0])); + rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; if( pTerm->prereqRight & pNew->maskSelf ) continue; @@ -4267,16 +4273,16 @@ static int whereLoopAddBtreeIndex( if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; + pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 46; assert( 46==whereCostFromInt(25) ); + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==whereCost(25) ); }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ - nIn = whereCostFromInt(pExpr->x.pList->nExpr); + nIn = whereCost(pExpr->x.pList->nExpr); } pNew->rRun += nIn; pNew->u.btree.nEq++; @@ -4297,7 +4303,8 @@ static int whereLoopAddBtreeIndex( }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; - nIn = 10; assert( 10==whereCostFromInt(2) ); + /* TUNING: IS NULL selects 2 rows */ + nIn = 10; assert( 10==whereCost(2) ); pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; @@ -4325,7 +4332,7 @@ static int whereLoopAddBtreeIndex( && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); } - pNew->nOut = whereCostFromInt(nOut); + pNew->nOut = whereCost(nOut); } #endif if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ @@ -4451,7 +4458,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = whereCostFromInt(pSrc->pTab->nRowEst); + rSize = whereCost(pSrc->pTab->nRowEst); rLogSize = estLog(rSize); /* Automatic indexes */ @@ -4473,9 +4480,12 @@ static int whereLoopAddBtree( pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; - assert( 43==whereCostFromInt(20) ); - pNew->rSetup = 43 + rLogSize + rSize; - pNew->nOut = 33; assert( 33==whereCostFromInt(10) ); + /* TUNING: One-time cost for computing the automatic index is + ** approximately 6*N*log2(N) where N is the number of rows in + ** the table being indexed. */ + pNew->rSetup = rLogSize + rSize + 26; assert( 26==whereCost(6) ); + /* TUNING: Each index lookup yields 10 rows in the table */ + pNew->nOut = 33; assert( 33==whereCost(10) ); pNew->rRun = whereCostAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_TEMP_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; @@ -4501,7 +4511,12 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 - b; + /* TUNING: Cost of full table scan is 3*(N + log2(N)). + ** + The extra 3 factor is to encourage the use of indexed lookups + ** over full scans. A smaller constant 2 is used for covering + ** index scans so that a covering index scan will be favored over + ** a table scan. */ + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; }else{ @@ -4517,8 +4532,22 @@ static int whereLoopAddBtree( ){ pNew->iSortIdx = b ? iSortIdx : 0; pNew->nOut = rSize; - pNew->rRun = whereCostAdd(rSize,rLogSize); - pNew->rRun += ((m!=0) ? rLogSize : 10) - b; + if( m==0 ){ + /* TUNING: Cost of a covering index scan is 2*(N + log2(N)). + ** + The extra 2 factor is to encourage the use of indexed lookups + ** over index scans. A table scan uses a factor of 3 so that + ** index scans are favored over table scans. + ** + If this covering index might also help satisfy the ORDER BY + ** clause, then the cost is fudged down slightly so that this + ** index is favored above other indices that have no hope of + ** helping with the ORDER BY. */ + pNew->rRun = 10 + whereCostAdd(rSize,rLogSize) - b; + }else{ + assert( b!=0 ); + /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) + ** which we will simplify to just N*log2(N) */ + pNew->rRun = rSize + rLogSize; + } rc = whereLoopInsert(pBuilder, pNew); if( rc ) break; } @@ -4675,7 +4704,8 @@ static int whereLoopAddVirtual( && pIdxInfo->orderByConsumed); pNew->rSetup = 0; pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost); - pNew->nOut = 46; assert( 46 == whereCostFromInt(25) ); + /* TUNING: Every virtual table query returns 25 rows */ + pNew->nOut = 46; assert( 46==whereCost(25) ); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -5066,6 +5096,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ @@ -5081,8 +5112,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ WhereLoop **pX; /* Used to divy up the pSpace memory */ char *pSpace; /* Temporary memory used by this routine */ - db = pWInfo->pParse->db; + pParse = pWInfo->pParse; + db = pParse->db; nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver\n")); @@ -5099,8 +5134,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pFrom->aLoop = pX; } - /* Seed the search with a single WherePath containing zero WhereLoops */ - aFrom[0].nRow = pWInfo->pParse->nQueryLoop; + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 25. If the cost + ** of computing an automatic index is not paid back within the first 25 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==whereCost(25) ); nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller @@ -5109,7 +5148,8 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrderedValid = 1; }else{ - /* Compute an estimate on the cost to sort the entire result set */ + /* TUNING: Estimated cost of sorting is N*log2(N) where N is the + ** number of output rows. */ rSortCost = nRowEst + estLog(nRowEst); WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } @@ -5254,7 +5294,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ } if( nFrom==0 ){ - sqlite3ErrorMsg(pWInfo->pParse, "no query solution"); + sqlite3ErrorMsg(pParse, "no query solution"); sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } @@ -5335,7 +5375,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; pLoop->u.btree.nEq = 1; - pLoop->rRun = 33; /* 33 == whereCostFromInt(10) */ + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==whereCost(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; @@ -5353,7 +5394,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->nLTerm = j; pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; - pLoop->rRun = 39; /* 39 == whereCostFromInt(15) */ + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==whereCost(15) */ break; } } diff --git a/test/autoindex1.test b/test/autoindex1.test index 9d67def05f..f7b0977fbc 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -78,6 +78,9 @@ do_test autoindex1-202 { do_test autoindex1-210 { db eval { PRAGMA automatic_index=ON; + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } } {11 911 22 922 33 933 44 944 55 955 66 966 77 977 88 988} @@ -143,6 +146,9 @@ do_test autoindex1-401 { do_execsql_test autoindex1-500 { CREATE TABLE t501(a INTEGER PRIMARY KEY, b); CREATE TABLE t502(x INTEGER PRIMARY KEY, y); + INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t501',null,'1000000'); + INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t502',null,'1000'); + ANALYZE sqlite_master; EXPLAIN QUERY PLAN SELECT b FROM t501 WHERE t501.a IN (SELECT x FROM t502 WHERE y=?);