diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 815a8993e4..e700389b6c 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -137,6 +137,16 @@ typedef union RtreeCoord RtreeCoord; */ #define HASHSIZE 128 +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 + /* ** An rtree virtual-table object. */ @@ -151,6 +161,7 @@ struct Rtree { char *zName; /* Name of r-tree table */ RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -1343,6 +1354,19 @@ static int rtreeFilter( return rc; } +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=308002 + if( sqlite3_libversion_number()>=300802 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -1378,13 +1402,14 @@ static int rtreeFilter( ** is 'a', the second from the left 'b' etc. */ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; int rc = SQLITE_OK; int ii; + i64 nRow; /* Estimated rows returned by this scan */ int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); - UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ @@ -1404,9 +1429,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ /* This strategy involves a two rowid lookups on an B-Tree structures ** and then a linear search of an R-Tree node. This should be ** considered almost as quick as a direct rowid lookup (for which - ** sqlite uses an internal cost of 0.0). + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. */ - pIdxInfo->estimatedCost = 10.0; + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); return SQLITE_OK; } @@ -1435,8 +1462,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ return SQLITE_NOMEM; } - assert( iIdx>=0 ); - pIdxInfo->estimatedCost = (2000000.0 / (double)(iIdx + 1)); + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + return rc; } @@ -2911,6 +2941,37 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zSql = "SELECT stat FROM sqlite_stat1 WHERE tbl= ? || '_rowid'"; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; + + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(p, 1, pRtree->zName, -1, SQLITE_STATIC); + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + + return rc; +} + static sqlite3_module rtreeModule = { 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ @@ -2996,6 +3057,7 @@ static int rtreeSqlInit( appStmt[7] = &pRtree->pWriteParent; appStmt[8] = &pRtree->pDeleteParent; + rc = rtreeQueryStat1(db, pRtree); for(i=0; i10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} - 0 1 1 {SCAN TABLE t2} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test new file mode 100644 index 0000000000..47e8581483 --- /dev/null +++ b/ext/rtree/rtreeC.test @@ -0,0 +1,112 @@ +# 2011 March 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Make sure the rtreenode() testing function can handle entries with +# 64-bit rowids. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } +set testprefix rtreeC + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE r_tree USING rtree(id, min_x, max_x, min_y, max_y); + CREATE TABLE t(x, y); +} + +do_eqp_test 1.1 { + SELECT * FROM r_tree, t + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 1 {SCAN TABLE t} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 1.2 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 1.3 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 1.5 { + SELECT * FROM t, r_tree +} { + 0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:} + 0 1 0 {SCAN TABLE t} +} + +do_execsql_test 2.0 { + INSERT INTO t VALUES(0, 0); + INSERT INTO t VALUES(0, 1); + INSERT INTO t VALUES(0, 2); + INSERT INTO t VALUES(0, 3); + INSERT INTO t VALUES(0, 4); + INSERT INTO t VALUES(0, 5); + INSERT INTO t VALUES(0, 6); + INSERT INTO t VALUES(0, 7); + INSERT INTO t VALUES(0, 8); + INSERT INTO t VALUES(0, 9); + + INSERT INTO t SELECT x+1, y FROM t; + INSERT INTO t SELECT x+2, y FROM t; + INSERT INTO t SELECT x+4, y FROM t; + INSERT INTO r_tree SELECT NULL, x-1, x+1, y-1, y+1 FROM t; + ANALYZE; +} + +db close +sqlite3 db test.db + +do_eqp_test 2.1 { + SELECT * FROM r_tree, t + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 1 {SCAN TABLE t} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 2.2 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 2.3 { + SELECT * FROM t, r_tree + WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y +} { + 0 0 0 {SCAN TABLE t} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} +} + +do_eqp_test 2.5 { + SELECT * FROM t, r_tree +} { + 0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:} + 0 1 0 {SCAN TABLE t} +} + +finish_test + diff --git a/manifest b/manifest index a1791039ae..4d0644314b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunreachable\scode,\sreplacing\sit\sin\smost\scases\swith\sassert()\sor\sNEVER()\nmacros. -D 2013-11-11T16:55:52.347 +C Add\sa\sway\sfor\svirtual\stables\sto\sreturn\sthe\sexpected\snumber\sof\srows\sfor\sa\sscan\s(not\sjust\sthe\soverall\scost)\sto\sSQLite.\sHave\sthe\srtree\smodule\smake\suse\sof\sthis. +D 2013-11-11T19:01:33.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d12e4455cf7a36e42d3949876c1c3b88ff70867a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,19 +120,20 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c db516d7e59a14c92df10b552789509f2b632df3a +F ext/rtree/rtree.c 1ec67e64ab28acd72d6a39416b24cb6b33a8bd9e F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test fb94b98c1145b7f44c72635d11492f35349ab27e +F ext/rtree/rtree6.test fe0bd377a21c68ce2826129d14354c884cb1f354 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 +F ext/rtree/rtreeC.test 0c3dcd379e012c76df5e59d53bc99a8b48f1603b F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h c34c1e41d1ab80bb8ad09aae402c9c956871a765 @@ -220,7 +221,7 @@ F src/resolve.c fc4673cc49b116e51e7f12de074c0acf8f2388f9 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c a040df82c4ac90b27114e1c21a05d77cc6dc6f80 F src/shell.c 03d8d9b4052430343ff30d646334621f980f1202 -F src/sqlite.h.in ecbd714ecc0ff710a149fc72db6552738148ad0c +F src/sqlite.h.in f73c5fa1f1ff7c82ad8dff4daee690fe0ef43638 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 3da1940a2ba05a663e9016d57f1ea1f79ffcb03e @@ -292,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 2c2a59315754521667e525e5f33c4238b7c7dcb1 +F src/where.c 85766647d693ed7cfba42dfce07e2cb9c3384bd9 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1135,7 +1136,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 569fedd6bb07bdd3430da8dc65de4a9fdfe204cc -R 749a13031920b9205045d3321716b16a -U drh -Z 7491e1f10541a194c40abc7098f10a18 +P 924d63b283a3d059838114c95d42c6feaf913529 +R d1d8a68221f9b1872aa5651bda60ad81 +U dan +Z 4d69165a9eae811b48725d315a514f08 diff --git a/manifest.uuid b/manifest.uuid index 6b1e24127e..a19bc65e5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -924d63b283a3d059838114c95d42c6feaf913529 \ No newline at end of file +5a3cfd747a85480d215784817c3821d87ecfa2f7 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 196186dd94..fae5a7af33 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5290,10 +5290,22 @@ struct sqlite3_module { ** the correct order to satisfy the ORDER BY clause so that no separate ** sorting step is required. ** -** ^The estimatedCost value is an estimate of the cost of doing the -** particular lookup. A full scan of a table with N entries should have -** a cost of N. A binary search of a table of N entries should have a -** cost of approximately log(N). +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite version 3.8.2. If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to included crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. */ struct sqlite3_index_info { /* Inputs */ @@ -5318,7 +5330,8 @@ struct sqlite3_index_info { char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ - double estimatedCost; /* Estimated cost of using this index */ + double estimatedCost; /* Estimated cost of using this index */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ }; /* diff --git a/src/where.c b/src/where.c index f741519a40..efbd310975 100644 --- a/src/where.c +++ b/src/where.c @@ -1978,6 +1978,7 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else #define TRACE_IDX_INPUTS(A) @@ -4787,6 +4788,7 @@ static int whereLoopAddVirtual( pIdxInfo->needToFreeIdxStr = 0; pIdxInfo->orderByConsumed = 0; pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; @@ -4846,8 +4848,7 @@ static int whereLoopAddVirtual( && pIdxInfo->orderByConsumed); pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); - /* TUNING: Every virtual table query returns 25 rows */ - pNew->nOut = 46; assert( 46==sqlite3LogEst(25) ); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr);