Add a way for virtual tables to return the expected number of rows for a scan (not just the overall cost) to SQLite. Have the rtree module make use of this.

FossilOrigin-Name: 5a3cfd747a85480d215784817c3821d87ecfa2f7

ext/rtree/rtree.c

@@ -137,6 +137,16 @@ typedef union RtreeCoord RtreeCoord;
 */
 #define HASHSIZE 128
 
+/* The xBestIndex method of this virtual table requires an estimate of
+** the number of rows in the virtual table to calculate the costs of
+** various strategies. If possible, this estimate is loaded from the
+** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum).
+** Otherwise, if no sqlite_stat1 entry is available, use 
+** RTREE_DEFAULT_ROWEST.
+*/
+#define RTREE_DEFAULT_ROWEST 1048576
+#define RTREE_MIN_ROWEST         100
+
 /* 
 ** An rtree virtual-table object.
 */
@@ -151,6 +161,7 @@ struct Rtree {
   char *zName;                /* Name of r-tree table */ 
   RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ 
   int nBusy;                  /* Current number of users of this structure */
+  i64 nRowEst;                /* Estimated number of rows in this table */
 
   /* List of nodes removed during a CondenseTree operation. List is
   ** linked together via the pointer normally used for hash chains -
@@ -1343,6 +1354,19 @@ static int rtreeFilter(
   return rc;
 }
 
+/*
+** Set the pIdxInfo->estimatedRows variable to nRow. Unless this
+** extension is currently being used by a version of SQLite too old to
+** support estimatedRows. In that case this function is a no-op.
+*/
+static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
+#if SQLITE_VERSION_NUMBER>=308002
+  if( sqlite3_libversion_number()>=300802 ){
+    pIdxInfo->estimatedRows = nRow;
+  }
+#endif
+}
+
 /*
 ** Rtree virtual table module xBestIndex method. There are three
 ** table scan strategies to choose from (in order from most to 
@@ -1378,13 +1402,14 @@ static int rtreeFilter(
 ** is 'a', the second from the left 'b' etc.
 */
 static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  Rtree *pRtree = (Rtree*)tab;
   int rc = SQLITE_OK;
   int ii;
+  i64 nRow;                       /* Estimated rows returned by this scan */
 
   int iIdx = 0;
   char zIdxStr[RTREE_MAX_DIMENSIONS*8+1];
   memset(zIdxStr, 0, sizeof(zIdxStr));
-  UNUSED_PARAMETER(tab);
 
   assert( pIdxInfo->idxStr==0 );
   for(ii=0; ii<pIdxInfo->nConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){
@@ -1404,9 +1429,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
       /* This strategy involves a two rowid lookups on an B-Tree structures
       ** and then a linear search of an R-Tree node. This should be 
       ** considered almost as quick as a direct rowid lookup (for which 
-      ** sqlite uses an internal cost of 0.0).
+      ** sqlite uses an internal cost of 0.0). It is expected to return
+      ** a single row.
       */ 
-      pIdxInfo->estimatedCost = 10.0;
+      pIdxInfo->estimatedCost = 30.0;
+      setEstimatedRows(pIdxInfo, 1);
       return SQLITE_OK;
     }
 
@@ -1435,8 +1462,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
   if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){
     return SQLITE_NOMEM;
   }
-  assert( iIdx>=0 );
+
-  pIdxInfo->estimatedCost = (2000000.0 / (double)(iIdx + 1));
+  nRow = pRtree->nRowEst / (iIdx + 1);
+  pIdxInfo->estimatedCost = (double)6.0 * (double)nRow;
+  setEstimatedRows(pIdxInfo, nRow);
+
   return rc;
 }
 
@@ -2911,6 +2941,37 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){
   return rc;
 }
 
+/*
+** This function populates the pRtree->nRowEst variable with an estimate
+** of the number of rows in the virtual table. If possible, this is based
+** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST.
+*/
+static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){
+  const char *zSql = "SELECT stat FROM sqlite_stat1 WHERE tbl= ? || '_rowid'";
+  sqlite3_stmt *p;
+  int rc;
+  i64 nRow = 0;
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_text(p, 1, pRtree->zName, -1, SQLITE_STATIC);
+    if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0);
+    rc = sqlite3_finalize(p);
+  }else if( rc!=SQLITE_NOMEM ){
+    rc = SQLITE_OK;
+  }
+
+  if( rc==SQLITE_OK ){
+    if( nRow==0 ){
+      pRtree->nRowEst = RTREE_DEFAULT_ROWEST;
+    }else{
+      pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST);
+    }
+  }
+
+  return rc;
+}
+
 static sqlite3_module rtreeModule = {
   0,                          /* iVersion */
   rtreeCreate,                /* xCreate - create a table */
@@ -2996,6 +3057,7 @@ static int rtreeSqlInit(
   appStmt[7] = &pRtree->pWriteParent;
   appStmt[8] = &pRtree->pDeleteParent;
 
+  rc = rtreeQueryStat1(db, pRtree);
   for(i=0; i<N_STATEMENT && rc==SQLITE_OK; i++){
     char *zSql = sqlite3_mprintf(azSql[i], zDb, zPrefix);
     if( zSql ){

ext/rtree/rtree6.test

@@ -96,7 +96,7 @@ do_eqp_test rtree6.2.4 {
   SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
 } {
   0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} 
-  0 1 1 {SCAN TABLE t2}
+  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
 }
 
 do_eqp_test rtree6.2.5 {

ext/rtree/rtreeC.test

@@ -0,0 +1,112 @@
+# 2011 March 2
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Make sure the rtreenode() testing function can handle entries with
+# 64-bit rowids.
+# 
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !rtree { finish_test ; return }
+set testprefix rtreeC
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE r_tree USING rtree(id, min_x, max_x, min_y, max_y);
+  CREATE TABLE t(x, y);
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM r_tree, t 
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
+} {
+  0 0 1 {SCAN TABLE t}
+  0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t, r_tree
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
+} {
+  0 0 0 {SCAN TABLE t}
+  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t, r_tree
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y
+} {
+  0 0 0 {SCAN TABLE t}
+  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 1.5 {
+  SELECT * FROM t, r_tree
+} {
+  0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:}
+  0 1 0 {SCAN TABLE t} 
+}
+
+do_execsql_test 2.0 {
+  INSERT INTO t VALUES(0, 0);
+  INSERT INTO t VALUES(0, 1);
+  INSERT INTO t VALUES(0, 2);
+  INSERT INTO t VALUES(0, 3);
+  INSERT INTO t VALUES(0, 4);
+  INSERT INTO t VALUES(0, 5);
+  INSERT INTO t VALUES(0, 6);
+  INSERT INTO t VALUES(0, 7);
+  INSERT INTO t VALUES(0, 8);
+  INSERT INTO t VALUES(0, 9);
+
+  INSERT INTO t SELECT x+1, y FROM t;
+  INSERT INTO t SELECT x+2, y FROM t;
+  INSERT INTO t SELECT x+4, y FROM t;
+  INSERT INTO r_tree SELECT NULL, x-1, x+1, y-1, y+1 FROM t;
+  ANALYZE;
+}
+
+db close
+sqlite3 db test.db
+
+do_eqp_test 2.1 {
+  SELECT * FROM r_tree, t 
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
+} {
+  0 0 1 {SCAN TABLE t}
+  0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 2.2 {
+  SELECT * FROM t, r_tree
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
+} {
+  0 0 0 {SCAN TABLE t}
+  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 2.3 {
+  SELECT * FROM t, r_tree
+  WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y
+} {
+  0 0 0 {SCAN TABLE t}
+  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa}
+}
+
+do_eqp_test 2.5 {
+  SELECT * FROM t, r_tree
+} {
+  0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:}
+  0 1 0 {SCAN TABLE t} 
+}
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Remove\sunreachable\scode,\sreplacing\sit\sin\smost\scases\swith\sassert()\sor\sNEVER()\nmacros.
+C Add\sa\sway\sfor\svirtual\stables\sto\sreturn\sthe\sexpected\snumber\sof\srows\sfor\sa\sscan\s(not\sjust\sthe\soverall\scost)\sto\sSQLite.\sHave\sthe\srtree\smodule\smake\suse\sof\sthis.
-D 2013-11-11T16:55:52.347
+D 2013-11-11T19:01:33.120
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in d12e4455cf7a36e42d3949876c1c3b88ff70867a
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -120,19 +120,20 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c db516d7e59a14c92df10b552789509f2b632df3a
+F ext/rtree/rtree.c 1ec67e64ab28acd72d6a39416b24cb6b33a8bd9e
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
 F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc
 F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0
 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e
-F ext/rtree/rtree6.test fb94b98c1145b7f44c72635d11492f35349ab27e
+F ext/rtree/rtree6.test fe0bd377a21c68ce2826129d14354c884cb1f354
 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971
 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a
 F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34
 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf
 F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2
+F ext/rtree/rtreeC.test 0c3dcd379e012c76df5e59d53bc99a8b48f1603b
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea
 F ext/rtree/sqlite3rtree.h c34c1e41d1ab80bb8ad09aae402c9c956871a765
@@ -220,7 +221,7 @@ F src/resolve.c fc4673cc49b116e51e7f12de074c0acf8f2388f9
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
 F src/select.c a040df82c4ac90b27114e1c21a05d77cc6dc6f80
 F src/shell.c 03d8d9b4052430343ff30d646334621f980f1202
-F src/sqlite.h.in ecbd714ecc0ff710a149fc72db6552738148ad0c
+F src/sqlite.h.in f73c5fa1f1ff7c82ad8dff4daee690fe0ef43638
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
 F src/sqliteInt.h 3da1940a2ba05a663e9016d57f1ea1f79ffcb03e
@@ -292,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d
 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74
-F src/where.c 2c2a59315754521667e525e5f33c4238b7c7dcb1
+F src/where.c 85766647d693ed7cfba42dfce07e2cb9c3384bd9
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1135,7 +1136,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 569fedd6bb07bdd3430da8dc65de4a9fdfe204cc
+P 924d63b283a3d059838114c95d42c6feaf913529
-R 749a13031920b9205045d3321716b16a
+R d1d8a68221f9b1872aa5651bda60ad81
-U drh
+U dan
-Z 7491e1f10541a194c40abc7098f10a18
+Z 4d69165a9eae811b48725d315a514f08

manifest.uuid

@@ -1 +1 @@
-924d63b283a3d059838114c95d42c6feaf913529
+5a3cfd747a85480d215784817c3821d87ecfa2f7

src/sqlite.h.in

@@ -5290,10 +5290,22 @@ struct sqlite3_module {
 ** the correct order to satisfy the ORDER BY clause so that no separate
 ** sorting step is required.
 **
-** ^The estimatedCost value is an estimate of the cost of doing the
+** ^The estimatedCost value is an estimate of the cost of a particular
-** particular lookup.  A full scan of a table with N entries should have
+** strategy. A cost of N indicates that the cost of the strategy is similar
-** a cost of N.  A binary search of a table of N entries should have a
+** to a linear scan of an SQLite table with N rows. A cost of log(N) 
-** cost of approximately log(N).
+** indicates that the expense of the operation is similar to that of a
+** binary search on a unique indexed field of an SQLite table with N rows.
+**
+** ^The estimatedRows value is an estimate of the number of rows that
+** will be returned by the strategy.
+**
+** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
+** structure for SQLite version 3.8.2. If a virtual table extension is
+** used with an SQLite version earlier than 3.8.2, the results of attempting 
+** to read or write the estimatedRows field are undefined (but are likely 
+** to included crashing the application). The estimatedRows field should
+** therefore only be used if [sqlite3_libversion_number()] returns a
+** value greater than or equal to 3008002.
 */
 struct sqlite3_index_info {
   /* Inputs */
@@ -5318,7 +5330,8 @@ struct sqlite3_index_info {
   char *idxStr;              /* String, possibly obtained from sqlite3_malloc */
   int needToFreeIdxStr;      /* Free idxStr using sqlite3_free() if true */
   int orderByConsumed;       /* True if output is already ordered */
-  double estimatedCost;      /* Estimated cost of using this index */
+  double estimatedCost;           /* Estimated cost of using this index */
+  sqlite3_int64 estimatedRows;    /* Estimated number of rows returned */
 };
 
 /*

src/where.c

@@ -1978,6 +1978,7 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){
   sqlite3DebugPrintf("  idxStr=%s\n", p->idxStr);
   sqlite3DebugPrintf("  orderByConsumed=%d\n", p->orderByConsumed);
   sqlite3DebugPrintf("  estimatedCost=%g\n", p->estimatedCost);
+  sqlite3DebugPrintf("  estimatedRows=%lld\n", p->estimatedRows);
 }
 #else
 #define TRACE_IDX_INPUTS(A)
@@ -4787,6 +4788,7 @@ static int whereLoopAddVirtual(
     pIdxInfo->needToFreeIdxStr = 0;
     pIdxInfo->orderByConsumed = 0;
     pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
+    pIdxInfo->estimatedRows = 25;
     rc = vtabBestIndex(pParse, pTab, pIdxInfo);
     if( rc ) goto whereLoopAddVtab_exit;
     pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
@@ -4846,8 +4848,7 @@ static int whereLoopAddVirtual(
                                      && pIdxInfo->orderByConsumed);
       pNew->rSetup = 0;
       pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
-      /* TUNING: Every virtual table query returns 25 rows */
+      pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
-      pNew->nOut = 46;  assert( 46==sqlite3LogEst(25) );
       whereLoopInsert(pBuilder, pNew);
       if( pNew->u.vtab.needFree ){
         sqlite3_free(pNew->u.vtab.idxStr);