Optimize queries of the form "SELECT count(*) FROM <tbl>" by adding a sqlite3BtreeCount() interface to the btree layer. (CVS 6316)

FossilOrigin-Name: d4aa6593183224b6868a322511511c0bbf63b598

manifest

@@ -1,5 +1,5 @@
-C Scan\san\sindex\sinstead\sof\sa\stable\sfor\s"SELECT\scount(*)\sFROM\s<tbl>"\squeries.\sBecause\san\sindex\sis\susually\ssmaller\sthan\sa\stable\son\sdisk,\sthis\ssaves\ssome\sIO.\s(CVS\s6315)
+C Optimize\squeries\sof\sthe\sform\s"SELECT\scount(*)\sFROM\s<tbl>"\sby\sadding\sa\ssqlite3BtreeCount()\sinterface\sto\sthe\sbtree\slayer.\s(CVS\s6316)
-D 2009-02-23T17:33:50
+D 2009-02-24T10:01:52
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in d64baddbf55cdf33ff030e14da837324711a4ef7
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -103,8 +103,8 @@ F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627
 F src/backup.c 2d3f31148d7b086c5c72d9edcd04fc2751b0aa6e
 F src/bitvec.c 44f7059ac1f874d364b34af31b9617e52223ba75
 F src/btmutex.c 63c5cc4ad5715690767ffcb741e185d7bc35ec1a
-F src/btree.c 086fdb4505aa00275d6873829aeb51bf57da8d16
+F src/btree.c e0178d6fb69c8f332f3fba96cfc0b08275ad5e76
-F src/btree.h 4eab72af6adf95f0b08b61a72ef9781bdb0bf63f
+F src/btree.h 96a019c9f28da38e79940512d7800e419cd8c702
 F src/btreeInt.h 0a4884e6152d7cae9c741e91b830064c19fd2c05
 F src/build.c a1db48aec62c95049d783f231195812ff97ae268
 F src/callback.c 5f10bca853e59a2c272bbfd5b720303f8b69e520
@@ -113,7 +113,7 @@ F src/date.c 0d804df3bbda46329946a01ff5c75c3f4f135218
 F src/delete.c 06e78b6eb53f27acc809a0f69178ea719748bb42
 F src/expr.c 97545fa4058f86c67eb7cacadf60d2964300b00c
 F src/fault.c dc88c821842157460750d2d61a8a8b4197d047ff
-F src/func.c 2fb36cd7cc24e16845db203187d1e52811b0fa9c
+F src/func.c de2eed7d96365210faecda877c5a12cf440bdf42
 F src/global.c 448419c44ce0701104c2121b0e06919b44514c0c
 F src/hash.c 5824e6ff7ba78cd34c8d6cd724367713583e5b55
 F src/hash.h 28f38ebb1006a5beedcb013bcdfe31befe7437ae
@@ -154,11 +154,11 @@ F src/printf.c 9866a9a9c4a90f6d4147407f373df3fd5d5f9b6f
 F src/random.c 676b9d7ac820fe81e6fb2394ac8c10cff7f38628
 F src/resolve.c 60a5f405540debee767d8c21ab78a5210b174fa2
 F src/rowset.c ba9375f37053d422dd76965a9c370a13b6e1aac4
-F src/select.c aa7328a23c0abe019f98bb7e1f4f63d62e20ba98
+F src/select.c 474557a5e4388c347f055c6759da1a7a4fc01e32
 F src/shell.c f109ebbb50132926ebbc173a6c2d8838d5d78527
 F src/sqlite.h.in 14f4d065bafed8500ea558a75a8e2be89c784d61
 F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17
-F src/sqliteInt.h ac53d3b963c0c98b8f8c6df652a9cde2fd00e987
+F src/sqliteInt.h b294711ad509e356aa75da9ef12334c19d86f64a
 F src/sqliteLimit.h ffe93f5a0c4e7bd13e70cd7bf84cfb5c3465f45d
 F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
 F src/table.c 332ab0ea691e63862e2a8bdfe2c0617ee61062a3
@@ -199,7 +199,7 @@ F src/update.c 9c11bc0bba520bcfce47e229a7235a9bc5f9121a
 F src/utf.c 1da9c832dba0fa8f865b5b902d93f420a1ee4245
 F src/util.c 1363f64351f3b544790f3c523439354c02f8c4e9
 F src/vacuum.c 4929a585ef0fb1dfaf46302f8a9c4aa30c2d9cf5
-F src/vdbe.c 23a620da910b7d3a60ffebc088b7f00f5a6cd247
+F src/vdbe.c 13194e2961ab92ec42016f05797f02a898d06729
 F src/vdbe.h d70a68bee196ab228914a3902c79dbd24342a0f2
 F src/vdbeInt.h d12bc259b34d3d610ebf05d648eb6346d48478c3
 F src/vdbeapi.c f94fe2eb6f48687e918f0df7eed1409cff9dcf58
@@ -673,7 +673,7 @@ F test/where6.test 42c4373595f4409d9c6a9987b4a60000ad664faf
 F test/where7.test 2487cda68faabf5edeb524289913f00f8d64e223
 F test/where8.test 1b9152a086408ee789166d0a954abc597372f868
 F test/where8m.test c1010d61826412ff66abd29bfb32e5d6b37d965c
-F test/where9.test 0e44fd96a838f7fa9ecd39a6569bfc4bd446faad
+F test/where9.test 12c1e46364fb245ff84253758dd76dacc7bfe619
 F test/whereA.test ef8d699d87934bd747119c75fbb4711b584a8b60
 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/zeroblob.test 792124852ec61458a2eb527b5091791215e0be95
@@ -701,7 +701,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P bc078e0007b6c3dc07722820bb53798b643212b3
+P 294ba6f743c9132dce0e73da480bd3c2071e7239
-R 19edfff27399a21c42d0d895b28d7f0b
+R 524bbf1793d6465eb6ab93cdb258f1df
 U danielk1977
-Z 309cb578835963c08d5f7f73ef6b305b
+Z cbf078ee778e00d0fbc4a96fcd97b1a6

manifest.uuid

@@ -1 +1 @@
-294ba6f743c9132dce0e73da480bd3c2071e7239
+d4aa6593183224b6868a322511511c0bbf63b598

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.566 2009/02/18 20:31:18 drh Exp $
+** $Id: btree.c,v 1.567 2009/02/24 10:01:52 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** See the header comment on "btreeInt.h" for additional information.
@@ -6751,6 +6751,75 @@ int sqlite3BtreeFlags(BtCursor *pCur){
   return pPage->aData[pPage->hdrOffset];
 }
 
+#ifndef SQLITE_OMIT_BTREECOUNT
+/*
+** The first argument, pCur, is a cursor opened on some b-tree. Count the
+** number of entries in the b-tree and write the result to *pnEntry.
+**
+** SQLITE_OK is returned if the operation is successfully executed. 
+** Otherwise, if an error is encountered (i.e. an IO error or database
+** corruption) an SQLite error code is returned.
+*/
+int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){
+  i64 nEntry = 0;                      /* Value to return in *pnEntry */
+  int rc;                              /* Return code */
+  rc = moveToRoot(pCur);
+
+  /* Unless an error occurs, the following loop runs one iteration for each
+  ** page in the B-Tree structure (not including overflow pages). 
+  */
+  while( rc==SQLITE_OK ){
+    int iIdx;                          /* Index of child node in parent */
+    MemPage *pPage;                    /* Current page of the b-tree */
+
+    /* If this is a leaf page or the tree is not an int-key tree, then 
+    ** this page contains countable entries. Increment the entry counter
+    ** accordingly.
+    */
+    pPage = pCur->apPage[pCur->iPage];
+    if( pPage->leaf || !pPage->intKey ){
+      nEntry += pPage->nCell;
+    }
+
+    /* pPage is a leaf node. This loop navigates the cursor so that it 
+    ** points to the first interior cell that it points to the parent of
+    ** the next page in the tree that has not yet been visited. The
+    ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell
+    ** of the page, or to the number of cells in the page if the next page
+    ** to visit is the right-child of its parent.
+    **
+    ** If all pages in the tree have been visited, return SQLITE_OK to the
+    ** caller.
+    */
+    if( pPage->leaf ){
+      do {
+        if( pCur->iPage==0 ){
+          /* All pages of the b-tree have been visited. Return successfully. */
+          *pnEntry = nEntry;
+          return SQLITE_OK;
+        }
+        sqlite3BtreeMoveToParent(pCur);
+      }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell );
+
+      pCur->aiIdx[pCur->iPage]++;
+      pPage = pCur->apPage[pCur->iPage];
+    }
+
+    /* Descend to the child node of the cell that the cursor currently 
+    ** points at. This is the right-child if (iIdx==pPage->nCell).
+    */
+    iIdx = pCur->aiIdx[pCur->iPage];
+    if( iIdx==pPage->nCell ){
+      rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8]));
+    }else{
+      rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx)));
+    }
+  }
+
+  /* An error has occured. Return an error code. */
+  return rc;
+}
+#endif
 
 /*
 ** Return the pager associated with a BTree.  This routine is used for

src/btree.h

@@ -13,7 +13,7 @@
 ** subsystem.  See comments in the source code for a detailed description
 ** of what each interface routine does.
 **
-** @(#) $Id: btree.h,v 1.108 2009/02/03 16:51:25 danielk1977 Exp $
+** @(#) $Id: btree.h,v 1.109 2009/02/24 10:01:52 danielk1977 Exp $
 */
 #ifndef _BTREE_H_
 #define _BTREE_H_
@@ -173,6 +173,10 @@ int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*);
 void sqlite3BtreeCacheOverflow(BtCursor *);
 void sqlite3BtreeClearCursor(BtCursor *);
 
+#ifndef SQLITE_OMIT_BTREECOUNT
+int sqlite3BtreeCount(BtCursor *, i64 *);
+#endif
+
 #ifdef SQLITE_TEST
 int sqlite3BtreeCursorInfo(BtCursor*, int*, int);
 void sqlite3BtreeCursorList(Btree*);

src/func.c

@@ -16,7 +16,7 @@
 ** sqliteRegisterBuildinFunctions() found at the bottom of the file.
 ** All other code has file scope.
 **
-** $Id: func.c,v 1.223 2009/02/19 14:39:25 danielk1977 Exp $
+** $Id: func.c,v 1.224 2009/02/24 10:01:52 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include <stdlib.h>
@@ -1413,7 +1413,8 @@ void sqlite3RegisterGlobalFunctions(void){
     AGGREGATE(sum,               1, 0, 0, sumStep,         sumFinalize    ),
     AGGREGATE(total,             1, 0, 0, sumStep,         totalFinalize    ),
     AGGREGATE(avg,               1, 0, 0, sumStep,         avgFinalize    ),
-    AGGREGATE(count,             0, 0, 0, countStep,       countFinalize  ),
+ /* AGGREGATE(count,             0, 0, 0, countStep,       countFinalize  ), */
+    {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0},
     AGGREGATE(count,             1, 0, 0, countStep,       countFinalize  ),
     AGGREGATE(group_concat,      1, 0, 0, groupConcatStep, groupConcatFinalize),
     AGGREGATE(group_concat,      2, 0, 0, groupConcatStep, groupConcatFinalize),

src/select.c

@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.501 2009/02/20 10:58:42 danielk1977 Exp $
+** $Id: select.c,v 1.502 2009/02/24 10:01:52 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 
@@ -2953,6 +2953,39 @@ static u8 minMaxQuery(Select *p){
   return WHERE_ORDERBY_NORMAL;
 }
 
+/*
+** The select statement passed as the first argument is an aggregate query.
+** The second argment is the associated aggregate-info object. This 
+** function tests if the SELECT is of the form:
+**
+**   SELECT count(*) FROM <tbl>
+**
+** where table is a database table, not a sub-select or view. If the query
+** does match this pattern, then a pointer to the Table object representing
+** <tbl> is returned. Otherwise, 0 is returned.
+*/
+static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){
+  Table *pTab;
+  Expr *pExpr;
+
+  assert( !p->pGroupBy );
+
+  if( p->pWhere || p->pHaving || p->pEList->nExpr!=1
+   || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect
+  ){
+    return 0;
+  }
+
+  pTab = p->pSrc->a[0].pTab;
+  pExpr = p->pEList->a[0].pExpr;
+  if( !pTab || pTab->pSelect || IsVirtual(pTab) ) return 0;
+  if( pExpr->op!=TK_AGG_FUNCTION ) return 0;
+  if( (pAggInfo->aFunc[0].pFunc->flags&SQLITE_FUNC_COUNT)==0 ) return 0;
+  if( pExpr->flags&EP_Distinct ) return 0;
+
+  return pTab;
+}
+
 /*
 ** If the source-list item passed as an argument was augmented with an
 ** INDEXED BY clause, then try to locate the specified index. If there
@@ -3998,10 +4031,64 @@ int sqlite3Select(
      
     } /* endif pGroupBy */
     else {
-      ExprList *pMinMax = 0;
       ExprList *pDel = 0;
-      u8 flag;
+#ifndef SQLITE_OMIT_BTREECOUNT
+      Table *pTab;
+      if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){
+        /* If isSimpleCount() returns a pointer to a Table structure, then
+        ** the SQL statement is of the form:
+        **
+        **   SELECT count(*) FROM <tbl>
+        **
+        ** where the Table structure returned represents table <tbl>.
+        **
+        ** This statement is so common that it is optimized specially. The
+        ** OP_Count instruction is executed either on the intkey table that
+        ** contains the data for table <tbl> or on one of its indexes. It
+        ** is better to execute the op on an index, as indexes are almost
+        ** always spread across less pages than their corresponding tables.
+        */
+        const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
+        const int iCsr = pParse->nTab++;     /* Cursor to scan b-tree */
+        Index *pIdx;                         /* Iterator variable */
+        KeyInfo *pKeyInfo = 0;               /* Keyinfo for scanned index */
+        Index *pBest = 0;                    /* Best index found so far */
+        int iRoot = pTab->tnum;              /* Root page of scanned b-tree */
 
+        sqlite3CodeVerifySchema(pParse, iDb);
+        sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
+
+        /* Search for the index that has the least amount of columns. If
+        ** there is such an index, and it has less columns than the table
+        ** does, then we can assume that it consumes less space on disk and
+        ** will therefore be cheaper to scan to determine the query result.
+        ** In this case set iRoot to the root page number of the index b-tree
+        ** and pKeyInfo to the KeyInfo structure required to navigate the
+        ** index.
+        **
+        ** In practice the KeyInfo structure will not be used. It is only 
+        ** passed to keep OP_OpenRead happy.
+        */
+        for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+          if( !pBest || pIdx->nColumn<pBest->nColumn ){
+            pBest = pIdx;
+          }
+        }
+        if( pBest && pBest->nColumn<pTab->nCol ){
+          iRoot = pBest->tnum;
+          pKeyInfo = sqlite3IndexKeyinfo(pParse, pBest);
+        }
+
+        /* Open a read-only cursor, execute the OP_Count, close the cursor. */
+        sqlite3VdbeAddOp3(v, OP_OpenRead, iCsr, iRoot, iDb);
+        if( pKeyInfo ){
+          sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO_HANDOFF);
+        }
+        sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem);
+        sqlite3VdbeAddOp1(v, OP_Close, iCsr);
+      }else
+#endif /* SQLITE_OMIT_BTREECOUNT */
+      {
         /* Check if the query is of one of the following forms:
         **
         **   SELECT min(x) FROM ...
@@ -4027,7 +4114,8 @@ int sqlite3Select(
         **     satisfying the 'ORDER BY' clause than it does in other cases.
         **     Refer to code and comments in where.c for details.
         */
-      flag = minMaxQuery(p);
+        ExprList *pMinMax = 0;
+        u8 flag = minMaxQuery(p);
         if( flag ){
           assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) );
           pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0);
@@ -4051,7 +4139,8 @@ int sqlite3Select(
         updateAccumulator(pParse, &sAggInfo);
         if( !pMinMax && flag ){
           sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak);
-        VdbeComment((v, "%s() by index",(flag==WHERE_ORDERBY_MIN?"min":"max")));
+          VdbeComment((v, "%s() by index",
+                (flag==WHERE_ORDERBY_MIN?"min":"max")));
         }
         sqlite3WhereEnd(pWInfo);
         finalizeAggFunctions(pParse, &sAggInfo);
@@ -4059,9 +4148,10 @@ int sqlite3Select(
         if( pHaving ){
           sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL);
         }
+      }
+
       selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, 
                       pDest, addrEnd, addrEnd);
-
       sqlite3ExprListDelete(db, pDel);
     }
     sqlite3VdbeResolveLabel(v, addrEnd);

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.836 2009/02/23 17:33:50 danielk1977 Exp $
+** @(#) $Id: sqliteInt.h,v 1.837 2009/02/24 10:01:52 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -887,6 +887,7 @@ struct FuncDef {
 #define SQLITE_FUNC_EPHEM    0x04 /* Ephemeral.  Delete with VDBE */
 #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
 #define SQLITE_FUNC_PRIVATE  0x10 /* Allowed for internal use only */
+#define SQLITE_FUNC_COUNT    0x20 /* Built-in count(*) aggregate */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are

src/vdbe.c

@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.821 2009/02/20 10:58:42 danielk1977 Exp $
+** $Id: vdbe.c,v 1.822 2009/02/24 10:01:52 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -2339,6 +2339,22 @@ case OP_MakeRecord: {
   break;
 }
 
+/* Opcode: Count P1 P2 * * *
+**
+** Store the number of entries (an integer value) in the table or index 
+** opened by cursor P1 in register P2
+*/
+#ifndef SQLITE_OMIT_BTREECOUNT
+case OP_Count: {         /* out2-prerelease */
+  i64 nEntry;
+  BtCursor *pCrsr = p->apCsr[pOp->p1]->pCursor;
+  rc = sqlite3BtreeCount(pCrsr, &nEntry);
+  pOut->flags = MEM_Int;
+  pOut->u.i = nEntry;
+  break;
+}
+#endif
+
 /* Opcode: Statement P1 * * * *
 **
 ** Begin an individual statement transaction which is part of a larger

test/where9.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the multi-index OR clause optimizer.
 #
-# $Id: where9.test,v 1.6 2009/02/23 17:33:50 danielk1977 Exp $
+# $Id: where9.test,v 1.7 2009/02/24 10:01:52 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -802,30 +802,4 @@ do_test where9-6.8.2 {
   }
 } {1 {cannot use index: t1b}}
 
-do_test where9-7.1 {
-  execsql {
-    CREATE TABLE t5(a, b, c);
-    EXPLAIN QUERY PLAN SELECT count(*) FROM t5;
-  }
-} {0 0 {TABLE t5}}
-do_test where9-7.2 {
-  execsql {
-    CREATE INDEX t5i1 ON t5(a, b);
-    EXPLAIN QUERY PLAN SELECT count(*) FROM t5;
-  }
-} {0 0 {TABLE t5 WITH INDEX t5i1}}
-do_test where9-7.3 {
-  execsql {
-    CREATE INDEX t5i2 ON t5(b);
-    EXPLAIN QUERY PLAN SELECT count(*) FROM t5;
-  }
-} {0 0 {TABLE t5 WITH INDEX t5i2}}
-do_test where9-7.4 {
-  execsql {
-    CREATE TABLE t6(a, b, c);
-    CREATE INDEX t6i1 ON t6(a, b, c);
-    EXPLAIN QUERY PLAN SELECT count(*) FROM t6;
-  }
-} {0 0 {TABLE t6}}
-
 finish_test