Work toward improving the NGQP's ability to optimize out ORDER BY clauses.

FossilOrigin-Name: 67367f1e1f0c3eb6be65eea9873910aa62b49884

manifest

@@ -1,5 +1,5 @@
-C Merge\sin\sall\strunk\schanges\sup\sthrough\sthe\s3.7.17\srelease.
+C Work\stoward\simproving\sthe\sNGQP's\sability\sto\soptimize\sout\sORDER\sBY\sclauses.
-D 2013-05-20T15:14:42.009
+D 2013-05-21T15:52:07.219
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -196,7 +196,7 @@ F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb
 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5
-F src/sqliteInt.h b9b6a2e97254fb50b6977ec55ad0f4e4ce0510d2
+F src/sqliteInt.h 3b89d4517d984363a96f6f7b348440e01f13ced3
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c 51d935c6593748ed004006ee0f2147a860d56e90
+F src/where.c a01d93b37fbce377326849fe810dbcd2334d40c6
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 9fe20292558bb9422de91e35648cb834cbf3b306 118a3b35693b134d56ebd780123b7fd6f1497668
+P 14ab6675e5eab3761256a06dad23d2b11220788a
-R b3fec2668cd66935dfd81d00e243b60d
+R fc1fe5e0f9d1ee7dd6bc32ff7b604dd6
 U drh
-Z e02385c0316003dbdd17c2ecf0ba1968
+Z 0d777b81f154f68ed82e3cd424f83c9e

manifest.uuid

@@ -1 +1 @@
-14ab6675e5eab3761256a06dad23d2b11220788a
+67367f1e1f0c3eb6be65eea9873910aa62b49884

src/sqliteInt.h

@@ -1919,8 +1919,8 @@ typedef u64 Bitmask;
 ** contains more than 63 columns and the 64-th or later column is used.
 */
 struct SrcList {
-  i16 nSrc;        /* Number of tables or subqueries in the FROM clause */
+  u8 nSrc;        /* Number of tables or subqueries in the FROM clause */
-  i16 nAlloc;      /* Number of entries allocated in a[] below */
+  u8 nAlloc;      /* Number of entries allocated in a[] below */
   struct SrcList_item {
     Schema *pSchema;  /* Schema to which this item is fixed */
     char *zDatabase;  /* Name of database holding this table */

src/where.c

@@ -56,7 +56,8 @@ typedef struct WhereVtabPlan WhereVtabPlan;
 struct WhereLoop {
   Bitmask prereq;       /* Bitmask of other loops that must run first */
   Bitmask maskSelf;     /* Bitmask identifying table iTab */
-  u16 iTab;             /* Index of the table coded by this loop */
+  u8 iTab;              /* Position in FROM clause of table coded by this loop */
+  u8 iSortIdx;          /* Sorting index number.  0==None */
   u16 nTerm;            /* Number of entries in aTerm[] */
   u32 wsFlags;          /* WHERE_* flags describing the plan */
   double rSetup;        /* One-time setup cost (ex: create transient index) */
@@ -1987,6 +1988,7 @@ static int termCanDriveIndex(
   if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
   if( (pTerm->eOperator & WO_EQ)==0 ) return 0;
   if( (pTerm->prereqRight & notReady)!=0 ) return 0;
+  if( pTerm->u.leftColumn<0 ) return 0;
   aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity;
   if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
   return 1;
@@ -5168,6 +5170,21 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
 ** and no insert will occur if an existing WhereLoop is faster and has
 ** fewer dependencies than the template.  Otherwise a new WhereLoop is
 ** added based no the template.
+**
+** If pBuilder->pBest is not NULL then we only care about the very
+** best template and that template should be stored in pBuilder->pBest.
+** If pBuilder->pBest is NULL then a list of the best templates are stored
+** in pBuilder->pWInfo->pLoops.
+**
+** When accumulating multiple loops (when pBuilder->pBest is NULL) we
+** still might overwrite similar loops with the new template if the
+** template is better.  Loops may be overwritten if the following 
+** conditions are met:
+**
+**    (1)  They have the same iTab.
+**    (2)  They have the same iSortIdx.
+**    (3)  The template has same or fewer dependencies than the current loop
+**    (4)  The template has the same or lower cost than the current loop
 */
 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0;
@@ -5175,6 +5192,11 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   sqlite3 *db = pBuilder->db;
   WhereInfo *pWInfo = pBuilder->pWInfo;
 
+  /* If pBuilder->pBest is defined, then only keep track of the single
+  ** best WhereLoop.  pBuilder->pBest->maskSelf==0 indicates that no
+  ** prior WhereLoops have been evaluated and that the current pTemplate
+  ** is therefore the first and hence the best and should be retained.
+  */
   if( (p = pBuilder->pBest)!=0 ){
     if( p->maskSelf!=0 ){
       if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){
@@ -5195,7 +5217,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   ** priority over pTemplate.
   */
   for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){
-    if( p->iTab!=pTemplate->iTab ) continue;
+    if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue;
     if( (p->prereq & pTemplate->prereq)==p->prereq
      && p->rSetup<=pTemplate->rSetup
      && p->rRun<=pTemplate->rRun
@@ -5238,7 +5260,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
     memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0]));
   }
   if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
-    if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){
+    if( p->u.btree.pIndex && p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){
       p->u.btree.pIndex = 0;
     }
   }else{
@@ -5342,6 +5364,36 @@ static int whereLoopAddBtreeIndex(
   return rc;
 }
 
+/*
+** Return True if it is possible that pIndex might be useful in
+** implementing the ORDER BY clause in pBuilder.
+**
+** Return False if pBuilder does not contain an ORDER BY clause or
+** if there is no way for pIndex to be useful in implementing that
+** ORDER BY clause.
+*/
+static int indexMightHelpWithOrderBy(
+  WhereLoopBuilder *pBuilder,
+  Index *pIndex,
+  int iCursor
+){
+  ExprList *pOB;
+  int iCol;
+  int ii;
+
+  if( (pOB = pBuilder->pOrderBy)==0 ) return 0;
+  iCol = pIndex->aiColumn[0];
+  for(ii=0; ii<pOB->nExpr; ii++){
+    Expr *pExpr = pOB->a[ii].pExpr;
+    if( pExpr->op!=TK_COLUMN ) return 0;
+    if( pExpr->iTable==iCursor ){
+      if( pExpr->iColumn==iCol ) return 1;
+      return 0;
+    }
+  }
+  return 0;
+}
+
 /*
 ** Add all WhereLoop objects a single table of the join were the table
 ** is idenfied by pBuilder->pNew->iTab.  That table is guaranteed to be
@@ -5358,9 +5410,11 @@ static int whereLoopAddBtree(
   struct SrcList_item *pSrc;  /* The FROM clause btree term to add */
   WhereLoop *pNew;            /* Template WhereLoop object */
   int rc = SQLITE_OK;         /* Return code */
+  int iSortIdx = 0;           /* Index number */
+  int b;                      /* A boolean value */
   double rSize;               /* number of rows in the table */
   double rLogSize;            /* Logarithm of the number of rows in the table */
-
+  
   pNew = pBuilder->pNew;
   pSrc = pBuilder->pTabList->a + pNew->iTab;
   assert( !IsVirtual(pSrc->pTab) );
@@ -5419,26 +5473,25 @@ static int whereLoopAddBtree(
     }
   }
 
-  /* Insert a full table scan */
-  pNew->u.btree.nEq = 0;
-  pNew->nTerm = 0;
-  pNew->rSetup = (double)0;
-  pNew->prereq = mExtra;
-  pNew->u.btree.pIndex = 0;
-  pNew->wsFlags = 0;
-  pNew->nOut = rSize;
-  pNew->rRun = rSize + rLogSize;
-  /* TBD: Reduce nOut using constraints */
-  rc = whereLoopInsert(pBuilder, pNew);
-
   /* Loop over all indices
   */
-  for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){
+  for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){
     pNew->u.btree.nEq = 0;
     pNew->nTerm = 0;
+    pNew->iSortIdx = 0;
+    pNew->rSetup = (double)0;
+    pNew->prereq = mExtra;
+    pNew->u.btree.pIndex = pProbe;
+    b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
     if( pProbe->tnum<=0 ){
       /* Integer primary key index */
       pNew->wsFlags = WHERE_IPK;
+
+      /* Full table scan */
+      pNew->nOut = rSize;
+      pNew->rRun = (rSize + rLogSize)*(3+b); /* 4x penalty for a full-scan */
+      rc = whereLoopInsert(pBuilder, pNew);
+      if( rc ) break;
     }else{
       Bitmask m = pSrc->colUsed;
       int j;
@@ -5448,10 +5501,17 @@ static int whereLoopAddBtree(
           m &= ~(((Bitmask)1)<<x);
         }
       }
-      pNew->wsFlags = m==0 ? WHERE_IDX_ONLY : 0;
+      pNew->wsFlags = (m==0) ? WHERE_IDX_ONLY : 0;
-    }
-    pNew->u.btree.pIndex = pProbe;
 
+      /* Full scan via index */
+      if( m==0 || b ){
+        pNew->iSortIdx = b ? iSortIdx : 0;
+        pNew->nOut = rSize;
+        pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize);
+        rc = whereLoopInsert(pBuilder, pNew);
+        if( rc ) break;
+      }
+    }
     rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1);
 
     /* If there was an INDEXED BY clause, then only that one index is
@@ -5681,6 +5741,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
       pNew->rRun = rTotal;
       pNew->nOut = nRow;
       pNew->prereq = prereq;
+      memset(&pNew->u, 0, sizeof(pNew->u));
       rc = whereLoopInsert(pBuilder, pNew);
     }
   }