From fe05af87f8df113033d017f6d0a8835b2ceeb386 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Jul 2005 03:14:59 +0000
Subject: [PATCH] In where.c, split out the code that selects an index into a
 separate subroutine. (CVS 2554)

FossilOrigin-Name: c30cbba9ead1b4d07f225b1e8a65d5d5230ea45d
---
 manifest           |  24 +-
 manifest.uuid      |   2 +-
 src/build.c        |  55 ++--
 src/expr.c         |   4 +-
 src/sqliteInt.h    |  46 +--
 src/where.c        | 797 ++++++++++++++++++++++++---------------------
 test/select2.test  |   3 +-
 test/sort.test     |   3 +-
 test/subquery.test |   4 +-
 9 files changed, 496 insertions(+), 442 deletions(-)

diff --git a/manifest b/manifest
index 9484f523dd..9398537b4f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Extra\smemory\susage\sinstrumentation\sadded.\s(CVS\s2553)
-D 2005-07-20T14:31:53
+C In\swhere.c,\ssplit\sout\sthe\scode\sthat\sselects\san\sindex\sinto\sa\sseparate\nsubroutine.\s(CVS\s2554)
+D 2005-07-21T03:15:00
 F Makefile.in 22ea9c0fe748f591712d8fe3c6d972c6c173a165
 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -33,12 +33,12 @@ F src/attach.c 3615dbe960cbee4aa5ea300b8a213dad36527b0f
 F src/auth.c 18c5a0befe20f3a58a41e3ddd78f372faeeefe1f
 F src/btree.c ec55bd70052cdd0958f3a0e79ad58d93561acb20
 F src/btree.h 41a71ce027db9ddee72cb43df2316bbe3a1d92af
-F src/build.c 1f40c07a11e0a4eed1cef1ad4e52cf3f9770f220
+F src/build.c a908365b4f900096f406f9028181550f818f59fd
 F src/callback.c 0910b611e0c158f107ee3ff86f8a371654971e2b
 F src/date.c 7444b0900a28da77e57e3337a636873cff0ae940
 F src/delete.c 250d436a68fe371b4ab403d1c0f6fdc9a6860c39
 F src/experimental.c 50c1e3b34f752f4ac10c36f287db095c2b61766d
-F src/expr.c 94dce12d5228af02fdafc23e56abfeae25f3b694
+F src/expr.c cf5146e8a0a1ce7261ac2f9ecb15e99eb98de7ac
 F src/func.c 2be0799df0c05066a29e589485ebee0b3f756a15
 F src/hash.c 2b1b13f7400e179631c83a1be0c664608c8f021f
 F src/hash.h 1b0c445e1c89ff2aaad9b4605ba61375af001e84
@@ -64,7 +64,7 @@ F src/random.c 90adff4e73a3b249eb4f1fc2a6ff9cf78c7233a4
 F src/select.c c611471052773b94af771693686bd5bcdbbb0dba
 F src/shell.c 25b3217d7c64e6497225439d261a253a23efff26
 F src/sqlite.h.in 838382ed6b48d392366a55e07f49d9d71263e1fe
-F src/sqliteInt.h 97d50f5714a5f5a8190b871305e33a96c4638a8a
+F src/sqliteInt.h a3252616131187e227268fc405c8c536b3be9fac
 F src/table.c 25b3ff2b39b7d87e8d4a5da0713d68dfc06cbee9
 F src/tclsqlite.c cccaf6b78c290d824cf8ea089b8b27377e545830
 F src/test1.c 722c1444b5774705eb6eb11163343fc94ffe17f7
@@ -85,7 +85,7 @@ F src/vdbeapi.c 7f392f0792d1258c958083d7de9eae7c3530c9a6
 F src/vdbeaux.c 3732a86566a6be4da4c606e9334baf3fd98667af
 F src/vdbefifo.c b8805850afe13b43f1de78d58088cb5d66f88e1e
 F src/vdbemem.c da8e8d6f29dd1323f782f000d7cd120027c9ff03
-F src/where.c 5a84161299ff90aeb67c885f6bf89c29c3efbb27
+F src/where.c bc0473f786f14970bd48415d5dd168cc7c9c72b7
 F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42
 F test/all.test 7f0988442ab811dfa41793b5b550f5828ce316f3
 F test/alter.test 9d6837a3d946b73df692b7cef2a7644d2e2f6bc6
@@ -188,14 +188,14 @@ F test/rowid.test 040a3bef06f970c45f5fcd14b2355f7f4d62f0cf
 F test/safety.test 907b64fee719554a3622853812af3886fddbbb4f
 F test/schema.test 21cbe7dac652f6d7eb058f3dec369bdbf46bbde6
 F test/select1.test ad700a2a1c325a23a7206ad4d189e33917de526f
-F test/select2.test 01b9cbc06e5ed662ce0289aa5f47314d54541e82
+F test/select2.test f3c2678c3a9f3cf08ec4988a3845bda64be6d9e3
 F test/select3.test 44dccad96a1b9940bb8c6f982d4d218dcd262760
 F test/select4.test c239f516aa31f42f2ef7c6d7cd01105f08f934ca
 F test/select5.test 2d414f712bff8e590091e08f9b7287600731be00
 F test/select6.test 6559d16ad16edb7d6864f7e74a3d204d0af72486
 F test/select7.test 1bf795b948c133a15a2a5e99d3270e652ec58ce6
-F test/sort.test 312eade533cb3c7667110ccfa6e818db1078fd6c
-F test/subquery.test 0e37f0f032799c28aa8fcc0dc04ee28a78e5ce8b
+F test/sort.test 8aaec2e01bf97691c96fd2f0294e635540bebcda
+F test/subquery.test d8364e0719ca09bf55c2bdadf761774c94c52633
 F test/subselect.test 3f3f7a940dc3195c3139f4d530385cb54665d614
 F test/table.test e87fb2211b97c6a3a367fbc116e8572091b53160
 F test/tableapi.test 6a66d58b37d46dc0f2b3c7d4bd2617d209399bd1
@@ -286,7 +286,7 @@ F www/tclsqlite.tcl 425be741b8ae664f55cb1ef2371aab0a75109cf9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b
 F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b
-P a35bd50af8961133adc66e40c38402e81a02bb56
-R 73dbc3e4bd94a516434db925353f0d59
+P ac669f56c0759a7e3eaa6f0018c8fb9d614e7d69
+R f4b09ad5038d1bd3854fa979c8cbf5db
 U drh
-Z b94984498572a6b7851afac6f92deee0
+Z 17d57d5ddd72afd69aa6a395d66fa40e
diff --git a/manifest.uuid b/manifest.uuid
index 36a376c70b..c9baa7159d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac669f56c0759a7e3eaa6f0018c8fb9d614e7d69
\ No newline at end of file
+c30cbba9ead1b4d07f225b1e8a65d5d5230ea45d
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index f03a40d6c8..3ad25d1d14 100644
--- a/src/build.c
+++ b/src/build.c
@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.330 2005/06/30 17:04:21 drh Exp $
+** $Id: build.c,v 1.331 2005/07/21 03:15:00 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -2476,6 +2476,32 @@ IdList *sqlite3IdListAppend(IdList *pList, Token *pToken){
   return pList;
 }
 
+/*
+** Delete an IdList.
+*/
+void sqlite3IdListDelete(IdList *pList){
+  int i;
+  if( pList==0 ) return;
+  for(i=0; i<pList->nId; i++){
+    sqliteFree(pList->a[i].zName);
+  }
+  sqliteFree(pList->a);
+  sqliteFree(pList);
+}
+
+/*
+** Return the index in pList of the identifier named zId.  Return -1
+** if not found.
+*/
+int sqlite3IdListIndex(IdList *pList, const char *zName){
+  int i;
+  if( pList==0 ) return -1;
+  for(i=0; i<pList->nId; i++){
+    if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i;
+  }
+  return -1;
+}
+
 /*
 ** Append a new table name to the given SrcList.  Create a new SrcList if
 ** need be.  A new entry is created in the SrcList even if pToken is NULL.
@@ -2560,32 +2586,6 @@ void sqlite3SrcListAddAlias(SrcList *pList, Token *pToken){
   }
 }
 
-/*
-** Delete an IdList.
-*/
-void sqlite3IdListDelete(IdList *pList){
-  int i;
-  if( pList==0 ) return;
-  for(i=0; i<pList->nId; i++){
-    sqliteFree(pList->a[i].zName);
-  }
-  sqliteFree(pList->a);
-  sqliteFree(pList);
-}
-
-/*
-** Return the index in pList of the identifier named zId.  Return -1
-** if not found.
-*/
-int sqlite3IdListIndex(IdList *pList, const char *zName){
-  int i;
-  if( pList==0 ) return -1;
-  for(i=0; i<pList->nId; i++){
-    if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i;
-  }
-  return -1;
-}
-
 /*
 ** Delete an entire SrcList including all its substructure.
 */
@@ -2601,6 +2601,7 @@ void sqlite3SrcListDelete(SrcList *pList){
     sqlite3SelectDelete(pItem->pSelect);
     sqlite3ExprDelete(pItem->pOn);
     sqlite3IdListDelete(pItem->pUsing);
+    sqlite3WhereIdxListDelete(pItem->pWIdx);
   }
   sqliteFree(pList);
 }
diff --git a/src/expr.c b/src/expr.c
index 33bb802376..5f797029f4 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -12,7 +12,7 @@
 ** This file contains routines used for analyzing expressions and
 ** for generating VDBE code that evaluates expressions in SQLite.
 **
-** $Id: expr.c,v 1.211 2005/07/08 18:25:26 drh Exp $
+** $Id: expr.c,v 1.212 2005/07/21 03:15:00 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -93,6 +93,7 @@ char sqlite3CompareAffinity(Expr *pExpr, char aff2){
     return SQLITE_AFF_NONE;
   }else{
     /* One side is a column, the other is not. Use the columns affinity. */
+    assert( aff1==0 || aff2==0 );
     return (aff1 + aff2);
   }
 }
@@ -488,6 +489,7 @@ SrcList *sqlite3SrcListDup(SrcList *p){
     pNewItem->pSelect = sqlite3SelectDup(pOldItem->pSelect);
     pNewItem->pOn = sqlite3ExprDup(pOldItem->pOn);
     pNewItem->pUsing = sqlite3IdListDup(pOldItem->pUsing);
+    pNewItem->pWIdx = 0;
     pNewItem->colUsed = pOldItem->colUsed;
   }
   return pNew;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 7af55d5b97..3425008c51 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.394 2005/07/19 17:38:23 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.395 2005/07/21 03:15:00 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -298,30 +298,31 @@ extern int sqlite3_iMallocReset; /* Set iMallocFail to this when it reaches 0 */
 /*
 ** Forward references to structures
 */
+typedef struct AggExpr AggExpr;
+typedef struct AuthContext AuthContext;
+typedef struct CollSeq CollSeq;
 typedef struct Column Column;
-typedef struct Table Table;
-typedef struct Index Index;
+typedef struct Db Db;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
-typedef struct Parse Parse;
-typedef struct Token Token;
-typedef struct IdList IdList;
-typedef struct SrcList SrcList;
-typedef struct WhereInfo WhereInfo;
-typedef struct WhereLevel WhereLevel;
-typedef struct Select Select;
-typedef struct AggExpr AggExpr;
-typedef struct FuncDef FuncDef;
-typedef struct Trigger Trigger;
-typedef struct TriggerStep TriggerStep;
-typedef struct TriggerStack TriggerStack;
 typedef struct FKey FKey;
-typedef struct Db Db;
-typedef struct AuthContext AuthContext;
+typedef struct FuncDef FuncDef;
+typedef struct IdList IdList;
+typedef struct Index Index;
 typedef struct KeyClass KeyClass;
-typedef struct CollSeq CollSeq;
 typedef struct KeyInfo KeyInfo;
 typedef struct NameContext NameContext;
+typedef struct Parse Parse;
+typedef struct Select Select;
+typedef struct SrcList SrcList;
+typedef struct Table Table;
+typedef struct Token Token;
+typedef struct TriggerStack TriggerStack;
+typedef struct TriggerStep TriggerStep;
+typedef struct Trigger Trigger;
+typedef struct WhereIdx WhereIdx;
+typedef struct WhereInfo WhereInfo;
+typedef struct WhereLevel WhereLevel;
 
 /*
 ** Each database file to be accessed by the system is an instance
@@ -924,10 +925,11 @@ struct SrcList {
     char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
     Table *pTab;      /* An SQL table corresponding to zName */
     Select *pSelect;  /* A SELECT statement used in place of a table name */
-    int jointype;     /* Type of join between this table and the next */
-    int iCursor;      /* The VDBE cursor number used to access this table */
+    u8 jointype;      /* Type of join between this table and the next */
+    i16 iCursor;      /* The VDBE cursor number used to access this table */
     Expr *pOn;        /* The ON clause of a join */
     IdList *pUsing;   /* The USING clause of a join */
+    WhereIdx *pWIdx;  /* List of structures used by the optimizer */
     Bitmask colUsed;  /* Bit N (1<<N) set if column N or pTab is used */
   } a[1];             /* One entry for each identifier on the list */
 };
@@ -953,14 +955,13 @@ struct WhereLevel {
   Index *pIdx;         /* Index used.  NULL if no index */
   int iTabCur;         /* The VDBE cursor used to access the table */
   int iIdxCur;         /* The VDBE cursor used to acesss pIdx */
-  int score;           /* How well this index scored */
   int brk;             /* Jump here to break out of the loop */
   int cont;            /* Jump here to continue with the next loop cycle */
   int op, p1, p2;      /* Opcode used to terminate the loop */
   int iLeftJoin;       /* Memory cell used to implement LEFT OUTER JOIN */
   int top;             /* First instruction of interior of the loop */
   int inOp, inP1, inP2;/* Opcode used to implement an IN operator */
-  int bRev;            /* Do the scan in the reverse direction */
+  int flags;           /* Flags associated with this level */
 };
 
 /*
@@ -1568,6 +1569,7 @@ CollSeq *sqlite3GetCollSeq(sqlite3*, CollSeq *, const char *, int);
 char sqlite3AffinityType(const Token*);
 void sqlite3Analyze(Parse*, Token*, Token*);
 int sqlite3InvokeBusyHandler(BusyHandler*);
+void sqlite3WhereIdxListDelete(WhereIdx*);
 
 #ifdef SQLITE_SSE
 #include "sseInt.h"
diff --git a/src/where.c b/src/where.c
index 1e5dc61612..e376543ca5 100644
--- a/src/where.c
+++ b/src/where.c
@@ -16,7 +16,7 @@
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.147 2005/07/19 22:22:13 drh Exp $
+** $Id: where.c,v 1.148 2005/07/21 03:15:00 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -34,6 +34,22 @@
 */
 typedef struct WhereClause WhereClause;
 
+/*
+** An instance of the following structure holds information about how well
+** a particular index helps in a search.  A list of such structures is
+** attached to each SrcList_item of a SrcList.
+*/
+struct WhereIdx {
+  Index *pIdx;      /* The index under consideration */
+  Bitmask prereq;   /* Prerequesite FROM clause elements for using this index */
+  int nEqTerm;      /* Number of Idx column constrainted by == or IN */
+  int nTerm;        /* Total number of Index Columns used */
+  int flags;        /* Flags.  See below */
+  double rRowEst;   /* Estimated number of rows selected */
+  double rScore;    /* Score of this index */
+  WhereIdx *pNext;  /* Next WhereIdx on the same FROM clause element */
+};
+
 /*
 ** The query generator uses an array of instances of this structure to
 ** help it analyze the subexpressions of the WHERE clause.  Each WHERE
@@ -72,6 +88,7 @@ struct WhereTerm {
   u16 flags;              /* Bit flags.  See below */
   i16 leftCursor;         /* Cursor number of X in "X <op> <expr>" */
   i16 leftColumn;         /* Column number of X in "X <op> <expr>" */
+  u8 operator;            /* A WO_xx value describing <op> */
   WhereClause *pWC;       /* The clause this term is part of */
   Bitmask prereqRight;    /* Bitmask of tables used by pRight */
   Bitmask prereqAll;      /* Bitmask of tables referenced by p */
@@ -89,6 +106,7 @@ struct WhereTerm {
 ** WHERE clause.  Mostly this is a container for one or more WhereTerms.
 */
 struct WhereClause {
+  Parse *pParse;           /* The parser context */
   int nTerm;               /* Number of terms */
   int nSlot;               /* Number of entries in a[] */
   WhereTerm *a;            /* Pointer to an array of terms */
@@ -131,7 +149,8 @@ struct ExprMaskSet {
 /*
 ** Initialize a preallocated WhereClause structure.
 */
-static void whereClauseInit(WhereClause *pWC){
+static void whereClauseInit(WhereClause *pWC, Parse *pParse){
+  pWC->pParse = pParse;
   pWC->nTerm = 0;
   pWC->nSlot = ARRAYSIZE(pWC->aStatic);
   pWC->a = pWC->aStatic;
@@ -294,7 +313,10 @@ static Bitmask exprListTableUsage(ExprMaskSet *pMaskSet, ExprList *pList){
 ** "=", "<", ">", "<=", ">=", and "IN".
 */
 static int allowedOp(int op){
-  assert( TK_GT==TK_LE-1 && TK_LE==TK_LT-1 && TK_LT==TK_GE-1 && TK_EQ==TK_GT-1);
+  assert( TK_GT>TK_EQ && TK_GT<TK_GE );
+  assert( TK_LT>TK_EQ && TK_LT<TK_GE );
+  assert( TK_LE>TK_EQ && TK_LE<TK_GE );
+  assert( TK_GE==TK_EQ+4 );
   return op==TK_IN || (op>=TK_EQ && op<=TK_GE);
 }
 
@@ -308,14 +330,7 @@ static int allowedOp(int op){
 ** are converted into "Y op X".
 */
 static void exprCommute(Expr *pExpr){
-  assert(
-     pExpr->op==TK_EQ ||
-     pExpr->op==TK_NE ||
-     pExpr->op==TK_LT ||
-     pExpr->op==TK_LE ||
-     pExpr->op==TK_GT ||
-     pExpr->op==TK_GE
-  );
+  assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN );
   SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl);
   SWAP(Expr*,pExpr->pRight,pExpr->pLeft);
   if( pExpr->op>=TK_GT ){
@@ -328,6 +343,80 @@ static void exprCommute(Expr *pExpr){
   }
 }
 
+/*
+** Bitmasks for the operators that indices are able to exploit.  An
+** OR-ed combination of these values can be used when searching for
+** terms in the where clause.
+*/
+#define WO_IN  1
+#define WO_EQ  2
+#define WO_LT  (2<<(TK_LT-TK_EQ))
+#define WO_LE  (2<<(TK_LE-TK_EQ))
+#define WO_GT  (2<<(TK_GT-TK_EQ))
+#define WO_GE  (2<<(TK_GE-TK_EQ))
+
+/*
+** Translate from TK_xx operator to WO_xx bitmask.
+*/
+static int operatorMask(int op){
+  assert( allowedOp(op) );
+  if( op==TK_IN ){
+    return WO_IN;
+  }else{
+    return 1<<(op+1-TK_EQ);
+  }
+}
+
+/*
+** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
+** where X is a reference to the iColumn of table iCur and <op> is one of
+** the WO_xx operator codes specified by the op parameter.
+** Return a pointer to the term.  Return 0 if not found.
+*/
+static WhereTerm *findTerm(
+  WhereClause *pWC,     /* The WHERE clause to be searched */
+  int iCur,             /* Cursor number of LHS */
+  int iColumn,          /* Column number of LHS */
+  Bitmask notReady,     /* RHS must not overlap with this mask */
+  u8 op,                /* Mask of WO_xx values describing operator */
+  Index *pIdx           /* Must be compatible with this index, if not NULL */
+){
+  WhereTerm *pTerm;
+  int k;
+  for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){
+    if( pTerm->leftCursor==iCur
+       && (pTerm->prereqRight & notReady)==0
+       && pTerm->leftColumn==iColumn
+       && (pTerm->operator & op)!=0
+    ){
+      if( iCur>=0 && pIdx ){
+        Expr *pX = pTerm->pExpr;
+        CollSeq *pColl;
+        char idxaff;
+        int k;
+        Parse *pParse = pWC->pParse;
+
+        idxaff = pIdx->pTable->aCol[iColumn].affinity;
+        if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue;
+        pColl = sqlite3ExprCollSeq(pParse, pX->pLeft);
+        if( !pColl ){
+          if( pX->pRight ){
+            pColl = sqlite3ExprCollSeq(pParse, pX->pRight);
+          }
+          if( !pColl ){
+            pColl = pParse->db->pDfltColl;
+          }
+        }
+        for(k=0; k<pIdx->nColumn && pIdx->aiColumn[k]!=iColumn; k++){}
+        assert( k<pIdx->nColumn );
+        if( pColl!=pIdx->keyInfo.aColl[k] ) continue;
+      }
+      return pTerm;
+    }
+  }
+  return 0;
+}
+
 /*
 ** The input to this routine is an WhereTerm structure with only the
 ** "p" field filled in.  The job of this routine is to analyze the
@@ -349,6 +438,7 @@ static void exprAnalyze(
   pTerm->prereqAll = prereqAll = exprTableUsage(pMaskSet, pExpr);
   pTerm->leftCursor = -1;
   pTerm->iPartner = -1;
+  pTerm->operator = 0;
   idxRight = -1;
   if( allowedOp(pExpr->op) && (pTerm->prereqRight & prereqLeft)==0 ){
     Expr *pLeft = pExpr->pLeft;
@@ -356,6 +446,7 @@ static void exprAnalyze(
     if( pLeft->op==TK_COLUMN ){
       pTerm->leftCursor = pLeft->iTable;
       pTerm->leftColumn = pLeft->iColumn;
+      pTerm->operator = operatorMask(pExpr->op);
     }
     if( pRight && pRight->op==TK_COLUMN ){
       WhereTerm *pNew;
@@ -375,6 +466,7 @@ static void exprAnalyze(
       pNew->leftColumn = pLeft->iColumn;
       pNew->prereqRight = prereqLeft;
       pNew->prereqAll = prereqAll;
+      pNew->operator = operatorMask(pDup->op);
     }
   }
 }
@@ -497,6 +589,191 @@ static int sortableByRowid(
   return 0;
 }
 
+/*
+** Value for flags returned by bestIndex()
+*/
+#define WHERE_ROWID_EQ       0x001    /* rowid=EXPR or rowid IN (...) */
+#define WHERE_ROWID_RANGE    0x002    /* rowid<EXPR and/or rowid>EXPR */
+#define WHERE_COLUMN_EQ      0x004    /* x=EXPR or x IN (...) */
+#define WHERE_COLUMN_RANGE   0x008    /* x<EXPR and/or x>EXPR */
+#define WHERE_SCAN           0x010    /* Do a full table scan */
+#define WHERE_REVERSE        0x020    /* Scan in reverse order */
+#define WHERE_ORDERBY        0x040    /* Output will appear in correct order */
+#define WHERE_IDX_ONLY       0x080    /* Use index only - omit table */
+#define WHERE_TOP_LIMIT      0x100    /* x<EXPR or x<=EXPR constraint */
+#define WHERE_BTM_LIMIT      0x200    /* x>EXPR or x>=EXPR constraint */
+
+/*
+** Find the best index for accessing a particular table.  Return the index,
+** flags that describe how the index should be used, and the "score" for
+** this index.
+*/
+static double bestIndex(
+  Parse *pParse,              /* The parsing context */
+  WhereClause *pWC,           /* The WHERE clause */
+  struct SrcList_item *pSrc,  /* The FROM clause term to search */
+  Bitmask notReady,           /* Mask of cursors that are not available */
+  ExprList *pOrderBy,         /* The order by clause */
+  Index **ppIndex,            /* Make *ppIndex point to the best index */
+  int *pFlags                 /* Put flags describing this choice in *pFlags */
+){
+  WhereTerm *pTerm;
+  Index *pProbe;
+  Index *bestIdx = 0;
+  double bestScore = 0.0;
+  int bestFlags = 0;
+  int iCur = pSrc->iCursor;
+  int rev;
+
+  /* Check for a rowid=EXPR or rowid IN (...) constraint
+  */
+  pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0);
+  if( pTerm ){
+    *ppIndex = 0;
+    if( pTerm->operator & WO_EQ ){
+      *pFlags = WHERE_ROWID_EQ;
+      if( pOrderBy ) *pFlags |= WHERE_ORDERBY;
+      return 1.0e10;
+    }else{
+      *pFlags = WHERE_ROWID_EQ;
+      return 1.0e9;
+    }
+  }
+
+  /* Check for constraints on a range of rowids
+  */
+  pTerm = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE|WO_GT|WO_GE, 0);
+  if( pTerm ){
+    int flags;
+    *ppIndex = 0;
+    if( pTerm->operator & (WO_LT|WO_LE) ){
+      flags = WHERE_ROWID_RANGE | WHERE_TOP_LIMIT;
+      if( findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0) ){
+        flags |= WHERE_BTM_LIMIT;
+      }
+    }else{
+      flags = WHERE_ROWID_RANGE | WHERE_BTM_LIMIT;
+      if( findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0) ){
+        flags |= WHERE_TOP_LIMIT;
+      }
+    }
+    if( pOrderBy && sortableByRowid(iCur, pOrderBy, &rev) ){
+      flags |= WHERE_ORDERBY;
+      if( rev ) flags |= WHERE_REVERSE;
+    }
+    bestScore = 99.0;
+    bestFlags = flags;
+  }
+
+  /* Look at each index.
+  */
+  for(pProbe=pSrc->pTab->pIndex; pProbe; pProbe=pProbe->pNext){
+    int i;
+    int nEq;
+    int usesIN = 0;
+    int flags;
+    double score = 0.0;
+
+    /* Count the number of columns in the index that are satisfied
+    ** by x=EXPR constraints or x IN (...) constraints.
+    */
+    for(i=0; i<pProbe->nColumn; i++){
+      int j = pProbe->aiColumn[i];
+      pTerm = findTerm(pWC, iCur, j, notReady, WO_EQ|WO_IN, pProbe);
+      if( pTerm==0 ) break;
+      if( pTerm->operator==WO_IN ){
+        if( i==0 ) usesIN = 1;
+        break;
+      }
+    }
+    nEq = i + usesIN;
+    score = i*100.0 + usesIN*50.0;
+
+    /* The optimization type is RANGE if there are no == or IN constraints
+    */
+    if( usesIN || nEq ){
+      flags = WHERE_COLUMN_EQ;
+    }else{
+      flags = WHERE_COLUMN_RANGE;
+    }
+
+    /* Look for range constraints
+    */
+    if( !usesIN && nEq<pProbe->nColumn ){
+      int j = pProbe->aiColumn[nEq];
+      pTerm = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pProbe);
+      if( pTerm ){
+        score += 20.0;
+        flags = WHERE_COLUMN_RANGE;
+        if( pTerm->operator & (WO_LT|WO_LE) ){
+          flags |= WHERE_TOP_LIMIT;
+          if( findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pProbe) ){
+            flags |= WHERE_BTM_LIMIT;
+            score += 20.0;
+          }
+        }else{
+          flags |= WHERE_BTM_LIMIT;
+          if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pProbe) ){
+            flags |= WHERE_TOP_LIMIT;
+            score += 20;
+          }
+        }
+      }
+    }
+
+    /* Add extra points if this index can be used to satisfy the ORDER BY
+    ** clause
+    */
+    if( pOrderBy && !usesIN &&
+        isSortingIndex(pParse, pProbe, pSrc->pTab, iCur, pOrderBy, nEq, &rev) ){
+      flags |= WHERE_ORDERBY;
+      score += 10.0;
+      if( rev ) flags |= WHERE_REVERSE;
+    }
+
+    /* Check to see if we can get away with using just the index without
+    ** ever reading the table.  If that is the case, then add one bonus
+    ** point to the score.
+    */
+    if( score>0.0 && pSrc->colUsed < (((Bitmask)1)<<(BMS-1)) ){
+      Bitmask m = pSrc->colUsed;
+      int j;
+      for(j=0; j<pProbe->nColumn; j++){
+        int x = pProbe->aiColumn[j];
+        if( x<BMS-1 ){
+          m &= ~(((Bitmask)1)<<x);
+        }
+      }
+      if( m==0 ){
+        flags |= WHERE_IDX_ONLY;
+        score += 5;
+      }
+    }
+
+    /* If this index has achieved the best score so far, then use it.
+    */
+    if( score>bestScore ){
+      bestIdx = pProbe;
+      bestScore = score;
+      bestFlags = flags;
+    }
+  }
+
+  /* Disable sorting if we are coming out in rowid order
+  */
+  if( bestIdx==0 && pOrderBy && sortableByRowid(iCur, pOrderBy, &rev) ){
+    bestFlags |= WHERE_ORDERBY;
+    if( rev ) bestFlags |= WHERE_REVERSE;
+  }
+
+
+  /* Report the best result
+  */
+  *ppIndex = bestIdx;
+  *pFlags = bestFlags;
+  return bestScore;
+}
+
 
 /*
 ** Disable a term in the WHERE clause.  Except, do not disable the term
@@ -551,34 +828,6 @@ static void buildIndexProbe(Vdbe *v, int nColumn, int brk, Index *pIdx){
   sqlite3IndexAffinityStr(v, pIdx);
 }
 
-/*
-** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
-** where X is a reference to the iColumn of table iCur and <op> is either
-** op1 or op2.  Return a pointer to the term.
-*/
-static WhereTerm *findTerm(
-  WhereClause *pWC,     /* The WHERE clause to be searched */
-  int iCur,             /* Cursor number of LHS */
-  int iColumn,          /* Column number of LHS */
-  Bitmask loopMask,     /* RHS must not overlap with this mask */
-  u8 op1, u8 op2        /* Expression must use either of these opcodes */
-){
-  WhereTerm *pTerm;
-  int k;
-  for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){
-    u8 op = pTerm->pExpr->op;
-    if( pTerm->leftCursor==iCur
-       && (pTerm->prereqRight & loopMask)==0
-       && pTerm->leftColumn==iColumn
-       && (op==op1 || op==op2)
-    ){
-      break;
-    }
-  }
-  assert( k>0 );  /* The search is always successful */
-  return pTerm;
-}
-
 
 /*
 ** Generate code for an equality term of the WHERE clause.  An equality
@@ -717,12 +966,9 @@ WhereInfo *sqlite3WhereBegin(
   WhereInfo *pWInfo;         /* Will become the return value of this function */
   Vdbe *v = pParse->pVdbe;   /* The virtual database engine */
   int brk, cont = 0;         /* Addresses used during code generation */
-  Bitmask loopMask;          /* One bit cleared for each outer loop */
+  Bitmask notReady;          /* Cursors that are not yet positioned */
   WhereTerm *pTerm;          /* A single term in the WHERE clause */
   ExprMaskSet maskSet;       /* The expression mask set */
-  int iDirectEq[BMS];        /* Term of the form ROWID==X for the N-th table */
-  int iDirectLt[BMS];        /* Term of the form ROWID<X or ROWID<=X */
-  int iDirectGt[BMS];        /* Term of the form ROWID>X or ROWID>=X */
   WhereClause wc;            /* The WHERE clause is divided into these terms */
   struct SrcList_item *pTabItem;  /* A single entry from pTabList */
   WhereLevel *pLevel;             /* A single level in the pWInfo list */
@@ -742,7 +988,7 @@ WhereInfo *sqlite3WhereBegin(
   ** contains additional unfactored AND operators.
   */
   initMaskSet(&maskSet);
-  whereClauseInit(&wc);
+  whereClauseInit(&wc, pParse);
   whereSplit(&wc, pWhere);
     
   /* Allocate and initialize the WhereInfo structure that will become the
@@ -775,261 +1021,28 @@ WhereInfo *sqlite3WhereBegin(
     exprAnalyze(pTabList, &maskSet, &wc.a[i]);
   }
 
-  /* Figure out what index to use (if any) for each nested loop.
-  ** Make pWInfo->a[i].pIdx point to the index to use for the i-th nested
-  ** loop where i==0 is the outer loop and i==pTabList->nSrc-1 is the inner
-  ** loop. 
-  **
-  ** If terms exist that use the ROWID of any table, then set the
-  ** iDirectEq[], iDirectLt[], or iDirectGt[] elements for that table
-  ** to the index of the term containing the ROWID.  We always prefer
-  ** to use a ROWID which can directly access a table rather than an
-  ** index which requires reading an index first to get the rowid then
-  ** doing a second read of the actual database table.
-  **
-  ** Actually, if there are more than 32 tables in the join, only the
-  ** first 32 tables are candidates for indices.  This is (again) due
-  ** to the limit of 32 bits in an integer bitmask.
+  /* Chose the best index to use for each table in the FROM clause
   */
-  loopMask = ~(Bitmask)0;
+  notReady = ~(Bitmask)0;
   pTabItem = pTabList->a;
   pLevel = pWInfo->a;
-  for(i=0; i<pTabList->nSrc && i<ARRAYSIZE(iDirectEq); i++,pTabItem++,pLevel++){
-    int j;
-    int iCur = pTabItem->iCursor;            /* The cursor for this table */
-    Bitmask mask = getMask(&maskSet, iCur);  /* Cursor mask for this table */
-    Table *pTab = pTabItem->pTab;
-    Index *pIdx;
-    Index *pBestIdx = 0;
-    int bestScore = 0;
-    int bestRev = 0;
-
-    /* Check to see if there is an expression that uses only the
-    ** ROWID field of this table.  For terms of the form ROWID==expr
-    ** set iDirectEq[i] to the index of the term.  For terms of the
-    ** form ROWID<expr or ROWID<=expr set iDirectLt[i] to the term index.
-    ** For terms like ROWID>expr or ROWID>=expr set iDirectGt[i].
-    **
-    ** (Added:) Treat ROWID IN expr like ROWID=expr.
-    */
-    pLevel->iIdxCur = -1;
-    iDirectEq[i] = -1;
-    iDirectLt[i] = -1;
-    iDirectGt[i] = -1;
-    for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
-      if( pTerm->leftCursor==iCur && pTerm->leftColumn<0
-            && (pTerm->prereqRight & loopMask)==0 ){
-        switch( pTerm->pExpr->op ){
-          case TK_IN:
-          case TK_EQ: iDirectEq[i] = j; break;
-          case TK_LE:
-          case TK_LT: iDirectLt[i] = j; break;
-          case TK_GE:
-          case TK_GT: iDirectGt[i] = j;  break;
-        }
-      }
+  for(i=0; i<pTabList->nSrc; i++, pTabItem++, pLevel++){
+    Index *pBest;
+    int flags;
+    bestIndex(pParse, &wc, pTabItem, notReady,
+              (i==0 && ppOrderBy) ? *ppOrderBy : 0,
+              &pBest, &flags);
+    if( flags & WHERE_ORDERBY ){
+      *ppOrderBy = 0;
     }
-
-    /* If we found a term that tests ROWID with == or IN, that term
-    ** will be used to locate the rows in the database table.  There
-    ** is no need to continue into the code below that looks for
-    ** an index.  We will always use the ROWID over an index.
-    */
-    if( iDirectEq[i]>=0 ){
-      loopMask &= ~mask;
-      pLevel->pIdx = 0;
-      continue;
-    }
-
-    /* Do a search for usable indices.  Leave pBestIdx pointing to
-    ** the "best" index.  pBestIdx is left set to NULL if no indices
-    ** are usable.
-    **
-    ** The best index is the one with the highest score.  The score
-    ** for the index is determined as follows.  For each of the
-    ** left-most terms that is fixed by an equality operator, add
-    ** 32 to the score.  The right-most term of the index may be
-    ** constrained by an inequality.  Add 4 if for an "x<..." constraint
-    ** and add 8 for an "x>..." constraint.  If both constraints
-    ** are present, add 12.
-    **
-    ** If the left-most term of the index uses an IN operator
-    ** (ex:  "x IN (...)")  then add 16 to the score.
-    **
-    ** If an index can be used for sorting, add 2 to the score.
-    ** If an index contains all the terms of a table that are ever
-    ** used by any expression in the SQL statement, then add 1 to
-    ** the score.
-    **
-    ** This scoring system is designed so that the score can later be
-    ** used to determine how the index is used.  If the score&0x1c is 0
-    ** then all constraints are equalities.  If score&0x4 is not 0 then
-    ** there is an inequality used as a termination key.  (ex: "x<...")
-    ** If score&0x8 is not 0 then there is an inequality used as the
-    ** start key.  (ex: "x>...").  A score or 0x10 is the special case
-    ** of an IN operator constraint.  (ex:  "x IN ...").
-    **
-    ** The IN operator (as in "<expr> IN (...)") is treated the same as
-    ** an equality comparison except that it can only be used on the
-    ** left-most column of an index and other terms of the WHERE clause
-    ** cannot be used in conjunction with the IN operator to help satisfy
-    ** other columns of the index.
-    */
-    for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-      Bitmask eqMask = 0;  /* Index columns covered by an x=... term */
-      Bitmask ltMask = 0;  /* Index columns covered by an x<... term */
-      Bitmask gtMask = 0;  /* Index columns covered by an x>... term */
-      Bitmask inMask = 0;  /* Index columns covered by an x IN .. term */
-      Bitmask m;
-      int nEq, score, bRev = 0;
-
-      if( pIdx->nColumn>sizeof(eqMask)*8 ){
-        continue;  /* Ignore indices with too many columns to analyze */
-      }
-      for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
-        Expr *pX = pTerm->pExpr;
-        CollSeq *pColl = sqlite3ExprCollSeq(pParse, pX->pLeft);
-        if( !pColl && pX->pRight ){
-          pColl = sqlite3ExprCollSeq(pParse, pX->pRight);
-        }
-        if( !pColl ){
-          pColl = pParse->db->pDfltColl;
-        }
-        if( pTerm->leftCursor==iCur && (pTerm->prereqRight & loopMask)==0 ){
-          int iColumn = pTerm->leftColumn;
-          int k;
-          char idxaff = iColumn>=0 ? pIdx->pTable->aCol[iColumn].affinity : 0; 
-          for(k=0; k<pIdx->nColumn; k++){
-            /* If the collating sequences or affinities don't match, 
-            ** ignore this index.  */
-            if( pColl!=pIdx->keyInfo.aColl[k] ) continue;
-            if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue;
-            if( pIdx->aiColumn[k]==iColumn ){
-              switch( pX->op ){
-                case TK_IN: {
-                  if( k==0 ) inMask |= 1;
-                  break;
-                }
-                case TK_EQ: {
-                  eqMask |= ((Bitmask)1)<<k;
-                  break;
-                }
-                case TK_LE:
-                case TK_LT: {
-                  ltMask |= ((Bitmask)1)<<k;
-                  break;
-                }
-                case TK_GE:
-                case TK_GT: {
-                  gtMask |= ((Bitmask)1)<<k;
-                  break;
-                }
-                default: {
-                  /* CANT_HAPPEN */
-                  assert( 0 );
-                  break;
-                }
-              }
-              break;
-            }
-          }
-        }
-      }
-
-      /* The following loop ends with nEq set to the number of columns
-      ** on the left of the index with == constraints.
-      */
-      for(nEq=0; nEq<pIdx->nColumn; nEq++){
-        m = (((Bitmask)1)<<(nEq+1))-1;
-        if( (m & eqMask)!=m ) break;
-      }
-
-      /* Begin assembling the score
-      */
-      score = nEq*32;   /* Base score is 32 times number of == constraints */
-      m = ((Bitmask)1)<<nEq;
-      if( m & ltMask ) score+=4;    /* Increase score for a < constraint */
-      if( m & gtMask ) score+=8;    /* Increase score for a > constraint */
-      if( score==0 && inMask ) score = 16; /* Default score for IN constraint */
-
-      /* Give bonus points if this index can be used for sorting
-      */
-      if( i==0 && score!=16 && ppOrderBy && *ppOrderBy ){
-        int base = pTabList->a[0].iCursor;
-        if( isSortingIndex(pParse, pIdx, pTab, base, *ppOrderBy, nEq, &bRev) ){
-          score += 2;
-        }
-      }
-
-      /* Check to see if we can get away with using just the index without
-      ** ever reading the table.  If that is the case, then add one bonus
-      ** point to the score.
-      */
-      if( score && pTabItem->colUsed < (((Bitmask)1)<<(BMS-1)) ){
-        for(m=0, j=0; j<pIdx->nColumn; j++){
-          int x = pIdx->aiColumn[j];
-          if( x<BMS-1 ){
-            m |= ((Bitmask)1)<<x;
-          }
-        }
-        if( (pTabItem->colUsed & m)==pTabItem->colUsed ){
-          score++;
-        }
-      }
-
-      /* If the score for this index is the best we have seen so far, then
-      ** save it
-      */
-      if( score>bestScore ){
-        pBestIdx = pIdx;
-        bestScore = score;
-        bestRev = bRev;
-      }
-    }
-    pLevel->pIdx = pBestIdx;
-    pLevel->score = bestScore;
-    pLevel->bRev = bestRev;
-    loopMask &= ~mask;
-    if( pBestIdx ){
+    pLevel->flags = flags;
+    pLevel->pIdx = pBest;
+    if( pBest ){
       pLevel->iIdxCur = pParse->nTab++;
+    }else{
+      pLevel->iIdxCur = -1;
     }
-  }
-
-  /* Check to see if the ORDER BY clause is or can be satisfied by the
-  ** use of an index on the first table.
-  */
-  if( ppOrderBy && *ppOrderBy && pTabList->nSrc>0 ){
-    Index *pIdx;             /* Index derived from the WHERE clause */
-    Table *pTab;             /* Left-most table in the FROM clause */
-    int bRev = 0;            /* True to reverse the output order */
-    int iCur;                /* Btree-cursor that will be used by pTab */
-    WhereLevel *pLevel0 = &pWInfo->a[0];
-
-    pTab = pTabList->a[0].pTab;
-    pIdx = pLevel0->pIdx;
-    iCur = pTabList->a[0].iCursor;
-    if( pIdx==0 && sortableByRowid(iCur, *ppOrderBy, &bRev) ){
-      /* The ORDER BY clause specifies ROWID order, which is what we
-      ** were going to be doing anyway...
-      */
-      *ppOrderBy = 0;
-      pLevel0->bRev = bRev;
-    }else if( pLevel0->score==16 ){
-      /* If there is already an IN index on the left-most table,
-      ** it will not give the correct sort order.
-      ** So, pretend that no suitable index is found.
-      */
-    }else if( iDirectEq[0]>=0 || iDirectLt[0]>=0 || iDirectGt[0]>=0 ){
-      /* If the left-most column is accessed using its ROWID, then do
-      ** not try to sort by index.  But do delete the ORDER BY clause
-      ** if it is redundant.
-      */
-    }else if( (pLevel0->score&2)!=0 ){
-      /* The index that was selected for searching will cause rows to
-      ** appear in sorted order.
-      */
-      *ppOrderBy = 0;
-    }
+    notReady &= ~getMask(&maskSet, pTabItem->iCursor);
   }
 
   /* Open all tables in the pTabList and any indices selected for
@@ -1044,7 +1057,7 @@ WhereInfo *sqlite3WhereBegin(
 
     pTab = pTabItem->pTab;
     if( pTab->isTransient || pTab->pSelect ) continue;
-    if( (pLevel->score & 1)==0 ){
+    if( (pLevel->flags & WHERE_IDX_ONLY)==0 ){
       sqlite3OpenTableForReading(v, pTabItem->iCursor, pTab);
     }
     pLevel->iTabCur = pTabItem->iCursor;
@@ -1053,7 +1066,7 @@ WhereInfo *sqlite3WhereBegin(
       sqlite3VdbeOp3(v, OP_OpenRead, iIdxCur, pIx->tnum,
                      (char*)&pIx->keyInfo, P3_KEYINFO);
     }
-    if( (pLevel->score & 1)!=0 ){
+    if( (pLevel->flags & WHERE_IDX_ONLY)!=0 ){
       sqlite3VdbeAddOp(v, OP_SetNumColumns, iIdxCur, pIx->nColumn+1);
     }
     sqlite3CodeVerifySchema(pParse, pTab->iDb);
@@ -1070,7 +1083,7 @@ WhereInfo *sqlite3WhereBegin(
       if( z==0 ) z = pTab->zName;
       n = strlen(z);
       if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){
-        if( (pLevel->score & 1)!=0 ){
+        if( pLevel->flags & WHERE_IDX_ONLY ){
           strcpy(&sqlite3_query_plan[nQPlan], "{}");
           nQPlan += 2;
         }else{
@@ -1107,11 +1120,11 @@ WhereInfo *sqlite3WhereBegin(
 
   /* Generate the code to do the search
   */
-  loopMask = ~(Bitmask)0;
+  notReady = ~(Bitmask)0;
   pLevel = pWInfo->a;
   pTabItem = pTabList->a;
   for(i=0; i<pTabList->nSrc; i++, pTabItem++, pLevel++){
-    int j, k;
+    int j;
     int iCur = pTabItem->iCursor;  /* The VDBE cursor for the table */
     Index *pIdx;       /* The index we will be using */
     int iIdxCur;       /* The VDBE cursor for the index */
@@ -1124,7 +1137,7 @@ WhereInfo *sqlite3WhereBegin(
     /* Check to see if it is appropriate to omit the use of the table
     ** here and use its index instead.
     */
-    omitTable = (pLevel->score&1)!=0;
+    omitTable = (pLevel->flags & WHERE_IDX_ONLY)!=0;
 
     /* If this is the right table of a LEFT OUTER JOIN, allocate and
     ** initialize a memory cell that records if this table matches any
@@ -1138,14 +1151,14 @@ WhereInfo *sqlite3WhereBegin(
       VdbeComment((v, "# init LEFT JOIN no-match flag"));
     }
 
-    if( i<ARRAYSIZE(iDirectEq) && (k = iDirectEq[i])>=0 ){
+    if( pLevel->flags & WHERE_ROWID_EQ ){
       /* Case 1:  We can directly reference a single row using an
       **          equality comparison against the ROWID field.  Or
       **          we reference multiple rows using a "rowid IN (...)"
       **          construct.
       */
-      assert( k<wc.nTerm );
-      pTerm = &wc.a[k];
+      pTerm = findTerm(&wc, iCur, -1, notReady, WO_EQ|WO_IN, 0);
+      assert( pTerm!=0 );
       assert( pTerm->pExpr!=0 );
       assert( pTerm->leftCursor==iCur );
       assert( omitTable==0 );
@@ -1156,23 +1169,30 @@ WhereInfo *sqlite3WhereBegin(
       sqlite3VdbeAddOp(v, OP_NotExists, iCur, brk);
       VdbeComment((v, "pk"));
       pLevel->op = OP_Noop;
-    }else if( pIdx!=0 && pLevel->score>3 && (pLevel->score&0x0c)==0 ){
+    }else if( pLevel->flags & WHERE_COLUMN_EQ ){
       /* Case 2:  There is an index and all terms of the WHERE clause that
       **          refer to the index using the "==" or "IN" operators.
       */
       int start;
-      int nColumn = (pLevel->score+16)/32;
+      int nColumn;
       brk = pLevel->brk = sqlite3VdbeMakeLabel(v);
 
       /* For each column of the index, find the term of the WHERE clause that
       ** constraints that column.  If the WHERE clause term is X=expr, then
       ** generate code to evaluate expr and leave the result on the stack */
-      for(j=0; j<nColumn; j++){
-        pTerm = findTerm(&wc, iCur, pIdx->aiColumn[j], loopMask, TK_EQ, TK_IN);
-        assert( pTerm!=0 );
+      for(j=0; 1; j++){
+        int k = pIdx->aiColumn[j];
+        pTerm = findTerm(&wc, iCur, k, notReady, WO_EQ|WO_IN, pIdx);
+        if( pTerm==0 ) break;
+        if( pTerm->operator==WO_IN && j>0 ) break;
         assert( (pTerm->flags & TERM_CODED)==0 );
         codeEqualityTerm(pParse, pTerm, brk, pLevel);
+        if( pTerm->operator==WO_IN ){
+          j++;
+          break;
+        }
       }
+      nColumn = j;
       pLevel->iMem = pParse->nMem++;
       cont = pLevel->cont = sqlite3VdbeMakeLabel(v);
       buildIndexProbe(v, nColumn, brk, pIdx);
@@ -1183,7 +1203,7 @@ WhereInfo *sqlite3WhereBegin(
       ** the last matching element of the table.  The code (1) is executed
       ** once to initialize the search, the code (2) is executed before each
       ** iteration of the scan to see if the scan has finished. */
-      if( pLevel->bRev ){
+      if( pLevel->flags & WHERE_REVERSE ){
         /* Scan in reverse order */
         sqlite3VdbeAddOp(v, OP_MoveLe, iIdxCur, brk);
         start = sqlite3VdbeAddOp(v, OP_MemLoad, pLevel->iMem, 0);
@@ -1204,45 +1224,53 @@ WhereInfo *sqlite3WhereBegin(
       }
       pLevel->p1 = iIdxCur;
       pLevel->p2 = start;
-    }else if( i<ARRAYSIZE(iDirectLt) && (iDirectLt[i]>=0 || iDirectGt[i]>=0) ){
+    }else if( pLevel->flags & WHERE_ROWID_RANGE ){
       /* Case 3:  We have an inequality comparison against the ROWID field.
       */
       int testOp = OP_Noop;
       int start;
-      int bRev = pLevel->bRev;
+      WhereTerm *pStart, *pEnd;
+      int bRev = (pLevel->flags & WHERE_REVERSE)!=0;
 
       assert( omitTable==0 );
       brk = pLevel->brk = sqlite3VdbeMakeLabel(v);
       cont = pLevel->cont = sqlite3VdbeMakeLabel(v);
-      if( bRev ){
-        int t = iDirectGt[i];
-        iDirectGt[i] = iDirectLt[i];
-        iDirectLt[i] = t;
+      if( pLevel->flags & WHERE_BTM_LIMIT ){
+        pStart = findTerm(&wc, iCur, -1, notReady, WO_GT|WO_GE, 0);
+        assert( pStart!=0 );
+      }else{
+        pStart = 0;
       }
-      if( iDirectGt[i]>=0 ){
+      if( pLevel->flags & WHERE_TOP_LIMIT ){
+        pEnd = findTerm(&wc, iCur, -1, notReady, WO_LT|WO_LE, 0);
+        assert( pEnd!=0 );
+      }else{
+        pEnd = 0;
+      }
+      assert( pStart!=0 || pEnd!=0 );
+      if( bRev ){
+        pTerm = pStart;
+        pStart = pEnd;
+        pEnd = pTerm;
+      }
+      if( pStart ){
         Expr *pX;
-        k = iDirectGt[i];
-        assert( k<wc.nTerm );
-        pTerm = &wc.a[k];
-        pX = pTerm->pExpr;
+        pX = pStart->pExpr;
         assert( pX!=0 );
-        assert( pTerm->leftCursor==iCur );
+        assert( pStart->leftCursor==iCur );
         sqlite3ExprCode(pParse, pX->pRight);
         sqlite3VdbeAddOp(v, OP_ForceInt, pX->op==TK_LE || pX->op==TK_GT, brk);
         sqlite3VdbeAddOp(v, bRev ? OP_MoveLt : OP_MoveGe, iCur, brk);
         VdbeComment((v, "pk"));
-        disableTerm(pLevel, pTerm);
+        disableTerm(pLevel, pStart);
       }else{
         sqlite3VdbeAddOp(v, bRev ? OP_Last : OP_Rewind, iCur, brk);
       }
-      if( iDirectLt[i]>=0 ){
+      if( pEnd ){
         Expr *pX;
-        k = iDirectLt[i];
-        assert( k<wc.nTerm );
-        pTerm = &wc.a[k];
-        pX = pTerm->pExpr;
+        pX = pEnd->pExpr;
         assert( pX!=0 );
-        assert( pTerm->leftCursor==iCur );
+        assert( pEnd->leftCursor==iCur );
         sqlite3ExprCode(pParse, pX->pRight);
         pLevel->iMem = pParse->nMem++;
         sqlite3VdbeAddOp(v, OP_MemStore, pLevel->iMem, 1);
@@ -1251,7 +1279,7 @@ WhereInfo *sqlite3WhereBegin(
         }else{
           testOp = bRev ? OP_Lt : OP_Gt;
         }
-        disableTerm(pLevel, pTerm);
+        disableTerm(pLevel, pEnd);
       }
       start = sqlite3VdbeCurrentAddr(v);
       pLevel->op = bRev ? OP_Prev : OP_Next;
@@ -1262,29 +1290,8 @@ WhereInfo *sqlite3WhereBegin(
         sqlite3VdbeAddOp(v, OP_MemLoad, pLevel->iMem, 0);
         sqlite3VdbeAddOp(v, testOp, 'n', brk);
       }
-    }else if( pIdx==0 ){
-      /* Case 4:  There is no usable index.  We must do a complete
-      **          scan of the entire database table.
-      */
-      int start;
-      int opRewind;
-
-      assert( omitTable==0 );
-      brk = pLevel->brk = sqlite3VdbeMakeLabel(v);
-      cont = pLevel->cont = sqlite3VdbeMakeLabel(v);
-      if( pLevel->bRev ){
-        opRewind = OP_Last;
-        pLevel->op = OP_Prev;
-      }else{
-        opRewind = OP_Rewind;
-        pLevel->op = OP_Next;
-      }
-      sqlite3VdbeAddOp(v, opRewind, iCur, brk);
-      start = sqlite3VdbeCurrentAddr(v);
-      pLevel->p1 = iCur;
-      pLevel->p2 = start;
-    }else{
-      /* Case 5: The WHERE clause term that refers to the right-most
+    }else if( pLevel->flags & WHERE_COLUMN_RANGE ){
+      /* Case 4: The WHERE clause term that refers to the right-most
       **         column of the index is an inequality.  For example, if
       **         the index is on (x,y,z) and the WHERE clause is of the
       **         form "x=5 AND y<10" then this case is used.  Only the
@@ -1295,21 +1302,25 @@ WhereInfo *sqlite3WhereBegin(
       **         constraints but an index is selected anyway, in order
       **         to force the output order to conform to an ORDER BY.
       */
-      int score = pLevel->score;
-      int nEqColumn = score/32;
+      int nEqColumn;
       int start;
       int leFlag=0, geFlag=0;
       int testOp;
+      int topLimit = (pLevel->flags & WHERE_TOP_LIMIT)!=0;
+      int btmLimit = (pLevel->flags & WHERE_BTM_LIMIT)!=0;
+      int bRev = (pLevel->flags & WHERE_REVERSE)!=0;
 
       /* Evaluate the equality constraints
       */
-      for(j=0; j<nEqColumn; j++){
-        pTerm = findTerm(&wc, iCur, pIdx->aiColumn[j], loopMask, TK_EQ, TK_EQ);
-        assert( pTerm!=0 );
+      for(j=0; 1; j++){
+        int k = pIdx->aiColumn[j];
+        pTerm = findTerm(&wc, iCur, k, notReady, WO_EQ, pIdx);
+        if( pTerm==0 ) break;
         assert( (pTerm->flags & TERM_CODED)==0 );
         sqlite3ExprCode(pParse, pTerm->pExpr->pRight);
         disableTerm(pLevel, pTerm);
       }
+      nEqColumn = j;
 
       /* Duplicate the equality term values because they will all be
       ** used twice: once to make the termination key and once to make the
@@ -1331,9 +1342,10 @@ WhereInfo *sqlite3WhereBegin(
       ** 2002-Dec-04: On a reverse-order scan, the so-called "termination"
       ** key computed here really ends up being the start key.
       */
-      if( (score & 4)!=0 ){
+      if( topLimit ){
         Expr *pX;
-        pTerm = findTerm(&wc, iCur, pIdx->aiColumn[j], loopMask, TK_LT, TK_LE);
+        int k = pIdx->aiColumn[j];
+        pTerm = findTerm(&wc, iCur, k, notReady, WO_LT|WO_LE, pIdx);
         assert( pTerm!=0 );
         pX = pTerm->pExpr;
         assert( (pTerm->flags & TERM_CODED)==0 );
@@ -1346,16 +1358,16 @@ WhereInfo *sqlite3WhereBegin(
         leFlag = 1;
       }
       if( testOp!=OP_Noop ){
-        int nCol = nEqColumn + ((score & 4)!=0);
+        int nCol = nEqColumn + topLimit;
         pLevel->iMem = pParse->nMem++;
         buildIndexProbe(v, nCol, brk, pIdx);
-        if( pLevel->bRev ){
+        if( bRev ){
           int op = leFlag ? OP_MoveLe : OP_MoveLt;
           sqlite3VdbeAddOp(v, op, iIdxCur, brk);
         }else{
           sqlite3VdbeAddOp(v, OP_MemStore, pLevel->iMem, 1);
         }
-      }else if( pLevel->bRev ){
+      }else if( bRev ){
         sqlite3VdbeAddOp(v, OP_Last, iIdxCur, brk);
       }
 
@@ -1368,9 +1380,10 @@ WhereInfo *sqlite3WhereBegin(
       ** 2002-Dec-04: In the case of a reverse-order search, the so-called
       ** "start" key really ends up being used as the termination key.
       */
-      if( (score & 8)!=0 ){
+      if( btmLimit ){
         Expr *pX;
-        pTerm = findTerm(&wc, iCur, pIdx->aiColumn[j], loopMask, TK_GT, TK_GE);
+        int k = pIdx->aiColumn[j];
+        pTerm = findTerm(&wc, iCur, k, notReady, WO_GT|WO_GE, pIdx);
         assert( pTerm!=0 );
         pX = pTerm->pExpr;
         assert( (pTerm->flags & TERM_CODED)==0 );
@@ -1380,10 +1393,10 @@ WhereInfo *sqlite3WhereBegin(
       }else{
         geFlag = 1;
       }
-      if( nEqColumn>0 || (score&8)!=0 ){
-        int nCol = nEqColumn + ((score&8)!=0);
+      if( nEqColumn>0 || btmLimit ){
+        int nCol = nEqColumn + btmLimit;
         buildIndexProbe(v, nCol, brk, pIdx);
-        if( pLevel->bRev ){
+        if( bRev ){
           pLevel->iMem = pParse->nMem++;
           sqlite3VdbeAddOp(v, OP_MemStore, pLevel->iMem, 1);
           testOp = OP_IdxLT;
@@ -1391,7 +1404,7 @@ WhereInfo *sqlite3WhereBegin(
           int op = geFlag ? OP_MoveGe : OP_MoveGt;
           sqlite3VdbeAddOp(v, op, iIdxCur, brk);
         }
-      }else if( pLevel->bRev ){
+      }else if( bRev ){
         testOp = OP_Noop;
       }else{
         sqlite3VdbeAddOp(v, OP_Rewind, iIdxCur, brk);
@@ -1405,12 +1418,12 @@ WhereInfo *sqlite3WhereBegin(
       if( testOp!=OP_Noop ){
         sqlite3VdbeAddOp(v, OP_MemLoad, pLevel->iMem, 0);
         sqlite3VdbeAddOp(v, testOp, iIdxCur, brk);
-        if( (leFlag && !pLevel->bRev) || (!geFlag && pLevel->bRev) ){
+        if( (leFlag && !bRev) || (!geFlag && bRev) ){
           sqlite3VdbeChangeP3(v, -1, "+", P3_STATIC);
         }
       }
       sqlite3VdbeAddOp(v, OP_RowKey, iIdxCur, 0);
-      sqlite3VdbeAddOp(v, OP_IdxIsNull, nEqColumn + ((score&4)!=0), cont);
+      sqlite3VdbeAddOp(v, OP_IdxIsNull, nEqColumn + topLimit, cont);
       if( !omitTable ){
         sqlite3VdbeAddOp(v, OP_IdxRowid, iIdxCur, 0);
         sqlite3VdbeAddOp(v, OP_MoveGe, iCur, 0);
@@ -1418,11 +1431,32 @@ WhereInfo *sqlite3WhereBegin(
 
       /* Record the instruction used to terminate the loop.
       */
-      pLevel->op = pLevel->bRev ? OP_Prev : OP_Next;
+      pLevel->op = bRev ? OP_Prev : OP_Next;
       pLevel->p1 = iIdxCur;
       pLevel->p2 = start;
+    }else{
+      /* Case 5:  There is no usable index.  We must do a complete
+      **          scan of the entire table.
+      */
+      int start;
+      int opRewind;
+
+      assert( omitTable==0 );
+      brk = pLevel->brk = sqlite3VdbeMakeLabel(v);
+      cont = pLevel->cont = sqlite3VdbeMakeLabel(v);
+      if( pLevel->flags & WHERE_REVERSE ){
+        opRewind = OP_Last;
+        pLevel->op = OP_Prev;
+      }else{
+        opRewind = OP_Rewind;
+        pLevel->op = OP_Next;
+      }
+      sqlite3VdbeAddOp(v, opRewind, iCur, brk);
+      start = sqlite3VdbeCurrentAddr(v);
+      pLevel->p1 = iCur;
+      pLevel->p2 = start;
     }
-    loopMask &= ~getMask(&maskSet, iCur);
+    notReady &= ~getMask(&maskSet, iCur);
 
     /* Insert code to test every subexpression that can be completely
     ** computed using the current set of tables.
@@ -1430,7 +1464,7 @@ WhereInfo *sqlite3WhereBegin(
     for(pTerm=wc.a, j=wc.nTerm; j>0; j--, pTerm++){
       Expr *pE;
       if( pTerm->flags & (TERM_VIRTUAL|TERM_CODED) ) continue;
-      if( (pTerm->prereqAll & loopMask)!=0 ) continue;
+      if( (pTerm->prereqAll & notReady)!=0 ) continue;
       pE = pTerm->pExpr;
       assert( pE!=0 );
       if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
@@ -1451,7 +1485,7 @@ WhereInfo *sqlite3WhereBegin(
       VdbeComment((v, "# record LEFT JOIN hit"));
       for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
         if( pTerm->flags & (TERM_VIRTUAL|TERM_CODED) ) continue;
-        if( (pTerm->prereqAll & loopMask)!=0 ) continue;
+        if( (pTerm->prereqAll & notReady)!=0 ) continue;
         assert( pTerm->pExpr );
         sqlite3ExprIfFalse(pParse, pTerm->pExpr, cont, 1);
         pTerm->flags |= TERM_CODED;
@@ -1512,7 +1546,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     Table *pTab = pTabItem->pTab;
     assert( pTab!=0 );
     if( pTab->isTransient || pTab->pSelect ) continue;
-    if( (pLevel->score & 1)==0 ){
+    if( (pLevel->flags & WHERE_IDX_ONLY)==0 ){
       sqlite3VdbeAddOp(v, OP_Close, pTabItem->iCursor, 0);
     }
     if( pLevel->pIdx!=0 ){
@@ -1528,7 +1562,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     ** that reference the table and converts them into opcodes that
     ** reference the index.
     */
-    if( pLevel->score & 1 ){
+    if( pLevel->flags & WHERE_IDX_ONLY ){
       int i, j, last;
       VdbeOp *pOp;
       Index *pIdx = pLevel->pIdx;
@@ -1561,3 +1595,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
   sqliteFree(pWInfo);
   return;
 }
+
+
+/*
+** Delete a list of WhereIdx structures.
+*/
+void sqlite3WhereIdxListDelete(WhereIdx *p){
+  WhereIdx *pNext;
+  while( p ){
+    pNext = p->pNext;
+    sqliteFree(p);
+    p = pNext;
+  }
+}
diff --git a/test/select2.test b/test/select2.test
index 52525a6d5a..1c6a5c893d 100644
--- a/test/select2.test
+++ b/test/select2.test
@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the SELECT statement.
 #
-# $Id: select2.test,v 1.24 2005/01/25 04:27:55 danielk1977 Exp $
+# $Id: select2.test,v 1.25 2005/07/21 03:15:01 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -117,6 +117,7 @@ do_test select2-3.2c {
 } {500}
 do_test select2-3.2d {
   set sqlite_search_count 0
+btree_breakpoint
   execsql {SELECT * FROM tbl2 WHERE 1000=f2}
   set sqlite_search_count
 } {3}
diff --git a/test/sort.test b/test/sort.test
index 6d527c33ce..af8430fe11 100644
--- a/test/sort.test
+++ b/test/sort.test
@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the CREATE TABLE statement.
 #
-# $Id: sort.test,v 1.19 2005/02/02 01:10:45 danielk1977 Exp $
+# $Id: sort.test,v 1.20 2005/07/21 03:15:01 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -432,4 +432,3 @@ do_test sort-10.3 {
 } {2 1}
 
 finish_test
-
diff --git a/test/subquery.test b/test/subquery.test
index 4b626ed83a..07b261bf58 100644
--- a/test/subquery.test
+++ b/test/subquery.test
@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing correlated subqueries
 #
-# $Id: subquery.test,v 1.9 2005/05/23 15:06:39 drh Exp $
+# $Id: subquery.test,v 1.10 2005/07/21 03:15:01 drh Exp $
 #
 
 set testdir [file dirname $argv0]
@@ -228,9 +228,11 @@ do_test subquery-2.5.2 {
 do_test subquery-2.5.3 {
   execsql {
     CREATE INDEX t4i ON t4(x);
+    --pragma vdbe_listing=on; pragma vdbe_trace=on;
     SELECT * FROM t4 WHERE x IN (SELECT a FROM t3);
   }
 } {10.0}
+#exit
 do_test subquery-2.5.4 {
   execsql {
     DROP TABLE t3;