Allow an unlimited number of terms in the WHERE clause. The old limit was 100. (CVS 2550)

FossilOrigin-Name: ca69f36832d57775e73ac5cdbe0a32d7b759432b

manifest

@@ -1,5 +1,5 @@
-C Fix\sbugs\sin\sthe\snew\squery\splan\sinstrumention\slogic.\s(CVS\s2549)
+C Allow\san\sunlimited\snumber\sof\sterms\sin\sthe\sWHERE\sclause.\s\sThe\sold\slimit\swas\s100.\s(CVS\s2550)
-D 2005-07-15T23:24:24
+D 2005-07-16T13:33:21
 F Makefile.in 22ea9c0fe748f591712d8fe3c6d972c6c173a165
 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -85,7 +85,7 @@ F src/vdbeapi.c 7f392f0792d1258c958083d7de9eae7c3530c9a6
 F src/vdbeaux.c 3732a86566a6be4da4c606e9334baf3fd98667af
 F src/vdbefifo.c b8805850afe13b43f1de78d58088cb5d66f88e1e
 F src/vdbemem.c da8e8d6f29dd1323f782f000d7cd120027c9ff03
-F src/where.c e6b7be981e79988b1586d8bcdebfd7840e6462f7
+F src/where.c 9b02e75ef86f057cee5e93150b10cbc1092d7de6
 F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42
 F test/all.test 7f0988442ab811dfa41793b5b550f5828ce316f3
 F test/alter.test 9d6837a3d946b73df692b7cef2a7644d2e2f6bc6
@@ -165,7 +165,7 @@ F test/malloc2.test 655b972372d2754a3f6c6ed54d7cfd18fde9bd32
 F test/memdb.test 1860e060be810bf0775bc57408a5b7c4954bcaea
 F test/memleak.test df2b2b96e77f8ba159a332299535b1e5f18e49ac
 F test/minmax.test 9429a06f1f93acf76fcacafd17160a4392e88526
-F test/misc1.test a4a36c19f05e4c8646efe7a0d7242ba645d07379
+F test/misc1.test a4fe87c71f756ee36b08a698da46ea9c3b2471e7
 F test/misc2.test 5c699af2fede2694736a9f45aea7e2f052686e15
 F test/misc3.test 7bd937e2c62bcc6be71939faf068d506467b1e03
 F test/misc4.test edd3e3adf5b6e3b995b29843565ca58dd602f9a7
@@ -286,7 +286,7 @@ F www/tclsqlite.tcl 425be741b8ae664f55cb1ef2371aab0a75109cf9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b
 F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b
-P dfd5fd77b0764853f847eeee3c1fe047d60fee7e
+P 578490c91331a386f84652db0b3bfd74c82046e1
-R 407e6021e72c837a936d4205fddd2503
+R 544e009ea316e2cccd816cc597a10aab
 U drh
-Z 0ce70505a05c8384502058350e244c6a
+Z 0b48c3e07c6ac902e7fb79f1e0cbf3a1

manifest.uuid

@@ -1 +1 @@
-578490c91331a386f84652db0b3bfd74c82046e1
+ca69f36832d57775e73ac5cdbe0a32d7b759432b

src/where.c

@@ -16,10 +16,21 @@
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.144 2005/07/15 23:24:24 drh Exp $
+** $Id: where.c,v 1.145 2005/07/16 13:33:21 drh Exp $
 */
 #include "sqliteInt.h"
 
+/*
+** The number of bits in a Bitmask.  "BMS" means "BitMask Size".
+*/
+#define BMS  (sizeof(Bitmask)*8-1)
+
+/*
+** Determine the number of elements in an array.
+*/
+#define ARRAYSIZE(X)  (sizeof(X)/sizeof(X[0]))
+
+
 /*
 ** The query generator uses an array of instances of this structure to
 ** help it analyze the subexpressions of the WHERE clause.  Each WHERE
@@ -27,7 +38,7 @@
 **
 ** The idxLeft and idxRight fields are the VDBE cursor numbers for the
 ** table that contains the column that appears on the left-hand and
-** right-hand side of ExprInfo.p.  If either side of ExprInfo.p is
+** right-hand side of WhereTerm.p.  If either side of WhereTerm.p is
 ** something other than a simple column reference, then idxLeft or
 ** idxRight are -1.  
 **
@@ -47,13 +58,13 @@
 ** would be mapped into integers 0 through 7.
 **
 ** prereqLeft tells us every VDBE cursor that is referenced on the
-** left-hand side of ExprInfo.p.  prereqRight does the same for the
+** left-hand side of WhereTerm.p.  prereqRight does the same for the
 ** right-hand side of the expression.  The following identity always
 ** holds:
 **
 **       prereqAll = prereqLeft | prereqRight
 **
-** The ExprInfo.indexable field is true if the ExprInfo.p expression
+** The WhereTerm.indexable field is true if the WhereTerm.p expression
 ** is of a form that might control an index.  Indexable expressions
 ** look like this:
 **
@@ -62,22 +73,41 @@
 ** Where <column> is a simple column name and <op> is on of the operators
 ** that allowedOp() recognizes.  
 */
-typedef struct ExprInfo ExprInfo;
+typedef struct WhereTerm WhereTerm;
-struct ExprInfo {
+struct WhereTerm {
   Expr *p;                /* Pointer to the subexpression */
+  u16 flags;              /* Bit flags.  See below */
   u8 indexable;           /* True if this subexprssion is usable by an index */
   short int idxLeft;      /* p->pLeft is a column in this table number. -1 if
-                          ** p->pLeft is not the column of any table */
+                          ** p->pLeft is not a column of any table */
   short int idxRight;     /* p->pRight is a column in this table number. -1 if
-                          ** p->pRight is not the column of any table */
+                          ** p->pRight is not a column of any table */
   Bitmask prereqLeft;     /* Bitmask of tables referenced by p->pLeft */
   Bitmask prereqRight;    /* Bitmask of tables referenced by p->pRight */
   Bitmask prereqAll;      /* Bitmask of tables referenced by p */
 };
 
+/*
+** Allowed values of WhereTerm.flags
+*/
+#define TERM_DYNAMIC    0x0001   /* Need to call sqlite3ExprDelete(p) */
+#define TERM_VIRTUAL    0x0002   /* Added by the optimizer.  Do not code */
+
+/*
+** An instance of the following structure holds all information about a
+** WHERE clause.  Mostly this is a container for one or more WhereTerms.
+*/
+typedef struct WhereClause WhereClause;
+struct WhereClause {
+  int nTerm;               /* Number of terms */
+  int nSlot;               /* Number of entries in a[] */
+  WhereTerm *a;            /* Pointer to an array of terms */
+  WhereTerm aStatic[10];   /* Initial static space for the terms */
+};
+
 /*
 ** An instance of the following structure keeps track of a mapping
-** between VDBE cursor numbers and bits of the bitmasks in ExprInfo.
+** between VDBE cursor numbers and bits of the bitmasks in WhereTerm.
 **
 ** The VDBE cursor numbers are small integers contained in 
 ** SrcList_item.iCursor and Expr.iTable fields.  For any given WHERE 
@@ -107,10 +137,53 @@ struct ExprMaskSet {
   int ix[sizeof(Bitmask)*8];    /* Cursor assigned to each bit */
 };
 
+
 /*
-** Determine the number of elements in an array.
+** Initialize a preallocated WhereClause structure.
 */
-#define ARRAYSIZE(X)  (sizeof(X)/sizeof(X[0]))
+static void whereClauseInit(WhereClause *pWC){
+  pWC->nTerm = 0;
+  pWC->nSlot = ARRAYSIZE(pWC->aStatic);
+  pWC->a = pWC->aStatic;
+}
+
+/*
+** Deallocate a WhereClause structure.  The WhereClause structure
+** itself is not freed.  This routine is the inverse of whereClauseInit().
+*/
+static void whereClauseClear(WhereClause *pWC){
+  int i;
+  WhereTerm *a;
+  for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){
+    if( a->flags & TERM_DYNAMIC ){
+      sqlite3ExprDelete(a->p);
+    }
+  }
+  if( pWC->a!=pWC->aStatic ){
+    sqliteFree(pWC->a);
+  }
+}
+
+/*
+** Add a new entries to the WhereClause structure.  Increase the allocated
+** space as necessary.
+*/
+static void whereClauseInsert(WhereClause *pWC, Expr *p, int flags){
+  WhereTerm *pTerm;
+  if( pWC->nTerm>=pWC->nSlot ){
+    WhereTerm *pOld = pWC->a;
+    pWC->a = sqliteMalloc( sizeof(pWC->a[0])*pWC->nSlot*2 );
+    if( pWC->a==0 ) return;
+    memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm);
+    if( pOld!=pWC->aStatic ){
+      sqliteFree(pOld);
+    }
+    pWC->nSlot *= 2;
+  }
+  pTerm = &pWC->a[pWC->nTerm++];
+  pTerm->p = p;
+  pTerm->flags = flags;
+}
 
 /*
 ** This routine identifies subexpressions in the WHERE clause where
@@ -129,21 +202,14 @@ struct ExprMaskSet {
 ** subexpressions as it can and puts pointers to those subexpressions
 ** into aSlot[] entries.  The return value is the number of slots filled.
 */
-static int exprSplit(int nSlot, ExprInfo *aSlot, Expr *pExpr){
+static void whereSplit(WhereClause *pWC, Expr *pExpr){
-  int cnt = 0;
+  if( pExpr==0 ) return;
-  if( pExpr==0 || nSlot<1 ) return 0;
+  if( pExpr->op!=TK_AND ){
-  if( nSlot==1 || pExpr->op!=TK_AND ){
+    whereClauseInsert(pWC, pExpr, 0);
-    aSlot[0].p = pExpr;
-    return 1;
-  }
-  if( pExpr->pLeft->op!=TK_AND ){
-    aSlot[0].p = pExpr->pLeft;
-    cnt = 1 + exprSplit(nSlot-1, &aSlot[1], pExpr->pRight);
   }else{
-    cnt = exprSplit(nSlot, aSlot, pExpr->pLeft);
+    whereSplit(pWC, pExpr->pLeft);
-    cnt += exprSplit(nSlot-cnt, &aSlot[cnt], pExpr->pRight);
+    whereSplit(pWC, pExpr->pRight);
   }
-  return cnt;
 }
 
 /*
@@ -257,12 +323,12 @@ static int tableOrder(SrcList *pList, int iCur){
 }
 
 /*
-** The input to this routine is an ExprInfo structure with only the
+** The input to this routine is an WhereTerm structure with only the
 ** "p" field filled in.  The job of this routine is to analyze the
-** subexpression and populate all the other fields of the ExprInfo
+** subexpression and populate all the other fields of the WhereTerm
 ** structure.
 */
-static void exprAnalyze(SrcList *pSrc, ExprMaskSet *pMaskSet, ExprInfo *pInfo){
+static void exprAnalyze(SrcList *pSrc, ExprMaskSet *pMaskSet, WhereTerm *pInfo){
   Expr *pExpr = pInfo->p;
   pInfo->prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft);
   pInfo->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight);
@@ -481,7 +547,7 @@ static void buildIndexProbe(Vdbe *v, int nColumn, int brk, Index *pIdx){
 */
 static void codeEqualityTerm(
   Parse *pParse,      /* The parsing context */
-  ExprInfo *pTerm,    /* The term of the WHERE clause to be coded */
+  WhereTerm *pTerm,    /* The term of the WHERE clause to be coded */
   int brk,            /* Jump here to abandon the loop */
   WhereLevel *pLevel  /* When level of the FROM clause we are working on */
 ){
@@ -506,11 +572,6 @@ static void codeEqualityTerm(
   disableTerm(pLevel, &pTerm->p);
 }
 
-/*
-** The number of bits in a Bitmask.  "BMS" means "BitMask Size".
-*/
-#define BMS  (sizeof(Bitmask)*8-1)
-
 #ifdef SQLITE_TEST
 /*
 ** The following variable holds a text description of query plan generated
@@ -617,14 +678,13 @@ WhereInfo *sqlite3WhereBegin(
   WhereInfo *pWInfo;         /* Will become the return value of this function */
   Vdbe *v = pParse->pVdbe;   /* The virtual database engine */
   int brk, cont = 0;         /* Addresses used during code generation */
-  int nExpr;           /* Number of subexpressions in the WHERE clause */
   Bitmask loopMask;          /* One bit set for each outer loop */
-  ExprInfo *pTerm;     /* A single term in the WHERE clause; ptr to aExpr[] */
+  WhereTerm *pTerm;          /* A single term in the WHERE clause */
   ExprMaskSet maskSet;       /* The expression mask set */
   int iDirectEq[BMS];        /* Term of the form ROWID==X for the N-th table */
   int iDirectLt[BMS];        /* Term of the form ROWID<X or ROWID<=X */
   int iDirectGt[BMS];        /* Term of the form ROWID>X or ROWID>=X */
-  ExprInfo aExpr[101]; /* The WHERE clause is divided into these terms */
+  WhereClause wc;            /* The WHERE clause is divided into these terms */
   struct SrcList_item *pTabItem;  /* A single entry from pTabList */
   WhereLevel *pLevel;             /* A single level in the pWInfo list */
 
@@ -638,18 +698,13 @@ WhereInfo *sqlite3WhereBegin(
   }
 
   /* Split the WHERE clause into separate subexpressions where each
-  ** subexpression is separated by an AND operator.  If the aExpr[]
+  ** subexpression is separated by an AND operator.  If the wc.a[]
   ** array fills up, the last entry might point to an expression which
   ** contains additional unfactored AND operators.
   */
   initMaskSet(&maskSet);
-  memset(aExpr, 0, sizeof(aExpr));
+  whereClauseInit(&wc);
-  nExpr = exprSplit(ARRAYSIZE(aExpr), aExpr, pWhere);
+  whereSplit(&wc, pWhere);
-  if( nExpr==ARRAYSIZE(aExpr) ){
-    sqlite3ErrorMsg(pParse, "WHERE clause too complex - no more "
-       "than %d terms allowed", (int)ARRAYSIZE(aExpr)-1);
-    return 0;
-  }
     
   /* Allocate and initialize the WhereInfo structure that will become the
   ** return value.
@@ -657,6 +712,7 @@ WhereInfo *sqlite3WhereBegin(
   pWInfo = sqliteMalloc( sizeof(WhereInfo) + pTabList->nSrc*sizeof(WhereLevel));
   if( sqlite3_malloc_failed ){
     sqliteFree(pWInfo); /* Avoid leaking memory when malloc fails */
+    whereClauseClear(&wc);
     return 0;
   }
   pWInfo->pParse = pParse;
@@ -676,7 +732,7 @@ WhereInfo *sqlite3WhereBegin(
   for(i=0; i<pTabList->nSrc; i++){
     createMask(&maskSet, pTabList->a[i].iCursor);
   }
-  for(pTerm=aExpr, i=0; i<nExpr; i++, pTerm++){
+  for(pTerm=wc.a, i=0; i<wc.nTerm; i++, pTerm++){
     exprAnalyze(pTabList, &maskSet, pTerm);
   }
 
@@ -721,7 +777,7 @@ WhereInfo *sqlite3WhereBegin(
     iDirectEq[i] = -1;
     iDirectLt[i] = -1;
     iDirectGt[i] = -1;
-    for(pTerm=aExpr, j=0; j<nExpr; j++, pTerm++){
+    for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
       Expr *pX = pTerm->p;
       if( pTerm->idxLeft==iCur && pX->pLeft->iColumn<0
             && (pTerm->prereqRight & loopMask)==pTerm->prereqRight ){
@@ -792,7 +848,7 @@ WhereInfo *sqlite3WhereBegin(
       if( pIdx->nColumn>sizeof(eqMask)*8 ){
         continue;  /* Ignore indices with too many columns to analyze */
       }
-      for(pTerm=aExpr, j=0; j<nExpr; j++, pTerm++){
+      for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
         Expr *pX = pTerm->p;
         CollSeq *pColl = sqlite3ExprCollSeq(pParse, pX->pLeft);
         if( !pColl && pX->pRight ){
@@ -1051,8 +1107,8 @@ WhereInfo *sqlite3WhereBegin(
       **          we reference multiple rows using a "rowid IN (...)"
       **          construct.
       */
-      assert( k<nExpr );
+      assert( k<wc.nTerm );
-      pTerm = &aExpr[k];
+      pTerm = &wc.a[k];
       assert( pTerm->p!=0 );
       assert( pTerm->idxLeft==iCur );
       assert( omitTable==0 );
@@ -1073,9 +1129,9 @@ WhereInfo *sqlite3WhereBegin(
 
       /* For each column of the index, find the term of the WHERE clause that
       ** constraints that column.  If the WHERE clause term is X=expr, then
-      ** evaluation expr and leave the result on the stack */
+      ** generate code to evaluate expr and leave the result on the stack */
       for(j=0; j<nColumn; j++){
-        for(pTerm=aExpr, k=0; k<nExpr; k++, pTerm++){
+        for(pTerm=wc.a, k=0; k<wc.nTerm; k++, pTerm++){
           Expr *pX = pTerm->p;
           if( pX==0 ) continue;
           if( pTerm->idxLeft==iCur
@@ -1140,8 +1196,8 @@ WhereInfo *sqlite3WhereBegin(
       if( iDirectGt[i]>=0 ){
         Expr *pX;
         k = iDirectGt[i];
-        assert( k<nExpr );
+        assert( k<wc.nTerm );
-        pTerm = &aExpr[k];
+        pTerm = &wc.a[k];
         pX = pTerm->p;
         assert( pX!=0 );
         assert( pTerm->idxLeft==iCur );
@@ -1156,8 +1212,8 @@ WhereInfo *sqlite3WhereBegin(
       if( iDirectLt[i]>=0 ){
         Expr *pX;
         k = iDirectLt[i];
-        assert( k<nExpr );
+        assert( k<wc.nTerm );
-        pTerm = &aExpr[k];
+        pTerm = &wc.a[k];
         pX = pTerm->p;
         assert( pX!=0 );
         assert( pTerm->idxLeft==iCur );
@@ -1223,7 +1279,7 @@ WhereInfo *sqlite3WhereBegin(
       */
       for(j=0; j<nEqColumn; j++){
         int iIdxCol = pIdx->aiColumn[j];
-        for(pTerm=aExpr, k=0; k<nExpr; k++, pTerm++){
+        for(pTerm=wc.a, k=0; k<wc.nTerm; k++, pTerm++){
           Expr *pX = pTerm->p;
           if( pX==0 ) continue;
           if( pTerm->idxLeft==iCur
@@ -1259,7 +1315,7 @@ WhereInfo *sqlite3WhereBegin(
       ** key computed here really ends up being the start key.
       */
       if( (score & 4)!=0 ){
-        for(pTerm=aExpr, k=0; k<nExpr; k++, pTerm++){
+        for(pTerm=wc.a, k=0; k<wc.nTerm; k++, pTerm++){
           Expr *pX = pTerm->p;
           if( pX==0 ) continue;
           if( pTerm->idxLeft==iCur
@@ -1302,7 +1358,7 @@ WhereInfo *sqlite3WhereBegin(
       ** "start" key really ends up being used as the termination key.
       */
       if( (score & 8)!=0 ){
-        for(pTerm=aExpr, k=0; k<nExpr; k++, pTerm++){
+        for(pTerm=wc.a, k=0; k<wc.nTerm; k++, pTerm++){
           Expr *pX = pTerm->p;
           if( pX==0 ) continue;
           if( pTerm->idxLeft==iCur
@@ -1366,7 +1422,7 @@ WhereInfo *sqlite3WhereBegin(
     /* Insert code to test every subexpression that can be completely
     ** computed using the current set of tables.
     */
-    for(pTerm=aExpr, j=0; j<nExpr; j++, pTerm++){
+    for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
       Expr *pE = pTerm->p;
       if( pE==0 || ExprHasProperty(pE, EP_OptOnly) ) continue;
       if( (pTerm->prereqAll & loopMask)!=pTerm->prereqAll ) continue;
@@ -1386,7 +1442,7 @@ WhereInfo *sqlite3WhereBegin(
       sqlite3VdbeAddOp(v, OP_Integer, 1, 0);
       sqlite3VdbeAddOp(v, OP_MemStore, pLevel->iLeftJoin, 1);
       VdbeComment((v, "# record LEFT JOIN hit"));
-      for(pTerm=aExpr, j=0; j<nExpr; j++, pTerm++){
+      for(pTerm=wc.a, j=0; j<wc.nTerm; j++, pTerm++){
         Expr *pE = pTerm->p;
         if( pE==0 || ExprHasProperty(pE, EP_OptOnly) ) continue;
         if( (pTerm->prereqAll & loopMask)!=pTerm->prereqAll ) continue;
@@ -1397,6 +1453,7 @@ WhereInfo *sqlite3WhereBegin(
   }
   pWInfo->iContinue = cont;
   freeMaskSet(&maskSet);
+  whereClauseClear(&wc);
   return pWInfo;
 }
 

test/misc1.test

@@ -13,7 +13,7 @@
 # This file implements tests for miscellanous features that were
 # left out of other test files.
 #
-# $Id: misc1.test,v 1.34 2005/03/29 03:11:00 danielk1977 Exp $
+# $Id: misc1.test,v 1.35 2005/07/16 13:33:21 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -280,6 +280,10 @@ do_test misc1-9.1 {
 # A WHERE clause is not allowed to contain more than 99 terms.  Check to
 # make sure this limit is enforced.
 #
+# 2005-07-16: There is no longer a limit on the number of terms in a
+# WHERE clause.  But keep these tests just so that we have some tests
+# that use a large number of terms in the WHERE clause.
+#
 do_test misc1-10.0 {
   execsql {SELECT count(*) FROM manycol}
 } {9}
@@ -292,7 +296,7 @@ do_test misc1-10.1 {
 } {0 9}
 do_test misc1-10.2 {
   catchsql "SELECT count(*) FROM manycol $::where AND rowid>0"
-} {1 {WHERE clause too complex - no more than 100 terms allowed}}
+} {0 9}
 do_test misc1-10.3 {
   regsub "x0>=0" $::where "x0=0" ::where
   catchsql "DELETE FROM manycol $::where"
@@ -302,7 +306,7 @@ do_test misc1-10.4 {
 } {8}
 do_test misc1-10.5 {
   catchsql "DELETE FROM manycol $::where AND rowid>0"
-} {1 {WHERE clause too complex - no more than 100 terms allowed}}
+} {0 {}}
 do_test misc1-10.6 {
   execsql {SELECT x1 FROM manycol WHERE x0=100}
 } {101}
@@ -315,10 +319,10 @@ do_test misc1-10.8 {
 } {102}
 do_test misc1-10.9 {
   catchsql "UPDATE manycol SET x1=x1+1 $::where AND rowid>0"
-} {1 {WHERE clause too complex - no more than 100 terms allowed}}
+} {0 {}}
 do_test misc1-10.10 {
   execsql {SELECT x1 FROM manycol WHERE x0=100}
-} {102}
+} {103}
 
 # Make sure the initialization works even if a database is opened while
 # another process has the database locked.