Make use of sqlite3DbMallocSize to maximize the size of growable buffers

after each reallocation.  Added new comments and testcase() macros to
where.c. (CVS 5981)

FossilOrigin-Name: 46f2d08959423e130a5b346138311649d92f0fde

manifest

@@ -1,5 +1,5 @@
-C Variable\sname\schanges\sin\sthe\squery\soptimizer\sfor\sdisambiguation\sand\nclarification.\s\sClear\sspace\sin\sboolean\svectors\sfor\snew\sbit\svalues\sto\nencode\snew\squery\splan\stemplates.\s(CVS\s5980)
-D 2008-12-05T02:36:34
+C Make\suse\sof\ssqlite3DbMallocSize\sto\smaximize\sthe\ssize\sof\sgrowable\sbuffers\nafter\seach\sreallocation.\s\sAdded\snew\scomments\sand\stestcase()\smacros\sto\nwhere.c.\s(CVS\s5981)
+D 2008-12-05T15:24:16
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in f7e4c81c347b04f7b0f1c1b081a168645d7b8af7
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -103,12 +103,12 @@ F src/btmutex.c 63c5cc4ad5715690767ffcb741e185d7bc35ec1a
 F src/btree.c 372c5b32dc919fed608be57f440d054a46d002fd
 F src/btree.h 179c3ea813780df78a289a8f5130db18e6d4616e
 F src/btreeInt.h 8d21590c97b6a2c00cce1f78ed5dc5756e835108
-F src/build.c a89e901ea24d8ec845286f9a1fbfd14572a7777e
+F src/build.c ce642b06016d94b0dbc34d379bac82b597baf8d5
 F src/callback.c e970e5beddbdb23f89a6d05cb1a6419d9f755624
 F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c
 F src/date.c 88898ae96a0d7f66711caffa40b2cf30e623a387
 F src/delete.c d60885716666e5ea0f177b8db73c22c67ccba2cb
-F src/expr.c 01b1cf0783a6d0093d72b799fcb22c86146362ef
+F src/expr.c 9ce4f2585325bf3550d24fe60883a87263e0f630
 F src/fault.c dc88c821842157460750d2d61a8a8b4197d047ff
 F src/func.c b4570eb73d873041b8e68f5cdbb4556ff13a94c3
 F src/global.c 20a3fe46c8287a01ba3a7442558f0eb70c66b19a
@@ -120,7 +120,7 @@ F src/journal.c cffd2cd214e58c0e99c3ff632b3bee6c7cbb260e
 F src/legacy.c aac57bd984e666059011ea01ec4383892a253be3
 F src/loadext.c 3872457afdf25bb174fd383cb4e3e0d2a9e60552
 F src/main.c d67d2eee7a643a4a894e5de91231a99ad88400b9
-F src/malloc.c 4e7bf5a3fcfd848a09f7ce7abb00f7b358128161
+F src/malloc.c 89c7c58fbec06b80101fdc6dcdf9ee849b7cd7ac
 F src/mem0.c f2f84062d1f35814d6535c9f9e33de3bfb3b132c
 F src/mem1.c bb8e26adde7d463270e961f045802c61dbabc8f1
 F src/mem2.c 5d9968f576ba1babc787adbfb613cf428ab484ec
@@ -198,12 +198,12 @@ F src/vdbe.c 3e849f1cffda97280d36b88e0980bdbe8128be7e
 F src/vdbe.h 03516f28bf5aca00a53c4dccd6c313f96adb94f6
 F src/vdbeInt.h 1df957ab0f6a129735513d528c930dddfb4b23ef
 F src/vdbeapi.c 20722164e7701a0747eaea03cddbbe0de5cb37bf
-F src/vdbeaux.c 33ba6b66bc595f5522e6a6995a86799bacd8961b
+F src/vdbeaux.c 9db6b1eb6732ee83afb201b655ff55ff40d3d885
 F src/vdbeblob.c b0dcebfafedcf9c0addc7901ad98f6f986c08935
 F src/vdbemem.c 360396ac77b2da36a8cfc280e7c055482f0254e8
 F src/vtab.c 02c51eac45dbff1a1d6e73f58febf92ecb563f7f
 F src/walker.c 488c2660e13224ff70c0c82761118efb547f8f0d
-F src/where.c ef2a149bff7491207d84f825666c3d0e64e9692f
+F src/where.c 78ecad37fdbb8df030c532a77d5c99b8c8f2b7ae
 F tclinstaller.tcl 4356d9d94d2b5ed5e68f9f0c80c4df3048dd7617
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 597662c5d777a122f9a3df0047ea5c5bd383a911
@@ -663,7 +663,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P 06d206ef7d5e433ccde347d63dfcd2177545e1fd
-R 8d3d72064076a4415cfffcdda016269c
+P 81bd0b5ce8a1cf057064c44e9b5371502cb8c58c
+R 53922c8edc023a196f42abe6cad4dc9c
 U drh
-Z 95f6da2cc96e9f1bc5fb1f19dd7d6dc1
+Z 347f5152aa243c9ea5c9c8cc7007f532

manifest.uuid

@@ -1 +1 @@
-81bd0b5ce8a1cf057064c44e9b5371502cb8c58c
+46f2d08959423e130a5b346138311649d92f0fde

src/build.c

@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.503 2008/11/17 19:18:55 danielk1977 Exp $
+** $Id: build.c,v 1.504 2008/12/05 15:24:16 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -2929,7 +2929,7 @@ void *sqlite3ArrayAllocate(
       *pIdx = -1;
       return pArray;
     }
-    *pnAlloc = newSize;
+    *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry;
     pArray = pNew;
   }
   z = (char*)pArray;
@@ -3034,6 +3034,7 @@ SrcList *sqlite3SrcListEnlarge(
   if( pSrc->nSrc+nExtra>pSrc->nAlloc ){
     SrcList *pNew;
     int nAlloc = pSrc->nSrc+nExtra;
+    int nGot;
     pNew = sqlite3DbRealloc(db, pSrc,
                sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) );
     if( pNew==0 ){
@@ -3041,7 +3042,8 @@ SrcList *sqlite3SrcListEnlarge(
       return pSrc;
     }
     pSrc = pNew;
-    pSrc->nAlloc = nAlloc;
+    nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1;
+    pSrc->nAlloc = nGot;
   }
 
   /* Move existing slots that come after the newly inserted slots

src/expr.c

@@ -12,7 +12,7 @@
 ** This file contains routines used for analyzing expressions and
 ** for generating VDBE code that evaluates expressions in SQLite.
 **
-** $Id: expr.c,v 1.404 2008/11/19 16:52:44 danielk1977 Exp $
+** $Id: expr.c,v 1.405 2008/12/05 15:24:17 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -835,7 +835,7 @@ ExprList *sqlite3ExprListAppend(
       goto no_mem;
     }
     pList->a = a;
-    pList->nAlloc = n;
+    pList->nAlloc = sqlite3DbMallocSize(db, a)/sizeof(a[0]);
   }
   assert( pList->a!=0 );
   if( pExpr || pName ){

src/malloc.c

@@ -12,7 +12,7 @@
 **
 ** Memory allocation functions used throughout sqlite.
 **
-** $Id: malloc.c,v 1.48 2008/11/19 09:05:27 danielk1977 Exp $
+** $Id: malloc.c,v 1.49 2008/12/05 15:24:17 drh Exp $
 */
 #include "sqliteInt.h"
 #include <stdarg.h>
@@ -505,7 +505,9 @@ int sqlite3MallocSize(void *p){
   return sqlite3GlobalConfig.m.xSize(p);
 }
 int sqlite3DbMallocSize(sqlite3 *db, void *p){
-  if( isLookaside(db, p) ){
+  if( p==0 ){
+    return 0;
+  }else if( isLookaside(db, p) ){
     return db->lookaside.sz;
   }else{
     return sqlite3GlobalConfig.m.xSize(p);

src/vdbeaux.c

@@ -14,7 +14,7 @@
 ** to version 2.8.7, all this code was combined into the vdbe.c source file.
 ** But that file was getting too big so this subroutines were split out.
 **
-** $Id: vdbeaux.c,v 1.422 2008/12/04 20:40:10 drh Exp $
+** $Id: vdbeaux.c,v 1.423 2008/12/05 15:24:17 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -213,9 +213,10 @@ int sqlite3VdbeMakeLabel(Vdbe *p){
   i = p->nLabel++;
   assert( p->magic==VDBE_MAGIC_INIT );
   if( i>=p->nLabelAlloc ){
-    p->nLabelAlloc = p->nLabelAlloc*2 + 10;
+    int n = p->nLabelAlloc*2 + 5;
     p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel,
-                                    p->nLabelAlloc*sizeof(p->aLabel[0]));
+                                       n*sizeof(p->aLabel[0]));
+    p->nLabelAlloc = sqlite3DbMallocSize(p->db, p->aLabel)/sizeof(p->aLabel[0]);
   }
   if( p->aLabel ){
     p->aLabel[i] = -1;

src/where.c

@@ -16,7 +16,7 @@
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.331 2008/12/05 02:36:34 drh Exp $
+** $Id: where.c,v 1.332 2008/12/05 15:24:17 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -40,7 +40,10 @@ typedef struct ExprMaskSet ExprMaskSet;
 /*
 ** The query generator uses an array of instances of this structure to
 ** help it analyze the subexpressions of the WHERE clause.  Each WHERE
-** clause subexpression is separated from the others by an AND operator.
+** clause subexpression is separated from the others by AND operators.
+** (Note: the same data structure is also reused to hold a group of terms
+** separated by OR operators.  But at the top-level, everything is AND
+** separated.)
 **
 ** All WhereTerms are collected into a single WhereClause structure.  
 ** The following identity holds:
@@ -69,6 +72,10 @@ typedef struct ExprMaskSet ExprMaskSet;
 ** beginning with 0 in order to make the best possible use of the available
 ** bits in the Bitmask.  So, in the example above, the cursor numbers
 ** would be mapped into integers 0 through 7.
+**
+** The number of terms in a join is limited by the number of bits
+** in prereqRight and prereqAll.  The default is 64 bits, hence SQLite
+** is only able to process joins with 64 or fewer tables.
 */
 typedef struct WhereTerm WhereTerm;
 struct WhereTerm {
@@ -213,11 +220,18 @@ static void whereClauseClear(WhereClause *pWC){
 }
 
 /*
-** Add a new entries to the WhereClause structure.  Increase the allocated
-** space as necessary.
+** Add a single new WhereTerm entry to the WhereClause object pWC.
+** The new WhereTerm object is constructed from Expr p and with wtFlags.
+** The index in pWC->a[] of the new WhereTerm is returned on success.
+** 0 is returned if the new WhereTerm could not be added due to a memory
+** allocation error.  The memory allocation failure will be recorded in
+** the db->mallocFailed flag so that higher-level functions can detect it.
+**
+** This routine will increase the size of the pWC->a[] array as necessary.
 **
 ** If the wtFlags argument includes TERM_DYNAMIC, then responsibility
-** for freeing the expression p is assumed by the WhereClause object.
+** for freeing the expression p is assumed by the WhereClause object pWC.
+** This is true even if this routine fails to allocate a new WhereTerm.
 **
 ** WARNING:  This routine might reallocate the space used to store
 ** WhereTerms.  All pointers to WhereTerms should be invalidated after
@@ -242,10 +256,9 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, int wtFlags){
     if( pOld!=pWC->aStatic ){
       sqlite3DbFree(db, pOld);
     }
-    pWC->nSlot *= 2;
+    pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]);
   }
-  pTerm = &pWC->a[idx = pWC->nTerm];
-  pWC->nTerm++;
+  pTerm = &pWC->a[idx = pWC->nTerm++];
   pTerm->pExpr = p;
   pTerm->wtFlags = wtFlags;
   pTerm->pWC = pWC;
@@ -818,6 +831,7 @@ static void exprAnalyze(
       pNewExpr = sqlite3Expr(db, ops[i], sqlite3ExprDup(db, pExpr->pLeft),
                              sqlite3ExprDup(db, pList->a[i].pExpr), 0);
       idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
+      testcase( idxNew==0 );
       exprAnalyze(pSrc, pWC, idxNew);
       pTerm = &pWC->a[idxTerm];
       pWC->a[idxNew].iParent = idxTerm;
@@ -889,6 +903,7 @@ static void exprAnalyze(
         transferJoinMarkings(pNew, pExpr);
         pNew->pList = pList;
         idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC);
+        testcase( idxNew==0 );
         exprAnalyze(pSrc, pWC, idxNew);
         pTerm = &pWC->a[idxTerm];
         pWC->a[idxNew].iParent = idxTerm;
@@ -941,9 +956,11 @@ or_not_possible:
     }
     pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprDup(db,pLeft), pStr1, 0);
     idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC);
+    testcase( idxNew1==0 );
     exprAnalyze(pSrc, pWC, idxNew1);
     pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprDup(db,pLeft), pStr2, 0);
     idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC);
+    testcase( idxNew2==0 );
     exprAnalyze(pSrc, pWC, idxNew2);
     pTerm = &pWC->a[idxTerm];
     if( isComplete ){
@@ -975,6 +992,7 @@ or_not_possible:
       Expr *pNewExpr;
       pNewExpr = sqlite3Expr(db, TK_MATCH, 0, sqlite3ExprDup(db, pRight), 0);
       idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
+      testcase( idxNew==0 );
       pNewTerm = &pWC->a[idxNew];
       pNewTerm->prereqRight = prereqExpr;
       pNewTerm->leftCursor = pLeft->iTable;