Modifications to reduce memory consumption. (CVS 2422)

FossilOrigin-Name: 0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2

manifest

@@ -1,5 +1,5 @@
-C Fix\ssome\smemory\sleaks\sthat\soccur\safter\sa\smalloc\sfailure.\s(CVS\s2421)
+C Modifications\sto\sreduce\smemory\sconsumption.\s(CVS\s2422)
-D 2005-03-28T03:39:56
+D 2005-03-28T08:44:07
 F Makefile.in 5c00d0037104de2a50ac7647a5f12769795957a3
 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -30,7 +30,7 @@ F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
 F src/alter.c 9570af388bc99471ea6e1258817fbf06e3120030
 F src/attach.c 3615dbe960cbee4aa5ea300b8a213dad36527b0f
 F src/auth.c 18c5a0befe20f3a58a41e3ddd78f372faeeefe1f
-F src/btree.c c33c0e6833eb8ac0f0941c1f8e722f7c70dbef57
+F src/btree.c 657fd61dfb4dd37a63416652b1a28700e84b36f7
 F src/btree.h 41a71ce027db9ddee72cb43df2316bbe3a1d92af
 F src/build.c 2589c2ffa263406526d0cc5728405c6c2f9638f6
 F src/date.c 2134ef4388256e8247405178df8a61bd60dc180a
@@ -52,7 +52,7 @@ F src/os_unix.c fba0167576f09e242afd4c4978e1d2944b1da8b5
 F src/os_unix.h 40b2fd1d02cfa45d6c3dea25316fd019cf9fcb0c
 F src/os_win.c 2bbbe6fbb010763c3fa79d5e951afca9b138c6b5
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c 4c1322dc8458652eb61d23405edd07a7a201160b
+F src/pager.c 221076cde9af787fc45eec1bfebf5f20e4faacf0
 F src/pager.h 9a417a1e04737c227ebbba3bdf8597d6dd51513a
 F src/parse.y 10c0ace9efce31d5a06e03488a4284b9d97abc56
 F src/pragma.c 4b20dbc0f4b97f412dc511853d3d0c2e0d4adedc
@@ -79,7 +79,7 @@ F src/vdbe.c c7973dc0ab52538646018620e3d3c68aa9e6d6c4
 F src/vdbe.h 7f586cb6d6b57764e5aac1f87107d6a95ddce24c
 F src/vdbeInt.h e80721cd8ff611789e20743eec43363a9fb5a48e
 F src/vdbeapi.c 467caa6e6fb9247528b1c7ab9132ae1b4748e8ac
-F src/vdbeaux.c 91013922626fed75ad091459b7f05f9e86581690
+F src/vdbeaux.c 0932f570d276992c7b3ee989589b6ff9056f97e7
 F src/vdbemem.c 4e853ce3151eaf7906150da85a1b3ce1fe5e8da8
 F src/where.c c4b227458e8993decb515ed9a2fe2d4f5f8e3125
 F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42
@@ -221,7 +221,7 @@ F tool/lemon.c c88936c67f6411608db8fa4254d254f509fa40f6
 F tool/lempar.c e8b0eb00a6b905ce2ebd55965ed243574482cd5f
 F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133
 F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8
-F tool/memleak3.tcl b8eb053190e95a55dc188896afb972e8108822d6
+F tool/memleak3.tcl 2b1ab290badf3b26f9ba433baf7fad8def14aea8
 F tool/mkkeywordhash.c 02ac5c523fd6d55364cd70aded5c36ba6651a6bf
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@@ -278,7 +278,7 @@ F www/tclsqlite.tcl e73f8f8e5f20e8277619433f7970060ab01088fc
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b
-P ccb9f4022b3ccb1cc2ab001628fd38becfbf8efe
+P bcb5d72ef146b1019c72220701d385c7b0b5d0bd
-R ceb3b5e5af33f6f582749ad4190a9e4b
+R bb1ad29712ee81032569962fcc3720e5
-U drh
+U danielk1977
-Z feff361f817d89fa1dffb88a9f3296a3
+Z f1d40042415d09b383776e4b14cf65b8

manifest.uuid

@@ -1 +1 @@
-bcb5d72ef146b1019c72220701d385c7b0b5d0bd
+0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.253 2005/03/21 04:04:02 danielk1977 Exp $
+** $Id: btree.c,v 1.254 2005/03/28 08:44:07 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -3757,7 +3757,8 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){
 static int balance_nonroot(MemPage *pPage){
   MemPage *pParent;            /* The parent of pPage */
   Btree *pBt;                  /* The whole database */
-  int nCell = 0;               /* Number of cells in aCell[] */
+  int nCell = 0;               /* Number of cells in apCell[] */
+  int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
   int nOld;                    /* Number of pages in apOld[] */
   int nNew;                    /* Number of pages in apNew[] */
   int nDiv;                    /* Number of cells in apDiv[] */
@@ -3771,7 +3772,6 @@ static int balance_nonroot(MemPage *pPage){
   int pageFlags;               /* Value of pPage->aData[0] */
   int subtotal;                /* Subtotal of bytes in cells on one page */
   int iSpace = 0;              /* First unused byte of aSpace[] */
-  int mxCellPerPage;           /* Maximum number of cells in one page */
   MemPage *apOld[NB];          /* pPage and up to two siblings */
   Pgno pgnoOld[NB];            /* Page numbers for each page in apOld[] */
   MemPage *apCopy[NB];         /* Private copies of apOld[] pages */
@@ -3825,31 +3825,6 @@ static int balance_nonroot(MemPage *pPage){
   }
 #endif
 
-  /*
-  ** Allocate space for memory structures
-  */
-  mxCellPerPage = MX_CELL(pBt);
-  apCell = sqliteMallocRaw( 
-       (mxCellPerPage+2)*NB*(sizeof(u8*)+sizeof(int))
-     + sizeof(MemPage)*NB
-     + pBt->psAligned*(5+NB)
-     + (ISAUTOVACUUM ? (mxCellPerPage+2)*NN*2 : 0)
-  );
-  if( apCell==0 ){
-    return SQLITE_NOMEM;
-  }
-  szCell = (int*)&apCell[(mxCellPerPage+2)*NB];
-  aCopy[0] = (u8*)&szCell[(mxCellPerPage+2)*NB];
-  for(i=1; i<NB; i++){
-    aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
-  }
-  aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
-#ifndef SQLITE_OMIT_AUTOVACUUM
-  if( pBt->autoVacuum ){
-    aFrom = &aSpace[5*pBt->psAligned];
-  }
-#endif
-  
   /*
   ** Find the cell in the parent page whose left child points back
   ** to pPage.  The "idx" variable is the index of that cell.  If pPage
@@ -3910,8 +3885,35 @@ static int balance_nonroot(MemPage *pPage){
     apCopy[i] = 0;
     assert( i==nOld );
     nOld++;
+    nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow;
   }
 
+  /*
+  ** Allocate space for memory structures
+  */
+  apCell = sqliteMallocRaw( 
+       nMaxCells*sizeof(u8*)                           /* apCell */
+     + nMaxCells*sizeof(int)                           /* szCell */
+     + sizeof(MemPage)*NB                              /* aCopy */
+     + pBt->psAligned*(5+NB)                           /* aSpace */
+     + (ISAUTOVACUUM ? nMaxCells : 0)     /* aFrom */
+  );
+  if( apCell==0 ){
+    rc = SQLITE_NOMEM;
+    goto balance_cleanup;
+  }
+  szCell = (int*)&apCell[nMaxCells];
+  aCopy[0] = (u8*)&szCell[nMaxCells];
+  for(i=1; i<NB; i++){
+    aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
+  }
+  aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
+#ifndef SQLITE_OMIT_AUTOVACUUM
+  if( pBt->autoVacuum ){
+    aFrom = &aSpace[5*pBt->psAligned];
+  }
+#endif
+  
   /*
   ** Make copies of the content of pPage and its siblings into aOld[].
   ** The rest of this function will use data from the copies rather
@@ -3948,6 +3950,7 @@ static int balance_nonroot(MemPage *pPage){
     MemPage *pOld = apCopy[i];
     int limit = pOld->nCell+pOld->nOverflow;
     for(j=0; j<limit; j++){
+      assert( nCell<nMaxCells );
       apCell[nCell] = findOverflowCell(pOld, j);
       szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
 #ifndef SQLITE_OMIT_AUTOVACUUM
@@ -3975,6 +3978,7 @@ static int balance_nonroot(MemPage *pPage){
         dropCell(pParent, nxDiv, sz);
       }else{
         u8 *pTemp;
+        assert( nCell<nMaxCells );
         szCell[nCell] = sz;
         pTemp = &aSpace[iSpace];
         iSpace += sz;
@@ -4020,6 +4024,7 @@ static int balance_nonroot(MemPage *pPage){
   */
   usableSpace = pBt->usableSize - 12 + leafCorrection;
   for(subtotal=k=i=0; i<nCell; i++){
+    assert( i<nMaxCells );
     subtotal += szCell[i] + 2;
     if( subtotal > usableSpace ){
       szNew[k] = subtotal - szCell[i];
@@ -4051,6 +4056,8 @@ static int balance_nonroot(MemPage *pPage){
 
     r = cntNew[i-1] - 1;
     d = r + 1 - leafData;
+    assert( d<nMaxCells );
+    assert( r<nMaxCells );
     while( szRight==0 || szRight+szCell[d]+2<=szLeft-(szCell[r]+2) ){
       szRight += szCell[d] + 2;
       szLeft -= szCell[r] + 2;
@@ -4146,6 +4153,7 @@ static int balance_nonroot(MemPage *pPage){
   j = 0;
   for(i=0; i<nNew; i++){
     /* Assemble the new sibling page. */
+    assert( j<nMaxCells );
     MemPage *pNew = apNew[i];
     assert( pNew->pgno==pgnoNew[i] );
     assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
@@ -4160,6 +4168,7 @@ static int balance_nonroot(MemPage *pPage){
     */
     if( pBt->autoVacuum ){
       for(k=j; k<cntNew[i]; k++){
+        assert( k<nMaxCells );
         if( aFrom[k]==0xFF || apCopy[aFrom[k]]->pgno!=pNew->pgno ){
           rc = ptrmapPutOvfl(pNew, k-j);
           if( rc!=SQLITE_OK ){
@@ -4179,6 +4188,8 @@ static int balance_nonroot(MemPage *pPage){
       u8 *pCell;
       u8 *pTemp;
       int sz;
+
+      assert( j<nMaxCells );
       pCell = apCell[j];
       sz = szCell[j] + leafCorrection;
       if( !pNew->leaf ){

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.199 2005/03/28 03:39:56 drh Exp $
+** @(#) $Id: pager.c,v 1.200 2005/03/28 08:44:07 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -213,9 +213,16 @@ struct PgHistory {
 
 /*
 ** How big to make the hash table used for locating in-memory pages
-** by page number.
+** by page number. This macro looks a little silly, but is evaluated
+** at compile-time, not run-time (at least for gcc this is true).
 */
-#define N_PG_HASH 2048
+#define N_PG_HASH (\
+  (MAX_PAGES>1024)?2048: \
+  (MAX_PAGES>512)?1024: \
+  (MAX_PAGES>256)?512: \
+  (MAX_PAGES>128)?256: \
+  (MAX_PAGES>64)?128:64 \
+)
 
 /*
 ** Hash a page number

src/vdbeaux.c

@@ -57,10 +57,16 @@ void sqlite3VdbeTrace(Vdbe *p, FILE *trace){
 
 /*
 ** Resize the Vdbe.aOp array so that it contains at least N
+** elements. If the Vdbe is in VDBE_MAGIC_RUN state, then
+** the Vdbe.aOp array will be sized to contain exactly N 
 ** elements.
 */
 static void resizeOpArray(Vdbe *p, int N){
-  if( p->nOpAlloc<N ){
+  if( p->magic==VDBE_MAGIC_RUN ){
+    assert( N==p->nOp );
+    p->nOpAlloc = N;
+    p->aOp = sqliteRealloc(p->aOp, N*sizeof(Op));
+  }else if( p->nOpAlloc<N ){
     int oldSize = p->nOpAlloc;
     p->nOpAlloc = N+100;
     p->aOp = sqliteRealloc(p->aOp, p->nOpAlloc*sizeof(Op));
@@ -166,19 +172,35 @@ void sqlite3VdbeResolveLabel(Vdbe *p, int x){
 ** value to its correct non-zero value.
 **
 ** This routine is called once after all opcodes have been inserted.
+**
+** Variable *pMaxFuncArgs is set to the maximum value of any P1 argument 
+** to an OP_Function or P2 to an OP_AggFunc opcode. This is used by 
+** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
 */
-static void resolveP2Values(Vdbe *p){
+static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
   int i;
+  int nMax = 0;
   Op *pOp;
   int *aLabel = p->aLabel;
   if( aLabel==0 ) return;
   for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
+    u8 opcode = pOp->opcode;
+
+    /* Todo: Maybe OP_AggFunc should change to use P1 in the same
+     * way as OP_Function. */
+    if( opcode==OP_Function ){
+      if( pOp->p1>nMax ) nMax = pOp->p1;
+    }else if( opcode==OP_AggFunc ){
+      if( pOp->p2>nMax ) nMax = pOp->p2;
+    }
+
     if( pOp->p2>=0 ) continue;
     assert( -1-pOp->p2<p->nLabel );
     pOp->p2 = aLabel[-1-pOp->p2];
   }
   sqliteFree(p->aLabel);
   p->aLabel = 0;
+  *pMaxFuncArgs = nMax;
 }
 
 /*
@@ -603,6 +625,13 @@ void sqlite3VdbeMakeReady(
   */
   assert( p->nOp>0 );
 
+  /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. This
+   * is because the call to resizeOpArray() below may shrink the
+   * p->aOp[] array to save memory if called when in VDBE_MAGIC_RUN 
+   * state.
+   */
+  p->magic = VDBE_MAGIC_RUN;
+
   /* No instruction ever pushes more than a single element onto the
   ** stack.  And the stack never grows on successive executions of the
   ** same loop.  So the total number of instructions is an upper bound
@@ -611,12 +640,14 @@ void sqlite3VdbeMakeReady(
   ** Allocation all the stack space we will ever need.
   */
   if( p->aStack==0 ){
-    resolveP2Values(p);
+    int nArg;       /* Maximum number of args passed to a user function. */
+    resolveP2Values(p, &nArg);
+    resizeOpArray(p, p->nOp);
     assert( nVar>=0 );
     n = isExplain ? 10 : p->nOp;
     p->aStack = sqliteMalloc(
         n*sizeof(p->aStack[0])         /* aStack */
-      + n*sizeof(Mem*)                 /* apArg */
+      + nArg*sizeof(Mem*)              /* apArg */
       + nVar*sizeof(Mem)               /* aVar */
       + nVar*sizeof(char*)             /* azVar */
       + nMem*sizeof(Mem)               /* aMem */
@@ -630,7 +661,7 @@ void sqlite3VdbeMakeReady(
       p->nVar = nVar;
       p->okVar = 0;
       p->apArg = (Mem**)&p->aVar[nVar];
-      p->azVar = (char**)&p->apArg[n];
+      p->azVar = (char**)&p->apArg[nArg];
       p->apCsr = (Cursor**)&p->azVar[nVar];
       if( nAgg>0 ){
         p->nAgg = nAgg;

tool/memleak3.tcl

@@ -25,23 +25,60 @@ If all goes well a summary of unfreed allocations is printed out. If the
 GNU C library is in use and SQLITE_DEBUG is 3 or greater a stack trace is
 printed out for each unmatched allocation.
 
+If the \"-r <n>\" option is passed, then the program stops and prints out
+the state of the heap immediately after the <n>th call to malloc() or
+realloc().
+
 Example:
 
 $ ./testfixture ../sqlite/test/select1.test 2> memtrace.out
-$ tclsh $argv0 ./testfixture memtrace.out
+$ tclsh $argv0 ?-r <malloc-number>? ./testfixture memtrace.out
 "
-
+if { [llength $argv]!=2 && [llength $argv]!=4 } {
-# If stack traces are enabled, the 'addr2line' program is called to
+  set prg [file tail $argv0]
-# translate a binary stack address into a human-readable form.
+  puts "Usage: $prg ?-r <malloc-number>? <binary file> <mem trace file>"
-set addr2line addr2line
-
-if { [llength $argv]!=2 } {
-  puts "Usage: $argv0 <binary file> <mem trace file>"
   puts ""
   puts [string trim $doco]
   exit -1
 }
 
+# If stack traces are enabled, the 'addr2line' program is called to
+# translate a binary stack address into a human-readable form.
+set addr2line addr2line
+
+# When the SQLITE_MEMDEBUG is set as described above, SQLite prints
+# out a line for each malloc(), realloc() or free() call that the
+# library makes. If SQLITE_MEMDEBUG is 3, then a stack trace is printed
+# out before each malloc() and realloc() line.
+#
+# This program parses each line the SQLite library outputs and updates
+# the following global Tcl variables to reflect the "current" state of
+# the heap used by SQLite.
+#
+set nBytes 0               ;# Total number of bytes currently allocated.
+set nMalloc 0              ;# Total number of malloc()/realloc() calls.
+set nPeak 0                ;# Peak of nBytes.
+set iPeak 0                ;# nMalloc when nPeak was set.
+#
+# More detailed state information is stored in the $memmap array. 
+# Each key in the memmap array is the address of a chunk of memory
+# currently allocated from the heap. The value is a list of the 
+# following form
+# 
+#     {<number-of-bytes> <malloc id> <stack trace>}
+#
+array unset memmap
+
+# The executable program being analyzed.
+if {[llength $argv]==2} {
+  set exe [lindex $argv 0]
+  set memfile [lindex $argv 1]
+  set report_at -1
+} else {
+  set exe [lindex $argv 2]
+  set memfile [lindex $argv 3]
+  set report_at [lindex $argv 1]
+}
 
 proc process_input {input_file array_name} {
   upvar $array_name mem 
@@ -68,6 +105,17 @@ proc process_input {input_file array_name} {
       set mem($addr) [list $bytes "malloc $mallocid" $stack]
       set stack ""
 
+      # Increase the current heap usage
+      incr ::nBytes $bytes
+
+      # Increase the number of malloc() calls
+      incr ::nMalloc
+
+      if {$::nBytes > $::nPeak} {
+        set ::nPeak $::nBytes
+        set ::iPeak $::nMalloc
+      }
+
     } elseif { [regexp $FREE $line dummy bytes addr] } {
       # If this is a 'free' line, remove the entry from the mem array. If the 
       # entry does not exist, or is the wrong number of bytes, announce a
@@ -78,31 +126,86 @@ proc process_input {input_file array_name} {
       }
       unset mem($addr) 
 
+      # Decrease the current heap usage
+      incr ::nBytes [expr -1 * $bytes]
+
     } elseif { [regexp $REALLOC $line dummy mallocid ob b oa a] } {
-      # If it is a realloc line, remove the old mem entry and add a new one.
+      # "free" the old allocation in the internal model:
+      incr ::nBytes [expr -1 * $ob]
       unset mem($oa);
+
+      # "malloc" the new allocation
       set mem($a) [list $b "realloc $mallocid" $stack]
+      incr ::nBytes $b
       set stack ""
+
+      # Increase the number of malloc() calls
+      incr ::nMalloc
+
+      if {$::nBytes > $::nPeak} {
+        set ::nPeak $::nBytes
+        set ::iPeak $::nMalloc
+      }
+
     } else {
       # puts "REJECT: $line"
     }
+
+    if {$::nMalloc==$::report_at} report
   }
 
   close $input
 }
 
-process_input [lindex $argv 1] mem
+proc printstack {stack} {
-set exe [lindex $argv 0]
+  set fcount 10
-
+  if {[llength $stack]<10} {
-foreach key [array names mem] {
+    set fcount [llength $stack]
-  set bytes [lindex $mem($key) 0]
+  }
-  set mallocid [lindex $mem($key) 1]
+  foreach frame [lrange $stack 1 $fcount] {
-  set stack [lindex $mem($key) 2]
+    foreach {f l} [split [exec $::addr2line -f --exe=$::exe $frame] \n] {}
-  puts "Leaked $bytes bytes at 0x$key: $mallocid"
-  foreach frame [lrange $stack 1 10] {
-    foreach {f l} [split [exec $addr2line -f --exe=$exe $frame] \n] {}
     puts [format "%-30s %s" $f $l]
   }
   if {[llength $stack]>0 } {puts ""}
 }
 
+proc report {} {
+
+  foreach key [array names ::memmap] {
+    set stack [lindex $::memmap($key) 2]
+    set bytes [lindex $::memmap($key) 0]
+    lappend summarymap($stack) $bytes
+  }
+
+  foreach stack [array names summarymap] {
+    set allocs $summarymap($stack)
+    set sum 0
+    foreach a $allocs {
+      incr sum $a
+    }
+    lappend sorted [list $sum $stack]
+  }
+
+  set sorted [lsort -integer -index 0 $sorted]
+  foreach s $sorted {
+    set sum [lindex $s 0]
+    set stack [lindex $s 1]
+    set allocs $summarymap($stack)
+    puts "$sum bytes in [llength $allocs] chunks ($allocs)"
+    printstack $stack
+  }
+
+  # Print out summary statistics
+  puts "Total allocations            : $::nMalloc"
+  puts "Total outstanding allocations: [array size ::memmap]" 
+  puts "Current heap usage           : $::nBytes bytes"
+  puts "Peak heap usage              : $::nPeak bytes (malloc #$::iPeak)"
+
+  exit
+}
+
+process_input $memfile memmap
+report
+
+
+