Add a debugging memory allocator. (CVS 4227)

FossilOrigin-Name: 8d2d1c4ff9dca61f75e3048107ee9712d346a28c

main.mk

@@ -58,7 +58,7 @@ TCCX = $(TCC) $(OPTS) $(THREADSAFE) $(USLEEP) -I. -I$(TOP)/src
 LIBOBJ+= alter.o analyze.o attach.o auth.o btree.o build.o \
          callback.o complete.o date.o delete.o \
          expr.o func.o hash.o insert.o loadext.o \
-         main.o malloc.o mem1.o mutex.o \
+         main.o malloc.o mem1.o mem2.o mutex.o \
          opcodes.o os.o os_os2.o os_unix.o os_win.o \
          pager.o parse.o pragma.o prepare.o printf.o random.o \
          select.o table.o tclsqlite.o tokenize.o trigger.o \
@@ -92,7 +92,8 @@ SRC = \
   $(TOP)/src/loadext.c \
   $(TOP)/src/main.c \
   $(TOP)/src/malloc.c \
-  $(TOP)/src/mem.c \
+  $(TOP)/src/mem1.c \
+  $(TOP)/src/mem2.c \
   $(TOP)/src/mutex.c \
   $(TOP)/src/os.c \
   $(TOP)/src/os_os2.c \
@@ -347,6 +348,9 @@ malloc.o:	$(TOP)/src/malloc.c $(HDR)
 mem1.o:	$(TOP)/src/mem1.c $(HDR)
 	$(TCCX) -c $(TOP)/src/mem1.c
 
+mem2.o:	$(TOP)/src/mem2.c $(HDR)
+	$(TCCX) -c $(TOP)/src/mem2.c
+
 mutex.o:	$(TOP)/src/mutex.c $(HDR)
 	$(TCCX) -c $(TOP)/src/mutex.c
 

manifest

@@ -1,5 +1,5 @@
-C Add\sinitial\simplementations\sof\smutex\sand\smemory\ssubsystem\smodules.\s(CVS\s4226)
+C Add\sa\sdebugging\smemory\sallocator.\s(CVS\s4227)
-D 2007-08-15T13:04:54
+D 2007-08-15T17:07:57
 F Makefile.in 0c0e53720f658c7a551046442dd7afba0b72bfbe
 F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -51,7 +51,7 @@ F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 61a345d8126686aa3487aa8d2d0f68abd655f7a4
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387
-F main.mk c9336cf989f5e2076e5fbda909fa57cca24b6515
+F main.mk 4b70d380336187ddfb3e2f24394fad74e6266f92
 F mkdll.sh 37fa8a7412e51b5ab2bc6d4276135f022a0feffb
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 1a866b53637dab137191341cc875575a5ca110fb
@@ -88,7 +88,8 @@ F src/loadext.c 6c24ee62adfe7fbfb2f2dd43ff18e5534b19010f
 F src/main.c f12d230c1226d3f43c1f4595af1c25ccbe3017c7
 F src/malloc.c 3850ab4a2edfb190ffee353c5674ebd8c6b4ccc7
 F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217
-F src/mem1.c f4127f6f6f183b66f7c4345e33c9fe19a43e9942
+F src/mem1.c ef73642a171d174cd556d0168f8edbceaf94933b
+F src/mem2.c b8173ddcca23e99829617185154a35a6f046b214
 F src/mutex.c 667dae0de95f8fb92a3ffc8c3f20c0d26115a1a6
 F src/os.c 1f10b47acc1177fb9225edb4f5f0d25ed716f9cb
 F src/os.h cea2e179bb33f4fc09dbb9fcd51b2246544bd2db
@@ -527,7 +528,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 174116f7c0ceeceb5e32868b29fabf8a6943cbf6
+P c0fa3769790af199a4c8715c80bb8ff900730520
-R 7f615db37ae8f52942fff60e5e86001f
+R afa459b840a3b96de70924b91a0e1dbb
 U drh
-Z 657d3d4bb780276ea6842b10c93c7b7e
+Z 9341f1481e29775e7a036e24546ed5fc

manifest.uuid

@@ -1 +1 @@
-c0fa3769790af199a4c8715c80bb8ff900730520
+8d2d1c4ff9dca61f75e3048107ee9712d346a28c

src/mem1.c

@@ -12,9 +12,16 @@
 ** This file contains the C functions that implement a memory
 ** allocation subsystem for use by SQLite.  
 **
-** $Id: mem1.c,v 1.1 2007/08/15 13:04:54 drh Exp $
+** $Id: mem1.c,v 1.2 2007/08/15 17:07:57 drh Exp $
 */
 
+/*
+** This version of the memory allocator is the default.  It is
+** used when no other memory allocator is specified using compile-time
+** macros.
+*/
+#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)
+
 /*
 ** We will eventually construct multiple memory allocation subsystems
 ** suitable for use in various contexts:
@@ -202,3 +209,5 @@ void *sqlite3_realloc(void *pPrior, unsigned int nBytes){
   sqlite3_mutex_leave(memMutex);
   return (void*)p;
 }
+
+#endif /* !SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */

src/mem2.c

@@ -0,0 +1,366 @@
+/*
+** 2007 August 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains the C functions that implement a memory
+** allocation subsystem for use by SQLite.  
+**
+** $Id: mem2.c,v 1.1 2007/08/15 17:07:57 drh Exp $
+*/
+
+/*
+** This version of the memory allocator is used only if the
+** SQLITE_MEMDEBUG macro is defined and SQLITE_OMIT_MEMORY_ALLOCATION
+** is not defined.
+*/
+#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)
+
+/*
+** We will eventually construct multiple memory allocation subsystems
+** suitable for use in various contexts:
+**
+**    *  Normal multi-threaded builds
+**    *  Normal single-threaded builds
+**    *  Debugging builds
+**
+** This version is suitable for use in debugging builds.
+**
+** Features:
+**
+**    * Every allocate has guards at both ends.
+**    * New allocations are initialized with randomness
+**    * Allocations are overwritten with randomness when freed
+**    * Optional logs of malloc activity generated
+**    * Summary of outstanding allocations with backtraces to the
+**      point of allocation.
+**    * The ability to simulate memory allocation failure
+*/
+#include "sqliteInt.h"
+#include <stdio.h>
+
+/*
+** The backtrace functionality is only available with GLIBC
+*/
+#ifdef __GLIBC__
+  extern int backtrace(void**,int);
+  extern void backtrace_symbols_fd(void*const*,int,int);
+#else
+# define backtrace(A,B) 0
+# define backtrace_symbols_fd(A,B,C)
+#endif
+
+
+/*
+** Mutex to control access to the memory allocation subsystem.
+*/
+static sqlite3_mutex *memMutex = 0;
+
+/*
+** Current allocation and high-water mark.
+*/
+static sqlite3_uint64 nowUsed = 0;
+static sqlite3_uint64 mxUsed = 0;
+
+/*
+** The alarm callback and its arguments.  The memMutex lock will
+** be held while the callback is running.  Recursive calls into
+** the memory subsystem are allowed, but no new callbacks will be
+** issued.  The alarmBusy variable is set to prevent recursive
+** callbacks.
+*/
+static void (*alarmCallback)(void*, sqlite3_uint64, unsigned) = 0;
+static void *alarmArg = 0;
+static sqlite3_uint64 alarmThreshold = (((sqlite3_uint64)1)<<63);
+static int alarmBusy = 0;
+
+
+/*
+** Return the amount of memory currently checked out.
+*/
+sqlite3_uint64 sqlite3_memory_used(void){
+  sqlite3_uint64 n;
+  if( memMutex==0 ){
+    memMutex = sqlite3_mutex_alloc(1);
+  }
+  sqlite3_mutex_enter(memMutex, 1);
+  n = nowUsed;
+  sqlite3_mutex_leave(memMutex);  
+  return n;
+}
+
+/*
+** Return the maximum amount of memory that has ever been
+** checked out since either the beginning of this process
+** or since the most recent reset.
+*/
+sqlite3_uint64 sqlite3_memory_highwater(int resetFlag){
+  sqlite3_uint64 n;
+  if( memMutex==0 ){
+    memMutex = sqlite3_mutex_alloc(1);
+  }
+  sqlite3_mutex_enter(memMutex, 1);
+  n = mxUsed;
+  if( resetFlag ){
+    mxUsed = nowUsed;
+  }
+  sqlite3_mutex_leave(memMutex);  
+  return n;
+}
+
+/*
+** Change the alarm callback
+*/
+int sqlite3_memory_alarm(
+  void(*xCallback)(void *pArg, sqlite3_uint64 used, unsigned int N),
+  void *pArg,
+  sqlite3_uint64 iThreshold
+){
+  if( memMutex==0 ){
+    memMutex = sqlite3_mutex_alloc(1);
+  }
+  sqlite3_mutex_enter(memMutex, 1);
+  alarmCallback = xCallback;
+  alarmArg = pArg;
+  alarmThreshold = iThreshold;
+  sqlite3_mutex_leave(memMutex);
+  return SQLITE_OK;
+}
+
+/*
+** Trigger the alarm 
+*/
+static void sqlite3MemsysAlarm(unsigned nByte){
+  if( alarmCallback==0 || alarmBusy  ) return;
+  alarmBusy = 1;
+  alarmCallback(alarmArg, nowUsed, nByte);
+  alarmBusy = 0;
+}
+
+/*
+** Each memory allocation looks like this:
+**
+**    ----------------------------------------------------------------
+**    |  backtrace pointers |  MemBlockHdr |  allocation |  EndGuard |
+**    ----------------------------------------------------------------
+**
+** The application code sees only a pointer to the allocation.  We have
+** to back up from the allocation pointer to find the MemBlockHdr.  The
+** MemBlockHdr tells us the size of the allocation and the number of
+** backtrace pointers.  There is also a guard word at the end of the
+** MemBlockHdr.
+*/
+struct MemBlockHdr {
+  struct MemBlockHdr *pNext, *pPrev;  /* Linked list of all unfreed memory */
+  unsigned int iSize;                 /* Size of this allocation */
+  unsigned short nBacktrace;          /* Number of backtraces on this alloc */
+  unsigned short nBacktraceSlots;     /* Available backtrace slots */
+  unsigned int iForeGuard;            /* Guard word for sanity */
+};
+
+/*
+** Guard words
+*/
+#define FOREGUARD 0x80F5E153
+#define REARGUARD 0xE4676B53
+
+/*
+** Head and tail of a linked list of all outstanding allocations
+*/
+static struct MemBlockHdr *pFirst = 0;
+static struct MemBlockHdr *pLast = 0;
+
+/*
+** The number of levels of backtrace to save in new allocations.
+*/
+static int backtraceLevels = 0;
+
+/*
+** Given an allocation, find the MemBlockHdr for that allocation.
+**
+** This routine checks the guards at either end of the allocation and
+** if they are incorrect it asserts.
+*/
+static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){
+  struct MemBlockHdr *p;
+  unsigned int *pInt;
+
+  p = (struct MemBlockHdr*)pAllocation;
+  p--;
+  assert( p->iForeGuard==FOREGUARD );
+  assert( (p->iSize & 3)==0 );
+  pInt = (unsigned int*)pAllocation;
+  assert( pInt[p->iSize/sizeof(unsigned int)]==REARGUARD );
+  return p;
+}
+
+/*
+** Allocate nByte of memory
+*/
+void *sqlite3_malloc(unsigned int nByte){
+  struct MemBlockHdr *pHdr;
+  void **pBt;
+  unsigned int *pInt;
+  void *p;
+  unsigned int totalSize;
+
+  if( memMutex==0 ){
+    memMutex = sqlite3_mutex_alloc(1);
+  }
+  sqlite3_mutex_enter(memMutex, 1);
+  if( nowUsed+nByte>=alarmThreshold ){
+    sqlite3MemsysAlarm(nByte);
+  }
+  nByte = (nByte+3)&~3;
+  totalSize = nByte + sizeof(*pHdr) + sizeof(unsigned int) +
+               backtraceLevels*sizeof(void*);
+  p = malloc(totalSize);
+  if( p==0 ){
+    sqlite3MemsysAlarm(nByte);
+    p = malloc(totalSize);
+  }
+  if( p ){
+    pBt = p;
+    pHdr = (struct MemBlockHdr*)&pBt[backtraceLevels];
+    pHdr->pNext = 0;
+    pHdr->pPrev = pLast;
+    if( pLast ){
+      pLast->pNext = pHdr;
+    }else{
+      pFirst = pHdr;
+    }
+    pLast = pHdr;
+    pHdr->iForeGuard = FOREGUARD;
+    pHdr->nBacktraceSlots = backtraceLevels;
+    if( backtraceLevels ){
+      pHdr->nBacktrace = backtrace(pBt, backtraceLevels);
+    }else{
+      pHdr->nBacktrace = 0;
+    }
+    pHdr->iSize = nByte;
+    pInt = (unsigned int *)&pHdr[1];
+    pInt[nByte/sizeof(unsigned int)] = REARGUARD;
+    memset(pInt, 0x65, nByte);
+    nowUsed += nByte;
+    if( nowUsed>mxUsed ){
+      mxUsed = nowUsed;
+    }
+    p = (void*)pInt;
+  }
+  sqlite3_mutex_leave(memMutex);
+  return p; 
+}
+
+/*
+** Free memory.
+*/
+void sqlite3_free(void *pPrior){
+  struct MemBlockHdr *pHdr;
+  void **pBt;
+  if( pPrior==0 ){
+    return;
+  }
+  assert( memMutex!=0 );
+  pHdr = sqlite3MemsysGetHeader(pPrior);
+  pBt = (void**)pHdr;
+  pBt -= pHdr->nBacktraceSlots;
+  sqlite3_mutex_enter(memMutex, 1);
+  nowUsed -= pHdr->iSize;
+  if( pHdr->pPrev ){
+    assert( pHdr->pPrev->pNext==pHdr );
+    pHdr->pPrev->pNext = pHdr->pNext;
+  }else{
+    assert( pFirst==pHdr );
+    pFirst = pHdr->pNext;
+  }
+  if( pHdr->pNext ){
+    assert( pHdr->pNext->pPrev==pHdr );
+    pHdr->pNext->pPrev = pHdr->pPrev;
+  }else{
+    assert( pLast==pHdr );
+    pLast = pHdr->pPrev;
+  }
+  memset(pBt, 0x2b, sizeof(void*)*pHdr->nBacktrace + sizeof(*pHdr) +
+                    pHdr->iSize + sizeof(unsigned int));
+  free(pBt);
+  sqlite3_mutex_leave(memMutex);  
+}
+
+/*
+** Change the size of an existing memory allocation.
+**
+** For this debugging implementation, we *always* make a copy of the
+** allocation into a new place in memory.  In this way, if the 
+** higher level code is using pointer to the old allocation, it is 
+** much more likely to break and we are much more liking to find
+** the error.
+*/
+void *sqlite3_realloc(void *pPrior, unsigned int nByte){
+  struct MemBlockHdr *pOldHdr;
+  void *pNew;
+  unsigned nOld;
+  if( pPrior==0 ){
+    return sqlite3_malloc(nByte);
+  }
+  if( nByte==0 ){
+    sqlite3_free(pPrior);
+    return;
+  }
+  pOldHdr = sqlite3MemsysGetHeader(pPrior);
+  pNew = sqlite3_malloc(nByte);
+  if( pNew ){
+    memcpy(pNew, pPrior, nByte<pOldHdr->iSize ? nByte : pOldHdr->iSize);
+    if( nByte>pOldHdr->iSize ){
+      memset(&((char*)pNew)[pOldHdr->iSize], 0x2b, nByte - pOldHdr->iSize);
+    }
+    sqlite3_free(pPrior);
+  }
+  return pNew;
+}
+
+/*
+** Set the number of backtrace levels kept for each allocation.
+** A value of zero turns of backtracing.  The number is always rounded
+** up to a multiple of 2.
+*/
+void sqlite3_memdebug_backtrace_depth(int depth){
+  if( depth<0 ){ depth = 0; }
+  if( depth>20 ){ depth = 20; }
+  depth = (depth+1)&~1;
+  backtraceLevels = depth;
+}
+
+/*
+** Open the file indicated and write a log of all unfreed memory 
+** allocations into that log.
+*/
+void sqlite3_memdebug_dump(const char *zFilename){
+  FILE *out;
+  struct MemBlockHdr *pHdr;
+  void **pBt;
+  out = fopen(zFilename, "w");
+  if( out==0 ){
+    fprintf(stderr, "** Unable to output memory debug output log: %s **\n",
+                    zFilename);
+    return;
+  }
+  for(pHdr=pFirst; pHdr; pHdr=pHdr->pNext){
+    fprintf(out, "**** %d bytes at %p ****\n", pHdr->iSize, &pHdr[1]);
+    if( pHdr->nBacktrace ){
+      fflush(out);
+      pBt = (void**)pHdr;
+      pBt -= pHdr->nBacktraceSlots;
+      backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out));
+      fprintf(out, "\n");
+    }
+  }
+  fclose(out);
+}
+
+#endif /* SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */