Refactor the implementation of the scratch memory allocator. Add the

SQLITE_TESTCTRL_SCRATCHMALLOC interface to facilitate testing.

FossilOrigin-Name: a3475ddfbe4526e6e0b334fd1376ee7c31508b80

manifest

@@ -1,8 +1,8 @@
 -----BEGIN PGP SIGNED MESSAGE-----
 Hash: SHA1
 
-C Remove\sunnecessary\scode\sfrom\smalloc.c.\s\sEnhance\spcache1.c\sso\sthat\sis\stries\nto\sreuse\sexisting\spages,\srather\sthan\screate\snew\spages,\swhen\sSQLite\sis\sunder\nmemory\spressure.\s\s"Memory\spressure"\smeans\sthat\sSQLITE_CONFIG_PAGECACHE\smemory\nis\snearly\sexhausted\sor\ssqlite3_soft_heap_limit()\shas\sbeen\sreached.
+C Refactor\sthe\simplementation\sof\sthe\sscratch\smemory\sallocator.\s\sAdd\sthe\nSQLITE_TESTCTRL_SCRATCHMALLOC\sinterface\sto\sfacilitate\stesting.
-D 2010-08-27T12:21:06
+D 2010-08-27T17:16:45
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 543f91f24cd7fee774ecc0a61c19704c0c3e78fd
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -139,8 +139,8 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
 F src/loadext.c 6d422ea91cf3d2d00408c5a8f2391cd458da85f8
-F src/main.c 13c874909c9e2eeb75fe3c7bf021d52a5888acb1
+F src/main.c bdd23be253843d3f34faa8f5bd3f92ebbf05b94b
-F src/malloc.c e81193f1b5ac8ff744477253f1a04e0ac579c597
+F src/malloc.c f34c9253326fcd2dad0041801992ccf18ddd6ab5
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2
 F src/mem2.c 9e5f72e38573db9598fe60d3fa530d473cc8714e
@@ -174,7 +174,7 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706
 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
 F src/select.c 8add6cab889fc02e1492eda8dba462ccf11f51dd
 F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056
-F src/sqlite.h.in 76e41cea494cc3b2397f02ad702be0bee0559bb6
+F src/sqlite.h.in 60a40abb36376b4cdf6a9ac7604ed5d3ea2d0bf8
 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89
 F src/sqliteInt.h 78ed6bd32777ad06a8b9910d17307aeeae555485
 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44
@@ -522,7 +522,7 @@ F test/malloc_common.tcl f4a04b7a733eb114a3da16eb39035cde2c851220
 F test/manydb.test b3d3bc4c25657e7f68d157f031eb4db7b3df0d3c
 F test/memdb.test 0825155b2290e900264daaaf0334b6dfe69ea498
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
-F test/memsubsys1.test 5cdd6aa68f17dafe38549f3c2e62bae1afbd96fd
+F test/memsubsys1.test ef3d2af85bd55b8136e7fffd3b66e38e7864f32c
 F test/memsubsys2.test 72a731225997ad5e8df89fdbeae9224616b6aecc
 F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d
 F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0
@@ -850,14 +850,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 9616df8c47ababc2d148f0ab6286efa84bd990fb
+P 51049479a8577e03cc353f71f6e13a10c8323d91
-R 0d182eef4d36dd7c8dd068fdaf1d817b
+R 9667b04fa2c6e7e750d5c74a6bbee707
 U drh
-Z 21746e7d1c6ff945c6448c119d837f57
+Z c2de87f687a677d94fa81a950a4c025c
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1.4.6 (GNU/Linux)
 
-iD8DBQFMd623oxKgR168RlERAmqPAJ0ZTg9D4msTmyXdWHI6l76bjutowgCdHnY4
+iD8DBQFMd/MAoxKgR168RlERAvLjAJ9ChwPUiFzrhTLciw7Q4AEhSu2EvACfZWP3
-cuKPRmabeLtMkcKg8J5zzDs=
+sBXwHRcpD05MSHO9+3nFRAQ=
-=/Jkm
+=R7/R
 -----END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-51049479a8577e03cc353f71f6e13a10c8323d91
+a3475ddfbe4526e6e0b334fd1376ee7c31508b80

src/main.c

@@ -2513,6 +2513,22 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 
+    /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree);
+    **
+    ** Pass pFree into sqlite3ScratchFree(). 
+    ** If sz>0 then allocate a scratch buffer into pNew.  
+    */
+    case SQLITE_TESTCTRL_SCRATCHMALLOC: {
+      void *pFree, **ppNew;
+      int sz;
+      sz = va_arg(ap, int);
+      ppNew = va_arg(ap, void**);
+      pFree = va_arg(ap, void*);
+      if( sz ) *ppNew = sqlite3ScratchMalloc(sz);
+      sqlite3ScratchFree(pFree);
+      break;
+    }
+
   }
   va_end(ap);
 #endif /* SQLITE_OMIT_BUILTIN_TEST */

src/malloc.c

@@ -71,14 +71,18 @@ int sqlite3_release_memory(int n){
 #endif
 }
 
+/*
+** An instance of the following object records the location of
+** each unused scratch buffer.
+*/
+typedef struct ScratchFreeslot {
+  struct ScratchFreeslot *pNext;   /* Next unused scratch buffer */
+} ScratchFreeslot;
+
 /*
 ** State information local to the memory allocation subsystem.
 */
 static SQLITE_WSD struct Mem0Global {
-  /* Number of free pages for scratch and page-cache memory */
-  u32 nScratchFree;
-  u32 nPageFree;
-
   sqlite3_mutex *mutex;         /* Mutex to serialize access */
 
   /*
@@ -92,18 +96,21 @@ static SQLITE_WSD struct Mem0Global {
   void *alarmArg;
 
   /*
-  ** Pointers to the end of sqlite3GlobalConfig.pScratch and
+  ** Pointers to the end of sqlite3GlobalConfig.pScratch memory
-  ** sqlite3GlobalConfig.pPage to a block of memory that records
+  ** (so that a range test can be used to determine if an allocation
-  ** which pages are available.
+  ** being freed came from pScratch) and a pointer to the list of
+  ** unused scratch allocations.
   */
-  u32 *aScratchFree;
+  void *pScratchEnd;
+  ScratchFreeslot *pScratchFree;
+  u32 nScratchFree;
 
   /*
   ** True if heap is nearly "full" where "full" is defined by the
   ** sqlite3_soft_heap_limit() setting.
   */
   int nearlyFull;
-} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 };
+} mem0 = { 0, 0, 0, 0, 0, 0, 0 };
 
 #define mem0 GLOBAL(struct Mem0Global, mem0)
 
@@ -120,15 +127,25 @@ int sqlite3MallocInit(void){
   }
   if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100
       && sqlite3GlobalConfig.nScratch>=0 ){
-    int i;
+    int i, n, sz;
-    sqlite3GlobalConfig.szScratch = ROUNDDOWN8(sqlite3GlobalConfig.szScratch-4);
+    ScratchFreeslot *pSlot;
-    mem0.aScratchFree = (u32*)&((char*)sqlite3GlobalConfig.pScratch)
+    sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch);
-                  [sqlite3GlobalConfig.szScratch*sqlite3GlobalConfig.nScratch];
+    sqlite3GlobalConfig.szScratch = sz;
-    for(i=0; i<sqlite3GlobalConfig.nScratch; i++){ mem0.aScratchFree[i] = i; }
+    pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch;
-    mem0.nScratchFree = sqlite3GlobalConfig.nScratch;
+    n = sqlite3GlobalConfig.nScratch;
+    mem0.pScratchFree = pSlot;
+    mem0.nScratchFree = n;
+    for(i=0; i<n-1; i++){
+      pSlot->pNext = (ScratchFreeslot*)(sz+(char*)pSlot);
+      pSlot = pSlot->pNext;
+    }
+    pSlot->pNext = 0;
+    mem0.pScratchEnd = (void*)&pSlot[1];
   }else{
+    mem0.pScratchEnd = 0;
     sqlite3GlobalConfig.pScratch = 0;
     sqlite3GlobalConfig.szScratch = 0;
+    sqlite3GlobalConfig.nScratch = 0;
   }
   if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512
       || sqlite3GlobalConfig.nPage<1 ){
@@ -327,59 +344,61 @@ void *sqlite3ScratchMalloc(int n){
   void *p;
   assert( n>0 );
 
+  sqlite3_mutex_enter(mem0.mutex);
+  if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){
+    p = mem0.pScratchFree;
+    mem0.pScratchFree = mem0.pScratchFree->pNext;
+    mem0.nScratchFree--;
+    sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1);
+    sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n);
+  }else{
+    if( sqlite3GlobalConfig.bMemstat ){
+      sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n);
+      n = mallocWithAlarm(n, &p);
+      if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n);
+    }else{
+      p = sqlite3GlobalConfig.m.xMalloc(n);
+    }
+    sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH);
+  }
+  sqlite3_mutex_leave(mem0.mutex);
+
+#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
+  /* Verify that no more than two scratch allocations per thread
+  ** are outstanding at one time.  (This is only checked in the
+  ** single-threaded case since checking in the multi-threaded case
+  ** would be much more complicated.) */
+  assert( scratchAllocOut<=1 );
+  if( p ) scratchAllocOut++;
+#endif
+
+  return p;
+}
+void sqlite3ScratchFree(void *p){
+  if( p ){
+
 #if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
     /* Verify that no more than two scratch allocation per thread
     ** is outstanding at one time.  (This is only checked in the
     ** single-threaded case since checking in the multi-threaded case
     ** would be much more complicated.) */
-  assert( scratchAllocOut<=1 );
+    assert( scratchAllocOut>=1 && scratchAllocOut<=2 );
+    scratchAllocOut--;
 #endif
 
-  if( sqlite3GlobalConfig.szScratch<n ){
+    if( p>=sqlite3GlobalConfig.pScratch && p<mem0.pScratchEnd ){
-    goto scratch_overflow;
+      /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */
-  }else{  
+      ScratchFreeslot *pSlot;
+      pSlot = (ScratchFreeslot*)p;
       sqlite3_mutex_enter(mem0.mutex);
-    if( mem0.nScratchFree==0 ){
+      pSlot->pNext = mem0.pScratchFree;
-      sqlite3_mutex_leave(mem0.mutex);
+      mem0.pScratchFree = pSlot;
-      goto scratch_overflow;
+      mem0.nScratchFree++;
-    }else{
+      assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch );
-      int i;
+      sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1);
-      i = mem0.aScratchFree[--mem0.nScratchFree];
-      i *= sqlite3GlobalConfig.szScratch;
-      sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1);
-      sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n);
-      sqlite3_mutex_leave(mem0.mutex);
-      p = (void*)&((char*)sqlite3GlobalConfig.pScratch)[i];
-      assert(  (((u8*)p - (u8*)0) & 7)==0 );
-    }
-  }
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-  scratchAllocOut = p!=0;
-#endif
-
-  return p;
-
-scratch_overflow:
-  if( sqlite3GlobalConfig.bMemstat ){
-    sqlite3_mutex_enter(mem0.mutex);
-    sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n);
-    n = mallocWithAlarm(n, &p);
-    if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n);
       sqlite3_mutex_leave(mem0.mutex);
     }else{
-    p = sqlite3GlobalConfig.m.xMalloc(n);
+      /* Release memory back to the heap */
-  }
-  sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH);
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-  scratchAllocOut = p!=0;
-#endif
-  return p;    
-}
-void sqlite3ScratchFree(void *p){
-  if( p ){
-    if( sqlite3GlobalConfig.pScratch==0
-           || p<sqlite3GlobalConfig.pScratch
-           || p>=(void*)mem0.aScratchFree ){
       assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) );
       assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) );
       sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
@@ -394,26 +413,6 @@ void sqlite3ScratchFree(void *p){
       }else{
         sqlite3GlobalConfig.m.xFree(p);
       }
-    }else{
-      int i;
-      i = (int)((u8*)p - (u8*)sqlite3GlobalConfig.pScratch);
-      i /= sqlite3GlobalConfig.szScratch;
-      assert( i>=0 && i<sqlite3GlobalConfig.nScratch );
-      sqlite3_mutex_enter(mem0.mutex);
-      assert( mem0.nScratchFree<(u32)sqlite3GlobalConfig.nScratch );
-      mem0.aScratchFree[mem0.nScratchFree++] = i;
-      sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1);
-      sqlite3_mutex_leave(mem0.mutex);
-
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-    /* Verify that no more than two scratch allocation per thread
-    ** is outstanding at one time.  (This is only checked in the
-    ** single-threaded case since checking in the multi-threaded case
-    ** would be much more complicated.) */
-    assert( scratchAllocOut>=1 && scratchAllocOut<=2 );
-    scratchAllocOut = 0;
-#endif
-
     }
   }
 }

src/sqlite.h.in

@@ -1257,15 +1257,14 @@ struct sqlite3_mem_methods {
 ** aligned memory buffer from which the scrach allocations will be
 ** drawn, the size of each scratch allocation (sz),
 ** and the maximum number of scratch allocations (N).  The sz
-** argument must be a multiple of 16. The sz parameter should be a few bytes
+** argument must be a multiple of 16.
-** larger than the actual scratch space required due to internal overhead.
 ** The first argument must be a pointer to an 8-byte aligned buffer
 ** of at least sz*N bytes of memory.
-** ^SQLite will use no more than one scratch buffer per thread.  So
+** ^SQLite will use no more than two scratch buffers per thread.  So
-** N should be set to the expected maximum number of threads.  ^SQLite will
+** N should be set to twice the expected maximum number of threads.
-** never require a scratch buffer that is more than 6 times the database
+** ^SQLite will never require a scratch buffer that is more than 6
-** page size. ^If SQLite needs needs additional scratch memory beyond 
+** times the database page size. ^If SQLite needs needs additional
-** what is provided by this configuration option, then 
+** scratch memory beyond what is provided by this configuration option, then 
 ** [sqlite3_malloc()] will be used to obtain the memory needed.</dd>
 **
 ** <dt>SQLITE_CONFIG_PAGECACHE</dt>
@@ -1285,8 +1284,7 @@ struct sqlite3_mem_methods {
 ** memory needs for the first N pages that it adds to cache.  ^If additional
 ** page cache memory is needed beyond what is provided by this option, then
 ** SQLite goes to [sqlite3_malloc()] for the additional storage space.
-** ^The implementation might use one or more of the N buffers to hold 
+** The pointer in the first argument must
-** memory accounting information. The pointer in the first argument must
 ** be aligned to an 8-byte boundary or subsequent behavior of SQLite
 ** will be undefined.</dd>
 **
@@ -5100,7 +5098,8 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16
 #define SQLITE_TESTCTRL_PGHDRSZ                 17
-#define SQLITE_TESTCTRL_LAST                    17
+#define SQLITE_TESTCTRL_SCRATCHMALLOC           18
+#define SQLITE_TESTCTRL_LAST                    18
 
 /*
 ** CAPI3REF: SQLite Runtime Status

test/memsubsys1.test

@@ -11,7 +11,6 @@
 #
 # This file contains tests of the memory allocation subsystem
 #
-# $Id: memsubsys1.test,v 1.17 2009/07/18 14:36:24 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl