Updates to mem6.c allocator. (CVS 5473)

FossilOrigin-Name: 43a4cae2acea33d1a17c0037516e9c27fb7e8e91

manifest

@@ -1,5 +1,5 @@
-C Reduce\sthe\ssize\sof\sthe\sparser\sallocation.\s\sAdd\sadditional\sinstrumentation\nto\smem2.\s\sspeed1*.test\suses\sscratch\smalloc.\s(CVS\s5472)
+C Updates\sto\smem6.c\sallocator.\s(CVS\s5473)
-D 2008-07-24T23:34:07
+D 2008-07-25T08:49:00
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 77ff156061bb870aa0a8b3d545c670d08070f7e6
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -115,15 +115,15 @@ F src/insert.c e8efc17d037346e4a4a6949e72aab850befe3d5d
 F src/journal.c cffd2cd214e58c0e99c3ff632b3bee6c7cbb260e
 F src/legacy.c 3626c71fb70912abec9a4312beba753a9ce800df
 F src/loadext.c ae0eed9fa96d74172d2a90ee63b5bc36d284295c
-F src/main.c 6aeae6ec84ad3e023eaa619651079d45900959bc
+F src/main.c 97b638ed3bab4b804439209e4a20fc8017fb859f
 F src/malloc.c c4b525896b8c188dab98609180d13dbeeeb33a84
 F src/md5.c 008216bbb5d34c6fbab5357aa68575ad8a31516a
-F src/mem1.c 08f95af5095ba5d1fae17deb0ea186e8cdefd8fa
+F src/mem1.c 3a7fe31d8290baa3bb203af72f7dfd6323966bcd
-F src/mem2.c 01726714af78c144b5f1f76338bd094deaab1789
+F src/mem2.c 87381b143530cc377592e868bd548e881c2498a3
 F src/mem3.c c73e935d0b900abc51d5fa45f880937b062f4a9f
 F src/mem4.c 6703adb1717b26d9d70a1c2586b4b7b7ffee7909
 F src/mem5.c 0b0ba1c2a02d86eb812dea6debacee841e3856f7
-F src/mem6.c af63943a8311ad87d12d9b4d4908f87b3f956773
+F src/mem6.c 1edd29bdd837682c54d0836fa426abfb8f838d67
 F src/mutex.c a485a0eac8ee2cd95f66e565b4c6696c18db968f
 F src/mutex.h e52ffa1dfc6a6077e8b1823d2c2b7dfcbcf85594
 F src/mutex_os2.c 9c5637aa4c307c552566d0f0b3bd206245b54a97
@@ -144,9 +144,9 @@ F src/printf.c 2174222bc346a11b1eac2a654ccc4f635355ae7e
 F src/random.c 5c754319d38abdd6acd74601ee0105504adc508a
 F src/select.c a152b1436d7117e25ce010453c61d1002214e337
 F src/shell.c 4b835fe734304ac22a3385868cd3790c1e4f7aa1
-F src/sqlite.h.in 30a57188b126a001dbc28955885fb9698a3b02e9
+F src/sqlite.h.in db24c33101337693d5930da8d31b0ca21e266447
 F src/sqlite3ext.h 1e3887c9bd3ae66cb599e922824b04cd0d0f2c3e
-F src/sqliteInt.h 424d3e4d843c4c95258dd38a920d8448831ec741
+F src/sqliteInt.h 17ada799d10670e20eb8549b848ca622b2e8bc26
 F src/sqliteLimit.h f435e728c6b620ef7312814d660a81f9356eb5c8
 F src/status.c 2076bc996618b1a112a797f05c1d62026ae01402
 F src/table.c 22744786199c9195720c15a7a42cb97b2e2728d8
@@ -168,7 +168,7 @@ F src/test_devsym.c 6012cb8e3acf812513511025a4fa5d626e0ba19b
 F src/test_func.c 24a556989685495013e08f311ae31c4ef86ddb8c
 F src/test_hexio.c 2f1122aa3f012fa0142ee3c36ce5c902a70cd12f
 F src/test_loadext.c df8ab3a6481ddebbdf0d28ebac5d9e0790f7860f
-F src/test_malloc.c e898fc34dc96286e87a23a60579c15db7967d23a
+F src/test_malloc.c cd90645ea37a5199b4cb3168223f73a29160cca9
 F src/test_md5.c 28209a4e2068711b5443c33104fe41f21d160071
 F src/test_mutex.c d3422d9f60cc1330249d102e74b333f0d24a0cb6
 F src/test_onefile.c 243157b10275251c5dc2d6619aee2ff9ae22379c
@@ -436,7 +436,7 @@ F test/pager2.test c025f91b75fe65e85febda64d9416428b8a5cab5
 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4
 F test/pageropt.test 6df72c441db0a037b7ec6990d16311c24fbea77b
 F test/pagesize.test e0a8b3fe80f8b8e808d94a00734c7a18c76c407e
-F test/permutations.test ea696b305916bb7a0cccb612b5585738746b0c32
+F test/permutations.test 30f4bcd3937ce8dcfaac68d1070fd94637b50c2d
 F test/pragma.test 2c675ed9a288094ed62bf55b35fbc749e25670fb
 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
 F test/printf.test 262a5acd3158f788e9bdf7f18d718f3af32ff6ef
@@ -597,7 +597,7 @@ F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8
 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf
 F tool/mkkeywordhash.c ef93810fc41fb3d3dbacf9a33a29be88ea99ffa9
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
-F tool/mksqlite3c.tcl d7fea82624d6f7c4bfee3be558038d7f27913189
+F tool/mksqlite3c.tcl dbc5c9a6752e729f09bf4422a04fac4b84ee61e4
 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87
 F tool/omittest.tcl e29ddad7f546c5aad0198dca389066c3bda53f2d
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@@ -612,7 +612,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 1dbced29de5f59ba2ebf877edcadf171540374d1
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P cb1876d8dc102be74be98dd57ac14ee67be8e8e2
+P 599a9dea8fc97d0e7f09e67c9954de8cc1b8748e
-R 71cb87040ca7c102d12bf7d55288c874
+R ce0b42684f54a70ca69065b71071d9d4
-U drh
+U danielk1977
-Z 3828cdd4b6a6e072ef07f4672692577c
+Z 80afc91241769621e2af3603082ce4f6

manifest.uuid

@@ -1 +1 @@
-599a9dea8fc97d0e7f09e67c9954de8cc1b8748e
+43a4cae2acea33d1a17c0037516e9c27fb7e8e91

src/main.c

@@ -14,7 +14,7 @@
 ** other files are for internal use by SQLite and should not be
 ** accessed by users of the library.
 **
-** $Id: main.c,v 1.481 2008/07/24 08:20:40 danielk1977 Exp $
+** $Id: main.c,v 1.482 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -269,6 +269,7 @@ int sqlite3_config(int op, ...){
 
 #if defined(SQLITE_ENABLE_MEMSYS6)
     case SQLITE_CONFIG_CHUNKALLOC: {
+      sqlite3Config.nSmall = va_arg(ap, int);
       sqlite3Config.m = *sqlite3MemGetMemsys6();
       break;
     }

src/mem1.c

@@ -17,7 +17,7 @@
 ** This file contains implementations of the low-level memory allocation
 ** routines specified in the sqlite3_mem_methods object.
 **
-** $Id: mem1.c,v 1.24 2008/07/24 08:20:40 danielk1977 Exp $
+** $Id: mem1.c,v 1.25 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 
@@ -120,7 +120,7 @@ static void sqlite3MemShutdown(void *NotUsed){
   return;
 }
 
-sqlite3_mem_methods *sqlite3MemGetDefault(void){
+const sqlite3_mem_methods *sqlite3MemGetDefault(void){
   static const sqlite3_mem_methods defaultMethods = {
      sqlite3MemMalloc,
      sqlite3MemFree,

src/mem2.c

@@ -19,7 +19,7 @@
 ** This file contains implementations of the low-level memory allocation
 ** routines specified in the sqlite3_mem_methods object.
 **
-** $Id: mem2.c,v 1.36 2008/07/24 23:34:07 drh Exp $
+** $Id: mem2.c,v 1.37 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 
@@ -324,7 +324,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){
 }
 
 
-sqlite3_mem_methods *sqlite3MemGetDefault(void){
+const sqlite3_mem_methods *sqlite3MemGetDefault(void){
   static const sqlite3_mem_methods defaultMethods = {
      sqlite3MemMalloc,
      sqlite3MemFree,

src/mem6.c

@@ -10,19 +10,52 @@
 **
 *************************************************************************
 **
-** $Id: mem6.c,v 1.2 2008/07/24 10:11:28 danielk1977 Exp $
+** This file contains an alternative memory allocation system for SQLite.
+** This system is implemented as a wrapper around the default memory
+** allocation system (usually the one found in mem1.c - system malloc).
+**
+** This system differentiates between requests for "small" allocations 
+** (by default those of 128 bytes or less) and "large" allocations (all
+** others). The 256 byte threshhold is configurable at runtime.
+**
+** All requests for large allocations are passed through to the
+** default memory allocation system.
+**
+** Requests for small allocations are met by allocating space within
+** one or more larger "chunks" of memory obtained from the default
+** memory allocation system. Chunks of memory are usually 64KB or 
+** larger. The algorithm used to manage space within each chunk is
+** the same as that used by mem5.c. 
+**
+** This strategy is designed to prevent the default memory allocation
+** system (usually the system malloc) from suffering from heap 
+** fragmentation. On some systems, heap fragmentation can cause a 
+** significant real-time slowdown.
+**
+** $Id: mem6.c,v 1.3 2008/07/25 08:49:00 danielk1977 Exp $
 */
 
 #ifdef SQLITE_ENABLE_MEMSYS6
 
+#include "sqliteInt.h"
+
 /*
-** Maximum size of any allocation is ((1<<LOGMAX)*Mem6Chunk.nAtom). Since
+** Maximum size of any "small" allocation is ((1<<LOGMAX)*Mem6Chunk.nAtom).
-** Mem6Chunk.nAtom is always at least 8, this is not really a practical
+** Mem6Chunk.nAtom is always at least 8, so this is not a practical
-** limitation.
+** limitation
 */
 #define LOGMAX 30
 
-#include "sqliteInt.h"
+/*
+** Default value for the "small" allocation size threshold.
+*/
+#define SMALL_MALLOC_DEFAULT_THRESHOLD 256
+
+/*
+** Minimum size for a memory chunk.
+*/
+#define MIN_CHUNKSIZE (1<<16)
+
 
 typedef struct Mem6Chunk Mem6Chunk;
 typedef struct Mem6Link Mem6Link;
@@ -278,12 +311,9 @@ static Mem6Chunk *chunkInit(u8 *zChunk, int nChunk, int nMinAlloc){
 
 struct Mem6Global {
   sqlite3_mem_methods parent;     /* Used to allocate chunks */
-  int nChunkSize;                 /* Size of each chunk, in bytes. */
   int nMinAlloc;                  /* Minimum allowed allocation size */
-
+  int nThreshold;                 /* Allocs larger than this go to parent */
   sqlite3_mutex *mutex;
-
-  /* This data structure will be fixed... */
   Mem6Chunk *pChunk;              /* Singly linked list of all memory chunks */
 } mem6;
 
@@ -296,6 +326,22 @@ static void mem6Leave(void){
   sqlite3_mutex_leave(mem6.mutex);
 }
 
+/*
+** Based on the number and size of the currently allocated chunks, return
+** the size of the next chunk to allocate, in bytes.
+*/
+static int nextChunkSize(void){
+  int iTotal = 0;
+  Mem6Chunk *p;
+  for(p=mem6.pChunk; p; p=p->pNext){
+    iTotal += mem6.parent.xSize((void *)p);
+  }
+  if( iTotal==0 ){
+    iTotal = MIN_CHUNKSIZE;
+  }
+  return iTotal;
+}
+
 /*
 ** The argument is a pointer that may or may not have been allocated from
 ** one of the Mem6Chunk objects managed within mem6. If it is, return
@@ -323,7 +369,7 @@ static void *memsys6Malloc(int nByte){
   void *p = 0;
 
   mem6Enter();
-  if( nByte>=mem6.nChunkSize/3 ){
+  if( nByte>mem6.nThreshold ){
     p = mem6.parent.xMalloc(nByte);
   }else{
     for(pChunk=mem6.pChunk; !p && pChunk; pChunk=pChunk->pNext){
@@ -331,9 +377,10 @@ static void *memsys6Malloc(int nByte){
     }
   
     if( !p ){
-      p = mem6.parent.xMalloc(mem6.nChunkSize);
+      int iSize = nextChunkSize();
+      p = mem6.parent.xMalloc(iSize);
       if( p ){
-        pChunk = chunkInit((u8 *)p, mem6.nChunkSize, mem6.nMinAlloc);
+        pChunk = chunkInit((u8 *)p, iSize, mem6.nMinAlloc);
         pChunk->pNext = mem6.pChunk;
         mem6.pChunk = pChunk;
         p = chunkMalloc(pChunk, nByte);
@@ -398,10 +445,12 @@ static int memsys6Roundup(int n){
 static int memsys6Init(void *pCtx){
   u8 bMemstat = sqlite3Config.bMemstat;
   mem6.parent = *sqlite3MemGetDefault();
-  mem6.nChunkSize = (1<<16);
   mem6.nMinAlloc = 16;
   mem6.pChunk = 0;
-
+  mem6.nThreshold = sqlite3Config.nSmall;
+  if( mem6.nThreshold<=0 ){
+    mem6.nThreshold = SMALL_MALLOC_DEFAULT_THRESHOLD;
+  }
   if( !bMemstat ){
     mem6.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM);
   }
@@ -419,6 +468,10 @@ static int memsys6Init(void *pCtx){
 }
 
 static void memsys6Shutdown(void *pCtx){
+  if( mem6.parent.xShutdown ){
+    mem6.parent.xShutdown(mem6.parent.pAppData);
+  }
+  memset(&mem6, 0, sizeof(mem6));
 }
 
 /*

src/sqlite.h.in

@@ -30,7 +30,7 @@
 ** the version number) and changes its name to "sqlite3.h" as
 ** part of the build process.
 **
-** @(#) $Id: sqlite.h.in,v 1.378 2008/07/24 08:20:40 danielk1977 Exp $
+** @(#) $Id: sqlite.h.in,v 1.379 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #ifndef _SQLITE3_H_
 #define _SQLITE3_H_
@@ -1157,8 +1157,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_CONFIG_MEMSTATUS     9  /* boolean */
 #define SQLITE_CONFIG_MUTEX        10  /* sqlite3_mutex_methods* */
 #define SQLITE_CONFIG_GETMUTEX     11  /* sqlite3_mutex_methods* */
-
+#define SQLITE_CONFIG_CHUNKALLOC   12  /* int threshold */
-#define SQLITE_CONFIG_CHUNKALLOC   12  /* nil */
 
 
 /*

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.744 2008/07/24 08:20:40 danielk1977 Exp $
+** @(#) $Id: sqliteInt.h,v 1.745 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -1789,6 +1789,7 @@ struct Sqlite3Config {
   int isInit;                       /* True after initialization has finished */
   int isMallocInit;                 /* True after malloc is initialized */
   sqlite3_mutex *pInitMutex;        /* Mutex used by sqlite3_initialize() */
+  int nSmall;                       /* alloc size threshold used by mem6.c */
 };
 
 /*
@@ -1841,7 +1842,7 @@ void sqlite3ScratchFree(void*);
 void *sqlite3PageMalloc(int);
 void sqlite3PageFree(void*);
 void sqlite3MemSetDefault(void);
-sqlite3_mem_methods *sqlite3MemGetDefault(void);
+const sqlite3_mem_methods *sqlite3MemGetDefault(void);
 const sqlite3_mem_methods *sqlite3MemGetMemsys5(void);
 const sqlite3_mem_methods *sqlite3MemGetMemsys3(void);
 const sqlite3_mem_methods *sqlite3MemGetMemsys6(void);

src/test_malloc.c

@@ -13,7 +13,7 @@
 ** This file contains code used to implement test interfaces to the
 ** memory allocation subsystem.
 **
-** $Id: test_malloc.c,v 1.39 2008/07/24 08:20:40 danielk1977 Exp $
+** $Id: test_malloc.c,v 1.40 2008/07/25 08:49:00 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -967,11 +967,13 @@ static int test_config_chunkalloc(
   Tcl_Obj *CONST objv[]
 ){
   int rc;
-  if( objc!=1 ){
+  int nThreshold;
-    Tcl_WrongNumArgs(interp, 1, objv, "");
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "THRESHOLD");
     return TCL_ERROR;
   }
-  rc = sqlite3_config(SQLITE_CONFIG_CHUNKALLOC);
+  if( Tcl_GetIntFromObj(interp, objv[1], &nThreshold) ) return TCL_ERROR;
+  rc = sqlite3_config(SQLITE_CONFIG_CHUNKALLOC, nThreshold);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return TCL_OK;
 }

test/permutations.test

@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# $Id: permutations.test,v 1.16 2008/07/24 10:32:31 danielk1977 Exp $
+# $Id: permutations.test,v 1.17 2008/07/25 08:49:01 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -447,7 +447,7 @@ run_tests "memsys6" -description {
   catch {db close}
   sqlite3_reset_auto_extension
   sqlite3_shutdown
-  sqlite3_config_chunkalloc
+  sqlite3_config_chunkalloc 0
   install_malloc_faultsim 1 
   sqlite3_initialize
   autoinstall_test_functions

tool/mksqlite3c.tcl

@@ -213,6 +213,7 @@ foreach file {
    mem2.c
    mem3.c
    mem5.c
+   mem6.c
    mutex.c
    mutex_os2.c
    mutex_unix.c