diff --git a/Makefile.in b/Makefile.in index f279f41d42..be96c415c0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -123,7 +123,7 @@ TCC += -DSQLITE_OMIT_LOAD_EXTENSION=1 LIBOBJ = alter.lo analyze.lo attach.lo auth.lo btmutex.lo btree.lo build.lo \ callback.lo complete.lo date.lo \ delete.lo expr.lo func.lo hash.lo journal.lo insert.lo loadext.lo \ - main.lo malloc.lo mem1.lo mem2.lo mutex.lo \ + main.lo malloc.lo mem1.lo mem2.lo mem3.lo mutex.lo \ mutex_os2.lo mutex_unix.lo mutex_w32.lo \ opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \ pager.lo parse.lo pragma.lo prepare.lo printf.lo random.lo \ @@ -159,6 +159,7 @@ SRC = \ $(TOP)/src/malloc.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ + $(TOP)/src/mem3.c \ $(TOP)/src/mutex.c \ $(TOP)/src/mutex_os2.c \ $(TOP)/src/mutex_unix.c \ @@ -411,6 +412,9 @@ mem1.lo: $(TOP)/src/mem1.c $(HDR) mem2.lo: $(TOP)/src/mem2.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem2.c +mem3.lo: $(TOP)/src/mem3.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem3.c + mutex.lo: $(TOP)/src/mutex.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex.c diff --git a/main.mk b/main.mk index 606e2c2b0e..4325f5a483 100644 --- a/main.mk +++ b/main.mk @@ -51,7 +51,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src LIBOBJ+= alter.o analyze.o attach.o auth.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o \ expr.o func.o hash.o insert.o journal.o loadext.o \ - main.o malloc.o mem1.o mem2.o mutex.o mutex_os2.o \ + main.o malloc.o mem1.o mem2.o mem3.o mutex.o mutex_os2.o \ mutex_unix.o mutex_w32.o \ opcodes.o os.o os_os2.o os_unix.o os_win.o \ pager.o parse.o pragma.o prepare.o printf.o random.o \ @@ -106,6 +106,7 @@ SRC = \ $(TOP)/src/malloc.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ + $(TOP)/src/mem3.c \ $(TOP)/src/mutex.c \ $(TOP)/src/mutex.h \ $(TOP)/src/mutex_os2.c \ diff --git a/manifest b/manifest index c8d3ccbf3a..75c69d630d 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Reorder\ssome\stests\sat\sthe\sbeginning\sof\ssqlite3_step()\sto\swork\saround\nmisuse\sby\spython.\s\sTicket\s#2732.\s(CVS\s4492) -D 2007-10-17T01:44:21 -F Makefile.in 75b729d562e9525d57d9890ec598b38e1a8b02bc +C Added\san\sexperimental\smalloc-free\smemory\sallocation\ssubsystem,\sintended\nfor\suse\son\sembedded\ssystems.\s\sRuns\s7%\sfaster\sthan\swhen\susing\ssystem\nmalloc()\son\sLinux.\s(CVS\s4493) +D 2007-10-19T17:47:25 +F Makefile.in 30c7e3ba426ddb253b8ef037d1873425da6009a8 F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 F VERSION 873b19f713b5e7481dd12648f2fa3178058dee00 @@ -63,7 +63,7 @@ F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/icu.c 61a345d8126686aa3487aa8d2d0f68abd655f7a4 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387 -F main.mk b260a68e26e9553afe8568bbf5554443858e7c49 +F main.mk 53f2973d5a5bf78bedabf6a881ea594ee4730975 F mkdll.sh 37fa8a7412e51b5ab2bc6d4276135f022a0feffb F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 1a866b53637dab137191341cc875575a5ca110fb @@ -102,8 +102,9 @@ F src/loadext.c 124e566563d1c03e68e1396cb44df9870612c6e9 F src/main.c 994a6b6914d91dc6dea5012667ec0a52e74d3bca F src/malloc.c de4e77fe70a9a0ac47a1c3a874422b107231bf31 F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217 -F src/mem1.c 232075b7da8c9b7f23159bbda25c7407168ab9db -F src/mem2.c 8651e5306c1d5c0a7ab91c027a653ced1ca3e6d6 +F src/mem1.c cacb202bc379da10d69aa66d497c0ea7bd9cd8a5 +F src/mem2.c 3f669b5e20975a5a2ca392aca891cd686e22b097 +F src/mem3.c 6c3cc2d3894acdc2eef5d65960a3bd4c8ba3aed7 F src/mutex.c 3259f62c2429967aee6dc112117a6d2f499ef061 F src/mutex.h 079fa6fe9da18ceb89e79012c010594c6672addb F src/mutex_os2.c 7fe4773e98ed74a63b2e54fc557929eb155f6269 @@ -148,10 +149,10 @@ F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f F src/test_async.c c5ea222c2bb0c3c33ab910d1b82622655dd50684 F src/test_autoext.c 855157d97aa28cf84233847548bfacda21807436 F src/test_btree.c c1308ba0b88ab577fa56c9e493a09829dfcded9c -F src/test_config.c 6fb459214b27952b143f45e35200d94096d54cc6 -F src/test_hexio.c 94a1efec4b19311eb7c4dc40e8496a3d8eadf18a +F src/test_config.c fd6ba4c62dd943e794f00f6ea1e9e32d97bf27f1 +F src/test_hexio.c 1a1cd8324d57585ea86b922f609fa1fbaaf9662d F src/test_loadext.c 22065d601a18878e5542191001f0eaa5d77c0ed8 -F src/test_malloc.c c34e7696dc4a5150c82452be28b87c7e38ba15ad +F src/test_malloc.c 72ceed192f7b229db34a2869ff9285b41a5cb796 F src/test_md5.c 34599caee5b1c73dcf86ca31f55846fab8c19ef7 F src/test_onefile.c d877baba46837587345933376c00c656f58d6fb6 F src/test_schema.c 12c9de7661d6294eec2d57afbb52e2af1128084f @@ -581,7 +582,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 329dd014b0d851f21ff6690b6149e9d73c92302d -R bf9fb86ed107ca6e77e9b8d6a0ec33f6 +P e8d591e8c369794921a4acbba5b17fddca730ee7 +R 57568b40e30acc759ef65998fb849eb6 U drh -Z 091afaa93f86f18bd96775701cef18ed +Z ffa57053e998becc58e159cdcfc3c99d diff --git a/manifest.uuid b/manifest.uuid index 8830869a9e..936ad3d37e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8d591e8c369794921a4acbba5b17fddca730ee7 \ No newline at end of file +8487ca82fade60b9fa63abf74e10f6ebcb48b98e \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index ece7b9d598..cfd29aa4e6 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -12,7 +12,7 @@ ** This file contains the C functions that implement a memory ** allocation subsystem for use by SQLite. ** -** $Id: mem1.c,v 1.11 2007/10/06 01:40:35 drh Exp $ +** $Id: mem1.c,v 1.12 2007/10/19 17:47:25 drh Exp $ */ /* @@ -20,7 +20,8 @@ ** used when no other memory allocator is specified using compile-time ** macros. */ -#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION) +#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION) \ + && !defined(SQLITE_MEMORY_SIZE) /* ** We will eventually construct multiple memory allocation subsystems diff --git a/src/mem2.c b/src/mem2.c index 5d96087761..0815a55ac4 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -12,7 +12,7 @@ ** This file contains the C functions that implement a memory ** allocation subsystem for use by SQLite. ** -** $Id: mem2.c,v 1.15 2007/10/15 19:34:32 drh Exp $ +** $Id: mem2.c,v 1.16 2007/10/19 17:47:25 drh Exp $ */ /* @@ -20,7 +20,8 @@ ** SQLITE_MEMDEBUG macro is defined and SQLITE_OMIT_MEMORY_ALLOCATION ** is not defined. */ -#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION) +#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION) \ + && !defined(SQLITE_MEMORY_SIZE) /* ** We will eventually construct multiple memory allocation subsystems @@ -87,7 +88,7 @@ struct MemBlockHdr { /* ** Number of malloc size increments to track. */ -#define NCSIZE 500 +#define NCSIZE 1000 /* ** All of the static variables used by this module are collected @@ -156,9 +157,9 @@ static struct { /* ** Gather statistics on the sizes of memory allocations. - ** sizeCnt[i] is the number of allocation attempts of i*4 + ** sizeCnt[i] is the number of allocation attempts of i*8 ** bytes. i==NCSIZE is the number of allocation attempts for - ** sizes more than NCSIZE*4 bytes. + ** sizes more than NCSIZE*8 bytes. */ int sizeCnt[NCSIZE]; diff --git a/src/mem3.c b/src/mem3.c new file mode 100644 index 0000000000..c7b0de3109 --- /dev/null +++ b/src/mem3.c @@ -0,0 +1,636 @@ +/* +** 2007 October 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement a memory +** allocation subsystem for use by SQLite. +** +** This version of the memory allocation subsystem omits all +** use of malloc(). All dynamically allocatable memory is +** contained in a static array, mem.aPool[]. The size of this +** fixed memory pool is SQLITE_MEMORY_SIZE bytes. +** +** This version of the memory allocation subsystem is used if +** and only if SQLITE_MEMORY_SIZE is defined. +** +** $Id: mem3.c,v 1.1 2007/10/19 17:47:25 drh Exp $ +*/ + +/* +** This version of the memory allocator is used only when +** SQLITE_MEMORY_SIZE is defined. +*/ +#if defined(SQLITE_MEMORY_SIZE) +#include "sqliteInt.h" + +/* +** Maximum size (in Mem3Blocks) of a "small" chunk. +*/ +#define MX_SMALL 10 + + +/* +** Number of freelist hash slots +*/ +#define N_HASH 61 + +/* +** A memory allocation (also called a "chunk") consists of two or +** more blocks where each block is 8 bytes. The first 8 bytes are +** a header that is not returned to the user. +** +** A chunk is two or more blocks that is either checked out or +** free. The first block has format u.hdr. u.hdr.size is the +** size of the allocation in blocks if the allocation is free. +** If the allocation is checked out, u.hdr.size is the negative +** of the size. Similarly, u.hdr.prevSize is the size of the +** immediately previous allocation. +** +** We often identify a chunk by its index in mem.aPool[]. When +** this is done, the chunk index refers to the second block of +** the chunk. In this way, the first chunk has an index of 1. +** A chunk index of 0 means "no such chunk" and is the equivalent +** of a NULL pointer. +** +** The second block of free chunks is of the form u.list. The +** two fields form a double-linked list of chunks of related sizes. +** Pointers to the head of the list are stored in mem.aiSmall[] +** for smaller chunks and mem.aiHash[] for larger chunks. +** +** The second block of a chunk is user data if the chunk is checked +** out. +*/ +typedef struct Mem3Block Mem3Block; +struct Mem3Block { + union { + struct { + int prevSize; /* Size of previous chunk in Mem3Block elements */ + int size; /* Size of current chunk in Mem3Block elements */ + } hdr; + struct { + int next; /* Index in mem.aPool[] of next free chunk */ + int prev; /* Index in mem.aPool[] of previous free chunk */ + } list; + } u; +}; + +/* +** All of the static variables used by this module are collected +** into a single structure named "mem". This is to keep the +** static variables organized and to reduce namespace pollution +** when this module is combined with other in the amalgamation. +*/ +static struct { + /* + ** The alarm callback and its arguments. The mem.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. The alarmBusy variable is set to prevent recursive + ** callbacks. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; + int alarmBusy; + + /* + ** Mutex to control access to the memory allocation subsystem. + */ + sqlite3_mutex *mutex; + + /* + ** Current allocation and high-water mark. + */ + sqlite3_int64 nowUsed; + sqlite3_int64 mxUsed; + + /* + ** iMaster is the index of the master chunk. Most new allocations + ** occur off of this chunk. szMaster is the size (in Mem3Blocks) + ** of the current master. iMaster is 0 if there is not master chunk. + ** The master chunk is not in either the aiHash[] or aiSmall[]. + */ + int iMaster; + int szMaster; + + /* + ** Array of lists of free blocks according to the block size + ** for smaller chunks, or a hash on the block size for larger + ** chunks. + */ + int aiSmall[MX_SMALL-1]; /* For sizes 2 through MX_SMALL, inclusive */ + int aiHash[N_HASH]; /* For sizes MX_SMALL+1 and larger */ + + /* + ** Memory available for allocation + */ + Mem3Block aPool[SQLITE_MEMORY_SIZE/sizeof(Mem3Block)+2]; +} mem; + +/* +** Unlink the chunk at mem.aPool[i] from list it is currently +** on. *pRoot is the list that i is a member of. +*/ +static void unlinkChunkFromList(int i, int *pRoot){ + int next = mem.aPool[i].u.list.next; + int prev = mem.aPool[i].u.list.prev; + if( prev==0 ){ + *pRoot = next; + }else{ + mem.aPool[prev].u.list.next = next; + } + if( next ){ + mem.aPool[next].u.list.prev = prev; + } + mem.aPool[i].u.list.next = 0; + mem.aPool[i].u.list.prev = 0; +} + +/* +** Unlink the chunk at index i from +** whatever list is currently a member of. +*/ +static void unlinkChunk(int i){ + int size, hash; + size = mem.aPool[i-1].u.hdr.size; + assert( size==mem.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + unlinkChunkFromList(i, &mem.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + unlinkChunkFromList(i, &mem.aiHash[hash]); + } +} + +/* +** Link the chunk at mem.aPool[i] so that is on the list rooted +** at *pRoot. +*/ +static void linkChunkIntoList(int i, int *pRoot){ + mem.aPool[i].u.list.next = *pRoot; + mem.aPool[i].u.list.prev = 0; + if( *pRoot ){ + mem.aPool[*pRoot].u.list.prev = i; + } + *pRoot = i; +} + +/* +** Link the chunk at index i into either the appropriate +** small chunk list, or into the large chunk hash table. +*/ +static void linkChunk(int i){ + int size, hash; + size = mem.aPool[i-1].u.hdr.size; + assert( size==mem.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + linkChunkIntoList(i, &mem.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + linkChunkIntoList(i, &mem.aiHash[hash]); + } +} + +/* +** Enter the mutex mem.mutex. Allocate it if it is not already allocated. +** +** Also: Initialize the memory allocation subsystem the first time +** this routine is called. +*/ +static void enterMem(void){ + if( mem.mutex==0 ){ + mem.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); + mem.aPool[0].u.hdr.size = SQLITE_MEMORY_SIZE/8; + mem.aPool[SQLITE_MEMORY_SIZE/8].u.hdr.prevSize = SQLITE_MEMORY_SIZE/8; + mem.iMaster = 1; + mem.szMaster = SQLITE_MEMORY_SIZE/8; + } + sqlite3_mutex_enter(mem.mutex); +} + +/* +** Return the amount of memory currently checked out. +*/ +sqlite3_int64 sqlite3_memory_used(void){ + sqlite3_int64 n; + enterMem(); + n = mem.nowUsed; + sqlite3_mutex_leave(mem.mutex); + return n; +} + +/* +** Return the maximum amount of memory that has ever been +** checked out since either the beginning of this process +** or since the most recent reset. +*/ +sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ + sqlite3_int64 n; + enterMem(); + n = mem.mxUsed; + if( resetFlag ){ + mem.mxUsed = mem.nowUsed; + } + sqlite3_mutex_leave(mem.mutex); + return n; +} + +/* +** Change the alarm callback +*/ +int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + enterMem(); + mem.alarmCallback = xCallback; + mem.alarmArg = pArg; + mem.alarmThreshold = iThreshold; + sqlite3_mutex_leave(mem.mutex); + return SQLITE_OK; +} + +/* +** Trigger the alarm +*/ +static void sqlite3MemsysAlarm(int nByte){ + void (*xCallback)(void*,sqlite3_int64,int); + sqlite3_int64 nowUsed; + void *pArg; + if( mem.alarmCallback==0 || mem.alarmBusy ) return; + mem.alarmBusy = 1; + xCallback = mem.alarmCallback; + nowUsed = mem.nowUsed; + pArg = mem.alarmArg; + sqlite3_mutex_leave(mem.mutex); + xCallback(pArg, nowUsed, nByte); + sqlite3_mutex_enter(mem.mutex); + mem.alarmBusy = 0; +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned includes the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int internal_size(void *p){ + Mem3Block *pBlock = (Mem3Block*)p; + assert( pBlock[-1].u.hdr.size<0 ); + return -pBlock[-1].u.hdr.size*8; +} + +/* +** Chunk i is a free chunk that has been unlinked. Adjust its +** size parameters for check-out and return a pointer to the +** user portion of the chunk. +*/ +static void *checkOutChunk(int i, int nBlock){ + assert( mem.aPool[i-1].u.hdr.size==nBlock ); + assert( mem.aPool[i+nBlock-1].u.hdr.prevSize==nBlock ); + mem.aPool[i-1].u.hdr.size = -nBlock; + mem.aPool[i+nBlock-1].u.hdr.prevSize = -nBlock; + return &mem.aPool[i]; +} + +/* +** Carve a piece off of the end of the mem.iMaster free chunk. +** Return a pointer to the new allocation. Or, if the master chunk +** is not large enough, return 0. +*/ +static void *internal_from_master(int nBlock){ + assert( mem.szMaster>=nBlock ); + if( nBlock>=mem.szMaster-1 ){ + /* Use the entire master */ + void *p = checkOutChunk(mem.iMaster, mem.szMaster); + mem.iMaster = 0; + mem.szMaster = 0; + return p; + }else{ + /* Split the master block. Return the tail. */ + int newi; + newi = mem.iMaster + mem.szMaster - nBlock; + assert( newi > mem.iMaster+1 ); + mem.aPool[mem.iMaster+mem.szMaster-1].u.hdr.prevSize = -nBlock; + mem.aPool[newi-1].u.hdr.size = -nBlock; + mem.szMaster -= nBlock; + mem.aPool[newi-1].u.hdr.prevSize = mem.szMaster; + mem.aPool[mem.iMaster-1].u.hdr.size = mem.szMaster; + return (void*)&mem.aPool[newi]; + } +} + +/* +** *pRoot is the head of a list of free chunks of the same size +** or same size hash. In other words, *pRoot is an entry in either +** mem.aiSmall[] or mem.aiHash[]. +** +** This routine examines all entries on the given list and tries +** to coalesce each entries with adjacent free chunks. +** +** If it sees a chunk that is larger than mem.iMaster, it replaces +** the current mem.iMaster with the new larger chunk. In order for +** this mem.iMaster replacement to work, the master chunk must be +** linked into the hash tables. That is not the normal state of +** affairs, of course. The calling routine must link the master +** chunk before invoking this routine, then must unlink the (possibly +** changed) master chunk once this routine has finished. +*/ +static void mergeChunks(int *pRoot){ + int iNext, prev, size, i; + + for(i=*pRoot; i>0; i=iNext){ + iNext = mem.aPool[i].u.list.next; + size = mem.aPool[i-1].u.hdr.size; + assert( size>0 ); + if( mem.aPool[i-1].u.hdr.prevSize>0 ){ + unlinkChunkFromList(i, pRoot); + prev = i - mem.aPool[i-1].u.hdr.prevSize; + assert( prev>=0 ); + if( prev==iNext ){ + iNext = mem.aPool[prev].u.list.next; + } + unlinkChunk(prev); + size = i + size - prev; + mem.aPool[prev-1].u.hdr.size = size; + mem.aPool[prev+size-1].u.hdr.prevSize = size; + linkChunk(prev); + i = prev; + } + if( size>mem.szMaster ){ + mem.iMaster = i; + mem.szMaster = size; + } + } +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +*/ +static void *internal_malloc(int nByte){ + int i; + int nBlock; + + assert( sizeof(Mem3Block)==8 ); + if( nByte<=0 ){ + nBlock = 2; + }else{ + nBlock = (nByte + 15)/8; + } + assert( nBlock >= 2 ); + + /* STEP 1: + ** Look for an entry of the correct size in either the small + ** chunk table or in the large chunk hash table. This is + ** successful most of the time (about 9 times out of 10). + */ + if( nBlock <= MX_SMALL ){ + i = mem.aiSmall[nBlock-2]; + if( i>0 ){ + unlinkChunkFromList(i, &mem.aiSmall[nBlock-2]); + return checkOutChunk(i, nBlock); + } + }else{ + int hash = nBlock % N_HASH; + for(i=mem.aiHash[hash]; i>0; i=mem.aPool[i].u.list.next){ + if( mem.aPool[i-1].u.hdr.size==nBlock ){ + unlinkChunkFromList(i, &mem.aiHash[hash]); + return checkOutChunk(i, nBlock); + } + } + } + + /* STEP 2: + ** Try to satisfy the allocation by carving a piece off of the end + ** of the master chunk. This step usually works if step 1 fails. + */ + if( mem.szMaster>=nBlock ){ + return internal_from_master(nBlock); + } + + + /* STEP 3: + ** Loop through the entire memory pool. Coalesce adjacent free + ** chunks. Recompute the master chunk as the largest free chunk. + ** Then try again to satisfy the allocation by carving a piece off + ** of the end of the master chunk. This step happens very + ** rarely (we hope!) + */ + if( mem.iMaster ){ + linkChunk(mem.iMaster); + mem.iMaster = 0; + mem.szMaster = 0; + } + for(i=0; i=nBlock ){ + return internal_from_master(nBlock); + } + } + + /* If none of the above worked, then we fail. */ + return 0; +} + +/* +** Free an outstanding memory allocation. +*/ +void internal_free(void *pOld){ + Mem3Block *p = (Mem3Block*)pOld; + int i; + int size; + assert( p>mem.aPool && p<&mem.aPool[SQLITE_MEMORY_SIZE/8] ); + i = p - mem.aPool; + size = -mem.aPool[i-1].u.hdr.size; + assert( size>=2 ); + assert( mem.aPool[i+size-1].u.hdr.prevSize==-size ); + mem.aPool[i-1].u.hdr.size = size; + mem.aPool[i+size-1].u.hdr.prevSize = size; + linkChunk(i); + + /* Try to expand the master using the newly freed chunk */ + if( mem.iMaster ){ + while( mem.aPool[mem.iMaster-1].u.hdr.prevSize>0 ){ + size = mem.aPool[mem.iMaster-1].u.hdr.prevSize; + mem.iMaster -= size; + mem.szMaster += size; + unlinkChunk(mem.iMaster); + mem.aPool[mem.iMaster-1].u.hdr.size = mem.szMaster; + mem.aPool[mem.iMaster+mem.szMaster-1].u.hdr.prevSize = mem.szMaster; + } + while( mem.aPool[mem.iMaster+mem.szMaster-1].u.hdr.size>0 ){ + unlinkChunk(mem.iMaster+mem.szMaster); + mem.szMaster += mem.aPool[mem.iMaster+mem.szMaster-1].u.hdr.size; + mem.aPool[mem.iMaster-1].u.hdr.size = mem.szMaster; + mem.aPool[mem.iMaster+mem.szMaster-1].u.hdr.prevSize = mem.szMaster; + } + } +} + +/* +** Allocate nBytes of memory +*/ +void *sqlite3_malloc(int nBytes){ + sqlite3_int64 *p = 0; + if( nBytes>0 ){ + enterMem(); + if( mem.alarmCallback!=0 && mem.nowUsed+nBytes>=mem.alarmThreshold ){ + sqlite3MemsysAlarm(nBytes); + } + p = internal_malloc(nBytes); + if( p==0 ){ + sqlite3MemsysAlarm(nBytes); + p = internal_malloc(nBytes); + } + if( p ){ + mem.nowUsed += internal_size(p); + if( mem.nowUsed>mem.mxUsed ){ + mem.mxUsed = mem.nowUsed; + } + } + sqlite3_mutex_leave(mem.mutex); + } + return (void*)p; +} + +/* +** Free memory. +*/ +void sqlite3_free(void *pPrior){ + if( pPrior==0 ){ + return; + } + assert( mem.mutex!=0 ); + sqlite3_mutex_enter(mem.mutex); + mem.nowUsed -= internal_size(pPrior); + internal_free(pPrior); + sqlite3_mutex_leave(mem.mutex); +} + +/* +** Change the size of an existing memory allocation +*/ +void *sqlite3_realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return sqlite3_malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pPrior); + return 0; + } + assert( mem.mutex!=0 ); + sqlite3_mutex_enter(mem.mutex); + nOld = internal_size(pPrior); + if( mem.alarmCallback!=0 && mem.nowUsed+nBytes-nOld>=mem.alarmThreshold ){ + sqlite3MemsysAlarm(nBytes-nOld); + } + p = internal_malloc(nBytes); + if( p==0 ){ + sqlite3MemsysAlarm(nBytes); + p = internal_malloc(nBytes); + if( p==0 ){ + return 0; + } + } + if( nOldmem.mxUsed ){ + mem.mxUsed = mem.nowUsed; + } + sqlite3_mutex_leave(mem.mutex); + return p; +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +void sqlite3_memdebug_dump(const char *zFilename){ +#ifdef SQLITE_DEBUG + FILE *out; + int i, j, size; + if( zFilename==0 || zFilename[0]==0 ){ + out = stdout; + }else{ + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + } + enterMem(); + fprintf(out, "CHUNKS:\n"); + for(i=1; i<=SQLITE_MEMORY_SIZE/8; i+=size){ + size = mem.aPool[i-1].u.hdr.size; + if( size>=-1 && size<=1 ){ + fprintf(out, "%p size error\n", &mem.aPool[i]); + assert( 0 ); + break; + } + if( mem.aPool[i+(size<0?-size:size)-1].u.hdr.prevSize!=size ){ + fprintf(out, "%p tail size does not match\n", &mem.aPool[i]); + assert( 0 ); + break; + } + if( size<0 ){ + size = -size; + fprintf(out, "%p %6d bytes checked out\n", &mem.aPool[i], size*8-8); + }else{ + fprintf(out, "%p %6d bytes free%s\n", &mem.aPool[i], size*8-8, + i==mem.iMaster ? " **master**" : ""); + } + } + for(i=0; i0; j=mem.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem.aPool[j], mem.aPool[j-1].u.hdr.size*8-8); + } + fprintf(out, "\n"); + } + for(i=0; i0; j=mem.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem.aPool[j], mem.aPool[j-1].u.hdr.size*8-8); + } + fprintf(out, "\n"); + } + fprintf(out, "master=%d\n", mem.iMaster); + fprintf(out, "nowUsed=%lld\n", mem.nowUsed); + fprintf(out, "mxUsed=%lld\n", mem.mxUsed); + sqlite3_mutex_leave(mem.mutex); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#endif +} + + +#endif /* !SQLITE_MEMORY_SIZE */ diff --git a/src/test_config.c b/src/test_config.c index cd3dce4d42..aa769e6d26 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -16,7 +16,7 @@ ** The focus of this file is providing the TCL testing layer ** access to compile-time constants. ** -** $Id: test_config.c,v 1.15 2007/09/03 15:26:21 drh Exp $ +** $Id: test_config.c,v 1.16 2007/10/19 17:47:25 drh Exp $ */ #include "sqliteLimit.h" @@ -80,6 +80,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_MEMORY_SIZE + Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mem3", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_ALTERTABLE Tcl_SetVar2(interp, "sqlite_options", "altertable", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/test_hexio.c b/src/test_hexio.c index 4ec2b82c7b..4dc83e9766 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -17,7 +17,7 @@ ** with historical versions of the "binary" command. So it seems ** easier and safer to build our own mechanism. ** -** $Id: test_hexio.c,v 1.5 2007/09/01 11:04:27 danielk1977 Exp $ +** $Id: test_hexio.c,v 1.6 2007/10/19 17:47:25 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -31,7 +31,7 @@ ** binary data. zBuf[] is 2*n+1 bytes long. Overwrite zBuf[] ** with a hexadecimal representation of its original binary input. */ -static void binToHex(unsigned char *zBuf, int N){ +void sqlite3TestBinToHex(unsigned char *zBuf, int N){ const unsigned char zHex[] = "0123456789ABCDEF"; int i, j; unsigned char c; @@ -51,7 +51,7 @@ static void binToHex(unsigned char *zBuf, int N){ ** the binary data. Spaces in the original input are ignored. ** Return the number of bytes of binary rendered. */ -static int hexToBin(const unsigned char *zIn, int N, unsigned char *aOut){ +int sqlite3TestHexToBin(const unsigned char *zIn, int N, unsigned char *aOut){ const unsigned char aMap[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -130,7 +130,7 @@ static int hexio_read( if( got<0 ){ got = 0; } - binToHex(zBuf, got); + sqlite3TestBinToHex(zBuf, got); Tcl_AppendResult(interp, zBuf, 0); sqlite3_free(zBuf); return TCL_OK; @@ -167,7 +167,7 @@ static int hexio_write( if( aOut==0 ){ return TCL_ERROR; } - nOut = hexToBin(zIn, nIn, aOut); + nOut = sqlite3TestHexToBin(zIn, nIn, aOut); out = fopen(zFile, "r+"); if( out==0 ){ Tcl_AppendResult(interp, "cannot open output file ", zFile, 0); @@ -209,7 +209,7 @@ static int hexio_get_int( if( aOut==0 ){ return TCL_ERROR; } - nOut = hexToBin(zIn, nIn, aOut); + nOut = sqlite3TestHexToBin(zIn, nIn, aOut); if( nOut>=4 ){ memcpy(aNum, aOut, 4); }else{ @@ -244,7 +244,7 @@ static int hexio_render_int16( if( Tcl_GetIntFromObj(interp, objv[1], &val) ) return TCL_ERROR; aNum[0] = val>>8; aNum[1] = val; - binToHex(aNum, 2); + sqlite3TestBinToHex(aNum, 2); Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)aNum, 4)); return TCL_OK; } @@ -273,7 +273,7 @@ static int hexio_render_int32( aNum[1] = val>>16; aNum[2] = val>>8; aNum[3] = val; - binToHex(aNum, 4); + sqlite3TestBinToHex(aNum, 4); Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)aNum, 8)); return TCL_OK; } @@ -302,10 +302,10 @@ static int utf8_to_utf8( } zOrig = (unsigned char *)Tcl_GetStringFromObj(objv[1], &n); z = sqlite3_malloc( n+3 ); - n = hexToBin(zOrig, n, z); + n = sqlite3TestHexToBin(zOrig, n, z); z[n] = 0; nOut = sqlite3Utf8To8(z); - binToHex(z,nOut); + sqlite3TestBinToHex(z,nOut); Tcl_AppendResult(interp, (char*)z, 0); sqlite3_free(z); #endif diff --git a/src/test_malloc.c b/src/test_malloc.c index df9d67db6c..5b777f7d81 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -13,7 +13,7 @@ ** This file contains code used to implement test interfaces to the ** memory allocation subsystem. ** -** $Id: test_malloc.c,v 1.8 2007/09/03 07:31:10 danielk1977 Exp $ +** $Id: test_malloc.c,v 1.9 2007/10/19 17:47:25 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -153,6 +153,106 @@ static int test_free( return TCL_OK; } +/* +** These routines are in test_hexio.c +*/ +int sqlite3TestHexToBin(const char *, int, char *); +int sqlite3TestBinToHex(char*,int); + +/* +** Usage: memset ADDRESS SIZE HEX +** +** Set a chunk of memory (obtained from malloc, probably) to a +** specified hex pattern. +*/ +static int test_memset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void *p; + int size, n, i; + char *zHex; + char *zOut; + char zBin[100]; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "ADDRESS SIZE HEX"); + return TCL_ERROR; + } + if( textToPointer(Tcl_GetString(objv[1]), &p) ){ + Tcl_AppendResult(interp, "bad pointer: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &size) ){ + return TCL_ERROR; + } + if( size<=0 ){ + Tcl_AppendResult(interp, "size must be positive", (char*)0); + return TCL_ERROR; + } + zHex = Tcl_GetStringFromObj(objv[3], &n); + if( n>sizeof(zBin)*2 ) n = sizeof(zBin)*2; + n = sqlite3TestHexToBin(zHex, n, zBin); + if( n==0 ){ + Tcl_AppendResult(interp, "no data", (char*)0); + return TCL_ERROR; + } + zOut = p; + for(i=0; i0 ){ + if( size>(sizeof(zHex)-1)/2 ){ + n = (sizeof(zHex)-1)/2; + }else{ + n = size; + } + memcpy(zHex, zBin, n); + zBin += n; + size -= n; + sqlite3TestBinToHex(zHex, n); + Tcl_AppendResult(interp, zHex, (char*)0); + } + return TCL_OK; +} + /* ** Usage: sqlite3_memory_used ** @@ -234,7 +334,7 @@ static int test_memdebug_dump( Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); return TCL_ERROR; } -#ifdef SQLITE_MEMDEBUG +#if defined(SQLITE_MEMDEBUG) || defined(SQLITE_MEMORY_SIZE) { extern void sqlite3_memdebug_dump(const char*); sqlite3_memdebug_dump(Tcl_GetString(objv[1])); @@ -395,6 +495,8 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_malloc", test_malloc }, { "sqlite3_realloc", test_realloc }, { "sqlite3_free", test_free }, + { "memset", test_memset }, + { "memget", test_memget }, { "sqlite3_memory_used", test_memory_used }, { "sqlite3_memory_highwater", test_memory_highwater }, { "sqlite3_memdebug_backtrace", test_memdebug_backtrace },