database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-07 02:42:48 +03:00
Code Activity

Modify sqlite3_release_memory() to use a global LRU list of pages. Untested. (CVS 4301)

Browse Source 
FossilOrigin-Name: 5626ce0b5e249d48b56fdc4561ef663941eb23dc
This commit is contained in:
danielk1977
2007-08-27 17:27:49 +00:00
parent 4b6b4ab0d2
commit 9f61c2f129
5 changed files with 213 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
manifest
View File

@@ -1,5 +1,5 @@
C Create\sa\sfresh\spthread_mutexattr_t\severy\stime\sa\srecursive\smutex\sis\nallocated.\s\sTicket\s#2588.\s(CVS\s4300)
D 2007-08-25T16:31:30
C Modify\ssqlite3_release_memory()\sto\suse\sa\sglobal\sLRU\slist\sof\spages.\sUntested.\s(CVS\s4301)
D 2007-08-27T17:27:49
F Makefile.in 938f2769921fa1b30c633548f153804021eb1512
F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -103,7 +103,7 @@ F src/malloc.c d4282f50964ab1ca31f504c97b7cf2fdb4d4195d
F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217
F src/mem1.c afe2fbf6d7e8247c6c9f69c1481358b1cad60c08
F src/mem2.c 1a2ca756a285b5365d667841508cc1f98938b8d8
F src/mutex.c 438d59f4ea7a69d8607c024702d3a137ee55bc5e
F src/mutex.c adbad5e138e8caef8d3763280c75bfe2167812f6
F src/os.c a8ed3c495161475dbce255f7003144144fb425f1
F src/os.h 2bfbbad126a775e4d8c7d59eb4d9585a5fd7dfb5
F src/os_common.h a5c446d3b93f09f369d13bf217de4bed3437dd1c
@@ -115,7 +115,7 @@ F src/os_unix.c 27b1fad58587bc949013a5a4df9fc20fce395648
F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
F src/os_win.c 4f840e97624dbde9cae3d020ce072a4f1d2a11b1
F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
F src/pager.c b076458d72dcda7517c49aae0f0fd43c9a1195e5
F src/pager.c cfa6dc38b797206549491de3ec7f0aea50611dda
F src/pager.h 53087c6fb9db01aed17c7fd044662a27507e89b8
F src/parse.y 2d2ce439dc6184621fb0b86f4fc5aca7f391a590
F src/pragma.c 9b989506a1b7c8aecd6befb8235e2f57a4aba7e5
@@ -125,7 +125,7 @@ F src/random.c 4a22746501bf36b0a088c66e38dde5daba6a35da
F src/select.c 98c367bce3f38c5adfcc97de9ab5c79b0e5dc2b2
F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96
F src/shell.c ac29402b538515fa4697282387be9c1205e6e9eb
F src/sqlite.h.in 50c8b68c7470b91653157d3554caa8a1287d90e3
F src/sqlite.h.in 2d45cd3fc1b6677d06c8e547bbe1b9a040a7f677
F src/sqlite3ext.h 9a26028378c288af500d8b94ed079666fed5806b
F src/sqliteInt.h 13c908f5f156a192fcd247f993ac513bfaf81f53
F src/sqliteLimit.h 1bcbbdfa856f8b71b561abb31edb864b0eca1d12
@@ -561,7 +561,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
P 7289079d6b4a7a160063e34c0f5e43637ef7476f
R d800e16d6642cfcaec4398d15460213b
U drh
Z 01a98a984e78c80696ffafb95f213436
P 3d746343add3feb9d208302a00b419d71d6ba246
R 95f58c79a104b4b1aa01178fe78f1fa0
U danielk1977
Z 8ec1ffcae0650e586d1badb059ab31f9

2
manifest.uuid
View File

@@ -1 +1 @@
3d746343add3feb9d208302a00b419d71d6ba246
5626ce0b5e249d48b56fdc4561ef663941eb23dc

6
src/mutex.c
View File

@@ -12,7 +12,7 @@
** This file contains the C functions that implement mutexes for
** use by the SQLite core.
**
** $Id: mutex.c,v 1.13 2007/08/25 16:31:30 drh Exp $
** $Id: mutex.c,v 1.14 2007/08/27 17:27:49 danielk1977 Exp $
*/
/*
** If SQLITE_MUTEX_APPDEF is defined, then this whole module is
@@ -253,6 +253,7 @@ struct sqlite3_mutex {
** <li> SQLITE_MUTEX_STATIC_MEM
** <li> SQLITE_MUTEX_STATIC_MEM2
** <li> SQLITE_MUTEX_STATIC_PRNG
** <li> SQLITE_MUTEX_STATIC_LRU
** </ul>
**
** The first two constants cause sqlite3_mutex_alloc() to create
@@ -285,6 +286,7 @@ sqlite3_mutex *sqlite3_mutex_alloc(int iType){
{ PTHREAD_MUTEX_INITIALIZER, },
{ PTHREAD_MUTEX_INITIALIZER, },
{ PTHREAD_MUTEX_INITIALIZER, },
{ PTHREAD_MUTEX_INITIALIZER, },
};
sqlite3_mutex *p;
switch( iType ){
@@ -474,7 +476,7 @@ sqlite3_mutex *sqlite3_mutex_alloc(int iType){
break;
}
default: {
static sqlite3_mutex staticMutexes[4];
static sqlite3_mutex staticMutexes[5];
static int isInit = 0;
while( !isInit ){
static long lock = 0;

255
src/pager.c
View File

@@ -18,7 +18,7 @@
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.376 2007/08/24 16:29:24 drh Exp $
** @(#) $Id: pager.c,v 1.377 2007/08/27 17:27:49 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
@@ -131,6 +131,13 @@
*/
#define FORCE_ALIGNMENT(X) (((X)+7)&~7)
typedef struct PgHdr PgHdr;
typedef struct PagerLruLink PagerLruLink;
struct PagerLruLink {
PgHdr *pNext;
PgHdr *pPrev;
};
/*
** Each in-memory image of a page begins with the following header.
** This header is only visible to this pager module. The client
@@ -221,12 +228,11 @@
** content is needed in the future, it should be read from the
** original database file.
*/
typedef struct PgHdr PgHdr;
struct PgHdr {
Pager *pPager; /* The pager to which this page belongs */
Pgno pgno; /* The page number for this page */
PgHdr *pNextHash, *pPrevHash; /* Hash collision chain for PgHdr.pgno */
PgHdr *pNextFree, *pPrevFree; /* Freelist of pages where nRef==0 */
PagerLruLink free; /* Next and previous free pages */
PgHdr *pNextAll; /* A list of all pages */
u8 inJournal; /* TRUE if has been written to journal */
u8 dirty; /* TRUE if we need to write back changes */
@@ -235,6 +241,9 @@ struct PgHdr {
u8 needRead; /* Read content if PagerWrite() is called */
short int nRef; /* Number of users of this page */
PgHdr *pDirty, *pPrevDirty; /* Dirty pages */
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
PagerLruLink gfree; /* Global list of nRef==0 pages */
#endif
u32 notUsed; /* Buffer space */
#ifdef SQLITE_CHECK_PAGES
u32 pageHash;
@@ -283,6 +292,13 @@ struct PgHistory {
#define PGHDR_TO_HIST(P,PGR) \
((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra])
typedef struct PagerLruList PagerLruList;
struct PagerLruList {
PgHdr *pFirst;
PgHdr *pLast;
PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */
};
/*
** A open page cache is an instance of the following structure.
**
@@ -338,8 +354,7 @@ struct Pager {
sqlite3_file *fd, *jfd; /* File descriptors for database and journal */
sqlite3_file *stfd; /* File descriptor for the statement subjournal*/
BusyHandler *pBusyHandler; /* Pointer to sqlite.busyHandler */
PgHdr *pFirst, *pLast; /* List of free pages */
PgHdr *pFirstSynced; /* First free page with PgHdr.needSync==0 */
PagerLruList lru; /* LRU list of free pages */
PgHdr *pAll; /* List of all pages */
PgHdr *pStmt; /* List of pages in the statement subjournal */
PgHdr *pDirty; /* List of all dirty pages */
@@ -394,6 +409,7 @@ int sqlite3_pager_pgfree_count = 0; /* Number of cache pages freed */
*/
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
static Pager *sqlite3PagerList = 0;
static PagerLruList sqlite3LruPageList = {0, 0, 0};
#endif
@@ -515,6 +531,99 @@ static const unsigned char aJournalMagic[] = {
# define REFINFO(X)
#endif
static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
pLink->pNext = 0;
pLink->pPrev = pList->pLast;
if( pList->pLast ){
int iOff = (char *)pLink - (char *)pPg;
PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]);
pLastLink->pNext = pPg;
}else{
assert(!pList->pFirst);
pList->pFirst = pPg;
}
pList->pLast = pPg;
if( !pList->pFirstSynced && pPg->needSync==0 ){
pList->pFirstSynced = pPg;
}
}
static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
int iOff = (char *)pLink - (char *)pPg;
if( pPg==pList->pFirst ){
pList->pFirst = pLink->pNext;
}
if( pPg==pList->pLast ){
pList->pLast = pLink->pPrev;
}
if( pLink->pPrev ){
PagerLruLink *pPrevLink = (PagerLruLink *)(&((u8 *)pLink->pPrev)[iOff]);
pPrevLink->pNext = pLink->pNext;
}
if( pLink->pNext ){
PagerLruLink *pNextLink = (PagerLruLink *)(&((u8 *)pLink->pNext)[iOff]);
pNextLink->pPrev = pLink->pPrev;
}
if( pPg==pList->pFirstSynced ){
PgHdr *p = pLink->pNext;
while( p && p->needSync ){
PagerLruLink *pL = (PagerLruLink *)(&((u8 *)p)[iOff]);
p = pL->pNext;
}
pList->pFirstSynced = p;
}
pLink->pNext = pLink->pPrev = 0;
}
/*
** Add page to the free-list
*/
static void lruListAdd(PgHdr *pPg){
listAdd(&pPg->pPager->lru, &pPg->free, pPg);
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
if( !pPg->pPager->memDb ){
sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
listAdd(&sqlite3LruPageList, &pPg->gfree, pPg);
sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
}
#endif
}
/*
** Remove page from free-list
*/
static void lruListRemove(PgHdr *pPg){
listRemove(&pPg->pPager->lru, &pPg->free, pPg);
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
if( !pPg->pPager->memDb ){
sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
listRemove(&sqlite3LruPageList, &pPg->gfree, pPg);
sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
}
#endif
}
/*
** Set the Pager.pFirstSynced variable
*/
static void lruListSetFirstSynced(Pager *pPager){
pPager->lru.pFirstSynced = pPager->lru.pFirst;
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
if( !pPager->memDb ){
PgHdr *p;
sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
for(p=sqlite3LruPageList.pFirst; p && p->needSync; p=p->gfree.pNext);
assert(p==pPager->lru.pFirstSynced || p==sqlite3LruPageList.pFirstSynced);
sqlite3LruPageList.pFirstSynced = p;
sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
}
#endif
}
/*
** Return true if page *pPg has already been written to the statement
** journal (or statement snapshot has been created, if *pPg is part
@@ -1081,12 +1190,13 @@ static void pager_reset(Pager *pPager){
IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
PAGER_INCR(sqlite3_pager_pgfree_count);
pNext = pPg->pNextAll;
lruListRemove(pPg);
sqlite3_free(pPg);
}
assert(pPager->lru.pFirst==0);
assert(pPager->lru.pFirstSynced==0);
assert(pPager->lru.pLast==0);
pPager->pStmt = 0;
pPager->pFirst = 0;
pPager->pFirstSynced = 0;
pPager->pLast = 0;
pPager->pAll = 0;
pPager->nHash = 0;
sqlite3_free(pPager->aHash);
@@ -1165,7 +1275,7 @@ static int pager_end_transaction(Pager *pPager){
pPager->origDbSize = 0;
pPager->setMaster = 0;
pPager->needSync = 0;
pPager->pFirstSynced = pPager->pFirst;
lruListSetFirstSynced(pPager);
pPager->dbSize = -1;
return (rc==SQLITE_OK?rc2:rc);
@@ -2284,27 +2394,8 @@ static void unlinkHashChain(Pager *pPager, PgHdr *pPg){
static void unlinkPage(PgHdr *pPg){
Pager *pPager = pPg->pPager;
/* Keep the pFirstSynced pointer pointing at the first synchronized page */
if( pPg==pPager->pFirstSynced ){
PgHdr *p = pPg->pNextFree;
while( p && p->needSync ){ p = p->pNextFree; }
pPager->pFirstSynced = p;
}
/* Unlink from the freelist */
if( pPg->pPrevFree ){
pPg->pPrevFree->pNextFree = pPg->pNextFree;
}else{
assert( pPager->pFirst==pPg );
pPager->pFirst = pPg->pNextFree;
}
if( pPg->pNextFree ){
pPg->pNextFree->pPrevFree = pPg->pPrevFree;
}else{
assert( pPager->pLast==pPg );
pPager->pLast = pPg->pPrevFree;
}
pPg->pNextFree = pPg->pPrevFree = 0;
/* Unlink from free page list */
lruListRemove(pPg);
/* Unlink from the pgno hash table */
unlinkHashChain(pPager, pPg);
@@ -2498,21 +2589,7 @@ Pgno sqlite3PagerPagenumber(DbPage *p){
static void _page_ref(PgHdr *pPg){
if( pPg->nRef==0 ){
/* The page is currently on the freelist. Remove it. */
if( pPg==pPg->pPager->pFirstSynced ){
PgHdr *p = pPg->pNextFree;
while( p && p->needSync ){ p = p->pNextFree; }
pPg->pPager->pFirstSynced = p;
}
if( pPg->pPrevFree ){
pPg->pPrevFree->pNextFree = pPg->pNextFree;
}else{
pPg->pPager->pFirst = pPg->pNextFree;
}
if( pPg->pNextFree ){
pPg->pNextFree->pPrevFree = pPg->pPrevFree;
}else{
pPg->pPager->pLast = pPg->pPrevFree;
}
lruListRemove(pPg);
pPg->pPager->nRef++;
}
pPg->nRef++;
@@ -2635,7 +2712,7 @@ static int syncJournal(Pager *pPager){
for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
pPg->needSync = 0;
}
pPager->pFirstSynced = pPager->pFirst;
lruListSetFirstSynced(pPager);
}
#ifndef NDEBUG
@@ -2647,7 +2724,7 @@ static int syncJournal(Pager *pPager){
for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
assert( pPg->needSync==0 );
}
assert( pPager->pFirstSynced==pPager->pFirst );
assert( pPager->lru.pFirstSynced==pPager->lru.pFirst );
}
#endif
@@ -2857,14 +2934,14 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
/* Find a page to recycle. Try to locate a page that does not
** require us to do an fsync() on the journal.
*/
pPg = pPager->pFirstSynced;
pPg = pPager->lru.pFirstSynced;
/* If we could not find a page that does not require an fsync()
** on the journal file then fsync the journal file. This is a
** very slow operation, so we work hard to avoid it. But sometimes
** it can't be helped.
*/
if( pPg==0 && pPager->pFirst && syncOk && !MEMDB){
if( pPg==0 && pPager->lru.pFirst && syncOk && !MEMDB){
int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
int rc = syncJournal(pPager);
if( rc!=0 ){
@@ -2885,7 +2962,7 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
return rc;
}
}
pPg = pPager->pFirst;
pPg = pPager->lru.pFirst;
}
if( pPg==0 ){
return SQLITE_OK;
@@ -2945,7 +3022,7 @@ int sqlite3PagerReleaseMemory(int nReq){
int nReleased = 0; /* Bytes of memory released so far */
sqlite3_mutex *mutex; /* The MEM2 mutex */
Pager *pPager; /* For looping over pagers */
int i; /* Passes over pagers */
int rc = SQLITE_OK;
/* Acquire the memory-management mutex
*/
@@ -2959,34 +3036,35 @@ int sqlite3PagerReleaseMemory(int nReq){
pPager->iInUseMM = 1;
}
/* Outermost loop runs for at most two iterations. First iteration we
** try to find memory that can be released without calling fsync(). Second
** iteration (which only runs if the first failed to free nReq bytes of
** memory) is permitted to call fsync(). This is of course much more
** expensive.
*/
for(i=0; i<=1; i++){
/* Loop through all the SQLite pagers opened by the current thread. */
Pager *pPager = sqlite3PagerList;
for( ; pPager && (nReq<0 || nReleased<nReq); pPager=pPager->pNext){
while( rc==SQLITE_OK && (nReq<0 || nReleased<nReq) ){
PgHdr *pPg;
int rc = SQLITE_OK;
PgHdr *pRecycled;
/* In-memory databases should not appear on the pager list */
assert( !MEMDB );
/* Skip pagers that are currently in use by the b-tree layer */
if( pPager->iInUseDB ) continue;
/* For each pager, try to free as many pages as possible (without
** calling fsync() if this is the first iteration of the outermost
** loop).
/* Try to find a page to recycle that does not require a sync(). If
** this is not possible, find one that does require a sync().
*/
while( (nReq<0 || nReleased<nReq) &&
SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) &&
pPg
) {
sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
pPg = sqlite3LruPageList.pFirstSynced;
while( pPg && (pPg->needSync || pPg->pPager->iInUseDB) ){
pPg = pPg->gfree.pNext;
}
if( !pPg ){
pPg = sqlite3LruPageList.pFirst;
while( pPg && pPg->pPager->iInUseDB ){
pPg = pPg->gfree.pNext;
}
}
sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
if( !pPg ) break;
pPager = pPg->pPager;
assert(!pPg->needSync || pPg==pPager->lru.pFirst);
assert(pPg->needSync || pPg==pPager->lru.pFirstSynced);
rc = pager_recycle(pPager, 1, &pRecycled);
assert(pRecycled==pPg || rc!=SQLITE_OK);
if( rc==SQLITE_OK ){
/* We've found a page to free. At this point the page has been
** removed from the page hash-table, free-list and synced-list
** (pFirstSynced). It is still in the all pages (pAll) list.
@@ -3011,9 +3089,7 @@ int sqlite3PagerReleaseMemory(int nReq){
IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno));
PAGER_INCR(sqlite3_pager_pgfree_count);
sqlite3_free(pPg);
}
if( rc!=SQLITE_OK ){
}else{
/* An error occured whilst writing to the database file or
** journal in pager_recycle(). The error is not returned to the
** caller of this function. Instead, set the Pager.errCode variable.
@@ -3029,7 +3105,6 @@ int sqlite3PagerReleaseMemory(int nReq){
pager_error(pPager, rc);
}
}
}
/* Clear the memory management flags and release the mutex
*/
@@ -3252,9 +3327,9 @@ static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){
/* Create a new PgHdr if any of the four conditions defined
** above are met: */
if( pPager->nPage<pPager->mxPage
|| pPager->pFirst==0
|| pPager->lru.pFirst==0
|| MEMDB
|| (pPager->pFirstSynced==0 && pPager->doNotSync)
|| (pPager->lru.pFirstSynced==0 && pPager->doNotSync)
){
if( pPager->nPage>=pPager->nHash ){
pager_resize_hash_table(pPager,
@@ -3541,19 +3616,9 @@ int sqlite3PagerUnref(DbPage *pPg){
** destructor and add the page to the freelist.
*/
if( pPg->nRef==0 ){
Pager *pPager;
pPager = pPg->pPager;
pPg->pNextFree = 0;
pPg->pPrevFree = pPager->pLast;
pPager->pLast = pPg;
if( pPg->pPrevFree ){
pPg->pPrevFree->pNextFree = pPg;
}else{
pPager->pFirst = pPg;
}
if( pPg->needSync==0 && pPager->pFirstSynced==0 ){
pPager->pFirstSynced = pPg;
}
Pager *pPager = pPg->pPager;
lruListAdd(pPg);
if( pPager->xDestructor ){
pPager->xDestructor(pPg, pPager->pageSize);
}

4
src/sqlite.h.in
View File

@@ -30,7 +30,7 @@
** the version number) and changes its name to "sqlite3.h" as
** part of the build process.
**
** @(#) $Id: sqlite.h.in,v 1.242 2007/08/25 16:21:30 drh Exp $
** @(#) $Id: sqlite.h.in,v 1.243 2007/08/27 17:27:49 danielk1977 Exp $
*/
#ifndef _SQLITE3_H_
#define _SQLITE3_H_
@@ -3283,6 +3283,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*);
** <li> SQLITE_MUTEX_STATIC_MEM
** <li> SQLITE_MUTEX_STATIC_MEM2
** <li> SQLITE_MUTEX_STATIC_PRNG
** <li> SQLITE_MUTEX_STATIC_LRU
** </ul>
**
** The first two constants cause sqlite3_mutex_alloc() to create
@@ -3387,6 +3388,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*);
#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */
#define SQLITE_MUTEX_STATIC_MEM2 4 /* sqlite3_release_memory() */
#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */
#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */
/*