From 84f786fcdd092e22aa1e0acecbe5ae459a5bd35e Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Tue, 28 Aug 2007 08:00:17 +0000 Subject: [PATCH] Add some comments and test-cases for the global lru page list (used by sqlite3_release_memory()). (CVS 4308) FossilOrigin-Name: 0b80168895993af5774369f839f284712d006f0a --- manifest | 16 +++---- manifest.uuid | 2 +- src/pager.c | 78 +++++++++++++++++++++++++----- test/malloc5.test | 120 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 194 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 9c206a56fd..5e305ca9f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\scorrect\sbtree\slocking\sin\sa\smultithreaded\senvironment.\s(CVS\s4307) -D 2007-08-28T02:27:52 +C Add\ssome\scomments\sand\stest-cases\sfor\sthe\sglobal\slru\spage\slist\s(used\sby\ssqlite3_release_memory()).\s(CVS\s4308) +D 2007-08-28T08:00:18 F Makefile.in e8296e112b8942a96c0ed504398bd0d43e3c67ce F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -116,7 +116,7 @@ F src/os_unix.c 89bf24aa2475048a7833c45c522e7c6a81b83bb8 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e F src/os_win.c 3ffd3aacff4cb69848284e29dcec0feff23b0752 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c cfa6dc38b797206549491de3ec7f0aea50611dda +F src/pager.c 51ca27639ab25c8838afc856d3cc6045a98752a7 F src/pager.h 53087c6fb9db01aed17c7fd044662a27507e89b8 F src/parse.y 2d2ce439dc6184621fb0b86f4fc5aca7f391a590 F src/pragma.c 9b989506a1b7c8aecd6befb8235e2f57a4aba7e5 @@ -345,7 +345,7 @@ F test/malloc.test dbfaedfca734283182db18a64416cf037c33648f F test/malloc2.test 96dfa6318d3b7bcd41d4fe7e24d35196d465bf58 F test/malloc3.test 65d323508c7c4e1fb5569d03a11070b0259ad2fe F test/malloc4.test cf1e856fbc8ebde562838cb47adce0ebc2d4f7d2 -F test/malloc5.test 6f7b96f1bea181d789a7140fd3fabfb0e333d8f5 +F test/malloc5.test 9db9205a5d444e23043368f961f92641c7770199 F test/malloc6.test 3733b9bd4e039c3239f869c40edbb88172025e2e F test/malloc7.test dd66d8f82916becf1d29b6640e4f4855485570f8 F test/malloc8.test 5ff95278bc73e815e295971afcdd175f6ba19258 @@ -563,7 +563,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 741d6fb096dcb232871d3a8468c386022afcf554 -R ff3e167d1bd072e0909067b1f2888076 -U drh -Z 6d8ca04435be2f950a1ddef0045b914b +P b8cc493b47e618648f645ab73eb0253739e03fcd +R 89ac354b82d38fc45db7b69d7f01c4df +U danielk1977 +Z 4eb080ba40501ea8bb2df84791a3ceaa diff --git a/manifest.uuid b/manifest.uuid index 5ea8d95fad..8a548f30cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8cc493b47e618648f645ab73eb0253739e03fcd \ No newline at end of file +0b80168895993af5774369f839f284712d006f0a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e6539e61a3..a2595243ba 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.377 2007/08/27 17:27:49 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.378 2007/08/28 08:00:18 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -132,6 +132,33 @@ #define FORCE_ALIGNMENT(X) (((X)+7)&~7) typedef struct PgHdr PgHdr; + +/* +** Each pager stores all currently unreferenced pages in a list sorted +** in least-recently-used (LRU) order (i.e. the first item on the list has +** not been referenced in a long time, the last item has been recently +** used). An instance of this structure is included as part of each +** pager structure for this purpose (variable Pager.lru). +** +** Additionally, if memory-management is enabled, all unreferenced pages +** are stored in a global LRU list (global variable sqlite3LruPageList). +** +** In both cases, the PagerLruList.pFirstSynced variable points to +** the first page in the corresponding list that does not require an +** fsync() operation before it's memory can be reclaimed. If no such +** page exists, PagerLruList.pFirstSynced is set to NULL. +*/ +typedef struct PagerLruList PagerLruList; +struct PagerLruList { + PgHdr *pFirst; /* First page in LRU list */ + PgHdr *pLast; /* Last page in LRU list (the most recently used) */ + PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */ +}; + +/* +** The following structure contains the next and previous pointers used +** to link a PgHdr structure into a PagerLruList linked list. +*/ typedef struct PagerLruLink PagerLruLink; struct PagerLruLink { PgHdr *pNext; @@ -292,13 +319,6 @@ struct PgHistory { #define PGHDR_TO_HIST(P,PGR) \ ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra]) -typedef struct PagerLruList PagerLruList; -struct PagerLruList { - PgHdr *pFirst; - PgHdr *pLast; - PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */ -}; - /* ** A open page cache is an instance of the following structure. ** @@ -531,10 +551,22 @@ static const unsigned char aJournalMagic[] = { # define REFINFO(X) #endif +/* +** Add page pPg to the end of the linked list managed by structure +** pList (pPg becomes the last entry in the list - the most recently +** used). Argument pLink should point to either pPg->free or pPg->gfree, +** depending on whether pPg is being added to the pager-specific or +** global LRU list. +*/ static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ pLink->pNext = 0; pLink->pPrev = pList->pLast; +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + assert(pLink==&pPg->free || pLink==&pPg->gfree); + assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); +#endif + if( pList->pLast ){ int iOff = (char *)pLink - (char *)pPg; PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]); @@ -550,9 +582,20 @@ static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ } } +/* +** Remove pPg from the list managed by the structure pointed to by pList. +** +** Argument pLink should point to either pPg->free or pPg->gfree, depending +** on whether pPg is being added to the pager-specific or global LRU list. +*/ static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ int iOff = (char *)pLink - (char *)pPg; +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + assert(pLink==&pPg->free || pLink==&pPg->gfree); + assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); +#endif + if( pPg==pList->pFirst ){ pList->pFirst = pLink->pNext; } @@ -580,7 +623,9 @@ static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ } /* -** Add page to the free-list +** Add page pPg to the list of free pages for the pager. If +** memory-management is enabled, also add the page to the global +** list of free pages. */ static void lruListAdd(PgHdr *pPg){ listAdd(&pPg->pPager->lru, &pPg->free, pPg); @@ -594,7 +639,9 @@ static void lruListAdd(PgHdr *pPg){ } /* -** Remove page from free-list +** Remove page pPg from the list of free pages for the associated pager. +** If memory-management is enabled, also remove pPg from the global list +** of free pages. */ static void lruListRemove(PgHdr *pPg){ listRemove(&pPg->pPager->lru, &pPg->free, pPg); @@ -608,7 +655,11 @@ static void lruListRemove(PgHdr *pPg){ } /* -** Set the Pager.pFirstSynced variable +** This function is called just after the needSync flag has been cleared +** from all pages managed by pPager (usually because the journal file +** has just been synced). It updates the pPager->lru.pFirstSynced variable +** and, if memory-management is enabled, the sqlite3LruPageList.pFirstSynced +** variable also. */ static void lruListSetFirstSynced(Pager *pPager){ pPager->lru.pFirstSynced = pPager->lru.pFirst; @@ -3056,6 +3107,10 @@ int sqlite3PagerReleaseMemory(int nReq){ } sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); + /* If pPg==0, then the block above has failed to find a page to + ** recycle. In this case return early - no further memory will + ** be released. + */ if( !pPg ) break; pPager = pPg->pPager; @@ -3089,6 +3144,7 @@ int sqlite3PagerReleaseMemory(int nReq){ IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno)); PAGER_INCR(sqlite3_pager_pgfree_count); sqlite3_free(pPg); + pPager->nPage--; }else{ /* An error occured whilst writing to the database file or ** journal in pager_recycle(). The error is not returned to the diff --git a/test/malloc5.test b/test/malloc5.test index a45bf8efcd..3d8af2d3fe 100644 --- a/test/malloc5.test +++ b/test/malloc5.test @@ -12,7 +12,7 @@ # This file contains test cases focused on the two memory-management APIs, # sqlite3_soft_heap_limit() and sqlite3_release_memory(). # -# $Id: malloc5.test,v 1.13 2007/08/22 22:04:37 drh Exp $ +# $Id: malloc5.test,v 1.14 2007/08/28 08:00:18 danielk1977 Exp $ #--------------------------------------------------------------------------- # NOTES ON EXPECTED BEHAVIOUR @@ -248,7 +248,7 @@ do_test malloc5-5.1 { } sqlite3_release_memory } 0 -do_test malloc5-5.1 { +do_test malloc5-5.2 { sqlite3_soft_heap_limit 5000 execsql { COMMIT; @@ -257,6 +257,122 @@ do_test malloc5-5.1 { } expr 1 } {1} +sqlite3_soft_heap_limit $::soft_limit + +#------------------------------------------------------------------------- +# The following test cases (malloc5-6.*) test the new global LRU list +# used to determine the pages to recycle when sqlite3_release_memory is +# called and there is more than one pager open. +# +proc nPage {db} { + set bt [btree_from_db $db] + array set stats [btree_pager_stats $bt] + set stats(page) +} +db close +file delete -force test.db test.db-journal test2.db test2.db-journal + +# This block of test-cases (malloc5-6.1.*) prepares two database files +# for the subsequent tests. +do_test malloc5-6.1.1 { + sqlite3 db test.db + execsql { + PRAGMA page_size=1024; + PRAGMA default_cache_size=10; + BEGIN; + CREATE TABLE abc(a PRIMARY KEY, b, c); + INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100)); + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + INSERT INTO abc + SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; + COMMIT; + } + copy_file test.db test2.db + sqlite3 db2 test2.db + list [expr [file size test.db]/1024] [expr [file size test2.db]/1024] +} {23 23} +do_test malloc5-6.1.2 { + list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] +} {10 10} + +do_test malloc5-6.2.1 { + execsql { SELECT * FROM abc } db2 + execsql {SELECT * FROM abc} db + list [nPage db] [nPage db2] +} {10 10} +do_test malloc5-6.2.2 { + # If we now try to reclaim some memory, it should come from the db2 cache. + sqlite3_release_memory 3000 + list [nPage db] [nPage db2] +} {10 7} +do_test malloc5-6.2.3 { + # Access the db2 cache again, so that all the db2 pages have been used + # more recently than all the db pages. Then try to reclaim 3000 bytes. + # This time, 3 pages should be pulled from the db cache. + execsql { SELECT * FROM abc } db2 + sqlite3_release_memory 3000 + list [nPage db] [nPage db2] +} {7 10} + + +do_test malloc5-6.3.1 { + # Now open a transaction and update 2 pages in the db2 cache. Then + # do a SELECT on the db cache so that all the db pages are more recently + # used than the db2 pages. When we try to free memory, SQLite should + # free the non-dirty db2 pages, then the db pages, then finally use + # sync() to free up the dirty db2 pages. The only page that cannot be + # freed is page1 of db2. Because there is an open transaction, the + # btree layer holds a reference to page 1 in the db2 cache. + execsql { + BEGIN; + UPDATE abc SET c = randstr(100,100) + WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc); + } db2 + execsql { SELECT * FROM abc } db + list [nPage db] [nPage db2] +} {10 10} +do_test malloc5-6.3.2 { + # Try to release 7700 bytes. This should release all the + # non-dirty pages held by db2. + sqlite3_release_memory [expr 7*1100] + list [nPage db] [nPage db2] +} {10 3} +do_test malloc5-6.3.3 { + # Try to release another 1000 bytes. This should come fromt the db + # cache, since all three pages held by db2 are either in-use or diry. + sqlite3_release_memory 1000 + list [nPage db] [nPage db2] +} {9 3} +do_test malloc5-6.3.4 { + # Now release 9900 more (about 9 pages worth). This should expunge + # the rest of the db cache. But the db2 cache remains intact, because + # SQLite tries to avoid calling sync(). + sqlite3_release_memory 9900 + list [nPage db] [nPage db2] +} {0 3} +do_test malloc5-6.3.5 { + # But if we are really insistent, SQLite will consent to call sync() + # if there is no other option. + sqlite3_release_memory 1000 + list [nPage db] [nPage db2] +} {0 2} +do_test malloc5-6.3.6 { + # The referenced page (page 1 of the db2 cache) will not be freed no + # matter how much memory we ask for: + sqlite3_release_memory 31459 + list [nPage db] [nPage db2] +} {0 1} + +db2 close sqlite3_soft_heap_limit $::soft_limit finish_test