Add some comments and test-cases for the global lru page list (used by sqlite3_release_memory()). (CVS 4308)

FossilOrigin-Name: 0b80168895993af5774369f839f284712d006f0a

manifest

@@ -1,5 +1,5 @@
-C Work\stoward\scorrect\sbtree\slocking\sin\sa\smultithreaded\senvironment.\s(CVS\s4307)
+C Add\ssome\scomments\sand\stest-cases\sfor\sthe\sglobal\slru\spage\slist\s(used\sby\ssqlite3_release_memory()).\s(CVS\s4308)
-D 2007-08-28T02:27:52
+D 2007-08-28T08:00:18
 F Makefile.in e8296e112b8942a96c0ed504398bd0d43e3c67ce
 F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -116,7 +116,7 @@ F src/os_unix.c 89bf24aa2475048a7833c45c522e7c6a81b83bb8
 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
 F src/os_win.c 3ffd3aacff4cb69848284e29dcec0feff23b0752
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c cfa6dc38b797206549491de3ec7f0aea50611dda
+F src/pager.c 51ca27639ab25c8838afc856d3cc6045a98752a7
 F src/pager.h 53087c6fb9db01aed17c7fd044662a27507e89b8
 F src/parse.y 2d2ce439dc6184621fb0b86f4fc5aca7f391a590
 F src/pragma.c 9b989506a1b7c8aecd6befb8235e2f57a4aba7e5
@@ -345,7 +345,7 @@ F test/malloc.test dbfaedfca734283182db18a64416cf037c33648f
 F test/malloc2.test 96dfa6318d3b7bcd41d4fe7e24d35196d465bf58
 F test/malloc3.test 65d323508c7c4e1fb5569d03a11070b0259ad2fe
 F test/malloc4.test cf1e856fbc8ebde562838cb47adce0ebc2d4f7d2
-F test/malloc5.test 6f7b96f1bea181d789a7140fd3fabfb0e333d8f5
+F test/malloc5.test 9db9205a5d444e23043368f961f92641c7770199
 F test/malloc6.test 3733b9bd4e039c3239f869c40edbb88172025e2e
 F test/malloc7.test dd66d8f82916becf1d29b6640e4f4855485570f8
 F test/malloc8.test 5ff95278bc73e815e295971afcdd175f6ba19258
@@ -563,7 +563,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 741d6fb096dcb232871d3a8468c386022afcf554
+P b8cc493b47e618648f645ab73eb0253739e03fcd
-R ff3e167d1bd072e0909067b1f2888076
+R 89ac354b82d38fc45db7b69d7f01c4df
-U drh
+U danielk1977
-Z 6d8ca04435be2f950a1ddef0045b914b
+Z 4eb080ba40501ea8bb2df84791a3ceaa

manifest.uuid

@@ -1 +1 @@
-b8cc493b47e618648f645ab73eb0253739e03fcd
+0b80168895993af5774369f839f284712d006f0a

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.377 2007/08/27 17:27:49 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.378 2007/08/28 08:00:18 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -132,6 +132,33 @@
 #define FORCE_ALIGNMENT(X)   (((X)+7)&~7)
 
 typedef struct PgHdr PgHdr;
+
+/*
+** Each pager stores all currently unreferenced pages in a list sorted
+** in least-recently-used (LRU) order (i.e. the first item on the list has 
+** not been referenced in a long time, the last item has been recently
+** used). An instance of this structure is included as part of each
+** pager structure for this purpose (variable Pager.lru).
+**
+** Additionally, if memory-management is enabled, all unreferenced pages 
+** are stored in a global LRU list (global variable sqlite3LruPageList).
+**
+** In both cases, the PagerLruList.pFirstSynced variable points to
+** the first page in the corresponding list that does not require an
+** fsync() operation before it's memory can be reclaimed. If no such
+** page exists, PagerLruList.pFirstSynced is set to NULL.
+*/
+typedef struct PagerLruList PagerLruList;
+struct PagerLruList {
+  PgHdr *pFirst;         /* First page in LRU list */
+  PgHdr *pLast;          /* Last page in LRU list (the most recently used) */
+  PgHdr *pFirstSynced;   /* First page in list with PgHdr.needSync==0 */
+};
+
+/*
+** The following structure contains the next and previous pointers used
+** to link a PgHdr structure into a PagerLruList linked list. 
+*/
 typedef struct PagerLruLink PagerLruLink;
 struct PagerLruLink {
   PgHdr *pNext;
@@ -292,13 +319,6 @@ struct PgHistory {
 #define PGHDR_TO_HIST(P,PGR)  \
             ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra])
 
-typedef struct PagerLruList PagerLruList;
-struct PagerLruList {
-  PgHdr *pFirst;
-  PgHdr *pLast;
-  PgHdr *pFirstSynced;   /* First page in list with PgHdr.needSync==0 */
-};
-
 /*
 ** A open page cache is an instance of the following structure.
 **
@@ -531,10 +551,22 @@ static const unsigned char aJournalMagic[] = {
 # define REFINFO(X)
 #endif
 
+/*
+** Add page pPg to the end of the linked list managed by structure
+** pList (pPg becomes the last entry in the list - the most recently 
+** used). Argument pLink should point to either pPg->free or pPg->gfree,
+** depending on whether pPg is being added to the pager-specific or
+** global LRU list.
+*/
 static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
   pLink->pNext = 0;
   pLink->pPrev = pList->pLast;
 
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
+  assert(pLink==&pPg->free || pLink==&pPg->gfree);
+  assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList);
+#endif
+
   if( pList->pLast ){
     int iOff = (char *)pLink - (char *)pPg;
     PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]);
@@ -550,9 +582,20 @@ static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
   }
 }
 
+/*
+** Remove pPg from the list managed by the structure pointed to by pList.
+**
+** Argument pLink should point to either pPg->free or pPg->gfree, depending 
+** on whether pPg is being added to the pager-specific or global LRU list.
+*/
 static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
   int iOff = (char *)pLink - (char *)pPg;
 
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
+  assert(pLink==&pPg->free || pLink==&pPg->gfree);
+  assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList);
+#endif
+
   if( pPg==pList->pFirst ){
     pList->pFirst = pLink->pNext;
   }
@@ -580,7 +623,9 @@ static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
 }
 
 /* 
-** Add page to the free-list 
+** Add page pPg to the list of free pages for the pager. If 
+** memory-management is enabled, also add the page to the global 
+** list of free pages.
 */
 static void lruListAdd(PgHdr *pPg){
   listAdd(&pPg->pPager->lru, &pPg->free, pPg);
@@ -594,7 +639,9 @@ static void lruListAdd(PgHdr *pPg){
 }
 
 /* 
-** Remove page from free-list 
+** Remove page pPg from the list of free pages for the associated pager.
+** If memory-management is enabled, also remove pPg from the global list
+** of free pages.
 */
 static void lruListRemove(PgHdr *pPg){
   listRemove(&pPg->pPager->lru, &pPg->free, pPg);
@@ -608,7 +655,11 @@ static void lruListRemove(PgHdr *pPg){
 }
 
 /* 
-** Set the Pager.pFirstSynced variable 
+** This function is called just after the needSync flag has been cleared
+** from all pages managed by pPager (usually because the journal file
+** has just been synced). It updates the pPager->lru.pFirstSynced variable
+** and, if memory-management is enabled, the sqlite3LruPageList.pFirstSynced
+** variable also.
 */
 static void lruListSetFirstSynced(Pager *pPager){
   pPager->lru.pFirstSynced = pPager->lru.pFirst;
@@ -3056,6 +3107,10 @@ int sqlite3PagerReleaseMemory(int nReq){
     }
     sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
 
+    /* If pPg==0, then the block above has failed to find a page to
+    ** recycle. In this case return early - no further memory will
+    ** be released.
+    */
     if( !pPg ) break;
 
     pPager = pPg->pPager;
@@ -3089,6 +3144,7 @@ int sqlite3PagerReleaseMemory(int nReq){
       IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno));
       PAGER_INCR(sqlite3_pager_pgfree_count);
       sqlite3_free(pPg);
+      pPager->nPage--;
     }else{
       /* An error occured whilst writing to the database file or 
       ** journal in pager_recycle(). The error is not returned to the 

test/malloc5.test

@@ -12,7 +12,7 @@
 # This file contains test cases focused on the two memory-management APIs, 
 # sqlite3_soft_heap_limit() and sqlite3_release_memory().
 #
-# $Id: malloc5.test,v 1.13 2007/08/22 22:04:37 drh Exp $
+# $Id: malloc5.test,v 1.14 2007/08/28 08:00:18 danielk1977 Exp $
 
 #---------------------------------------------------------------------------
 # NOTES ON EXPECTED BEHAVIOUR
@@ -248,7 +248,7 @@ do_test malloc5-5.1 {
   }
   sqlite3_release_memory
 } 0
-do_test malloc5-5.1 {
+do_test malloc5-5.2 {
   sqlite3_soft_heap_limit 5000
   execsql {
     COMMIT;
@@ -257,6 +257,122 @@ do_test malloc5-5.1 {
   }
   expr 1
 } {1}
+sqlite3_soft_heap_limit $::soft_limit
+
+#-------------------------------------------------------------------------
+# The following test cases (malloc5-6.*) test the new global LRU list
+# used to determine the pages to recycle when sqlite3_release_memory is
+# called and there is more than one pager open.
+#
+proc nPage {db} {
+  set bt [btree_from_db $db]
+  array set stats [btree_pager_stats $bt]
+  set stats(page)
+}
+db close
+file delete -force test.db test.db-journal test2.db test2.db-journal
+
+# This block of test-cases (malloc5-6.1.*) prepares two database files
+# for the subsequent tests.
+do_test malloc5-6.1.1 {
+  sqlite3 db test.db
+  execsql {
+    PRAGMA page_size=1024;
+    PRAGMA default_cache_size=10;
+    BEGIN;
+    CREATE TABLE abc(a PRIMARY KEY, b, c);
+    INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    INSERT INTO abc 
+        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
+    COMMIT;
+  } 
+  copy_file test.db test2.db
+  sqlite3 db2 test2.db
+  list [expr [file size test.db]/1024] [expr [file size test2.db]/1024]
+} {23 23}
+do_test malloc5-6.1.2 {
+  list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
+} {10 10}
+
+do_test malloc5-6.2.1 {
+  execsql { SELECT * FROM abc } db2
+  execsql {SELECT * FROM abc} db
+  list [nPage db] [nPage db2]
+} {10 10}
+do_test malloc5-6.2.2 {
+  # If we now try to reclaim some memory, it should come from the db2 cache.
+  sqlite3_release_memory 3000
+  list [nPage db] [nPage db2]
+} {10 7}
+do_test malloc5-6.2.3 {
+  # Access the db2 cache again, so that all the db2 pages have been used
+  # more recently than all the db pages. Then try to reclaim 3000 bytes.
+  # This time, 3 pages should be pulled from the db cache.
+  execsql { SELECT * FROM abc } db2
+  sqlite3_release_memory 3000
+  list [nPage db] [nPage db2]
+} {7 10}
+
+
+do_test malloc5-6.3.1 {
+  # Now open a transaction and update 2 pages in the db2 cache. Then
+  # do a SELECT on the db cache so that all the db pages are more recently
+  # used than the db2 pages. When we try to free memory, SQLite should
+  # free the non-dirty db2 pages, then the db pages, then finally use
+  # sync() to free up the dirty db2 pages. The only page that cannot be
+  # freed is page1 of db2. Because there is an open transaction, the
+  # btree layer holds a reference to page 1 in the db2 cache.
+  execsql {
+    BEGIN;
+    UPDATE abc SET c = randstr(100,100) 
+    WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);
+  } db2
+  execsql { SELECT * FROM abc } db
+  list [nPage db] [nPage db2]
+} {10 10}
+do_test malloc5-6.3.2 {
+  # Try to release 7700 bytes. This should release all the 
+  # non-dirty pages held by db2.
+  sqlite3_release_memory [expr 7*1100]
+  list [nPage db] [nPage db2]
+} {10 3}
+do_test malloc5-6.3.3 {
+  # Try to release another 1000 bytes. This should come fromt the db
+  # cache, since all three pages held by db2 are either in-use or diry.
+  sqlite3_release_memory 1000
+  list [nPage db] [nPage db2]
+} {9 3}
+do_test malloc5-6.3.4 {
+  # Now release 9900 more (about 9 pages worth). This should expunge
+  # the rest of the db cache. But the db2 cache remains intact, because
+  # SQLite tries to avoid calling sync().
+  sqlite3_release_memory 9900
+  list [nPage db] [nPage db2]
+} {0 3}
+do_test malloc5-6.3.5 {
+  # But if we are really insistent, SQLite will consent to call sync()
+  # if there is no other option.
+  sqlite3_release_memory 1000
+  list [nPage db] [nPage db2]
+} {0 2}
+do_test malloc5-6.3.6 {
+  # The referenced page (page 1 of the db2 cache) will not be freed no
+  # matter how much memory we ask for:
+  sqlite3_release_memory 31459
+  list [nPage db] [nPage db2]
+} {0 1}
+
+db2 close
 
 sqlite3_soft_heap_limit $::soft_limit
 finish_test