Discard the contents of the pager-cache only when the change-counter indicates that it is stale. (CVS 3711)

FossilOrigin-Name: 07b56965f3227c9f78680728b955395295c4aa49

manifest

@@ -1,5 +1,5 @@
-C Add\sa\scomment\sto\sthe\sOsSectorSize()\sfunction.\s(CVS\s3710)
+C Discard\sthe\scontents\sof\sthe\spager-cache\sonly\swhen\sthe\schange-counter\sindicates\sthat\sit\sis\sstale.\s(CVS\s3711)
-D 2007-03-23T10:08:39
+D 2007-03-23T18:12:07
 F Makefile.in 1fe3d0b46e40fd684e1e61f8e8056cefed16de9f
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -86,7 +86,7 @@ F src/os_unix.c 4642f23ed0c1ae0f1440db1d2b4231348af69360
 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
 F src/os_win.c 1d1d0989b0f235751504292c2f28e81044be0d70
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c dec2b25851190ab2d5b7d44065fae625c566dda5
+F src/pager.c da3c5e90b2c99bdba6129f568102c24cbd438452
 F src/pager.h 9f6b5ef42c761deec8a9b1966b32e9a9dc89a631
 F src/parse.y bcfe366c1fd61cfc40e5344eb69a31997a821af0
 F src/pragma.c a3fe1dacdbf320ad99d4125a60a5bce8f1808bc8
@@ -270,8 +270,8 @@ F test/misc6.test 3de55ec5cadf466ada587173faa5d6a4790a8bb7
 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33
 F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82
 F test/null.test 9503e1f63e959544c006d9f01709c5b5eab67d54
-F test/pager.test 6ee95e90ee8295e376e39a6c6566ef6df993601a
+F test/pager.test ceeef3caac4bf2046c6c06827f1a87dc1dec30f3
-F test/pager2.test 49c0f57c7da0b060f0486b85fdd074025caa694e
+F test/pager2.test c025f91b75fe65e85febda64d9416428b8a5cab5
 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4
 F test/pagesize.test 05c74ea49f790734ec1e9ab765d9bf1cce79b8f2
 F test/pragma.test 91739ef06ab9ecf91e90d25951858caba71d6fe7
@@ -437,7 +437,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P 177cd92910d01c97eb3133a59fad417edbb1aa92
+P 0fd9983a98d8d61654f252f1708a4d7232a96b53
-R a47ce4776623e0598aaf36f17e96e9f0
+R ec284a76fe569250a4f6f20542fb07fe
 U danielk1977
-Z 7cc396a55ca909ea4a46a72e6fbe0bfd
+Z c13a7ab87a49cb94c1a31349a8579dca

manifest.uuid

@@ -1 +1 @@
-0fd9983a98d8d61654f252f1708a4d7232a96b53
+07b56965f3227c9f78680728b955395295c4aa49

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.292 2007/03/19 17:44:27 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.293 2007/03/23 18:12:07 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -285,7 +285,8 @@ struct Pager {
   Pager *pNext;               /* Linked list of pagers in this thread */
 #endif
   char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
-  int doNotSync;
+  int doNotSync;              /* While true, do not spill the cache */
+  u32 iChangeCount;           /* Db change-counter for which cache is valid */
 };
 
 /*
@@ -851,10 +852,6 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){
 
 /*
 ** Unlock the database file.
-**
-** Once all locks have been removed from the database file, other
-** processes or threads might change the file.  So make sure all of
-** our internal cache is invalidated.
 */
 static void pager_unlock(Pager *pPager){
   if( !MEMDB ){
@@ -863,7 +860,20 @@ static void pager_unlock(Pager *pPager){
     IOTRACE(("UNLOCK %p\n", pPager))
   }
   pPager->state = PAGER_UNLOCK;
-  assert( pPager->pAll==0 );
+}
+
+/*
+** Execute a rollback if a transaction is active and unlock the 
+** database file. This is a no-op if the pager has already entered
+** the error-state.
+*/
+static void pagerUnlockAndRollback(Pager *pPager){
+  if( pPager->errCode ) return;
+  if( pPager->state>=PAGER_RESERVED ){
+    sqlite3PagerRollback(pPager);
+  }
+  pager_unlock(pPager);
+  assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) );
 }
 
 
@@ -889,12 +899,29 @@ static void pager_reset(Pager *pPager){
   sqliteFree(pPager->aHash);
   pPager->nPage = 0;
   pPager->aHash = 0;
-  if( pPager->state>=PAGER_RESERVED ){
-    sqlite3PagerRollback(pPager);
-  }
-  pager_unlock(pPager);
   pPager->nRef = 0;
-  assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) );
+}
+
+/*
+** This function resets the various pager flags to their initial
+** state but does not discard the cached content.
+*/
+static void pagerSoftReset(Pager *pPager){
+  PgHdr *pPg;
+
+  assert(pPager->pStmt==0);
+  assert(pPager->nRef==0);
+  assert(pPager->pFirstSynced==pPager->pFirst);
+  assert(pPager->aInJournal==0);
+
+  for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
+    assert( pPg->nRef==0 );
+    pPg->inJournal = 0;
+    pPg->inStmt = 0;
+    pPg->dirty = 0;
+    pPg->needSync = 0;
+    pPg->alwaysRollback = 0;
+  }
 }
 
 /*
@@ -1278,7 +1305,7 @@ static int pager_truncate(Pager *pPager, int nPage){
 ** If an I/O or malloc() error occurs, the journal-file is not deleted
 ** and an error code is returned.
 */
-static int pager_playback(Pager *pPager){
+static int pager_playback(Pager *pPager, int isHot){
   i64 szJ;                 /* Size of the journal file in bytes */
   u32 nRec;                /* Number of Records in the journal */
   int i;                   /* Loop counter */
@@ -1338,6 +1365,14 @@ static int pager_playback(Pager *pPager){
       nRec = (szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager);
     }
 
+    /* If nRec is 0 and this rollback is of a transaction created by this
+    ** process. In this case the rest of the journal file consists of
+    ** journalled copies of pages that need to be read back into the cache.
+    */
+    if( nRec==0 && !isHot ){
+      nRec = (szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager);
+    }
+
     /* If this is the first header read from the journal, truncate the
     ** database file back to it's original size.
     */
@@ -1361,10 +1396,6 @@ static int pager_playback(Pager *pPager){
           pPager->journalOff = szJ;
           break;
         }else{
-          /* If we are unable to rollback a hot journal, then the database
-          ** is probably not recoverable.  Return CORRUPT.
-          */
-          rc = SQLITE_CORRUPT;
           goto end_playback;
         }
       }
@@ -2096,6 +2127,7 @@ int sqlite3PagerClose(Pager *pPager){
   disable_simulated_io_errors();
   pPager->errCode = 0;
   pager_reset(pPager);
+  pagerUnlockAndRollback(pPager);
   enable_simulated_io_errors();
   TRACE2("CLOSE %d\n", PAGERID(pPager));
   IOTRACE(("CLOSE %p\n", pPager))
@@ -2615,6 +2647,106 @@ int sqlite3PagerReleaseMemory(int nReq){
 }
 #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */
 
+/*
+** This function is called to obtain the shared lock required before
+** data may be read from the pager cache. If the shared lock has already
+** been obtained, this function is a no-op.
+*/
+static int pagerSharedLock(Pager *pPager){
+  int rc = SQLITE_OK;
+
+  if( pPager->state==PAGER_UNLOCK ){
+    if( !MEMDB ){
+      assert( pPager->nRef==0 );
+      if( !pPager->noReadlock ){
+        rc = pager_wait_on_lock(pPager, SHARED_LOCK);
+        if( rc!=SQLITE_OK ){
+          return pager_error(pPager, rc);
+        }
+        assert( pPager->state>=SHARED_LOCK );
+      }
+  
+      /* If a journal file exists, and there is no RESERVED lock on the
+      ** database file, then it either needs to be played back or deleted.
+      */
+      if( hasHotJournal(pPager) ){
+        /* Get an EXCLUSIVE lock on the database file. At this point it is
+        ** important that a RESERVED lock is not obtained on the way to the
+        ** EXCLUSIVE lock. If it were, another process might open the
+        ** database file, detect the RESERVED lock, and conclude that the
+        ** database is safe to read while this process is still rolling it 
+        ** back.
+        ** 
+        ** Because the intermediate RESERVED lock is not requested, the
+        ** second process will get to this point in the code and fail to
+        ** obtain it's own EXCLUSIVE lock on the database file.
+        */
+        rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK);
+        if( rc!=SQLITE_OK ){
+          pager_unlock(pPager);
+          return pager_error(pPager, rc);
+        }
+        pPager->state = PAGER_EXCLUSIVE;
+ 
+        /* Open the journal for reading only.  Return SQLITE_BUSY if
+        ** we are unable to open the journal file. 
+        **
+        ** The journal file does not need to be locked itself.  The
+        ** journal file is never open unless the main database file holds
+        ** a write lock, so there is never any chance of two or more
+        ** processes opening the journal at the same time.
+        */
+        rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd);
+        if( rc!=SQLITE_OK ){
+          pager_unlock(pPager);
+          return SQLITE_BUSY;
+        }
+        pPager->journalOpen = 1;
+        pPager->journalStarted = 0;
+        pPager->journalOff = 0;
+        pPager->setMaster = 0;
+        pPager->journalHdr = 0;
+ 
+        /* Playback and delete the journal.  Drop the database write
+        ** lock and reacquire the read lock.
+        */
+        rc = pager_playback(pPager, 1);
+        if( rc!=SQLITE_OK ){
+          return pager_error(pPager, rc);
+        }
+      }
+
+      if( pPager->pAll ){
+        PgHdr *pPage1 = pager_lookup(pPager, 1);
+        if( pPage1 ){
+          unlinkHashChain(pPager, pPage1);
+        }
+
+        assert( !pager_lookup(pPager, 1) );
+        rc = sqlite3PagerAcquire(pPager, 1, &pPage1, 0);
+        if( rc==SQLITE_OK ){
+	  /* The change-counter is stored at offset 24. See also
+          ** pager_incr_changecounter().
+          */
+          u32 iChangeCount = retrieve32bits(pPage1, 24);
+          pPager->nRef++;
+          sqlite3PagerUnref(pPage1);
+          pPager->nRef--;
+          if( iChangeCount!=pPager->iChangeCount ){
+            pager_reset(pPager);
+          }else{
+            pagerSoftReset(pPager);
+          }
+          pPager->iChangeCount = iChangeCount;
+        }
+      }
+    }
+    pPager->state = PAGER_SHARED;
+  }
+
+  return rc;
+}
+
 /*
 ** Acquire a page.
 **
@@ -2647,6 +2779,8 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
   PgHdr *pPg;
   int rc;
 
+  assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 );
+
   /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page
   ** number greater than this, or zero, is requested.
   */
@@ -2665,71 +2799,13 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
   /* If this is the first page accessed, then get a SHARED lock
   ** on the database file.
   */
-  if( pPager->nRef==0 && !MEMDB ){
+  rc = pagerSharedLock(pPager);
-    if( !pPager->noReadlock ){
+  if( rc!=SQLITE_OK ){
-      rc = pager_wait_on_lock(pPager, SHARED_LOCK);
+    return rc;
-      if( rc!=SQLITE_OK ){
-        return pager_error(pPager, rc);
-      }
-    }
-
-    /* If a journal file exists, and there is no RESERVED lock on the
-    ** database file, then it either needs to be played back or deleted.
-    */
-    if( hasHotJournal(pPager) ){
-       /* Get an EXCLUSIVE lock on the database file. At this point it is
-       ** important that a RESERVED lock is not obtained on the way to the
-       ** EXCLUSIVE lock. If it were, another process might open the
-       ** database file, detect the RESERVED lock, and conclude that the
-       ** database is safe to read while this process is still rolling it 
-       ** back.
-       ** 
-       ** Because the intermediate RESERVED lock is not requested, the
-       ** second process will get to this point in the code and fail to
-       ** obtain it's own EXCLUSIVE lock on the database file.
-       */
-       rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK);
-       if( rc!=SQLITE_OK ){
-         pager_unlock(pPager);
-         return pager_error(pPager, rc);
-       }
-       pPager->state = PAGER_EXCLUSIVE;
-
-       /* Open the journal for reading only.  Return SQLITE_BUSY if
-       ** we are unable to open the journal file. 
-       **
-       ** The journal file does not need to be locked itself.  The
-       ** journal file is never open unless the main database file holds
-       ** a write lock, so there is never any chance of two or more
-       ** processes opening the journal at the same time.
-       */
-       rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd);
-       if( rc!=SQLITE_OK ){
-         pager_unlock(pPager);
-         return SQLITE_BUSY;
-       }
-       pPager->journalOpen = 1;
-       pPager->journalStarted = 0;
-       pPager->journalOff = 0;
-       pPager->setMaster = 0;
-       pPager->journalHdr = 0;
-
-       /* Playback and delete the journal.  Drop the database write
-       ** lock and reacquire the read lock.
-       */
-       rc = pager_playback(pPager);
-       if( rc!=SQLITE_OK ){
-         return pager_error(pPager, rc);
-       }
-    }
-    pPg = 0;
-  }else{
-    /* Search for page in cache */
-    pPg = pager_lookup(pPager, pgno);
-    if( MEMDB && pPager->state==PAGER_UNLOCK ){
-      pPager->state = PAGER_SHARED;
-    }
   }
+  assert( pPager->state!=PAGER_UNLOCK );
+
+  pPg = pager_lookup(pPager, pgno);
   if( pPg==0 ){
     /* The requested page is not in the page cache. */
     int h;
@@ -2768,7 +2844,8 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
       if( rc!=SQLITE_OK ){
         return rc;
       }
-      assert(pPg) ;
+      assert( pPager->state>=SHARED_LOCK );
+      assert(pPg);
     }
     pPg->pgno = pgno;
     if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
@@ -2841,6 +2918,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
 #endif
   }else{
     /* The requested page is in the page cache. */
+    assert(pPager->nRef>0 || pgno==1);
     TEST_INCR(pPager->nHit);
     page_ref(pPg);
   }
@@ -2864,7 +2942,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){
 
   assert( pPager!=0 );
   assert( pgno!=0 );
-  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
+
+  if( pPager->state==PAGER_UNLOCK ){
+    return 0;
+  }
+  if( (pPager->errCode && pPager->errCode!=SQLITE_FULL) ){
     return 0;
   }
   pPg = pager_lookup(pPager, pgno);
@@ -2917,8 +2999,9 @@ int sqlite3PagerUnref(DbPage *pPg){
     */
     pPager->nRef--;
     assert( pPager->nRef>=0 );
-    if( pPager->nRef==0 && !MEMDB ){
+    if( pPager->nRef==0 ){
-      pager_reset(pPager);
+      /* pager_reset(pPager); */
+      pagerUnlockAndRollback(pPager);
     }
   }
   return SQLITE_OK;
@@ -3564,19 +3647,20 @@ int sqlite3PagerRollback(Pager *pPager){
 
   if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
     if( pPager->state>=PAGER_EXCLUSIVE ){
-      pager_playback(pPager);
+      pager_playback(pPager, 0);
     }
     return pPager->errCode;
   }
   if( pPager->state==PAGER_RESERVED ){
     int rc2;
-    rc = pager_reload_cache(pPager);
+    /* rc = pager_reload_cache(pPager); */
+    rc = pager_playback(pPager, 0);
     rc2 = pager_unwritelock(pPager);
     if( rc==SQLITE_OK ){
       rc = rc2;
     }
   }else{
-    rc = pager_playback(pPager);
+    rc = pager_playback(pPager, 0);
   }
   pPager->dbSize = -1;
 
@@ -3802,6 +3886,7 @@ static int pager_incr_changecounter(Pager *pPager){
   /* Increment the value just read and write it back to byte 24. */
   change_counter++;
   put32bits(((char*)PGHDR_TO_DATA(pPgHdr))+24, change_counter);
+  pPager->iChangeCount = change_counter;
 
   /* Release the page reference. */
   sqlite3PagerUnref(pPgHdr);

test/pager.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is page cache subsystem.
 #
-# $Id: pager.test,v 1.25 2006/01/23 15:25:48 danielk1977 Exp $
+# $Id: pager.test,v 1.26 2007/03/23 18:12:07 danielk1977 Exp $
 
 
 set testdir [file dirname $argv0]
@@ -103,9 +103,13 @@ do_test pager-2.8 {
 do_test pager-2.9 {
   page_unref $::g1
 } {}
+
+# Update 24/03/2007: Even though the ref-count has dropped to zero, the
+# pager-cache still contains some pages. Previously, it was always true
+# that if there were no references to a pager it was empty.
 do_test pager-2.10 {
   pager_stats $::p1
-} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 1 ovfl 0}
+} {ref 0 page 1 max 10 size -1 state 0 err 0 hit 0 miss 1 ovfl 0}
 do_test pager-2.11 {
   set ::g1 [page_get $::p1 1]
   expr {$::g1!=0}
@@ -115,7 +119,7 @@ do_test pager-2.12 {
 } {1}
 do_test pager-2.13 {
   pager_stats $::p1
-} {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 2 ovfl 0}
+} {ref 1 page 2 max 10 size 0 state 1 err 0 hit 1 miss 2 ovfl 0}
 do_test pager-2.14 {
   set v [catch {
     page_write $::g1 "Page-One"
@@ -124,7 +128,7 @@ do_test pager-2.14 {
 } {0 {}}
 do_test pager-2.15 {
   pager_stats $::p1
-} {ref 1 page 1 max 10 size 1 state 2 err 0 hit 0 miss 2 ovfl 0}
+} {ref 1 page 2 max 10 size 1 state 2 err 0 hit 1 miss 2 ovfl 0}
 do_test pager-2.16 {
   page_read $::g1
 } {Page-One}
@@ -136,19 +140,19 @@ do_test pager-2.17 {
 } {0 {}}
 do_test pager-2.20 {
   pager_stats $::p1
-} {ref 1 page 1 max 10 size -1 state 1 err 0 hit 1 miss 2 ovfl 0}
+} {ref 1 page 2 max 10 size -1 state 1 err 0 hit 2 miss 2 ovfl 0}
 do_test pager-2.19 {
   pager_pagecount $::p1
 } {1}
 do_test pager-2.21 {
   pager_stats $::p1
-} {ref 1 page 1 max 10 size 1 state 1 err 0 hit 1 miss 2 ovfl 0}
+} {ref 1 page 2 max 10 size 1 state 1 err 0 hit 2 miss 2 ovfl 0}
 do_test pager-2.22 {
   page_unref $::g1
 } {}
 do_test pager-2.23 {
   pager_stats $::p1
-} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 1 miss 2 ovfl 0}
+} {ref 0 page 2 max 10 size -1 state 0 err 0 hit 2 miss 2 ovfl 0}
 do_test pager-2.24 {
   set v [catch {
     page_get $::p1 1
@@ -412,10 +416,10 @@ ifcapable memorydb {
     for {set i 1} {$i<5} {incr i} {
       set p [page_get $::p2 $i]
       page_write $p "Page $i"
-      page_unref $p
       pager_commit $::p2
+      page_unref $p
     }
-    pager_truncate $::p2 3
+    # pager_truncate $::p2 3
   } {}
   do_test pager-4.6.4 {
     pager_close $::p2

test/pager2.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is page cache subsystem.
 #
-# $Id: pager2.test,v 1.5 2004/11/22 05:26:28 danielk1977 Exp $
+# $Id: pager2.test,v 1.6 2007/03/23 18:12:07 danielk1977 Exp $
 
 
 set testdir [file dirname $argv0]
@@ -107,7 +107,7 @@ do_test pager2-2.9 {
 } {}
 do_test pager2-2.10 {
   pager_stats $::p1
-} {ref 0 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0}
+} {ref 0 page 1 max 10 size 0 state 0 err 0 hit 0 miss 1 ovfl 0}
 do_test pager2-2.11 {
   set ::g1 [page_get $::p1 1]
   expr {$::g1!=0}
@@ -150,7 +150,7 @@ do_test pager2-2.22 {
 } {}
 do_test pager2-2.23 {
   pager_stats $::p1
-} {ref 0 page 1 max 10 size 1 state 1 err 0 hit 1 miss 1 ovfl 0}
+} {ref 0 page 1 max 10 size 1 state 0 err 0 hit 1 miss 1 ovfl 0}
 do_test pager2-2.24 {
   set v [catch {
     page_get $::p1 1