From e277be054507c4f199f7a302d31472bbeecd511a Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Fri, 23 Mar 2007 18:12:06 +0000 Subject: [PATCH] Discard the contents of the pager-cache only when the change-counter indicates that it is stale. (CVS 3711) FossilOrigin-Name: 07b56965f3227c9f78680728b955395295c4aa49 --- manifest | 16 +-- manifest.uuid | 2 +- src/pager.c | 261 +++++++++++++++++++++++++++++++---------------- test/pager.test | 22 ++-- test/pager2.test | 6 +- 5 files changed, 198 insertions(+), 109 deletions(-) diff --git a/manifest b/manifest index 57de99d562..5b8424da45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sthe\sOsSectorSize()\sfunction.\s(CVS\s3710) -D 2007-03-23T10:08:39 +C Discard\sthe\scontents\sof\sthe\spager-cache\sonly\swhen\sthe\schange-counter\sindicates\sthat\sit\sis\sstale.\s(CVS\s3711) +D 2007-03-23T18:12:07 F Makefile.in 1fe3d0b46e40fd684e1e61f8e8056cefed16de9f F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -86,7 +86,7 @@ F src/os_unix.c 4642f23ed0c1ae0f1440db1d2b4231348af69360 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e F src/os_win.c 1d1d0989b0f235751504292c2f28e81044be0d70 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c dec2b25851190ab2d5b7d44065fae625c566dda5 +F src/pager.c da3c5e90b2c99bdba6129f568102c24cbd438452 F src/pager.h 9f6b5ef42c761deec8a9b1966b32e9a9dc89a631 F src/parse.y bcfe366c1fd61cfc40e5344eb69a31997a821af0 F src/pragma.c a3fe1dacdbf320ad99d4125a60a5bce8f1808bc8 @@ -270,8 +270,8 @@ F test/misc6.test 3de55ec5cadf466ada587173faa5d6a4790a8bb7 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82 F test/null.test 9503e1f63e959544c006d9f01709c5b5eab67d54 -F test/pager.test 6ee95e90ee8295e376e39a6c6566ef6df993601a -F test/pager2.test 49c0f57c7da0b060f0486b85fdd074025caa694e +F test/pager.test ceeef3caac4bf2046c6c06827f1a87dc1dec30f3 +F test/pager2.test c025f91b75fe65e85febda64d9416428b8a5cab5 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4 F test/pagesize.test 05c74ea49f790734ec1e9ab765d9bf1cce79b8f2 F test/pragma.test 91739ef06ab9ecf91e90d25951858caba71d6fe7 @@ -437,7 +437,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 177cd92910d01c97eb3133a59fad417edbb1aa92 -R a47ce4776623e0598aaf36f17e96e9f0 +P 0fd9983a98d8d61654f252f1708a4d7232a96b53 +R ec284a76fe569250a4f6f20542fb07fe U danielk1977 -Z 7cc396a55ca909ea4a46a72e6fbe0bfd +Z c13a7ab87a49cb94c1a31349a8579dca diff --git a/manifest.uuid b/manifest.uuid index 5909d31080..020cae7232 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fd9983a98d8d61654f252f1708a4d7232a96b53 \ No newline at end of file +07b56965f3227c9f78680728b955395295c4aa49 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index bf29bdb034..2b43d474a5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.292 2007/03/19 17:44:27 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.293 2007/03/23 18:12:07 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -285,7 +285,8 @@ struct Pager { Pager *pNext; /* Linked list of pagers in this thread */ #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ - int doNotSync; + int doNotSync; /* While true, do not spill the cache */ + u32 iChangeCount; /* Db change-counter for which cache is valid */ }; /* @@ -851,10 +852,6 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ /* ** Unlock the database file. -** -** Once all locks have been removed from the database file, other -** processes or threads might change the file. So make sure all of -** our internal cache is invalidated. */ static void pager_unlock(Pager *pPager){ if( !MEMDB ){ @@ -863,7 +860,20 @@ static void pager_unlock(Pager *pPager){ IOTRACE(("UNLOCK %p\n", pPager)) } pPager->state = PAGER_UNLOCK; - assert( pPager->pAll==0 ); +} + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. This is a no-op if the pager has already entered +** the error-state. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->errCode ) return; + if( pPager->state>=PAGER_RESERVED ){ + sqlite3PagerRollback(pPager); + } + pager_unlock(pPager); + assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) ); } @@ -889,12 +899,29 @@ static void pager_reset(Pager *pPager){ sqliteFree(pPager->aHash); pPager->nPage = 0; pPager->aHash = 0; - if( pPager->state>=PAGER_RESERVED ){ - sqlite3PagerRollback(pPager); - } - pager_unlock(pPager); pPager->nRef = 0; - assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) ); +} + +/* +** This function resets the various pager flags to their initial +** state but does not discard the cached content. +*/ +static void pagerSoftReset(Pager *pPager){ + PgHdr *pPg; + + assert(pPager->pStmt==0); + assert(pPager->nRef==0); + assert(pPager->pFirstSynced==pPager->pFirst); + assert(pPager->aInJournal==0); + + for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ + assert( pPg->nRef==0 ); + pPg->inJournal = 0; + pPg->inStmt = 0; + pPg->dirty = 0; + pPg->needSync = 0; + pPg->alwaysRollback = 0; + } } /* @@ -1278,7 +1305,7 @@ static int pager_truncate(Pager *pPager, int nPage){ ** If an I/O or malloc() error occurs, the journal-file is not deleted ** and an error code is returned. */ -static int pager_playback(Pager *pPager){ +static int pager_playback(Pager *pPager, int isHot){ i64 szJ; /* Size of the journal file in bytes */ u32 nRec; /* Number of Records in the journal */ int i; /* Loop counter */ @@ -1338,6 +1365,14 @@ static int pager_playback(Pager *pPager){ nRec = (szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager); } + /* If nRec is 0 and this rollback is of a transaction created by this + ** process. In this case the rest of the journal file consists of + ** journalled copies of pages that need to be read back into the cache. + */ + if( nRec==0 && !isHot ){ + nRec = (szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager); + } + /* If this is the first header read from the journal, truncate the ** database file back to it's original size. */ @@ -1361,10 +1396,6 @@ static int pager_playback(Pager *pPager){ pPager->journalOff = szJ; break; }else{ - /* If we are unable to rollback a hot journal, then the database - ** is probably not recoverable. Return CORRUPT. - */ - rc = SQLITE_CORRUPT; goto end_playback; } } @@ -2096,6 +2127,7 @@ int sqlite3PagerClose(Pager *pPager){ disable_simulated_io_errors(); pPager->errCode = 0; pager_reset(pPager); + pagerUnlockAndRollback(pPager); enable_simulated_io_errors(); TRACE2("CLOSE %d\n", PAGERID(pPager)); IOTRACE(("CLOSE %p\n", pPager)) @@ -2615,6 +2647,106 @@ int sqlite3PagerReleaseMemory(int nReq){ } #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ +/* +** This function is called to obtain the shared lock required before +** data may be read from the pager cache. If the shared lock has already +** been obtained, this function is a no-op. +*/ +static int pagerSharedLock(Pager *pPager){ + int rc = SQLITE_OK; + + if( pPager->state==PAGER_UNLOCK ){ + if( !MEMDB ){ + assert( pPager->nRef==0 ); + if( !pPager->noReadlock ){ + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + return pager_error(pPager, rc); + } + assert( pPager->state>=SHARED_LOCK ); + } + + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( hasHotJournal(pPager) ){ + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling it + ** back. + ** + ** Because the intermediate RESERVED lock is not requested, the + ** second process will get to this point in the code and fail to + ** obtain it's own EXCLUSIVE lock on the database file. + */ + rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + pager_unlock(pPager); + return pager_error(pPager, rc); + } + pPager->state = PAGER_EXCLUSIVE; + + /* Open the journal for reading only. Return SQLITE_BUSY if + ** we are unable to open the journal file. + ** + ** The journal file does not need to be locked itself. The + ** journal file is never open unless the main database file holds + ** a write lock, so there is never any chance of two or more + ** processes opening the journal at the same time. + */ + rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd); + if( rc!=SQLITE_OK ){ + pager_unlock(pPager); + return SQLITE_BUSY; + } + pPager->journalOpen = 1; + pPager->journalStarted = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. + */ + rc = pager_playback(pPager, 1); + if( rc!=SQLITE_OK ){ + return pager_error(pPager, rc); + } + } + + if( pPager->pAll ){ + PgHdr *pPage1 = pager_lookup(pPager, 1); + if( pPage1 ){ + unlinkHashChain(pPager, pPage1); + } + + assert( !pager_lookup(pPager, 1) ); + rc = sqlite3PagerAcquire(pPager, 1, &pPage1, 0); + if( rc==SQLITE_OK ){ + /* The change-counter is stored at offset 24. See also + ** pager_incr_changecounter(). + */ + u32 iChangeCount = retrieve32bits(pPage1, 24); + pPager->nRef++; + sqlite3PagerUnref(pPage1); + pPager->nRef--; + if( iChangeCount!=pPager->iChangeCount ){ + pager_reset(pPager); + }else{ + pagerSoftReset(pPager); + } + pPager->iChangeCount = iChangeCount; + } + } + } + pPager->state = PAGER_SHARED; + } + + return rc; +} + /* ** Acquire a page. ** @@ -2647,6 +2779,8 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ PgHdr *pPg; int rc; + assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 ); + /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page ** number greater than this, or zero, is requested. */ @@ -2665,71 +2799,13 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ /* If this is the first page accessed, then get a SHARED lock ** on the database file. */ - if( pPager->nRef==0 && !MEMDB ){ - if( !pPager->noReadlock ){ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - return pager_error(pPager, rc); - } - } - - /* If a journal file exists, and there is no RESERVED lock on the - ** database file, then it either needs to be played back or deleted. - */ - if( hasHotJournal(pPager) ){ - /* Get an EXCLUSIVE lock on the database file. At this point it is - ** important that a RESERVED lock is not obtained on the way to the - ** EXCLUSIVE lock. If it were, another process might open the - ** database file, detect the RESERVED lock, and conclude that the - ** database is safe to read while this process is still rolling it - ** back. - ** - ** Because the intermediate RESERVED lock is not requested, the - ** second process will get to this point in the code and fail to - ** obtain it's own EXCLUSIVE lock on the database file. - */ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - pager_unlock(pPager); - return pager_error(pPager, rc); - } - pPager->state = PAGER_EXCLUSIVE; - - /* Open the journal for reading only. Return SQLITE_BUSY if - ** we are unable to open the journal file. - ** - ** The journal file does not need to be locked itself. The - ** journal file is never open unless the main database file holds - ** a write lock, so there is never any chance of two or more - ** processes opening the journal at the same time. - */ - rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd); - if( rc!=SQLITE_OK ){ - pager_unlock(pPager); - return SQLITE_BUSY; - } - pPager->journalOpen = 1; - pPager->journalStarted = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - - /* Playback and delete the journal. Drop the database write - ** lock and reacquire the read lock. - */ - rc = pager_playback(pPager); - if( rc!=SQLITE_OK ){ - return pager_error(pPager, rc); - } - } - pPg = 0; - }else{ - /* Search for page in cache */ - pPg = pager_lookup(pPager, pgno); - if( MEMDB && pPager->state==PAGER_UNLOCK ){ - pPager->state = PAGER_SHARED; - } + rc = pagerSharedLock(pPager); + if( rc!=SQLITE_OK ){ + return rc; } + assert( pPager->state!=PAGER_UNLOCK ); + + pPg = pager_lookup(pPager, pgno); if( pPg==0 ){ /* The requested page is not in the page cache. */ int h; @@ -2768,7 +2844,8 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ if( rc!=SQLITE_OK ){ return rc; } - assert(pPg) ; + assert( pPager->state>=SHARED_LOCK ); + assert(pPg); } pPg->pgno = pgno; if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){ @@ -2841,6 +2918,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ #endif }else{ /* The requested page is in the page cache. */ + assert(pPager->nRef>0 || pgno==1); TEST_INCR(pPager->nHit); page_ref(pPg); } @@ -2864,7 +2942,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pPager!=0 ); assert( pgno!=0 ); - if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ + + if( pPager->state==PAGER_UNLOCK ){ + return 0; + } + if( (pPager->errCode && pPager->errCode!=SQLITE_FULL) ){ return 0; } pPg = pager_lookup(pPager, pgno); @@ -2917,8 +2999,9 @@ int sqlite3PagerUnref(DbPage *pPg){ */ pPager->nRef--; assert( pPager->nRef>=0 ); - if( pPager->nRef==0 && !MEMDB ){ - pager_reset(pPager); + if( pPager->nRef==0 ){ + /* pager_reset(pPager); */ + pagerUnlockAndRollback(pPager); } } return SQLITE_OK; @@ -3564,19 +3647,20 @@ int sqlite3PagerRollback(Pager *pPager){ if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ if( pPager->state>=PAGER_EXCLUSIVE ){ - pager_playback(pPager); + pager_playback(pPager, 0); } return pPager->errCode; } if( pPager->state==PAGER_RESERVED ){ int rc2; - rc = pager_reload_cache(pPager); + /* rc = pager_reload_cache(pPager); */ + rc = pager_playback(pPager, 0); rc2 = pager_unwritelock(pPager); if( rc==SQLITE_OK ){ rc = rc2; } }else{ - rc = pager_playback(pPager); + rc = pager_playback(pPager, 0); } pPager->dbSize = -1; @@ -3802,6 +3886,7 @@ static int pager_incr_changecounter(Pager *pPager){ /* Increment the value just read and write it back to byte 24. */ change_counter++; put32bits(((char*)PGHDR_TO_DATA(pPgHdr))+24, change_counter); + pPager->iChangeCount = change_counter; /* Release the page reference. */ sqlite3PagerUnref(pPgHdr); diff --git a/test/pager.test b/test/pager.test index 2363321185..176e00d7e6 100644 --- a/test/pager.test +++ b/test/pager.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # -# $Id: pager.test,v 1.25 2006/01/23 15:25:48 danielk1977 Exp $ +# $Id: pager.test,v 1.26 2007/03/23 18:12:07 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -103,9 +103,13 @@ do_test pager-2.8 { do_test pager-2.9 { page_unref $::g1 } {} + +# Update 24/03/2007: Even though the ref-count has dropped to zero, the +# pager-cache still contains some pages. Previously, it was always true +# that if there were no references to a pager it was empty. do_test pager-2.10 { pager_stats $::p1 -} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 1 ovfl 0} +} {ref 0 page 1 max 10 size -1 state 0 err 0 hit 0 miss 1 ovfl 0} do_test pager-2.11 { set ::g1 [page_get $::p1 1] expr {$::g1!=0} @@ -115,7 +119,7 @@ do_test pager-2.12 { } {1} do_test pager-2.13 { pager_stats $::p1 -} {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 2 ovfl 0} +} {ref 1 page 2 max 10 size 0 state 1 err 0 hit 1 miss 2 ovfl 0} do_test pager-2.14 { set v [catch { page_write $::g1 "Page-One" @@ -124,7 +128,7 @@ do_test pager-2.14 { } {0 {}} do_test pager-2.15 { pager_stats $::p1 -} {ref 1 page 1 max 10 size 1 state 2 err 0 hit 0 miss 2 ovfl 0} +} {ref 1 page 2 max 10 size 1 state 2 err 0 hit 1 miss 2 ovfl 0} do_test pager-2.16 { page_read $::g1 } {Page-One} @@ -136,19 +140,19 @@ do_test pager-2.17 { } {0 {}} do_test pager-2.20 { pager_stats $::p1 -} {ref 1 page 1 max 10 size -1 state 1 err 0 hit 1 miss 2 ovfl 0} +} {ref 1 page 2 max 10 size -1 state 1 err 0 hit 2 miss 2 ovfl 0} do_test pager-2.19 { pager_pagecount $::p1 } {1} do_test pager-2.21 { pager_stats $::p1 -} {ref 1 page 1 max 10 size 1 state 1 err 0 hit 1 miss 2 ovfl 0} +} {ref 1 page 2 max 10 size 1 state 1 err 0 hit 2 miss 2 ovfl 0} do_test pager-2.22 { page_unref $::g1 } {} do_test pager-2.23 { pager_stats $::p1 -} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 1 miss 2 ovfl 0} +} {ref 0 page 2 max 10 size -1 state 0 err 0 hit 2 miss 2 ovfl 0} do_test pager-2.24 { set v [catch { page_get $::p1 1 @@ -412,10 +416,10 @@ ifcapable memorydb { for {set i 1} {$i<5} {incr i} { set p [page_get $::p2 $i] page_write $p "Page $i" - page_unref $p pager_commit $::p2 + page_unref $p } - pager_truncate $::p2 3 + # pager_truncate $::p2 3 } {} do_test pager-4.6.4 { pager_close $::p2 diff --git a/test/pager2.test b/test/pager2.test index 3907257473..52dfe73e55 100644 --- a/test/pager2.test +++ b/test/pager2.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # -# $Id: pager2.test,v 1.5 2004/11/22 05:26:28 danielk1977 Exp $ +# $Id: pager2.test,v 1.6 2007/03/23 18:12:07 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -107,7 +107,7 @@ do_test pager2-2.9 { } {} do_test pager2-2.10 { pager_stats $::p1 -} {ref 0 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} +} {ref 0 page 1 max 10 size 0 state 0 err 0 hit 0 miss 1 ovfl 0} do_test pager2-2.11 { set ::g1 [page_get $::p1 1] expr {$::g1!=0} @@ -150,7 +150,7 @@ do_test pager2-2.22 { } {} do_test pager2-2.23 { pager_stats $::p1 -} {ref 0 page 1 max 10 size 1 state 1 err 0 hit 1 miss 1 ovfl 0} +} {ref 0 page 1 max 10 size 1 state 0 err 0 hit 1 miss 1 ovfl 0} do_test pager2-2.24 { set v [catch { page_get $::p1 1