Fix a problem with savepoint and incremental-vacuum. (CVS 6066)

FossilOrigin-Name: 08352f9ea9d2a1759320efc46e418079000855cb

manifest

@@ -1,5 +1,5 @@
-C Reset\sthe\scolumn\scache\sbefore\scoding\seach\sstep\sof\sa\strigger\sprogram.\sCandidate\sfix\sfor\s#3554.\s(CVS\s6065)
+C Fix\sa\sproblem\swith\ssavepoint\sand\sincremental-vacuum.\s(CVS\s6066)
-D 2008-12-26T07:56:39
+D 2008-12-27T15:23:13
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 77635d0909c2067cee03889a1e04ce910d8fb809
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -103,9 +103,9 @@ F src/attach.c 1c35f95da3c62d19de75b44cfefd12c81c1791b3
 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627
 F src/bitvec.c 4300d311b17fb3c1476623fd895a8feac02a0b08
 F src/btmutex.c 63c5cc4ad5715690767ffcb741e185d7bc35ec1a
-F src/btree.c f695109b39fc104bd2f904b1a2d483d44faf9c86
+F src/btree.c 581fdccd7b6539a8e37e843f9b45e0557b3b272f
 F src/btree.h 4f141cf748d2ee7c6d7fc175f64f87a45cd44113
-F src/btreeInt.h 7ef2c872371d7508657f8d7a4efe651c741d6ee6
+F src/btreeInt.h 8fea5cd7021cb8848fc2183a3e909469659daa0a
 F src/build.c 92335a6c6a7c119580be605c5dd1439602d6cf5d
 F src/callback.c bee8949d619b1b7b1e4dfac8a19c5116ae1dd12a
 F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c
@@ -142,8 +142,8 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60
 F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5
 F src/os_unix.c e6eacc7ec735ded605fefcbaf250058baa8feb12
 F src/os_win.c 496e3ceb499aedc63622a89ef76f7af2dd902709
-F src/pager.c 67af495bcd0723a1574dcdefebeac8edc3fee492
+F src/pager.c ec486337fae32dfcaac41f4471c70a0385fe8487
-F src/pager.h 7191294438881eb4d13eedade97891e8dc993905
+F src/pager.h 0579740d4c18826b46124c82330467b41f407eb1
 F src/parse.y 4d0e33a702dc3ea7b69d8ae1914b3fbd32e46057
 F src/pcache.c 16dc8da6e6ba6250f8dfd9ee46036db1cbceedc6
 F src/pcache.h f20c3e82dd6da622c3fe296170cb1801f9a2d75a
@@ -385,7 +385,7 @@ F test/in4.test 9bfd9226a82ac832046abc93acecf42627ebb45a
 F test/incrblob.test 4b9437bbb38724343dadbbcca6356bc2a9b435d1
 F test/incrblob2.test 5cca1c3cb29064c504b3b0cc3e2cd43e8053cfdf
 F test/incrblob_err.test c577c91d4ed9e8336cdb188b15d6ee2a6fe9604e
-F test/incrvacuum.test 9a6346c56ffa141024054ae7ba6c8655edf2d137
+F test/incrvacuum.test 6ef5877f26d1e7bc9bc137ea8f8f069a260ff9c6
 F test/incrvacuum2.test 46ef65f377e3937cfd1ba66e818309dab46f590d
 F test/incrvacuum_ioerr.test 57d2f5777ab13fa03b87b262a4ea1bad5cfc0291
 F test/index.test cbf301cdb2da43e4eac636c3400c2439af1834ad
@@ -492,7 +492,7 @@ F test/rollback.test 1f70ab4301d8d105d41438a436cad1fc8897f5e5
 F test/rowid.test 1c8fc43c60d273e6ea44dfb992db587f3164312c
 F test/rtree.test b85fd4f0861a40ca366ac195e363be2528dcfadf
 F test/safety.test b69e2b2dd5d52a3f78e216967086884bbc1a09c6
-F test/savepoint.test 5d43369333ab373c4bde69f090a1659c8d2275de
+F test/savepoint.test d00fe3f82773266410799aad0080426a6a936170
 F test/savepoint2.test 18f6c75d5c133b93838019df8988b8cdf379d3de
 F test/savepoint3.test b3c9aa5af3f777ccb8b9e15597c75c93eb5bc369
 F test/savepoint4.test fd8850063e3c40565545f5c291e7f79a30591670
@@ -686,7 +686,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P c6fd3b8f29927c0fc634f82885f144c78f0105d9
+P a1b1f6cd7d2c060bd75ce39347e1220b872806ed
-R c25735f44357d80a804bf8dbd5007586
+R 1a9ec7089b4909b30df347c1e2ee32e6
 U danielk1977
-Z e5dd7e980e4e7bfee46bc37a1c56ba2d
+Z 75549ad37a08cbb85b58f7a2c8f6829a

manifest.uuid

@@ -1 +1 @@
-a1b1f6cd7d2c060bd75ce39347e1220b872806ed
+08352f9ea9d2a1759320efc46e418079000855cb

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.552 2008/12/23 15:58:06 drh Exp $
+** $Id: btree.c,v 1.553 2008/12/27 15:23:13 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** See the header comment on "btreeInt.h" for additional information.
@@ -2280,15 +2280,10 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8);
 ** number of pages the database file will contain after this 
 ** process is complete.
 */
-static int incrVacuumStep(BtShared *pBt, Pgno nFin){
+static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
-  Pgno iLastPg;             /* Last page in the database */
   Pgno nFreeList;           /* Number of pages still on the free-list */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  iLastPg = pBt->nTrunc;
-  if( iLastPg==0 ){
-    iLastPg = pagerPagecount(pBt);
-  }
 
   if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){
     int rc;
@@ -2362,9 +2357,12 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin){
     }
   }
 
-  pBt->nTrunc = iLastPg - 1;
+  if( nFin==0 ){
-  while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){
+    iLastPg--;
-    pBt->nTrunc--;
+    while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){
+      iLastPg--;
+    }
+    sqlite3PagerTruncateImage(pBt->pPager, iLastPg);
   }
   return SQLITE_OK;
 }
@@ -2388,7 +2386,7 @@ int sqlite3BtreeIncrVacuum(Btree *p){
     rc = SQLITE_DONE;
   }else{
     invalidateAllOverflowCache(pBt);
-    rc = incrVacuumStep(pBt, 0);
+    rc = incrVacuumStep(pBt, 0, sqlite3PagerImageSize(pBt->pPager));
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -2403,7 +2401,7 @@ int sqlite3BtreeIncrVacuum(Btree *p){
 ** i.e. the database has been reorganized so that only the first *pnTrunc
 ** pages are in use.
 */
-static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){
+static int autoVacuumCommit(BtShared *pBt){
   int rc = SQLITE_OK;
   Pager *pPager = pBt->pPager;
   VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) );
@@ -2412,53 +2410,44 @@ static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){
   invalidateAllOverflowCache(pBt);
   assert(pBt->autoVacuum);
   if( !pBt->incrVacuum ){
-    Pgno nFin = 0;
+    Pgno nFin;
+    Pgno nFree;
+    Pgno nPtrmap;
+    Pgno iFree;
+    const int pgsz = pBt->pageSize;
+    Pgno nOrig = pagerPagecount(pBt);
 
-    if( pBt->nTrunc==0 ){
+    if( PTRMAP_ISPAGE(pBt, nOrig) ){
-      Pgno nFree;
+      return SQLITE_CORRUPT_BKPT;
-      Pgno nPtrmap;
+    }
-      const int pgsz = pBt->pageSize;
+    if( nOrig==PENDING_BYTE_PAGE(pBt) ){
-      Pgno nOrig = pagerPagecount(pBt);
+      nOrig--;
-
+    }
-      if( PTRMAP_ISPAGE(pBt, nOrig) ){
+    nFree = get4byte(&pBt->pPage1->aData[36]);
-        return SQLITE_CORRUPT_BKPT;
+    nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
-      }
+    nFin = nOrig - nFree - nPtrmap;
-      if( nOrig==PENDING_BYTE_PAGE(pBt) ){
+    if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
-        nOrig--;
+      nFin--;
-      }
+    }
-      nFree = get4byte(&pBt->pPage1->aData[36]);
+    while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
-      nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
+      nFin--;
-      nFin = nOrig - nFree - nPtrmap;
-      if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
-        nFin--;
-      }
-      while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
-        nFin--;
-      }
     }
 
-    while( rc==SQLITE_OK ){
+    for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){
-      rc = incrVacuumStep(pBt, nFin);
+      rc = incrVacuumStep(pBt, nFin, iFree);
     }
-    if( rc==SQLITE_DONE ){
+    if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){
-      assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc);
       rc = SQLITE_OK;
-      if( pBt->nTrunc && nFin ){
+      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
-        rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
+      put4byte(&pBt->pPage1->aData[32], 0);
-        put4byte(&pBt->pPage1->aData[32], 0);
+      put4byte(&pBt->pPage1->aData[36], 0);
-        put4byte(&pBt->pPage1->aData[36], 0);
+      sqlite3PagerTruncateImage(pBt->pPager, nFin);
-        pBt->nTrunc = nFin;
-      }
     }
     if( rc!=SQLITE_OK ){
       sqlite3PagerRollback(pPager);
     }
   }
 
-  if( rc==SQLITE_OK ){
-    *pnTrunc = pBt->nTrunc;
-    pBt->nTrunc = 0;
-  }
   assert( nRef==sqlite3PagerRefcount(pPager) );
   return rc;
 }
@@ -2500,7 +2489,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
     pBt->db = p->db;
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
-      rc = autoVacuumCommit(pBt, &nTrunc); 
+      rc = autoVacuumCommit(pBt);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
         return rc;
@@ -2677,10 +2666,6 @@ int sqlite3BtreeRollback(Btree *p){
   if( p->inTrans==TRANS_WRITE ){
     int rc2;
 
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    pBt->nTrunc = 0;
-#endif
-
     assert( TRANS_WRITE==pBt->inTransaction );
     rc2 = sqlite3PagerRollback(pBt->pPager);
     if( rc2!=SQLITE_OK ){
@@ -4329,16 +4314,6 @@ static int allocateBtreePage(
     *pPgno = nPage + 1;
 
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pBt->nTrunc ){
-      /* An incr-vacuum has already run within this transaction. So the
-      ** page to allocate is not from the physical end of the file, but
-      ** at pBt->nTrunc. 
-      */
-      *pPgno = pBt->nTrunc+1;
-      if( *pPgno==PENDING_BYTE_PAGE(pBt) ){
-        (*pPgno)++;
-      }
-    }
     if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){
       /* If *pPgno refers to a pointer-map page, allocate two new pages
       ** at the end of the file instead of one. The first allocated page
@@ -4349,9 +4324,6 @@ static int allocateBtreePage(
       (*pPgno)++;
       if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; }
     }
-    if( pBt->nTrunc ){
-      pBt->nTrunc = *pPgno;
-    }
 #endif
 
     assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
@@ -7059,11 +7031,6 @@ char *sqlite3BtreeIntegrityCheck(
   sCheck.nErr = 0;
   sCheck.mallocFailed = 0;
   *pnErr = 0;
-#ifndef SQLITE_OMIT_AUTOVACUUM
-  if( pBt->nTrunc!=0 ){
-    sCheck.nPage = pBt->nTrunc;
-  }
-#endif
   if( sCheck.nPage==0 ){
     unlockBtreeIfUnused(pBt);
     sqlite3BtreeLeave(p);

src/btreeInt.h

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btreeInt.h,v 1.37 2008/12/10 16:45:51 drh Exp $
+** $Id: btreeInt.h,v 1.38 2008/12/27 15:23:13 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -371,7 +371,6 @@ struct BtShared {
 #ifndef SQLITE_OMIT_AUTOVACUUM
   u8 autoVacuum;        /* True if auto-vacuum is enabled */
   u8 incrVacuum;        /* True if incr-vacuum is enabled */
-  Pgno nTrunc;          /* Non-zero if the db will be truncated (incr vacuum) */
 #endif
   u16 pageSize;         /* Total number of bytes on a page */
   u16 usableSize;       /* Number of usable bytes on each page */

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.523 2008/12/23 19:15:57 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.524 2008/12/27 15:23:13 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -172,6 +172,18 @@ struct PagerSavepoint {
 ** next successful rollback is performed on the pager cache. Also,
 ** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup()
 ** APIs, they may still be used successfully.
+**
+** Managing the size of the database file in pages is a little complicated.
+** The variable Pager.dbSize contains the number of pages that the database
+** image currently contains. As the database image grows or shrinks this
+** variable is updated. The variable Pager.dbFileSize contains the number
+** of pages in the database file. This may be different from Pager.dbSize
+** if some pages have been appended to the database image but not yet written
+** out from the cache to the actual file on disk. Or if the image has been
+** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable
+** contains the number of pages in the database image when the current
+** transaction was opened. The contents of all three of these variables is
+** only guaranteed to be correct if the boolean Pager.dbSizeValid is true.
 */
 struct Pager {
   sqlite3_vfs *pVfs;          /* OS functions to use for IO */
@@ -196,10 +208,11 @@ struct Pager {
   u8 dbModified;              /* True if there are any changes to the Db */
   u8 changeCountDone;         /* Set after incrementing the change-counter */
   u8 dbSizeValid;             /* Set when dbSize is correct */
+  Pgno dbSize;                /* Number of pages in the database */
+  Pgno dbOrigSize;            /* dbSize before the current transaction */
+  Pgno dbFileSize;            /* Number of pages in the database file */
   u32 vfsFlags;               /* Flags for sqlite3_vfs.xOpen() */
   int errCode;                /* One of several kinds of errors */
-  Pgno dbSize;                /* Number of pages in the file */
-  Pgno origDbSize;            /* dbSize before the current change */
   int nRec;                   /* Number of pages written to the journal */
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   int stmtNRec;               /* Number of records in stmt subjournal */
@@ -322,28 +335,30 @@ static const unsigned char aJournalMagic[] = {
 #define PAGER_MAX_PGNO 2147483647
 
 /*
-** Return false if it is necessary to write page *pPg into the sub-journal.
+** Return true if it is necessary to write page *pPg into the sub-journal.
-** More accurately, true is returned if either:
+** A page needs to be written into the sub-journal if there exists one
+** or more open savepoints for which:
 **
-**   * No savepoints are open, or
+**   * The page-number is less than or equal to PagerSavepoint.nOrig, and
-**   * The page has been saved to the sub-journal since the most recent
+**   * The bit corresponding to the page-number is not set in
-**     savepoint was opened.
+**     PagerSavepoint.pInSavepoint.
-**
-** TODO: There's a bug here. To do with PagerSavepoint.nOrig. Also consider
-**       the idea that the page may not be required by the outermost savepoint
-**       but may be required by some earlier savepoint, due to an incremental
-**       vacuum operation.
 */
-static int pageInSavepoint(PgHdr *pPg){
+static int subjRequiresPage(PgHdr *pPg){
+  Pgno pgno = pPg->pgno;
   Pager *pPager = pPg->pPager;
-  if( pPager->nSavepoint==0 ){
+  int i;
-    return 1;
+  for(i=0; i<pPager->nSavepoint; i++){
+    PagerSavepoint *p = &pPager->aSavepoint[i];
+    if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){
+      return 1;
+    }
   }
-  return sqlite3BitvecTest(
+  return 0;
-      pPager->aSavepoint[pPager->nSavepoint-1].pInSavepoint, pPg->pgno
-  );
 }
 
+/*
+** Return true if the page is already in the journal file.
+*/
 static int pageInJournal(PgHdr *pPg){
   return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno);
 }
@@ -715,7 +730,7 @@ static int writeJournalHdr(Pager *pPager){
   sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
   put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
   /* The initial database size */
-  put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbSize);
+  put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize);
   /* The assumed sector size for this process */
   put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize);
 
@@ -996,7 +1011,7 @@ static void pager_unlock(Pager *pPager){
       releaseAllSavepoint(pPager);
       pPager->journalOff = 0;
       pPager->journalStarted = 0;
-      pPager->origDbSize = 0;
+      pPager->dbOrigSize = 0;
     }
 
     pPager->state = PAGER_UNLOCK;
@@ -1089,7 +1104,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){
   }else if( pPager->state==PAGER_SYNCED ){
     pPager->state = PAGER_EXCLUSIVE;
   }
-  pPager->origDbSize = 0;
+  pPager->dbOrigSize = 0;
   pPager->setMaster = 0;
   pPager->needSync = 0;
   /* lruListSetFirstSynced(pPager); */
@@ -1232,6 +1247,9 @@ static int pager_playback_one_page(
   ){
     i64 ofst = (pgno-1)*(i64)pPager->pageSize;
     rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
+    if( pgno>pPager->dbFileSize ){
+      pPager->dbFileSize = pgno;
+    }
   }
   if( pPg ){
     /* No page should ever be explicitly rolled back that is in use, except
@@ -1417,6 +1435,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
       }else{
         rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
       }
+      if( rc==SQLITE_OK ){
+        pPager->dbFileSize = nPage;
+      }
     }
   }
   if( rc==SQLITE_OK ){
@@ -1654,7 +1675,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){
   /* Truncate the database back to the size it was before the 
   ** savepoint being reverted was opened.
   */
-  rc = pager_truncate(pPager, pSavepoint?pSavepoint->nOrig:pPager->origDbSize);
+  rc = pager_truncate(pPager, pSavepoint?pSavepoint->nOrig:pPager->dbOrigSize);
   assert( pPager->state>=PAGER_SHARED );
 
   /* Now roll back all main journal file records that occur after byte
@@ -2177,7 +2198,8 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){
       n /= pPager->pageSize;
     }
     if( pPager->state!=PAGER_UNLOCK ){
-      pPager->dbSize = (int)n;
+      pPager->dbSize = (Pgno)n;
+      pPager->dbFileSize = (Pgno)n;
       pPager->dbSizeValid = 1;
     }
   }
@@ -2248,7 +2270,11 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){
 }
 
 /*
-** Truncate the file to the number of pages specified.
+** Truncate the file to the number of pages specified. 
+**
+** Unless an IO error occurs, this function is guaranteed to modify the 
+** database file itself. If an exclusive lock is not held when this function
+** is called, one is obtained before truncating the file.
 */
 int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
   int rc = SQLITE_OK;
@@ -2257,7 +2283,7 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
   sqlite3PagerPagecount(pPager, 0);
   if( pPager->errCode ){
     rc = pPager->errCode;
-  }else if( nPage<pPager->dbSize ){
+  }else if( nPage<pPager->dbFileSize ){
     rc = syncJournal(pPager);
     if( rc==SQLITE_OK ){
       /* Get an exclusive lock on the database before truncating. */
@@ -2271,6 +2297,36 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
   return rc;
 }
 
+#ifndef SQLITE_OMIT_AUTOVACUUM
+/*
+** Truncate the in-memory database file image to nPage pages. Unlike
+** sqlite3PagerTruncate(), this function does not actually modify the
+** database file on disk. It just sets the internal state of the pager
+** object so that the truncation will be done when the current 
+** transaction is committed.
+*/
+void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){
+  assert( pPager->dbSizeValid );
+  assert( pPager->dbSize>=nPage );
+  pPager->dbSize = nPage;
+}
+
+/*
+** Return the current size of the database file image in pages. This
+** function differs from sqlite3PagerPagecount() in two ways:
+**
+**  a) It may only be called when at least one reference to a database
+**     page is held. This guarantees that the database size is already
+**     known and a call to sqlite3OsFileSize() is not required.
+**
+**  b) The return value is not adjusted for the locking page.
+*/
+Pgno sqlite3PagerImageSize(Pager *pPager){
+  assert( pPager->dbSizeValid );
+  return pPager->dbSize;
+}
+#endif  /* ifndef SQLITE_OMIT_AUTOVACUUM */
+
 /*
 ** Shutdown the page cache.  Free all memory and close all files.
 **
@@ -2480,6 +2536,9 @@ static int pager_write_pagelist(PgHdr *pList){
       if( pList->pgno==1 ){
         memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
       }
+      if( pList->pgno>pPager->dbFileSize ){
+        pPager->dbFileSize = pList->pgno;
+      }
     }
 #ifndef NDEBUG
     else{
@@ -3076,7 +3135,7 @@ static int pager_open_journal(Pager *pPager){
     rc = pPager->errCode;
     goto failed_to_open_journal;
   }
-  pPager->origDbSize = pPager->dbSize;
+  pPager->dbOrigSize = pPager->dbSize;
 
   rc = writeJournalHdr(pPager);
 
@@ -3156,14 +3215,14 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){
     ** overwritten with zeros.
     */
     assert( pPager->nRec==0 );
-    assert( pPager->origDbSize==0 );
+    assert( pPager->dbOrigSize==0 );
     assert( pPager->pInJournal==0 );
     sqlite3PagerPagecount(pPager, 0);
     pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize );
     if( !pPager->pInJournal ){
       rc = SQLITE_NOMEM;
     }else{
-      pPager->origDbSize = pPager->dbSize;
+      pPager->dbOrigSize = pPager->dbSize;
       rc = writeJournalHdr(pPager);
     }
   }
@@ -3223,7 +3282,7 @@ static int pager_write(PgHdr *pPg){
   ** to the journal then we can return right away.
   */
   sqlite3PcacheMakeDirty(pPg);
-  if( pageInJournal(pPg) && pageInSavepoint(pPg) ){
+  if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){
     pPager->dirtyCache = 1;
     pPager->dbModified = 1;
   }else{
@@ -3254,7 +3313,7 @@ static int pager_write(PgHdr *pPg){
     ** the transaction journal if it is not there already.
     */
     if( !pageInJournal(pPg) && pPager->journalOpen ){
-      if( pPg->pgno<=pPager->origDbSize ){
+      if( pPg->pgno<=pPager->dbOrigSize ){
         u32 cksum;
         char *pData2;
 
@@ -3320,10 +3379,10 @@ static int pager_write(PgHdr *pPg){
     ** the statement journal format differs from the standard journal format
     ** in that it omits the checksums and the header.
     */
-    if( !pageInSavepoint(pPg) ){
+    if( subjRequiresPage(pPg) ){
       i64 offset = pPager->stmtNRec*(4+pPager->pageSize);
       char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7);
-      assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize );
+      assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
       rc = write32bits(pPager->sjfd, offset, pPg->pgno);
       if( rc==SQLITE_OK ){
         rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4);
@@ -3488,12 +3547,12 @@ int sqlite3PagerDontWrite(DbPage *pDbPage){
   Pager *pPager = pPg->pPager;
   int rc;
 
-  if( pPg->pgno>pPager->origDbSize ){
+  if( pPg->pgno>pPager->dbOrigSize ){
     return SQLITE_OK;
   }
   if( pPager->pAlwaysRollback==0 ){
     assert( pPager->pInJournal );
-    pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->origDbSize);
+    pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->dbOrigSize);
     if( !pPager->pAlwaysRollback ){
       return SQLITE_NOMEM;
     }
@@ -3502,7 +3561,7 @@ int sqlite3PagerDontWrite(DbPage *pDbPage){
 
   if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){
     assert( pPager->state>=PAGER_SHARED );
-    if( pPager->dbSize==pPg->pgno && pPager->origDbSize<pPager->dbSize ){
+    if( pPager->dbSize==pPg->pgno && pPager->dbOrigSize<pPager->dbSize ){
       /* If this pages is the last page in the file and the file has grown
       ** during the current transaction, then do NOT mark the page as clean.
       ** When the database file grows, we must make sure that the last page
@@ -3545,14 +3604,14 @@ void sqlite3PagerDontRollback(DbPage *pPg){
   */
   if( pPager->journalOpen==0 
    || sqlite3BitvecTest(pPager->pAlwaysRollback, pPg->pgno)
-   || pPg->pgno>pPager->origDbSize
+   || pPg->pgno>pPager->dbOrigSize
   ){
     return;
   }
 
 #ifdef SQLITE_SECURE_DELETE
   if( sqlite3BitvecTest(pPager->pInJournal, pPg->pgno)!=0
-   || pPg->pgno>pPager->origDbSize ){
+   || pPg->pgno>pPager->dbOrigSize ){
     return;
   }
 #endif
@@ -3567,7 +3626,7 @@ void sqlite3PagerDontRollback(DbPage *pPg){
   ** pages on the freelist (ex: corrupt9.test) then the following is not
   ** necessarily true:
   */
-  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ); */
+  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->dbOrigSize ); */
 
   assert( pPager->pInJournal!=0 );
   sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
@@ -3611,6 +3670,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirect){
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
     if( isDirect && pPager->fd->pMethods ){
       const void *zBuf = pPgHdr->pData;
+      assert( pPager->dbFileSize>0 );
       rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0);
     }
 #endif
@@ -3678,8 +3738,8 @@ int sqlite3PagerCommitPhaseOne(
     return SQLITE_OK;
   }
 
-  PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", 
+  PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
-      pPager->zFilename, zMaster, nTrunc);
+      pPager->zFilename, zMaster, pPager->dbSize);
 
   /* If this is an in-memory db, or no pages have been written to, or this
   ** function has already been called, it is a no-op.
@@ -3705,7 +3765,7 @@ int sqlite3PagerCommitPhaseOne(
         !zMaster && 
         pPager->journalOpen &&
         pPager->journalOff==jrnlBufferSize(pPager) && 
-        nTrunc==0 && 
+        pPager->dbSize>=pPager->dbFileSize && 
         (pPg==0 || pPg->pDirty==0)
     );
     assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF );
@@ -3742,14 +3802,15 @@ int sqlite3PagerCommitPhaseOne(
       if( rc!=SQLITE_OK ) goto sync_exit;
       if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){
 #ifndef SQLITE_OMIT_AUTOVACUUM
-        if( nTrunc!=0 ){
+        if( pPager->dbSize<pPager->dbOrigSize ){
           /* If this transaction has made the database smaller, then all pages
           ** being discarded by the truncation must be written to the journal
           ** file.
           */
           Pgno i;
           Pgno iSkip = PAGER_MJ_PGNO(pPager);
-          for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){
+          Pgno dbSize = pPager->dbSize;
+          for( i=pPager->dbSize+1; i<=pPager->dbOrigSize; i++ ){
             if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){
               rc = sqlite3PagerGet(pPager, i, &pPg);
               if( rc!=SQLITE_OK ) goto sync_exit;
@@ -3758,6 +3819,7 @@ int sqlite3PagerCommitPhaseOne(
               if( rc!=SQLITE_OK ) goto sync_exit;
             }
           } 
+          pPager->dbSize = dbSize;
         }
 #endif
         rc = writeMasterJournal(pPager, zMaster);
@@ -3768,8 +3830,8 @@ int sqlite3PagerCommitPhaseOne(
     if( rc!=SQLITE_OK ) goto sync_exit;
 
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( nTrunc!=0 ){
+    if( pPager->dbSize<pPager->dbFileSize ){
-      rc = sqlite3PagerTruncate(pPager, nTrunc);
+      rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
       if( rc!=SQLITE_OK ) goto sync_exit;
     }
 #endif
@@ -3797,8 +3859,8 @@ int sqlite3PagerCommitPhaseOne(
     IOTRACE(("DBSYNC %p\n", pPager))
 
     pPager->state = PAGER_SYNCED;
-  }else if( MEMDB && nTrunc!=0 ){
+  }else if( MEMDB && pPager->dbSize<pPager->dbFileSize ){
-    rc = sqlite3PagerTruncate(pPager, nTrunc);
+    rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
   }
 
 sync_exit:
@@ -4133,7 +4195,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
   */
   if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){
     needSyncPgno = pPg->pgno;
-    assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize );
+    assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
     assert( pPg->flags&PGHDR_DIRTY );
     assert( pPager->needSync );
   }
@@ -4182,7 +4244,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
     assert( pPager->needSync );
     rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
     if( rc!=SQLITE_OK ){
-      if( pPager->pInJournal && needSyncPgno<=pPager->origDbSize ){
+      if( pPager->pInJournal && needSyncPgno<=pPager->dbOrigSize ){
         sqlite3BitvecClear(pPager->pInJournal, needSyncPgno);
       }
       return rc;

src/pager.h

@@ -13,7 +13,7 @@
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.89 2008/12/17 17:30:26 danielk1977 Exp $
+** @(#) $Id: pager.h,v 1.90 2008/12/27 15:23:13 danielk1977 Exp $
 */
 
 #ifndef _PAGER_H_
@@ -119,6 +119,11 @@ int sqlite3PagerSync(Pager *pPager);
 int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 
+#ifndef SQLITE_OMIT_AUTOVACUUM
+  void sqlite3PagerTruncateImage(Pager*,Pgno);
+  Pgno sqlite3PagerImageSize(Pager *);
+#endif
+
 #ifdef SQLITE_HAS_CODEC
   void sqlite3PagerSetCodec(Pager*,void*(*)(void*,void*,Pgno,int),void*);
 #endif

test/incrvacuum.test

@@ -14,7 +14,7 @@
 # Note: There are also some tests for incremental vacuum and IO 
 # errors in incrvacuum_ioerr.test.
 #
-# $Id: incrvacuum.test,v 1.20 2008/09/10 10:57:28 danielk1977 Exp $
+# $Id: incrvacuum.test,v 1.21 2008/12/27 15:23:13 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -732,6 +732,48 @@ do_test incrvacuum-14.1 {
   } db3
 } {1 {file is encrypted or is not a database}}
 
+do_test incrvacuum-15.1 {
+  db close
+  file delete -force test.db
+  sqlite3 db test.db
+
+  set str [string repeat "abcdefghij" 500]
+
+  execsql {
+    PRAGMA cache_size = 10;
+    PRAGMA auto_vacuum = incremental;
+    CREATE TABLE t1(x, y);
+    INSERT INTO t1 VALUES('a', $str);
+    INSERT INTO t1 VALUES('b', $str);
+    INSERT INTO t1 VALUES('c', $str);
+    INSERT INTO t1 VALUES('d', $str);
+    INSERT INTO t1 VALUES('e', $str);
+    INSERT INTO t1 VALUES('f', $str);
+    INSERT INTO t1 VALUES('g', $str);
+    INSERT INTO t1 VALUES('h', $str);
+    INSERT INTO t1 VALUES('i', $str);
+    INSERT INTO t1 VALUES('j', $str);
+    INSERT INTO t1 VALUES('j', $str);
+
+    CREATE TABLE t2(x PRIMARY KEY, y);
+    INSERT INTO t2 VALUES('a', $str);
+    INSERT INTO t2 VALUES('b', $str);
+    INSERT INTO t2 VALUES('c', $str);
+    INSERT INTO t2 VALUES('d', $str);
+
+    BEGIN;
+      DELETE FROM t2;
+      PRAGMA incremental_vacuum;
+  }
+
+  catchsql {INSERT INTO t2 SELECT * FROM t1}
+
+  execsql { 
+    COMMIT;
+    PRAGMA integrity_check;
+  }
+} {ok}
+
 db2 close
 db3 close
 finish_test

test/savepoint.test

@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# $Id: savepoint.test,v 1.3 2008/12/23 11:46:28 danielk1977 Exp $
+# $Id: savepoint.test,v 1.4 2008/12/27 15:23:13 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -376,5 +376,98 @@ ifcapable {autovacuum && pragma} {
   integrity_check savepoint-6.4
 }
 
+#-------------------------------------------------------------------------
+# The following tests, savepoint-7.*, attempt to break the logic 
+# surrounding savepoints by growing and shrinking the database file.
+#
+db close
+file delete -force test.db
+sqlite3 db test.db
+
+do_test savepoint-7.1 {
+  execsql {
+    PRAGMA auto_vacuum = incremental;
+    PRAGMA cache_size = 10;
+    BEGIN;
+    CREATE TABLE t1(a PRIMARY KEY, b);
+      INSERT INTO t1(a) VALUES('alligator');
+      INSERT INTO t1(a) VALUES('angelfish');
+      INSERT INTO t1(a) VALUES('ant');
+      INSERT INTO t1(a) VALUES('antelope');
+      INSERT INTO t1(a) VALUES('ape');
+      INSERT INTO t1(a) VALUES('baboon');
+      INSERT INTO t1(a) VALUES('badger');
+      INSERT INTO t1(a) VALUES('bear');
+      INSERT INTO t1(a) VALUES('beetle');
+      INSERT INTO t1(a) VALUES('bird');
+      INSERT INTO t1(a) VALUES('bison');
+      UPDATE t1 SET b =    randstr(1000,1000);
+      UPDATE t1 SET b = b||randstr(1000,1000);
+      UPDATE t1 SET b = b||randstr(1000,1000);
+      UPDATE t1 SET b = b||randstr(10,1000);
+    COMMIT;
+  }
+  expr ([execsql { PRAGMA page_count }] > 20)
+} {1}
+do_test savepoint-7.2.1 {
+  execsql {
+    BEGIN;
+      SAVEPOINT one;
+      CREATE TABLE t2(a, b);
+      INSERT INTO t2 SELECT a, b FROM t1;
+      ROLLBACK TO one;
+  }
+  execsql {
+    PRAGMA integrity_check;
+  }
+} {ok}
+do_test savepoint-7.2.2 {
+  execsql {
+    COMMIT;
+    PRAGMA integrity_check;
+  }
+} {ok}
+
+do_test savepoint-7.3.1 {
+  execsql {
+    CREATE TABLE t2(a, b);
+    INSERT INTO t2 SELECT a, b FROM t1;
+  }
+} {}
+do_test savepoint-7.3.2 {
+  execsql {
+    BEGIN;
+      SAVEPOINT one;
+        DELETE FROM t2;
+        PRAGMA incremental_vacuum;
+        SAVEPOINT two;
+          INSERT INTO t2 SELECT a, b FROM t1;
+        ROLLBACK TO two;
+    COMMIT;
+  }
+  execsql { PRAGMA integrity_check }
+} {ok}
+
+do_test savepoint-7.4.1 {
+  db close
+  file delete -force test.db
+  sqlite3 db test.db
+  execsql {
+    PRAGMA auto_vacuum = incremental;
+    CREATE TABLE t1(a, b, PRIMARY KEY(a, b));
+    INSERT INTO t1 VALUES(randstr(1000,1000), randstr(1000,1000));
+    BEGIN;
+      DELETE FROM t1;
+      SAVEPOINT one;
+      PRAGMA incremental_vacuum;
+      ROLLBACK TO one;
+    COMMIT;
+  }
+
+  execsql { PRAGMA integrity_check }
+} {ok}
+
+
+
 finish_test