Add largely untested code for the incremental vacuum function. (CVS 3876)

FossilOrigin-Name: f6a6d2b8872c05089810b1e095f39011f3035408

manifest

@@ -1,5 +1,5 @@
-C In\sthe\spager,\sload\sthe\scontent\sof\spages\swhich\swere\sinitialized\swith\nnoContent==1\sif\sthey\sare\ssubsequently\srequested\swith\snoContent==0.\s(CVS\s3875)
+C Add\slargely\suntested\scode\sfor\sthe\sincremental\svacuum\sfunction.\s(CVS\s3876)
-D 2007-04-26T12:11:28
+D 2007-04-26T14:42:35
 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -59,9 +59,9 @@ F src/alter.c 2c79ec40f65e33deaf90ca493422c74586e481a3
 F src/analyze.c 4bbf5ddf9680587c6d4917e02e378b6037be3651
 F src/attach.c a16ada4a4654a0d126b8223ec9494ebb81bc5c3c
 F src/auth.c 902f4722661c796b97f007d9606bd7529c02597f
-F src/btree.c 960bf64baa4d2bdb96019698e60d0b7763bf4e7e
+F src/btree.c d08db3a8207bf884bd891829cab84c5e4cf18d99
-F src/btree.h 9b2cc0d113c0bc2d37d244b9a394d56948c9acbf
+F src/btree.h 4c0b5855cef3e4e6627358aa69541d21a2015947
-F src/build.c 1880da163d9aa404016242b8b76d69907f682cd8
+F src/build.c 02e01ec7907c7d947ab3041fda0e81eaed05db42
 F src/callback.c 6414ed32d55859d0f65067aa5b88d2da27b3af9e
 F src/complete.c 7d1a44be8f37de125fcafd3d3a018690b3799675
 F src/date.c 94a6777df13d2aaacd19de080d9e8d3444364133
@@ -89,8 +89,8 @@ F src/os_win.c e94903c7dc1c0599c8ddce42efa0b6928068ddc5
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
 F src/pager.c cd2770b0f8bd1900b46121009336e7ad03fb274f
 F src/pager.h d652ddf092d2318d00e41f8539760fe8e57c157c
-F src/parse.y b6cfbadb6d5b21b5087d30698ee5af0ebb098767
+F src/parse.y a3940369e12c69c4968aa580cdc74cf73a664980
-F src/pragma.c 3b992b5b2640d6ae25cef05aa6a42cd1d6c43234
+F src/pragma.c 4fdefc03c3fd0ee87f8aad82bf80ba9bf1cdf416
 F src/prepare.c 4cb9c9eb926e8baf5652ca4b4f2416f53f5b5370
 F src/printf.c 0c6f40648770831341ac45ab32423a80b4c87f05
 F src/random.c 6119474a6f6917f708c1dee25b9a8e519a620e88
@@ -99,7 +99,7 @@ F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96
 F src/shell.c 3ae4654560e91220a95738a73d135d91d937cda1
 F src/sqlite.h.in e429f66f9245c7f8675db24b230c950b8672ad1c
 F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890
-F src/sqliteInt.h 047af0e4c38bbb8652836f72adc9e9199c51a1ba
+F src/sqliteInt.h 0b14d0eae083aafca0562d2261a404e5e5abc5f0
 F src/table.c 6d0da66dde26ee75614ed8f584a1996467088d06
 F src/tclsqlite.c ec69eb9ad56d03fbf7570ca1ca5ea947d1ec4b6f
 F src/test1.c 53b7eb5cba0012f592b5860f6ad3b5a3f887eb1e
@@ -125,7 +125,7 @@ F src/update.c 3359041db390a8f856d67272f299600e2104f350
 F src/utf.c e64a48bc21aa973eb622dd47da87d56a4cdcf528
 F src/util.c b6344325378e75b9e18175d8b6aed1723d73dad9
 F src/vacuum.c 8bd895d29e7074e78d4e80f948e35ddc9cf2beef
-F src/vdbe.c 814dab208a156250bc5e77f827f4e0c8ad734820
+F src/vdbe.c a3cf3792fdbd382f756eb7eb50006b2f3f8d4283
 F src/vdbe.h 0025259af1939fb264a545816c69e4b5b8d52691
 F src/vdbeInt.h 4b19fd8febad3fd14c4c97adaefc06754d323132
 F src/vdbeapi.c 245263aa2d70d87b1201753cddc881996f219843
@@ -238,6 +238,7 @@ F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51
 F test/func.test 865febfd5b968f62b85c841c6a305b20346f7f44
 F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a
 F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d
+F test/incrvacuum.test ee05edff95770f211fab28132291c175cd282e0c
 F test/index.test e65df12bed94b2903ee89987115e1578687e9266
 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
 F test/index3.test f66718cd92ce1216819d47e6a156755e4b2c4ca1
@@ -394,7 +395,7 @@ F tool/lempar.c 8f998bf8d08e2123149c2cc5d0597cd5d5d1abdd
 F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133
 F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8
 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf
-F tool/mkkeywordhash.c c6f797bfc698803d2afbcbfb6b42f2239b074e29
+F tool/mkkeywordhash.c e119bdc04305adcada8856d73ad7d837c4ec123c
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
 F tool/mksqlite3c.tcl 2d204fc271b2e2a2139e360527dd845385c4dffa
 F tool/mksqlite3internalh.tcl a85bb0c812db1a060e6e6dfab4e4c817f53d194b
@@ -462,7 +463,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 9cb0ed6ee9827bc6884a0195044d5b6ad0de698e
+P d0745a43b6e037d16e1ec38c7c4d961a80d1ef48
-R 654420a2ec3eb1f548ed11aa685883b0
+R 9a772b637e3a7b43df139e14c80416c0
-U drh
+U danielk1977
-Z 352203b1a39e1bdcedb9d02c8002561c
+Z d70505e165a313929d975eb44dd8ca1a

manifest.uuid

@@ -1 +1 @@
-d0745a43b6e037d16e1ec38c7c4d961a80d1ef48
+f6a6d2b8872c05089810b1e095f39011f3035408

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.358 2007/04/24 17:35:59 drh Exp $
+** $Id: btree.c,v 1.359 2007/04/26 14:42:35 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -336,7 +336,9 @@ struct BtShared {
   u8 minLeafFrac;       /* Minimum leaf payload as % of total page size */
   u8 pageSizeFixed;     /* True if the page size can no longer be changed */
 #ifndef SQLITE_OMIT_AUTOVACUUM
-  u8 autoVacuum;        /* True if database supports auto-vacuum */
+  u8 autoVacuum;        /* True if auto-vacuum is enabled */
+  u8 incrVacuum;        /* True if incr-vacuum is enabled */
+  Pgno nTrunc;          /* Non-zero if the db will be truncated (incr vacuum) */
 #endif
   u16 pageSize;         /* Total number of bytes on a page */
   u16 usableSize;       /* Number of usable bytes on each page */
@@ -510,7 +512,6 @@ struct BtLock {
   #define unlockAllTables(a)
 #else
 
-
 /*
 ** Query to see if btree handle p may obtain a lock of type eLock 
 ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return
@@ -1506,7 +1507,7 @@ int sqlite3BtreeOpen(
 ){
   BtShared *pBt;          /* Shared part of btree structure */
   Btree *p;               /* Handle to return */
-  int rc;
+  int rc = SQLITE_OK;
   int nReserve;
   unsigned char zDbHeader[100];
 #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO)
@@ -1569,22 +1570,15 @@ int sqlite3BtreeOpen(
 
   pBt = sqliteMalloc( sizeof(*pBt) );
   if( pBt==0 ){
-    *ppBtree = 0;
+    rc = SQLITE_NOMEM;
-    sqliteFree(p);
+    goto btree_open_out;
-    return SQLITE_NOMEM;
   }
   rc = sqlite3PagerOpen(&pBt->pPager, zFilename, EXTRA_SIZE, flags);
   if( rc==SQLITE_OK ){
     rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader);
   }
   if( rc!=SQLITE_OK ){
-    if( pBt->pPager ){
+    goto btree_open_out;
-      sqlite3PagerClose(pBt->pPager);
-    }
-    sqliteFree(pBt);
-    sqliteFree(p);
-    *ppBtree = 0;
-    return rc;
   }
   p->pBt = pBt;
 
@@ -1602,13 +1596,14 @@ int sqlite3BtreeOpen(
     pBt->minLeafFrac = 32;    /* 12.5% */
 #ifndef SQLITE_OMIT_AUTOVACUUM
     /* If the magic name ":memory:" will create an in-memory database, then
-    ** do not set the auto-vacuum flag, even if SQLITE_DEFAULT_AUTOVACUUM
+    ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if
-    ** is true. On the other hand, if SQLITE_OMIT_MEMORYDB has been defined,
+    ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if
-    ** then ":memory:" is just a regular file-name. Respect the auto-vacuum
+    ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a
-    ** default in this case.
+    ** regular file-name. In this case the auto-vacuum applies as per normal.
     */
     if( zFilename && !isMemdb ){
-      pBt->autoVacuum = SQLITE_DEFAULT_AUTOVACUUM;
+      pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0);
+      pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0);
     }
 #endif
     nReserve = 0;
@@ -1639,7 +1634,17 @@ int sqlite3BtreeOpen(
 #endif
   pBt->nRef = 1;
   *ppBtree = p;
-  return SQLITE_OK;
+
+btree_open_out:
+  if( rc!=SQLITE_OK ){
+    if( pBt && pBt->pPager ){
+      sqlite3PagerClose(pBt->pPager);
+    }
+    sqliteFree(pBt);
+    sqliteFree(p);
+    *ppBtree = 0;
+  }
+  return rc;
 }
 
 /*
@@ -1819,14 +1824,17 @@ int sqlite3BtreeGetReserve(Btree *p){
 ** determined by the SQLITE_DEFAULT_AUTOVACUUM macro.
 */
 int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){
-  BtShared *pBt = p->pBt;;
 #ifdef SQLITE_OMIT_AUTOVACUUM
   return SQLITE_READONLY;
 #else
-  if( pBt->pageSizeFixed ){
+  BtShared *pBt = p->pBt;
+  int av = (autoVacuum?1:0);
+  int iv = (autoVacuum==BTREE_AUTOVACUUM_INCR?1:0);
+  if( pBt->pageSizeFixed && av!=pBt->autoVacuum ){
     return SQLITE_READONLY;
   }
-  pBt->autoVacuum = (autoVacuum?1:0);
+  pBt->autoVacuum = av;
+  pBt->incrVacuum = iv;
   return SQLITE_OK;
 #endif
 }
@@ -1837,9 +1845,13 @@ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){
 */
 int sqlite3BtreeGetAutoVacuum(Btree *p){
 #ifdef SQLITE_OMIT_AUTOVACUUM
-  return 0;
+  return BTREE_AUTOVACUUM_NONE;
 #else
-  return p->pBt->autoVacuum;
+  return (
+    (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE:
+    (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL:
+    BTREE_AUTOVACUUM_INCR
+  );
 #endif
 }
 
@@ -1998,9 +2010,8 @@ static int newDatabase(BtShared *pBt){
   zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA );
   pBt->pageSizeFixed = 1;
 #ifndef SQLITE_OMIT_AUTOVACUUM
-  if( pBt->autoVacuum ){
+  assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 );
-    put4byte(&data[36 + 4*4], 1);
+  put4byte(&data[36 + 4*4], pBt->autoVacuum);
-  }
 #endif
   return SQLITE_OK;
 }
@@ -2285,9 +2296,121 @@ static int relocatePage(
   return rc;
 }
 
-/* Forward declaration required by autoVacuumCommit(). */
+/* Forward declaration required by incrVacuumStep(). */
 static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8);
 
+/*
+** Perform a single step of an incremental-vacuum. If successful,
+** return SQLITE_OK. If there is no work to do (and therefore no
+** point in calling this function again), return SQLITE_DONE.
+**
+** More specificly, this function attempts to re-organize the 
+** database so that the last page of the file currently in use
+** is no longer in use.
+**
+** If the nFin parameter is non-zero, the implementation assumes
+** that the caller will keep calling incrVacuumStep() until
+** it returns SQLITE_DONE or an error, and that nFin is the
+** number of pages the database file will contain after this 
+** process is complete.
+*/
+static int incrVacuumStep(BtShared *pBt, Pgno nFin){
+  Pgno iLastPg;             /* Last page in the database */
+  Pgno nFreeList;           /* Number of pages still on the free-list */
+
+  iLastPg = pBt->nTrunc;
+  if( iLastPg==0 ){
+    iLastPg = sqlite3PagerPagecount(pBt->pPager);
+  }
+
+  if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){
+    int rc;
+    u8 eType;
+    Pgno iPtrPage;
+
+    nFreeList = get4byte(&pBt->pPage1->aData[36]);
+    if( nFreeList==0 || nFin==iLastPg ){
+      return SQLITE_DONE;
+    }
+
+    rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
+    if( eType==PTRMAP_ROOTPAGE ){
+      return SQLITE_CORRUPT_BKPT;
+    }
+
+    if( eType==PTRMAP_FREEPAGE ){
+      if( nFin==0 ){
+        /* Remove the page from the files free-list. This is not required
+        ** if nFin is non-zero. In this case, the free-list will be
+        ** truncated to zero after this function returns, so it doesn't 
+        ** matter if it still contains some garbage entries.
+        */
+        Pgno iFreePg;
+        MemPage *pFreePg;
+        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+        assert( iFreePg==iLastPg );
+        releasePage(pFreePg);
+      }
+    } else {
+      Pgno iFreePg;             /* Index of free page to move pLastPg to */
+      MemPage *pLastPg;
+
+      rc = getPage(pBt, iLastPg, &pLastPg, 0);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+
+      do {
+        MemPage *pFreePg;
+        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
+        if( rc!=SQLITE_OK ){
+          releasePage(pLastPg);
+          return rc;
+        }
+        releasePage(pFreePg);
+      }while( nFin!=0 && iFreePg>nFin );
+      assert( iFreePg<iLastPg );
+
+      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg);
+      releasePage(pLastPg);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      } 
+    }
+  }
+
+  pBt->nTrunc = iLastPg - 1;
+  while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){
+    pBt->nTrunc--;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** A write-transaction must be opened before calling this function.
+** It performs a single unit of work towards an incremental vacuum.
+**
+** If the incremental vacuum is finished after this function has run,
+** SQLITE_DONE is returned. If it is not finished, but no error occured,
+** SQLITE_OK is returned. Otherwise an SQLite error code. 
+*/
+int sqlite3BtreeIncrVacuum(Btree *p){
+  BtShared *pBt = p->pBt;
+
+  assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE );
+  if( !pBt->autoVacuum ){
+    return SQLITE_DONE;
+  }
+
+  return incrVacuumStep(p->pBt, 0);
+}
+
 /*
 ** This routine is called prior to sqlite3PagerCommit when a transaction
 ** is commited for an auto-vacuum database.
@@ -2298,135 +2421,65 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8);
 ** pages are in use.
 */
 static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){
+  int rc = SQLITE_OK;
   Pager *pPager = pBt->pPager;
-  Pgno nFreeList;            /* Number of pages remaining on the free-list. */
-  int nPtrMap;               /* Number of pointer-map pages deallocated */
-  Pgno origSize;             /* Pages in the database file */
-  Pgno finSize;              /* Pages in the database file after truncation */
-  int rc;                    /* Return code */
-  u8 eType;
-  int pgsz = pBt->pageSize;  /* Page size for this database */
-  Pgno iDbPage;              /* The database page to move */
-  MemPage *pDbMemPage = 0;   /* "" */
-  Pgno iPtrPage;             /* The page that contains a pointer to iDbPage */
-  Pgno iFreePage;            /* The free-list page to move iDbPage to */
-  MemPage *pFreeMemPage = 0; /* "" */
-
 #ifndef NDEBUG
   int nRef = sqlite3PagerRefcount(pPager);
 #endif
 
-  assert( pBt->autoVacuum );
   if( PTRMAP_ISPAGE(pBt, sqlite3PagerPagecount(pPager)) ){
     return SQLITE_CORRUPT_BKPT;
   }
 
-  /* Figure out how many free-pages are in the database. If there are no
+  assert(pBt->autoVacuum);
-  ** free pages, then auto-vacuum is a no-op.
+  if( !pBt->incrVacuum ){
-  */
+    Pgno nFin = 0;
-  nFreeList = get4byte(&pBt->pPage1->aData[36]);
+
-  if( nFreeList==0 ){
+    if( pBt->nTrunc==0 ){
-    *pnTrunc = 0;
+      Pgno nFree;
-    return SQLITE_OK;
+      Pgno nPtrmap;
+      const int pgsz = pBt->pageSize;
+      Pgno nOrig = sqlite3PagerPagecount(pBt->pPager);
+      if( nOrig==PENDING_BYTE_PAGE(pBt) ){
+        nOrig--;
+      }
+      nFree = get4byte(&pBt->pPage1->aData[36]);
+      nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
+      nFin = nOrig - nFree - nPtrmap;
+      if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+        nFin--;
+      }
+      while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
+        nFin--;
+      }
     }
 
-  /* This block figures out how many pages there are in the database
+    while( rc==SQLITE_OK ){
-  ** now (variable origSize), and how many there will be after the
+      rc = incrVacuumStep(pBt, nFin);
-  ** truncation (variable finSize).
-  **
-  ** The final size is the original size, less the number of free pages
-  ** in the database, less any pointer-map pages that will no longer
-  ** be required, less 1 if the pending-byte page was part of the database
-  ** but is not after the truncation.
-  **/
-  origSize = sqlite3PagerPagecount(pPager);
-  if( origSize==PENDING_BYTE_PAGE(pBt) ){
-    origSize--;
     }
-  nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pBt, origSize)+pgsz/5)/(pgsz/5);
+    if( rc==SQLITE_DONE ){
-  finSize = origSize - nFreeList - nPtrMap;
+      assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc);
-  if( origSize>PENDING_BYTE_PAGE(pBt) && finSize<=PENDING_BYTE_PAGE(pBt) ){
+      rc = SQLITE_OK;
-    finSize--;
+      if( pBt->nTrunc ){
-  }
+        sqlite3PagerWrite(pBt->pPage1->pDbPage);
-  while( PTRMAP_ISPAGE(pBt, finSize) || finSize==PENDING_BYTE_PAGE(pBt) ){
-    finSize--;
-  }
-  TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize));
-
-  /* Variable 'finSize' will be the size of the file in pages after
-  ** the auto-vacuum has completed (the current file size minus the number
-  ** of pages on the free list). Loop through the pages that lie beyond
-  ** this mark, and if they are not already on the free list, move them
-  ** to a free page earlier in the file (somewhere before finSize).
-  */
-  for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){
-    /* If iDbPage is a pointer map page, or the pending-byte page, skip it. */
-    if( PTRMAP_ISPAGE(pBt, iDbPage) || iDbPage==PENDING_BYTE_PAGE(pBt) ){
-      continue;
-    }
-
-    rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage);
-    if( rc!=SQLITE_OK ) goto autovacuum_out;
-    if( eType==PTRMAP_ROOTPAGE ){
-      rc = SQLITE_CORRUPT_BKPT;
-      goto autovacuum_out;
-    }
-
-    /* If iDbPage is free, do not swap it.  */
-    if( eType==PTRMAP_FREEPAGE ){
-      continue;
-    }
-    rc = getPage(pBt, iDbPage, &pDbMemPage, 0);
-    if( rc!=SQLITE_OK ) goto autovacuum_out;
-
-    /* Find the next page in the free-list that is not already at the end 
-    ** of the file. A page can be pulled off the free list using the 
-    ** allocateBtreePage() routine.
-    */
-    do{
-      if( pFreeMemPage ){
-        releasePage(pFreeMemPage);
-        pFreeMemPage = 0;
-      }
-      rc = allocateBtreePage(pBt, &pFreeMemPage, &iFreePage, 0, 0);
-      if( rc!=SQLITE_OK ){
-        releasePage(pDbMemPage);
-        goto autovacuum_out;
-      }
-      assert( iFreePage<=origSize );
-    }while( iFreePage>finSize );
-    releasePage(pFreeMemPage);
-    pFreeMemPage = 0;
-
-    /* Relocate the page into the body of the file. Note that although the 
-    ** page has moved within the database file, the pDbMemPage pointer 
-    ** remains valid. This means that this function can run without
-    ** invalidating cursors open on the btree. This is important in 
-    ** shared-cache mode.
-    */
-    rc = relocatePage(pBt, pDbMemPage, eType, iPtrPage, iFreePage);
-    releasePage(pDbMemPage);
-    if( rc!=SQLITE_OK ) goto autovacuum_out;
-  }
-
-  /* The entire free-list has been swapped to the end of the file. So
-  ** truncate the database file to finSize pages and consider the
-  ** free-list empty.
-  */
-  rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
-  if( rc!=SQLITE_OK ) goto autovacuum_out;
         put4byte(&pBt->pPage1->aData[32], 0);
         put4byte(&pBt->pPage1->aData[36], 0);
-  *pnTrunc = finSize;
+        pBt->nTrunc = nFin;
-  assert( finSize!=PENDING_BYTE_PAGE(pBt) );
+      }
-
+    }
-autovacuum_out:
-  assert( nRef==sqlite3PagerRefcount(pPager) );
     if( rc!=SQLITE_OK ){
       sqlite3PagerRollback(pPager);
     }
+  }
+
+  if( rc==SQLITE_OK ){
+    *pnTrunc = pBt->nTrunc;
+    pBt->nTrunc = 0;
+  }
+  assert( nRef==sqlite3PagerRefcount(pPager) );
   return rc;
 }
+
 #endif
 
 /*
@@ -2615,6 +2668,10 @@ int sqlite3BtreeRollback(Btree *p){
   if( p->inTrans==TRANS_WRITE ){
     int rc2;
 
+#ifndef SQLITE_OMIT_AUTOVACUUM
+    pBt->nTrunc = 0;
+#endif
+
     assert( TRANS_WRITE==pBt->inTransaction );
     rc2 = sqlite3PagerRollback(pBt->pPager);
     if( rc2!=SQLITE_OK ){
@@ -3854,6 +3911,16 @@ static int allocateBtreePage(
     *pPgno = sqlite3PagerPagecount(pBt->pPager) + 1;
 
 #ifndef SQLITE_OMIT_AUTOVACUUM
+    if( pBt->nTrunc ){
+      /* An incr-vacuum has already run within this transaction. So the
+      ** page to allocate is not from the physical end of the file, but
+      ** at pBt->nTrunc. 
+      */
+      *pPgno = pBt->nTrunc+1;
+      if( *pPgno==PENDING_BYTE_PAGE(pBt) ){
+        (*pPgno)++;
+      }
+    }
     if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){
       /* If *pPgno refers to a pointer-map page, allocate two new pages
       ** at the end of the file instead of one. The first allocated page
@@ -3863,6 +3930,9 @@ static int allocateBtreePage(
       assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
       (*pPgno)++;
     }
+    if( pBt->nTrunc ){
+      pBt->nTrunc = *pPgno;
+    }
 #endif
 
     assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );

src/btree.h

@@ -13,7 +13,7 @@
 ** subsystem.  See comments in the source code for a detailed description
 ** of what each interface routine does.
 **
-** @(#) $Id: btree.h,v 1.74 2007/03/30 14:06:34 drh Exp $
+** @(#) $Id: btree.h,v 1.75 2007/04/26 14:42:36 danielk1977 Exp $
 */
 #ifndef _BTREE_H_
 #define _BTREE_H_
@@ -31,6 +31,10 @@
   #define SQLITE_DEFAULT_AUTOVACUUM 0
 #endif
 
+#define BTREE_AUTOVACUUM_NONE 0        /* Do not do auto-vacuum */
+#define BTREE_AUTOVACUUM_FULL 1        /* Do full auto-vacuum */
+#define BTREE_AUTOVACUUM_INCR 2        /* Incremental vacuum */
+
 /*
 ** Forward declarations of structure
 */
@@ -87,6 +91,8 @@ const char *sqlite3BtreeGetDirname(Btree *);
 const char *sqlite3BtreeGetJournalname(Btree *);
 int sqlite3BtreeCopyFile(Btree *, Btree *);
 
+int sqlite3BtreeIncrVacuum(Btree *);
+
 /* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR
 ** of the following flags:
 */

src/build.c

@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.421 2007/04/18 14:47:24 danielk1977 Exp $
+** $Id: build.c,v 1.422 2007/04/26 14:42:36 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -3349,3 +3349,20 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){
   }
   return pKey;
 }
+
+#ifndef SQLITE_OMIT_AUTOVACUUM
+/*
+** This is called to compile a statement of the form "INCREMENTAL VACUUM".
+*/
+void sqlite3IncrVacuum(Parse *pParse){
+  Vdbe *v = sqlite3GetVdbe(pParse);
+  if( v ){
+    int addr;
+    sqlite3BeginWriteOperation(pParse, 0, 0);
+    addr = sqlite3VdbeCurrentAddr(v);
+    sqlite3VdbeAddOp(v, OP_IncrVacuum, 0, addr+3);
+    sqlite3VdbeAddOp(v, OP_Callback, 0, 0);
+    sqlite3VdbeAddOp(v, OP_Goto, 0, addr);
+  }
+}
+#endif /* #ifndef SQLITE_OMIT_AUTOVACUUM */

src/parse.y

@@ -14,7 +14,7 @@
 ** the parser.  Lemon will also generate a header file containing
 ** numeric codes for all of the tokens.
 **
-** @(#) $Id: parse.y,v 1.218 2007/04/06 15:02:14 drh Exp $
+** @(#) $Id: parse.y,v 1.219 2007/04/26 14:42:36 danielk1977 Exp $
 */
 
 // All token codes are small integers with #defines that begin with "TK_"
@@ -903,6 +903,10 @@ cmd ::= VACUUM nm.             {sqlite3Vacuum(pParse);}
 %endif  SQLITE_OMIT_ATTACH
 %endif  SQLITE_OMIT_VACUUM
 
+%ifndef  SQLITE_OMIT_AUTOVACUUM
+cmd ::= INCREMENTAL VACUUM.    {sqlite3IncrVacuum(pParse);}
+%endif
+
 ///////////////////////////// The PRAGMA command /////////////////////////////
 //
 %ifndef SQLITE_OMIT_PRAGMA

src/pragma.c

@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file contains code used to implement the PRAGMA command.
 **
-** $Id: pragma.c,v 1.132 2007/03/30 17:11:13 danielk1977 Exp $
+** $Id: pragma.c,v 1.133 2007/04/26 14:42:36 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -73,6 +73,23 @@ static int getLockingMode(const char *z){
   return PAGER_LOCKINGMODE_QUERY;
 }
 
+#ifndef SQLITE_OMIT_AUTOVACUUM
+/*
+** Interpret the given string as an auto-vacuum mode value.
+**
+** The following strings, "none", "full" and "incremental" are 
+** acceptable, as are their numeric equivalents: 0, 1 and 2 respectively.
+*/
+static int getAutoVacuum(const char *z){
+  int i;
+  if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE;
+  if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL;
+  if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR;
+  i = atoi(z);
+  return ((i>=0&&i<=2)?i:0);
+}
+#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */
+
 #ifndef SQLITE_OMIT_PAGER_PRAGMAS
 /*
 ** Interpret the given string as a temp db location. Return 1 for file
@@ -389,7 +406,10 @@ void sqlite3Pragma(
           pBt ? sqlite3BtreeGetAutoVacuum(pBt) : SQLITE_DEFAULT_AUTOVACUUM;
       returnSingleInt(pParse, "auto_vacuum", auto_vacuum);
     }else{
-      sqlite3BtreeSetAutoVacuum(pBt, getBoolean(zRight));
+      int eAuto = getAutoVacuum(zRight);
+      if( eAuto>=0 ){
+        sqlite3BtreeSetAutoVacuum(pBt, eAuto);
+      }
     }
   }else
 #endif

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.552 2007/04/16 15:06:25 danielk1977 Exp $
+** @(#) $Id: sqliteInt.h,v 1.553 2007/04/26 14:42:36 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -1913,6 +1913,7 @@ int sqlite3VtabBegin(sqlite3 *, sqlite3_vtab *);
 FuncDef *sqlite3VtabOverloadFunction(FuncDef*, int nArg, Expr*);
 void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**);
 int sqlite3Reprepare(Vdbe*);
+void sqlite3IncrVacuum(Parse *pParse);
 
 #ifdef SQLITE_SSE
 #include "sseInt.h"

src/vdbe.c

@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.601 2007/04/18 16:45:24 drh Exp $
+** $Id: vdbe.c,v 1.602 2007/04/26 14:42:36 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -4554,6 +4554,24 @@ case OP_Vacuum: {        /* no-push */
 }
 #endif
 
+#if !defined(SQLITE_OMIT_AUTOVACUUM)
+/* Opcode: IncrVacuum * P2 *
+**
+** Perform a single step of the incremental vacuum procedure on
+** the main database. If the vacuum has finished, jump to instruction
+** P2. Otherwise, fall through to the next instruction.
+*/
+case OP_IncrVacuum: {        /* no-push */
+  Btree *pBt = db->aDb[0].pBt;
+  rc = sqlite3BtreeIncrVacuum(pBt);
+  if( rc==SQLITE_DONE ){
+    pc = pOp->p2 - 1;
+    rc = SQLITE_OK;
+  }
+  break;
+}
+#endif
+
 /* Opcode: Expire P1 * *
 **
 ** Cause precompiled statements to become expired. An expired statement

test/incrvacuum.test

@@ -0,0 +1,174 @@
+# 2007 April 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the incremental vacuum feature.
+#
+# $Id: incrvacuum.test,v 1.1 2007/04/26 14:42:36 danielk1977 Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If this build of the library does not support auto-vacuum, omit this
+# whole file.
+ifcapable {!autovacuum || !pragma} {
+  finish_test
+  return
+}
+
+#---------------------------------------------------------------------
+# Test the pragma on an empty database.
+#
+do_test incrvacuum-1.1 {
+  execsql {
+    pragma auto_vacuum;
+  }
+} {0}
+do_test incrvacuum-1.2 {
+  execsql {
+    pragma auto_vacuum = 'full';
+    pragma auto_vacuum;
+  }
+} {1}
+do_test incrvacuum-1.3 {
+  execsql {
+    pragma auto_vacuum = 'incremental';
+    pragma auto_vacuum;
+  }
+} {2}
+do_test incrvacuum-1.4 {
+  execsql {
+    pragma auto_vacuum = 'invalid';
+    pragma auto_vacuum;
+  }
+} {0}
+do_test incrvacuum-1.5 {
+  execsql {
+    pragma auto_vacuum = 1;
+    pragma auto_vacuum;
+  }
+} {1}
+do_test incrvacuum-1.6 {
+  execsql {
+    pragma auto_vacuum = '2';
+    pragma auto_vacuum;
+  }
+} {2}
+do_test incrvacuum-1.7 {
+  execsql {
+    pragma auto_vacuum = 5;
+    pragma auto_vacuum;
+  }
+} {0}
+
+#---------------------------------------------------------------------
+# Test the pragma on a non-empty database. It is possible to toggle
+# the connection between "full" and "incremental" mode, but not to
+# change from either of these to "none", or from "none" to "full" or
+# "incremental".
+#
+do_test incrvacuum-2.1 {
+  execsql {
+    pragma auto_vacuum = 1;
+    CREATE TABLE abc(a, b, c);
+  }
+} {}
+do_test incrvacuum-2.2 {
+  execsql {
+    pragma auto_vacuum = 'none';
+    pragma auto_vacuum;
+  }
+} {1}
+do_test incrvacuum-2.3 {
+  execsql {
+    pragma auto_vacuum = 'incremental';
+    pragma auto_vacuum;
+  }
+} {2}
+do_test incrvacuum-2.4 {
+  execsql {
+    pragma auto_vacuum = 'full';
+    pragma auto_vacuum;
+  }
+} {1}
+
+#---------------------------------------------------------------------
+# Test that when the auto_vacuum mode is "incremental", the database
+# does not shrink when pages are removed from it. But it does if
+# the mode is set to "full".
+#
+do_test incrvacuum-3.1 {
+  execsql {
+    pragma auto_vacuum;
+  }
+} {1}
+do_test incrvacuum-3.2 {
+  set ::str [string repeat 1234567890 110]
+  execsql {
+    PRAGMA auto_vacuum = 2;
+    BEGIN;
+    CREATE TABLE tbl2(str);
+    INSERT INTO tbl2 VALUES($::str);
+    COMMIT;
+  }
+  # 5 pages:
+  #
+  #   1 -> database header
+  #   2 -> first back-pointer page
+  #   3 -> table abc
+  #   4 -> table tbl2
+  #   5 -> table tbl2 overflow page.
+  #
+  expr {[file size test.db] / 1024}
+} {5}
+do_test incrvacuum-3.3 {
+  execsql {
+    DROP TABLE abc;
+    DELETE FROM tbl2;
+  }
+  expr {[file size test.db] / 1024}
+} {5}
+do_test incrvacuum-3.4 {
+  execsql {
+    PRAGMA auto_vacuum = 1;
+    INSERT INTO tbl2 VALUES('hello world');
+  }
+  expr {[file size test.db] / 1024}
+} {3}
+
+#---------------------------------------------------------------------
+# Try to run a simple incremental vacuum.
+#
+do_test incrvacuum-4.1 {
+  set ::str [string repeat 1234567890 110]
+  execsql {
+    PRAGMA auto_vacuum = 2;
+    INSERT INTO tbl2 VALUES($::str);
+    CREATE TABLE tbl1(a, b, c);
+  }
+  expr {[file size test.db] / 1024}
+} {5}
+do_test incrvacuum-4.2 {
+  execsql {
+    DELETE FROM tbl2;
+    DROP TABLE tbl1;
+  }
+  expr {[file size test.db] / 1024}
+} {5}
+do_test incrvacuum-4.3 {
+  set ::nStep 0
+  db eval {INCREMENTAL VACUUM} {
+    incr ::nStep
+  }
+  list [expr {[file size test.db] / 1024}] $::nStep
+} {3 2}
+
+finish_test
+

tool/mkkeywordhash.c

@@ -15,7 +15,7 @@ static const char zHdr[] =
   "**\n"
   "** The code in this file has been automatically generated by\n"
   "**\n"
-  "**     $Header: /home/drh/sqlite/trans/cvs/sqlite/sqlite/tool/mkkeywordhash.c,v 1.27 2007/04/06 11:26:00 drh Exp $\n"
+  "**     $Header: /home/drh/sqlite/trans/cvs/sqlite/sqlite/tool/mkkeywordhash.c,v 1.28 2007/04/26 14:42:36 danielk1977 Exp $\n"
   "**\n"
   "** The code in this file implements a function that determines whether\n"
   "** or not a given identifier is really an SQL keyword.  The same thing\n"
@@ -114,7 +114,8 @@ struct Keyword {
 #else
 #  define TRIGGER    0x00002000
 #endif
-#if defined(SQLITE_OMIT_VACUUM) || defined(SQLITE_OMIT_ATTACH)
+#if defined(SQLITE_OMIT_AUTOVACUUM) && \
+    (defined(SQLITE_OMIT_VACUUM) || defined(SQLITE_OMIT_ATTACH))
 #  define VACUUM     0
 #else
 #  define VACUUM     0x00004000
@@ -129,6 +130,11 @@ struct Keyword {
 #else
 #  define VTAB       0x00010000
 #endif
+#ifdef SQLITE_OMIT_AUTOVACUUM
+#  define AUTOVACUUM 0
+#else
+#  define AUTOVACUUM 0x00020000
+#endif
 
 /*
 ** These are the keywords
@@ -192,6 +198,7 @@ static Keyword aKeywordTable[] = {
   { "IGNORE",           "TK_IGNORE",       CONFLICT|TRIGGER       },
   { "IMMEDIATE",        "TK_IMMEDIATE",    ALWAYS                 },
   { "IN",               "TK_IN",           ALWAYS                 },
+  { "INCREMENTAL",      "TK_INCREMENTAL",  AUTOVACUUM             },
   { "INDEX",            "TK_INDEX",        ALWAYS                 },
   { "INITIALLY",        "TK_INITIALLY",    FKEY                   },
   { "INNER",            "TK_JOIN_KW",      ALWAYS                 },