Code to auto-vacuum the database if all root pages happen to be in the right place. Not active by default and largely untested. (CVS 2037)

FossilOrigin-Name: d12481f09cbe51c7ea499bc22afec5de3af14ad4
2025-08-07 02:42:48 +03:00 · 2004-11-02 12:56:41 +00:00
parent a82ffef115
commit 687566d786
7 changed files with 578 additions and 62 deletions
--- a/21
+++ b/21
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\ssupport.html\spage.\s(CVS\s2036)
-D 2004-11-01T16:03:12
+C Code\sto\sauto-vacuum\sthe\sdatabase\sif\sall\sroot\spages\shappen\sto\sbe\sin\sthe\sright\splace.\sNot\sactive\sby\sdefault\sand\slargely\suntested.\s(CVS\s2037)
+D 2004-11-02T12:56:41
 F Makefile.in 9e90c685d69f09039015a6b1f3b0a48e9738c9e5
 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1
@@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863
 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
 F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689
 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea
-F src/btree.c 231a0e7a00b96bafd4a7a0c774eece1510aec2fd
+F src/btree.c 92713a104c11abe0826a33590f1d4147332fd9a5
 F src/btree.h 94dfec0a1722d33359b23e7e310f2b64ffedf029
 F src/build.c bb896c5f85ab749d17ae5d730235134c12c08033
 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad
@@ -52,8 +52,8 @@ F src/os_unix.c 5824b22ba41fe9d514ef9169aac1b5fde73af229
 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13
 F src/os_win.c 9482dfc92f289b68205bb2c9315757c7e3946bfb
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c 6e19f9a64a9fae60bcf00140cecb5981765f3d95
-F src/pager.h 774d1973acbda341827d21b0da0150575d69f7d9
+F src/pager.c 6b00c0d5aac601b9f556b8fdba25e69438245f1a
+F src/pager.h cbe4ba356d9dd3f30260f322b3dc77408164df14
 F src/parse.y 08f4971f89e651f47b3f83fe7369c7edde254331
 F src/pragma.c 44e192eb5928157bdb015926f858a7c6e3ef6c98
 F src/printf.c 7a92adc00b758cd5ce087dae80181a8bbdb70ed2
@@ -87,6 +87,7 @@ F test/attach.test 6ad560eb3e77751a4faecd77da09deac9e38cc41
 F test/attach2.test f7795123d3051ace1672b6d23973da6435de3745
 F test/attach3.test 6d060986ff004ebb89e1876a331d96c6bb62269e
 F test/auth.test 1cc252d9e7b3bdc1314199cbf3a0d3c5ed026c21
+F test/autovacuum.test 77eec318b1be7764b8dcb3198c035edc30cd319f
 F test/bigfile.test d3744a8821ce9abb8697f2826a3e3d22b719e89f
 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747
 F test/bind.test a8682ba41433b93bb36a4213a43f282ca9aec5a9
@@ -157,7 +158,7 @@ F test/pagesize.test 56d11f4d6df9949d646bf87da1d6d995ed37cd78
 F test/pragma.test 66a66b7f3b273b93325c9a5794acb418f52fdcbf
 F test/printf.test 92ba4c510b4fc61120ffa4a01820446ed917ae57
 F test/progress.test 5ddba78cb6011fba36093973cfb3ac473b8fb96a x
-F test/quick.test 212a9cd4c40c72c7c4780fef1c2fbe5d1cb34ce6
+F test/quick.test 2dca186ebd5c418a7699944ba3b5e437d765eddd
 F test/quote.test 6d75cf635d93ba2484dc9cb378d88cbae9dc2c62
 F test/rollback.test 4097328d44510277244ef4fa51b22b2f11d7ef4c
 F test/rowid.test b3d059f5c8d8874fa1c31030e0636f67405d20ea
@@ -251,7 +252,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c
-P bebd967f3627220c3ce0352c8ca9c7c17b722ce6
-R 05340acf98e3d142b518cfefba2b2bd8
-U drh
-Z ee6b8d03ade5cb666f8adaa6773746cd
+P 5515accee348c6364cd58903a19029519797e123
+R d09732475e8e7fd32816b925d21968c5
+U danielk1977
+Z 3ba6d76f77fb940670c05d63aad699f9
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5515accee348c6364cd58903a19029519797e123
+d12481f09cbe51c7ea499bc22afec5de3af14ad4
--- a/src/btree.c
+++ b/src/btree.c
@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.195 2004/10/31 16:25:43 danielk1977 Exp $
+** $Id: btree.c,v 1.196 2004/11/02 12:56:41 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -414,23 +414,42 @@ static void put4byte(unsigned char *p, u32 v){
 #define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5)

 /*
-** The first byte of each 5-byte pointer map entry identifies the type
-** of page that the following 4-byte page number refers to (either a
-** regular btree page or an overflow page).
+** The pointer map is a lookup table that contains an entry for each database
+** page in the file except for page 1. In this context 'database page' refers
+** to any page that is not part of the pointer map itself.  Each pointer map
+** entry consists of a single byte 'type' and a 4 byte page number. The
+** PTRMAP_XXX identifiers below are the valid types. The interpretation
+** of the page-number depends on the type, as follows:
 **
-** If the type is PTRMAP_OVERFLOW, then the page is an overflow page.
-** In this case the pointer is always the first 4 bytes of the page.
+** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not
+**                  used in this case.
 **
-** If the type is PTRMAP_BTREE, then the page is a btree page. In this
-** case the pointer may be a 'left-pointer' (stored following a cell-header), 
-** a pointer to an overflow page (stored after a cell's data payload), 
-** or the 'right pointer' of a btree page.
+** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number 
+**                  is not used in this case.
+**
+** PTRMAP_OVERFLOW1: The database page is the first page in a list of 
+**                   overflow pages. The page number identifies the page that
+**                   contains the cell with a pointer to this overflow page.
+**
+** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of
+**                   overflow pages. The page-number identifies the previous
+**                   page in the overflow page list.
+**
+** PTRMAP_BTREE: The database page is a non-root btree page. The page number
+**               identifies the parent page in the btree.
 */
-#define PTRMAP_BTREE 1
-#define PTRMAP_OVERFLOW 2
+#define PTRMAP_ROOTPAGE 1
+#define PTRMAP_FREEPAGE 2
+#define PTRMAP_OVERFLOW1 3
+#define PTRMAP_OVERFLOW2 4
+#define PTRMAP_BTREE 5

 /*
 ** Write an entry into the pointer map.
+**
+** This routine updates the pointer map entry for page number 'key'
+** so that it maps to type 'eType' and parent page number 'pgno'.
+** An error code is returned if something goes wrong, otherwise SQLITE_OK.
 */
 static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){
  u8 *pPtrmap;    /* The pointer map page */
@@ -440,7 +459,7 @@ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){

  iPtrmap = PTRMAP_PAGENO(pBt->pageSize, key);
  rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap);
-  if( rc!=0 ){
+  if( rc!=SQLITE_OK ){
    return rc;
  }
  offset = PTRMAP_PTROFFSET(pBt->pageSize, key);
@@ -460,6 +479,10 @@ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){

 /*
 ** Read an entry from the pointer map.
+**
+** This routine retrieves the pointer map entry for page 'key', writing
+** the type and parent page number to *pEType and *pPgno respectively.
+** An error code is returned if something goes wrong, otherwise SQLITE_OK.
 */
 static int ptrmapGet(Btree *pBt, Pgno key, u8 *pEType, Pgno *pPgno){
  int iPtrmap;       /* Pointer map page index */
@@ -474,8 +497,8 @@ static int ptrmapGet(Btree *pBt, Pgno key, u8 *pEType, Pgno *pPgno){
  }

  offset = PTRMAP_PTROFFSET(pBt->pageSize, key);
-  *pEType = pPtrmap[offset];
-  *pPgno = get4byte(&pPtrmap[offset+1]);
+  if( pEType ) *pEType = pPtrmap[offset];
+  if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]);

  sqlite3pager_unref(pPtrmap);
  return SQLITE_OK;
@@ -1181,7 +1204,9 @@ int sqlite3BtreeOpen(
  *ppBtree = pBt;
 #ifdef SQLITE_AUTOVACUUM
  /* Note: This is temporary code for use during development of auto-vacuum. */
+  if( zFilename && 0!=strcmp(zFilename, ":memory:") ){
    pBt->autoVacuum = 1;
+  }
 #endif
  return SQLITE_OK;
 }
@@ -1453,6 +1478,272 @@ int sqlite3BtreeBeginTrans(Btree *pBt, int wrflag){
  return rc;
 }

+/*
+** The TRACE macro will print high-level status information about the
+** btree operation when the global variable sqlite3_btree_trace is
+** enabled.
+*/
+#if SQLITE_TEST
+# define TRACE(X)   if( sqlite3_btree_trace )\
+                        { sqlite3DebugPrintf X; fflush(stdout); }
+#else
+# define TRACE(X)
+#endif
+int sqlite3_btree_trace=0;  /* True to enable tracing */
+
+#ifndef SQLITE_OMIT_AUTOVACUUM
+
+/*
+** Set the pointer-map entries for all children of page pPage. Also, if
+** pPage contains cells that point to overflow pages, set the pointer
+** map entries for the overflow pages as well.
+*/
+static int setChildPtrmaps(MemPage *pPage){
+  int i;                             /* Counter variable */
+  int nCell;                         /* Number of cells in page pPage */
+  int rc = SQLITE_OK;                /* Return code */
+  Btree *pBt = pPage->pBt;
+  int isInitOrig = pPage->isInit;
+  Pgno pgno = pPage->pgno;
+
+  initPage(pPage, 0);
+  nCell = pPage->nCell;
+
+  for(i=0; i<nCell; i++){
+    CellInfo info;
+    u8 *pCell = findCell(pPage, i);
+
+    parseCellPtr(pPage, pCell, &info);
+    if( info.iOverflow ){
+      Pgno ovflPgno = get4byte(&pCell[info.iOverflow]);
+      rc = ptrmapPut(pBt, ovflPgno, PTRMAP_OVERFLOW1, pgno);
+      if( rc!=SQLITE_OK ) goto set_child_ptrmaps_out;
+    }
+    if( !pPage->leaf ){
+      Pgno childPgno = get4byte(pCell);
+      rc = ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno);
+      if( rc!=SQLITE_OK ) goto set_child_ptrmaps_out;
+    }
+  }
+
+  if( !pPage->leaf ){
+    Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
+    rc = ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno);
+  }
+
+set_child_ptrmaps_out:
+  pPage->isInit = isInitOrig;
+  return rc;
+}
+
+/*
+** Somewhere on pPage, which is guarenteed to be a btree page, not an overflow
+** page, is a pointer to page iFrom. Modify this pointer so that it points to
+** iTo. Parameter eType describes the type of pointer to be modified, as 
+** follows:
+**
+** PTRMAP_BTREE:     pPage is a btree-page. The pointer points at a child 
+**                   page of pPage.
+**
+** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow
+**                   page pointed to by one of the cells on pPage.
+**
+** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next
+**                   overflow page in the list.
+*/
+static void modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
+
+  if( eType==PTRMAP_OVERFLOW2 ){
+    assert( get4byte(pPage->aData)==iFrom );
+    put4byte(pPage->aData, iFrom);
+  }else{
+    int isInitOrig = pPage->isInit;
+    int i;
+    int nCell;
+
+    initPage(pPage, 0);
+    nCell = pPage->nCell;
+
+   /*  assert( !pPage->leaf && eType==PTRMAP_BTREE ); */
+
+    for(i=0; i<nCell; i++){
+      u8 *pCell = findCell(pPage, i);
+      if( eType==PTRMAP_OVERFLOW1 ){
+        CellInfo info;
+        parseCellPtr(pPage, pCell, &info);
+        if( info.iOverflow ){
+          if( iFrom==get4byte(&pCell[info.iOverflow]) ){
+            put4byte(&pCell[info.iOverflow], iTo);
+            break;
+          }
+        }
+      }else{
+        if( get4byte(pCell)==iFrom ){
+          put4byte(pCell, iTo);
+          break;
+        }
+      }
+    }
+  
+    if( i==nCell ){
+      assert( eType==PTRMAP_BTREE );
+      assert( get4byte(&pPage->aData[pPage->hdrOffset+8])==iFrom );
+      put4byte(&pPage->aData[pPage->hdrOffset+8], iTo);
+    }
+
+    pPage->isInit = isInitOrig;
+  }
+}
+
+/* Forward declaration required by autoVacuumCommit(). */
+static int allocatePage(Btree *, MemPage **, Pgno *, Pgno);
+
+/*
+** This routine is called prior to sqlite3pager_commit when a transaction
+** is commited for an auto-vacuum database.
+*/
+static int autoVacuumCommit(Btree *pBt){
+  Pager *pPager = pBt->pPager;
+  Pgno nFreeList;   /* Number of pages remaining on the free-list. */
+  Pgno origDbSize;  /* Pages in the database file */
+  Pgno finDbSize;   /* Pages in the database file after truncation */
+  int i;            /* Counter variable */
+  int rc;           /* Return code */
+  u8 eType;
+
+  Pgno iDbPage;              /* The database page to move */
+  u8 *pDbPage = 0;           /* "" */
+  MemPage *pDbMemPage = 0;   /* "" */
+  Pgno iPtrPage;             /* The page that contains a pointer to iDbPage */
+  MemPage *pPtrMemPage = 0;  /* "" */
+  Pgno iFreePage;            /* The free-list page to move iDbPage to */
+  MemPage *pFreeMemPage = 0; /* "" */
+
+#ifndef NDEBUG
+  int nRef = *sqlite3pager_stats(pPager);
+#endif
+
+  assert( pBt->autoVacuum );
+
+  /* Figure out how many free-pages are in the database. If there are no
+  ** free pages, then auto-vacuum is a no-op.
+  */
+  nFreeList = get4byte(&pBt->pPage1->aData[36]);
+  if( nFreeList==0 ) return SQLITE_OK;
+
+  origDbSize = sqlite3pager_pagecount(pPager);
+  finDbSize = origDbSize - nFreeList;
+  TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origDbSize, finDbSize));
+
+  /* Note: This is temporary code for use during development of auto-vacuum. 
+  **
+  ** Inspect the pointer map to make sure there are no root pages with a
+  ** page number greater than finDbSize. If so, the auto-vacuum cannot
+  ** proceed. This limitation will be fixed when root pages are automatically
+  ** allocated at the start of the database file.
+  */
+  for( i=finDbSize+1; i<=origDbSize; i++ ){
+    rc = ptrmapGet(pBt, i, &eType, 0);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    if( eType==PTRMAP_ROOTPAGE ){
+      TRACE(("AUTOVACUUM: Cannot proceed due to root-page on page %d\n", i));
+      return SQLITE_OK;
+    }
+  }
+
+  /* Variable 'finDbSize' will be the size of the file in pages after
+  ** the auto-vacuum has completed (the current file size minus the number
+  ** of pages on the free list). Loop through the pages that lie beyond
+  ** this mark, and if they are not already on the free list, move them
+  ** to a free page earlier in the file (somewhere before finDbSize).
+  */
+  for( iDbPage=finDbSize+1; iDbPage<=origDbSize; iDbPage++ ){
+    rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    assert( eType!=PTRMAP_ROOTPAGE );
+
+    /* If iDbPage is already on the free-list, do not swap it. */
+    if( eType==PTRMAP_FREEPAGE ){
+      continue;
+    }
+    rc = getPage(pBt, iDbPage, &pDbMemPage);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    pDbPage = pDbMemPage->aData;
+
+    /* Find the next page in the free-list that is not already at the end 
+    ** of the file. A page can be pulled off the free list using the 
+    ** allocatePage() routine.
+    */
+    do{
+      if( pFreeMemPage ){
+        releasePage(pFreeMemPage);
+        pFreeMemPage = 0;
+      }
+      rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0);
+      if( rc!=SQLITE_OK ) goto autovacuum_out;
+      assert( iFreePage<=origDbSize );
+    }while( iFreePage>finDbSize );
+
+    /* Move page iDbPage from it's current location to page number iFreePage */
+    TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d)\n", 
+        iDbPage, iFreePage, iPtrPage));
+    releasePage(pFreeMemPage);
+    pFreeMemPage = 0;
+    rc = sqlite3pager_movepage(pPager, pDbPage, iFreePage);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    pDbMemPage->pgno = iFreePage;
+
+    /* If pDbPage was a btree-page, then it may have child pages and/or cells
+    ** that point to overflow pages. The pointer map entries for all these
+    ** pages need to be changed.
+    */
+    if( eType==PTRMAP_BTREE ){
+      rc = setChildPtrmaps(pDbMemPage);
+      if( rc!=SQLITE_OK ) goto autovacuum_out;
+    }
+    releasePage(pDbMemPage);
+    pDbMemPage = 0;
+
+    /* Fix the database pointer on page iPtrPage that pointed at iDbPage so
+    ** that it points at iFreePage. Also fix the pointer map entry for
+    ** iPtrPage.
+    */
+    rc = getPage(pBt, iPtrPage, &pPtrMemPage);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    rc = sqlite3pager_write(pPtrMemPage->aData);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    modifyPagePointer(pPtrMemPage, iDbPage, iFreePage, eType);
+    rc = ptrmapPut(pBt, iFreePage, eType, iPtrPage);
+    if( rc!=SQLITE_OK ) goto autovacuum_out;
+    releasePage(pPtrMemPage);
+  }
+
+  /* The entire free-list has been swapped to the end of the file. So
+  ** truncate the database file to finDbSize pages and consider the
+  ** free-list empty.
+  */
+  rc = sqlite3pager_write(pBt->pPage1->aData);
+  if( rc!=SQLITE_OK ) goto autovacuum_out;
+  put4byte(&pBt->pPage1->aData[32], 0);
+  put4byte(&pBt->pPage1->aData[36], 0);
+  rc = sqlite3pager_truncate(pBt->pPager, finDbSize);
+  if( rc!=SQLITE_OK ) goto autovacuum_out;
+
+autovacuum_out:
+  /* TODO: A goto autovacuum_out; will fail to call releasePage() on 
+  ** outstanding references. Fix.
+  */
+  if( nRef!=*sqlite3pager_stats(pPager) ){
+    sqlite3pager_refdump(pPager);
+  }
+  assert( nRef==*sqlite3pager_stats(pPager) );
+  if( rc!=SQLITE_OK ){
+    sqlite3pager_rollback(pPager);
+  }
+  return rc;
+}
+#endif
+
 /*
 ** Commit the transaction currently in progress.
 **
@@ -2472,19 +2763,6 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){
  return rc;
 }

-/*
-** The TRACE macro will print high-level status information about the
-** btree operation when the global variable sqlite3_btree_trace is
-** enabled.
-*/
-#if SQLITE_TEST
-# define TRACE(X)   if( sqlite3_btree_trace )\
-                        { sqlite3DebugPrintf X; fflush(stdout); }
-#else
-# define TRACE(X)
-#endif
-int sqlite3_btree_trace=0;  /* True to enable tracing */
-
 /*
 ** Allocate a new page from the database file.
 **
@@ -2615,6 +2893,15 @@ static int freePage(MemPage *pPage){
  n = get4byte(&pPage1->aData[36]);
  put4byte(&pPage1->aData[36], n+1);

+#ifndef SQLITE_OMIT_AUTOVACUUM
+  /* If the database supports auto-vacuum, write an entry in the pointer-map
+  ** to indicate that the page is free.
+  */
+  if( pBt->autoVacuum ){
+    rc = ptrmapPut(pBt, pPage->pgno, PTRMAP_FREEPAGE, 0);
+  }
+#endif
+
  if( n==0 ){
    /* This is the first free page */
    rc = sqlite3pager_write(pPage->aData);
@@ -2757,9 +3044,9 @@ static int fillInCell(
      */
      if( pBt->autoVacuum && rc==0 ){
        if( pgnoPtrmap!=0 ){
-          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW, pgnoPtrmap);
+          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW2, pgnoPtrmap);
        }else{
-          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_BTREE, pPage->pgno);
+          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno);
        }
      }
 #endif
@@ -2867,7 +3154,7 @@ static int reparentChildPages(MemPage *pPage){
      parseCellPtr(pPage, pCell, &info);
      if( info.iOverflow ){
        Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
-        rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_BTREE, pPage->pgno);
+        rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno);
        if( rc!=SQLITE_OK ) return rc;
      }
    }
@@ -3649,7 +3936,7 @@ static int balance(MemPage *pPage){
    if( pPage->nOverflow>0 ){
      rc = balance_deeper(pPage);
    }
-    if( pPage->nCell==0 ){
+    if( rc==SQLITE_OK && pPage->nCell==0 ){
      rc = balance_shallower(pPage);
    }
  }else{
@@ -3762,7 +4049,9 @@ int sqlite3BtreeInsert(
  rc = balance(pPage);
  /* sqlite3BtreePageDump(pCur->pBt, pCur->pgnoRoot, 1); */
  /* fflush(stdout); */
+  if( rc==SQLITE_OK ){
    moveToRoot(pCur);
+  }
 end_insert:
  sqliteFree(newCell);
  return rc;
@@ -3878,6 +4167,18 @@ int sqlite3BtreeCreateTable(Btree *pBt, int *piTable, int flags){
  }
  rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1);
  if( rc ) return rc;
+#ifndef SQLITE_OMIT_AUTOVACUUM
+  /* Note: This is temporary code for use during development of auto-vacuum. 
+  ** There should be no need for a pointer map entry for root-pages.
+  */
+  if( pBt->autoVacuum ){
+    rc = ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0);
+    if( rc ){
+      sqlite3pager_unref(pRoot->aData);
+      return rc;
+    }
+  }
+#endif
  assert( sqlite3pager_iswriteable(pRoot->aData) );
  zeroPage(pRoot, flags | PTF_LEAF);
  sqlite3pager_unref(pRoot->aData);
@@ -4342,25 +4643,38 @@ static void checkList(
    }
    if( isFreeList ){
      int n = get4byte(&pOvfl[4]);
+#ifndef SQLITE_OMIT_AUTOVACUUM
+      if( pCheck->pBt->autoVacuum ){
+        checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext);
+      }
+#endif
      if( n>pCheck->pBt->usableSize/4-8 ){
        checkAppendMsg(pCheck, zContext,
           "freelist leaf count too big on page %d", iPage);
        N--;
      }else{
        for(i=0; i<n; i++){
-          checkRef(pCheck, get4byte(&pOvfl[8+i*4]), zContext);
+          Pgno iFreePage = get4byte(&pOvfl[8+i*4]);
+#ifndef SQLITE_OMIT_AUTOVACUUM
+          if( pCheck->pBt->autoVacuum ){
+            checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext);
+          }
+#endif
+          checkRef(pCheck, iFreePage, zContext);
        }
        N -= n;
      }
    }
 #ifndef SQLITE_OMIT_AUTOVACUUM
+    else{
      /* If this database supports auto-vacuum and iPage is not the last
      ** page in this overflow list, check that the pointer-map entry for
      ** the following page matches iPage.
      */
-    if( pCheck->pBt->autoVacuum && !isFreeList && N>0 ){
+      if( pCheck->pBt->autoVacuum && N>0 ){
        i = get4byte(pOvfl);
-      checkPtrmap(pCheck, i, PTRMAP_OVERFLOW, iPage, zContext);
+        checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext);
+      }
    }
 #endif
    iPage = get4byte(pOvfl);
@@ -4448,7 +4762,7 @@ static int checkTreePage(
      Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
 #ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
-        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_BTREE, iPage, zContext);
+        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext);
      }
 #endif
      checkList(pCheck, 0, pgnoOvfl, nPage, zContext);
@@ -4475,7 +4789,7 @@ static int checkTreePage(
    sprintf(zContext, "On page %d at right child: ", iPage);
 #ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum ){
-      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext);
+      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, 0);
    }
 #endif
    checkTreePage(pCheck, pgno, pPage, zContext,0,0,0,0);
@@ -4569,6 +4883,12 @@ char *sqlite3BtreeIntegrityCheck(Btree *pBt, int *aRoot, int nRoot){
  */
  for(i=0; i<nRoot; i++){
    if( aRoot[i]==0 ) continue;
+#ifndef SQLITE_OMIT_AUTOVACUUM
+/* Note: This is temporary code for use during development of auto-vacuum. */
+    if( pBt->autoVacuum && aRoot[i]>1 ){
+      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0);
+    }
+#endif
    checkTreePage(&sCheck, aRoot[i], 0, "List of tree roots: ", 0,0,0,0);
  }

@@ -4711,6 +5031,12 @@ int sqlite3BtreeIsInStmt(Btree *pBt){
 */
 int sqlite3BtreeSync(Btree *pBt, const char *zMaster){
  if( pBt->inTrans==TRANS_WRITE ){
+#ifndef SQLITE_OMIT_AUTOVACUUM
+    if( pBt->autoVacuum ){
+      int rc = autoVacuumCommit(pBt); 
+      if( rc!=SQLITE_OK ) return rc;
+    }
+#endif
    return sqlite3pager_sync(pBt->pPager, zMaster);
  }
  return SQLITE_OK;
--- a/src/pager.c
+++ b/src/pager.c
@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.169 2004/10/31 02:22:49 drh Exp $
+** @(#) $Id: pager.c,v 1.170 2004/11/02 12:56:41 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -2017,10 +2017,22 @@ static int pager_write_pagelist(PgHdr *pList){
  while( pList ){
    assert( pList->dirty );
    sqlite3OsSeek(&pPager->fd, (pList->pgno-1)*(i64)pPager->pageSize);
+    /* If there are dirty pages in the page cache with page numbers greater
+    ** than Pager.dbSize, this means sqlite3pager_truncate() was called to
+    ** make the file smaller (presumably by auto-vacuum code). Do not write
+    ** any such pages to the file.
+    */
+    if( pList->pgno<=pPager->dbSize ){
      CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6);
      TRACE3("STORE %d page %d\n", pPager->fd.h, pList->pgno);
      rc = sqlite3OsWrite(&pPager->fd, PGHDR_TO_DATA(pList), pPager->pageSize);
      CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 0);
+    }
+#ifndef NDEBUG
+    else{
+      TRACE3("NOSTORE %d page %d\n", pPager->fd.h, pList->pgno);
+    }
+#endif
    if( rc ) return rc;
    pList->dirty = 0;
    pList = pList->pDirty;
@@ -3202,6 +3214,81 @@ sync_exit:
  return rc;
 }

+#ifndef SQLITE_OMIT_AUTOVACUUM
+/*
+** Move the page identified by pData to location pgno in the file. 
+**
+** There must be no references to the current page pgno. If current page
+** pgno is not already in the rollback journal, it is not written there by
+** by this routine. The same applies to the page pData refers to on entry to
+** this routine.
+**
+** References to the page refered to by pData remain valid. Updating any
+** meta-data associated with page pData (i.e. data stored in the nExtra bytes
+** allocated along with the page) is the responsibility of the caller.
+**
+** A transaction must be active when this routine is called, however it is 
+** illegal to call this routine if a statment transaction is active.
+*/
+int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
+  PgHdr *pPg = DATA_TO_PGHDR(pData);
+  PgHdr *pPgOld; 
+
+  assert( !pPager->stmtInUse );
+  /* assert( pPg->pNextFree==0 && pPg->pPrevFree==0 && pPg->nRef>0 ); */
+  assert( pPg->nRef>0 );
+
+  /* Unlink pPg from it's hash-chain */
+  if( pPg->pNextHash ){
+    pPg->pNextHash->pPrevHash = pPg->pPrevHash;
+  }
+  if( pPg->pPrevHash ){
+    pPg->pPrevHash->pNextHash = pPg->pNextHash;
+  }else{
+    int h = pager_hash(pPg->pgno);
+    assert( pPager->aHash[h]==pPg );
+    pPager->aHash[h] = pPg->pNextHash;
+  }
+
+  /* Change the page number for pPg */
+  pPg->pgno = pgno;
+
+  pPgOld = pager_lookup(pPager, pgno);
+  if( pPgOld ){
+    /* Remove pPgOld from the page number hash-chain and insert pPg. */
+    assert(pPgOld->nRef==0 && !pPgOld->pNextStmt && !pPgOld->pPrevStmt );
+    if( pPgOld->pNextHash ){
+      pPgOld->pNextHash->pPrevHash = pPg;
+    }
+    if( pPgOld->pPrevHash ){
+      pPgOld->pPrevHash->pNextHash = pPg;
+    }else{
+      int h = pager_hash(pgno);
+      assert( pPager->aHash[h]==pPgOld );
+      pPager->aHash[h] = pPg;
+    }
+    pPgOld->pNextHash = pPgOld->pPrevHash = 0;
+  }else{
+    /* Insert pPg into it's new hash-chain. */
+    int h = pager_hash(pgno);
+    if( pPager->aHash[h] ){
+      pPager->aHash[h]->pNextHash = pPg;
+    }
+    pPg->pNextHash = pPager->aHash[h];
+    pPg->pPrevHash = 0;
+  }
+
+  /* Don't write the old page when sqlite3pager_sync() is called. Do write
+  ** the new one. 
+  */
+  pPgOld->dirty = 0;
+  pPg->dirty = 1;
+  pPager->dirtyCache = 1;
+
+  return SQLITE_OK;
+}
+#endif
+
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
 /*
 ** Return the current state of the file lock for the given pager.
--- a/src/pager.h
+++ b/src/pager.h
@@ -13,7 +13,7 @@
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.38 2004/10/05 02:41:43 drh Exp $
+** @(#) $Id: pager.h,v 1.39 2004/11/02 12:56:41 danielk1977 Exp $
 */

 /*
@@ -91,6 +91,7 @@ const char *sqlite3pager_dirname(Pager*);
 const char *sqlite3pager_journalname(Pager*);
 int sqlite3pager_rename(Pager*, const char *zNewName);
 void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*);
+int sqlite3pager_movepage(Pager*,void*,Pgno);

 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
 int sqlite3pager_lockstate(Pager*);
--- a/test/autovacuum.test
+++ b/test/autovacuum.test
@@ -0,0 +1,99 @@
+# 2001 September 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SELECT statement.
+#
+# $Id: autovacuum.test,v 1.1 2004/11/02 12:56:41 danielk1977 Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+proc make_str {char len} {
+  set str [string repeat $char. $len]
+  return [string range $str 0 [expr $len-1]]
+}
+
+proc file_pages {} {
+  return [expr [file size test.db] / 1024]
+}
+
+do_test autovacuum-1.1 {
+  execsql {
+    CREATE TABLE av1(a);
+  }
+} {}
+
+set ENTRY_LEN 1100
+
+set delete_orders [list]
+lappend delete_orders {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20}
+lappend delete_orders {20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} 
+lappend delete_orders {8 18 2 4 14 11 13 3 10 7 9 5 12 17 19 15 20 6 16 1}
+lappend delete_orders {10 3 11 17 19 20 7 4 13 6 1 14 16 12 9 18 8 15 5 2}
+
+lappend delete_orders {{1 2 3 4 5 6 7 8 9 10} {11 12 13 14 15 16 17 18 19 20}}
+lappend delete_orders \
+        {{19 8 17 15} {16 11 9 14} {18 5 3 1} {13 20 7 2} {6 12 4 10}}
+
+
+set tn 0
+foreach delete_order $delete_orders {
+  incr tn
+
+  # Set up the table.
+  set ::tbl_data [list]
+  foreach i [lsort -integer [eval concat $delete_order]] {
+    execsql "INSERT INTO av1 (oid, a) VALUES($i, '[make_str $i $ENTRY_LEN]')"
+    lappend ::tbl_data [make_str $i $ENTRY_LEN]
+  }
+
+# puts "File has [file_pages] pages"
+
+  do_test autovacuum-1.$tn.1 {
+    execsql {
+      pragma integrity_check
+    }
+  } {ok}
+
+  foreach delete $delete_order {
+# if {$delete==6} { set btree_trace 1 ; breakpoint }
+    do_test autovacuum-1.$tn.($delete).1 {
+      execsql "
+        DELETE FROM av1 WHERE oid IN ([join $delete ,])
+      "
+    } {}
+set btree_trace 0
+
+    do_test autovacuum-1.$tn.($delete).2 {
+      execsql {
+        pragma integrity_check
+      }
+    } {ok}
+
+    foreach d $delete {
+      set idx [lsearch $::tbl_data [make_str $d $ENTRY_LEN]]
+      set ::tbl_data [lreplace $::tbl_data $idx $idx]
+    }
+    do_test autovacuum-1.$tn.($delete).3 {
+      execsql {
+        select a from av1
+      }
+    } $::tbl_data
+# if {$::nErr>0} finish_test
+  }
+
+  do_test autovacuum-1.$tn.3 {
+    file_pages
+  } {3}
+}
+
+finish_test
+
--- a/test/quick.test
+++ b/test/quick.test
@@ -10,7 +10,7 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: quick.test,v 1.30 2004/09/02 14:57:09 drh Exp $
+# $Id: quick.test,v 1.31 2004/11/02 12:56:41 danielk1977 Exp $

 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -32,6 +32,8 @@ set EXCLUDE {
  misuse.test
  quick.test
  utf16.test
+
+  autovacuum.test
 }

 if {[sqlite3 -has-codec]} {