For filesystem databases, do not store a list of pages in the statement journal in main memory. (CVS 3825)

FossilOrigin-Name: 0af764a02695281b0a7d70ef4e7f8229fd3d6a30

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\stest\sscripts\sto\ssupport\salternative\sconfigurations.\s(CVS\s3824)
+C For\sfilesystem\sdatabases,\sdo\snot\sstore\sa\slist\sof\spages\sin\sthe\sstatement\sjournal\sin\smain\smemory.\s(CVS\s3825)
-D 2007-04-06T21:42:22
+D 2007-04-07T15:03:17
 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -58,7 +58,7 @@ F src/alter.c 2c79ec40f65e33deaf90ca493422c74586e481a3
 F src/analyze.c 4bbf5ddf9680587c6d4917e02e378b6037be3651
 F src/attach.c a16ada4a4654a0d126b8223ec9494ebb81bc5c3c
 F src/auth.c 902f4722661c796b97f007d9606bd7529c02597f
-F src/btree.c 5a0f425843ff81f3308fd083e606344b48c332f8
+F src/btree.c ea98c5549ef0b3d13e0d22e0d043fd07df972096
 F src/btree.h 9b2cc0d113c0bc2d37d244b9a394d56948c9acbf
 F src/build.c 7c2efa468f0c404ef5aa648d43c383318390937f
 F src/callback.c 31d22b4919c7645cbcbb1591ce2453e8c677c558
@@ -86,7 +86,7 @@ F src/os_unix.c 426b4c03c304ad78746d65d9ba101e0b72e18e23
 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
 F src/os_win.c c9a99524d6b2bdec636264cad1b67553925e3309
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c e1ee86ae8d09c84bcd91900ade250776d83e3c72
+F src/pager.c d0bd990ade9062ec1884a4c4e138f464e18e9250
 F src/pager.h e79a24cf200b8771366217f5bca414f5b7823f42
 F src/parse.y b6cfbadb6d5b21b5087d30698ee5af0ebb098767
 F src/pragma.c 3b992b5b2640d6ae25cef05aa6a42cd1d6c43234
@@ -150,7 +150,7 @@ F test/attachmalloc.test 03eeddd06e685ddbe975efd51824e4941847e5f4
 F test/auth.test 66923137cf78475f5671b5e6e6274935e055aea0
 F test/auth2.test 8da06f0ffcfd98154dda78e0f3b35a6503c27b64
 F test/autoinc.test 60005a676e3e4e17dfa9dbd08aa0b76587ff97e3
-F test/autovacuum.test 05f528c3bf98d086df7d86fddeb9d85037254913
+F test/autovacuum.test 4339e66003b9cf813dd667a83aed2dee27c4c36d
 F test/autovacuum_crash.test 05a63b8805b20cfba7ace82856ce4ccdda075a31
 F test/autovacuum_ioerr.test c46a76869cb6eddbbb40b419b2b6c4c001766b1f
 F test/autovacuum_ioerr2.test 2f8a3fb31f833fd0ca86ad4ad98913c73e807572
@@ -455,7 +455,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P 26b2e1aede3f776134b2d6e941d17a907843e650
+P 3471a2269fb6b3769b59b70992e6da3bdebea7df
-R a613e3c3d27dabfc3ee74ed8f3cd493b
+R fbb3aab61f7f8db520ad1a1642867d6f
-U drh
+U danielk1977
-Z 3f5436e4db1d58055f5d0cc5ce8f8601
+Z 424d41e4ea9fa8fe57d130ff951bd83e

manifest.uuid

@@ -1 +1 @@
-3471a2269fb6b3769b59b70992e6da3bdebea7df
+0af764a02695281b0a7d70ef4e7f8229fd3d6a30

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.352 2007/04/06 15:02:14 drh Exp $
+** $Id: btree.c,v 1.353 2007/04/07 15:03:17 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -5530,10 +5530,18 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){
     }
 
     if( pgnoMove!=pgnoRoot ){
+      /* pgnoRoot is the page that will be used for the root-page of
+      ** the new table (assuming an error did not occur). But we were
+      ** allocated pgnoMove. If required (i.e. if it was not allocated
+      ** by extending the file), the current page at position pgnoMove
+      ** is already journaled.
+      */
       u8 eType;
       Pgno iPtrPage;
 
       releasePage(pPageMove);
+
+      /* Move the page currently at pgnoRoot to pgnoMove. */
       rc = getPage(pBt, pgnoRoot, &pRoot, 0);
       if( rc!=SQLITE_OK ){
         return rc;
@@ -5552,6 +5560,8 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){
       }
       rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove);
       releasePage(pRoot);
+
+      /* Obtain the page at pgnoRoot */
       if( rc!=SQLITE_OK ){
         return rc;
       }

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.324 2007/04/06 18:23:18 drh Exp $
+** @(#) $Id: pager.c,v 1.325 2007/04/07 15:03:17 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -158,9 +158,7 @@ struct PgHdr {
   PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
   PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
   PgHdr *pNextAll;               /* A list of all pages */
-  PgHdr *pNextStmt, *pPrevStmt;  /* List of pages in the statement journal */
   u8 inJournal;                  /* TRUE if has been written to journal */
-  u8 inStmt;                     /* TRUE if in the statement subjournal */
   u8 dirty;                      /* TRUE if we need to write back changes */
   u8 needSync;                   /* Sync journal before writing this page */
   u8 alwaysRollback;             /* Disable DontRollback() for this page */
@@ -189,6 +187,8 @@ typedef struct PgHistory PgHistory;
 struct PgHistory {
   u8 *pOrig;     /* Original page text.  Restore to this on a full rollback */
   u8 *pStmt;     /* Text as it was at the beginning of the current statement */
+  PgHdr *pNextStmt, *pPrevStmt;  /* List of pages in the statement journal */
+  u8 inStmt;                     /* TRUE if in the statement subjournal */
 };
 
 /*
@@ -392,6 +392,21 @@ static const unsigned char aJournalMagic[] = {
 # define REFINFO(X)
 #endif
 
+/*
+** Return true if page *pPg has already been written to the statement
+** journal (or statement snapshot has been created, if *pPg is part
+** of an in-memory database).
+*/
+static int pageInStatement(PgHdr *pPg){
+  Pager *pPager = pPg->pPager;
+  if( MEMDB ){
+    return PGHDR_TO_HIST(pPg, pPager)->inStmt;
+  }else{
+    Pgno pgno = pPg->pgno;
+    u8 *a = pPager->aInStmt;
+    return (a && (int)pgno<=pPager->stmtSize && (a[pgno/8] & (1<<(pgno&7))));
+  }
+}
 
 /*
 ** Change the size of the pager hash table to N.  N must be a power
@@ -814,32 +829,17 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){
 */
 static void page_add_to_stmt_list(PgHdr *pPg){
   Pager *pPager = pPg->pPager;
-  if( pPg->inStmt ) return;
+  PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
-  assert( pPg->pPrevStmt==0 && pPg->pNextStmt==0 );
+  assert( MEMDB );
-  pPg->pPrevStmt = 0;
+  if( !pHist->inStmt ){
-  if( pPager->pStmt ){
+    assert( pHist->pPrevStmt==0 && pHist->pNextStmt==0 );
-    pPager->pStmt->pPrevStmt = pPg;
+    if( pPager->pStmt ){
+      PGHDR_TO_HIST(pPager->pStmt, pPager)->pPrevStmt = pPg;
+    }
+    pHist->pNextStmt = pPager->pStmt;
+    pPager->pStmt = pPg;
+    pHist->inStmt = 1;
   }
-  pPg->pNextStmt = pPager->pStmt;
-  pPager->pStmt = pPg;
-  pPg->inStmt = 1;
-}
-static void page_remove_from_stmt_list(PgHdr *pPg){
-  if( !pPg->inStmt ) return;
-  if( pPg->pPrevStmt ){
-    assert( pPg->pPrevStmt->pNextStmt==pPg );
-    pPg->pPrevStmt->pNextStmt = pPg->pNextStmt;
-  }else{
-    assert( pPg->pPager->pStmt==pPg );
-    pPg->pPager->pStmt = pPg->pNextStmt;
-  }
-  if( pPg->pNextStmt ){
-    assert( pPg->pNextStmt->pPrevStmt==pPg );
-    pPg->pNextStmt->pPrevStmt = pPg->pPrevStmt;
-  }
-  pPg->pNextStmt = 0;
-  pPg->pPrevStmt = 0;
-  pPg->inStmt = 0;
 }
 
 /*
@@ -2667,7 +2667,6 @@ int sqlite3PagerReleaseMemory(int nReq){
         */
         PgHdr *pTmp;
         assert( pPg );
-        page_remove_from_stmt_list(pPg);
         if( pPg==pPager->pAll ){
            pPager->pAll = pPg->pNextAll;
         }else{
@@ -3008,6 +3007,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
     }
 
     pPg->pgno = pgno;
+    assert( !MEMDB || pgno>pPager->stmtSize );
     if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
       sqlite3CheckMemory(pPager->aInJournal, pgno/8);
       assert( pPager->journalOpen );
@@ -3017,12 +3017,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){
       pPg->inJournal = 0;
       pPg->needSync = 0;
     }
-    if( pPager->aInStmt && (int)pgno<=pPager->stmtSize
+
-             && (pPager->aInStmt[pgno/8] & (1<<(pgno&7)))!=0 ){
-      page_add_to_stmt_list(pPg);
-    }else{
-      page_remove_from_stmt_list(pPg);
-    }
     makeClean(pPg);
     pPg->nRef = 1;
     REFINFO(pPg);
@@ -3374,7 +3369,7 @@ static int pager_write(PgHdr *pPg){
   ** to the journal then we can return right away.
   */
   makeDirty(pPg);
-  if( pPg->inJournal && (pPg->inStmt || pPager->stmtInUse==0) ){
+  if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){
     pPager->dirtyCache = 1;
   }else{
 
@@ -3449,7 +3444,6 @@ static int pager_write(PgHdr *pPg){
           pPg->needSync = !pPager->noSync;
           if( pPager->stmtInUse ){
             pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
-            page_add_to_stmt_list(pPg);
           }
         }
       }else{
@@ -3468,7 +3462,10 @@ static int pager_write(PgHdr *pPg){
     ** the statement journal format differs from the standard journal format
     ** in that it omits the checksums and the header.
     */
-    if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){
+    if( pPager->stmtInUse 
+     && !pageInStatement(pPg) 
+     && (int)pPg->pgno<=pPager->stmtSize 
+    ){
       assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
       if( MEMDB ){
         PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
@@ -3478,6 +3475,7 @@ static int pager_write(PgHdr *pPg){
           memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize);
         }
         PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
+        page_add_to_stmt_list(pPg);
       }else{
         char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7)-4;
         put32bits(pData2, pPg->pgno);
@@ -3490,7 +3488,6 @@ static int pager_write(PgHdr *pPg){
         assert( pPager->aInStmt!=0 );
         pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
       }
-      page_add_to_stmt_list(pPg);
     }
   }
 
@@ -3681,16 +3678,17 @@ void sqlite3PagerDontRollback(DbPage *pPg){
     pPg->inJournal = 1;
     if( pPager->stmtInUse ){
       pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
-      page_add_to_stmt_list(pPg);
     }
     PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
     IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
   }
-  if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){
+  if( pPager->stmtInUse 
+   && !pageInStatement(pPg) 
+   && (int)pPg->pgno<=pPager->stmtSize 
+  ){
     assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
     assert( pPager->aInStmt!=0 );
     pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
-    page_add_to_stmt_list(pPg);
   }
 }
 
@@ -3840,12 +3838,13 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){
   if( MEMDB ){
     pPg = pager_get_all_dirty_pages(pPager);
     while( pPg ){
-      clearHistory(PGHDR_TO_HIST(pPg, pPager));
+      PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
+      clearHistory(pHist);
       pPg->dirty = 0;
       pPg->inJournal = 0;
-      pPg->inStmt = 0;
+      pHist->inStmt = 0;
       pPg->needSync = 0;
-      pPg->pPrevStmt = pPg->pNextStmt = 0;
+      pHist->pPrevStmt = pHist->pNextStmt = 0;
       pPg = pPg->pDirty;
     }
     pPager->pDirty = 0;
@@ -3903,8 +3902,8 @@ int sqlite3PagerRollback(Pager *pPager){
       clearHistory(pHist);
       p->dirty = 0;
       p->inJournal = 0;
-      p->inStmt = 0;
+      pHist->inStmt = 0;
-      p->pPrevStmt = p->pNextStmt = 0;
+      pHist->pPrevStmt = pHist->pNextStmt = 0;
       if( pPager->xReiniter ){
         pPager->xReiniter(p, pPager->pageSize);
       }
@@ -4051,14 +4050,13 @@ int sqlite3PagerStmtCommit(Pager *pPager){
       /* sqlite3OsTruncate(pPager->stfd, 0); */
       sqliteFree( pPager->aInStmt );
       pPager->aInStmt = 0;
-    }
+    }else{
-    for(pPg=pPager->pStmt; pPg; pPg=pNext){
+      for(pPg=pPager->pStmt; pPg; pPg=pNext){
-      pNext = pPg->pNextStmt;
-      assert( pPg->inStmt );
-      pPg->inStmt = 0;
-      pPg->pPrevStmt = pPg->pNextStmt = 0;
-      if( MEMDB ){
         PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
+        pNext = pHist->pNextStmt;
+        assert( pHist->inStmt );
+        pHist->inStmt = 0;
+        pHist->pPrevStmt = pHist->pNextStmt = 0;
         sqliteFree(pHist->pStmt);
         pHist->pStmt = 0;
       }
@@ -4080,8 +4078,9 @@ int sqlite3PagerStmtRollback(Pager *pPager){
     PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
     if( MEMDB ){
       PgHdr *pPg;
-      for(pPg=pPager->pStmt; pPg; pPg=pPg->pNextStmt){
+      PgHistory *pHist;
-        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
+      for(pPg=pPager->pStmt; pPg; pPg=pHist->pNextStmt){
+        pHist = PGHDR_TO_HIST(pPg, pPager);
         if( pHist->pStmt ){
           memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize);
           sqliteFree(pHist->pStmt);

test/autovacuum.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the SELECT statement.
 #
-# $Id: autovacuum.test,v 1.25 2007/04/02 14:19:16 danielk1977 Exp $
+# $Id: autovacuum.test,v 1.26 2007/04/07 15:03:17 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -492,10 +492,23 @@ do_test autovacuum-3.7 {
   file_pages
 } [expr $AUTOVACUUM?1:2]
 
+
 #-----------------------------------------------------------------------
 # Test that if a statement transaction around a CREATE INDEX statement is
 # rolled back no corruption occurs.
 #
+do_test autovacuum-4.0 {
+  # The last round of tests may have left the db in non-autovacuum mode.
+  # Reset everything just in case.
+  #
+  db close
+  file delete -force test.db test.db-journal
+  sqlite3 db test.db
+  execsql {
+    PRAGMA auto_vacuum = 1;
+    PRAGMA auto_vacuum;
+  }
+} {1}
 do_test autovacuum-4.1 {
   execsql {
     CREATE TABLE av1(a, b);