Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077)

FossilOrigin-Name: 839ad771a6e781426c0fd624a4d1c91a5fcf8546

manifest

@@ -1,5 +1,5 @@
-C Auto-vacuum:\sAccount\sfor\sthe\spage\sreserved\sfor\swindows\slocking\s(PENDING_BYTE).\s(CVS\s2076)
+C Test\sauto-vacuum\smode\sfor\scrash-proofness.\sAlso\sfix\sa\sbug\srelated\sto\sthe\ssame.\s(CVS\s2077)
-D 2004-11-08T07:13:14
+D 2004-11-08T09:26:09
 F Makefile.in c4d2416860f472a1e3393714d0372074197565df
 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1
@@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863
 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
 F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689
 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea
-F src/btree.c e88077ac041e153ba130488e5cef9475dd656966
+F src/btree.c 63a84350a18f6ca68f16e2a12018b5041444a2df
 F src/btree.h 861e40b759a195ba63819740e484390012cf81ab
 F src/build.c f01c2f9b3ad334a301e97ee4f299c36228ead110
 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad
@@ -52,7 +52,7 @@ F src/os_unix.c 5824b22ba41fe9d514ef9169aac1b5fde73af229
 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13
 F src/os_win.c 9482dfc92f289b68205bb2c9315757c7e3946bfb
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c 942cbb9d1de5e198d26979f992eba1b4e527ac08
+F src/pager.c 2653787b86267c079283b8128541095f0febac4f
 F src/pager.h 9eba8c53dd91eae7f3f90743b2ee242da02a9862
 F src/parse.y 8456726833755ecd6dac9bcd8af205c8dc419d01
 F src/pragma.c 6a0ae7721e614c5a921e918ab5206d5e654f1a6f
@@ -109,7 +109,7 @@ F test/collate5.test 1dd5f0f508c46667f9d4606c7950c414b0bdc0d5
 F test/collate6.test 6c9470d1606ee3e564675b229653e320c49ec638
 F test/conflict.test c5b849b01cfbe0a4f63a90cba6f68e2fe3a75f87
 F test/corrupt.test 0080ddcece23e8ba47c44608c4fb73fd4d1d8ce2
-F test/crash.test a3f6d27f7cb7f7bd752461db1e14f7c781ecedc3
+F test/crash.test 9982d2c1fe45126a96c57d16310caf949d21d818
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/date.test dda578ec1857837156bd8b32f8e09d81d7d7881c
 F test/delete.test fc29491f6a7ac899ce29f4549a104809e245d9a6
@@ -158,7 +158,7 @@ F test/pagesize.test 2050bb574c43f9143d283d6bcd37fa1eb73f0e49
 F test/pragma.test bde1271384bc415af04d9dd736c073cf7ef33177
 F test/printf.test 92ba4c510b4fc61120ffa4a01820446ed917ae57
 F test/progress.test 5ddba78cb6011fba36093973cfb3ac473b8fb96a x
-F test/quick.test 2dca186ebd5c418a7699944ba3b5e437d765eddd
+F test/quick.test 9e968949a20b5ed5990c03dc45df3781a03c4b1a
 F test/quote.test 6d75cf635d93ba2484dc9cb378d88cbae9dc2c62
 F test/reindex.test 3552c6b944a3fab28cfd3049c04c65cb79419757
 F test/rollback.test 4097328d44510277244ef4fa51b22b2f11d7ef4c
@@ -253,7 +253,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c
-P ad433ec2b6bd34e33dfe119668f38fbb978e889d
+P d6335698696c7b651bbc436c5177d87eb57a8934
-R f5ced09dda22d2b1aa5518cabc940350
+R 631fbffc9d3d389ea9443eeeaba04e6a
 U danielk1977
-Z febc6549f1b02ac6b75a38defec39cf5
+Z 86fea8841f4dc937802f0e07bab9a3cf

manifest.uuid

@@ -1 +1 @@
-d6335698696c7b651bbc436c5177d87eb57a8934
+839ad771a6e781426c0fd624a4d1c91a5fcf8546

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.214 2004/11/08 07:13:14 danielk1977 Exp $
+** $Id: btree.c,v 1.215 2004/11/08 09:26:09 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -1825,9 +1825,6 @@ static int autoVacuumCommit(Btree *pBt, Pgno *nTrunc){
   *nTrunc = finSize;
 
 autovacuum_out:
-  /* TODO: A goto autovacuum_out; will fail to call releasePage() on 
-  ** outstanding references. Fix.
-  */
   assert( nRef==*sqlite3pager_stats(pPager) );
   if( rc!=SQLITE_OK ){
     sqlite3pager_rollback(pPager);
@@ -5267,7 +5264,6 @@ char *sqlite3BtreeIntegrityCheck(Btree *pBt, int *aRoot, int nRoot){
   for(i=0; i<nRoot; i++){
     if( aRoot[i]==0 ) continue;
 #ifndef SQLITE_OMIT_AUTOVACUUM
-/* Note: This is temporary code for use during development of auto-vacuum. */
     if( pBt->autoVacuum && aRoot[i]>1 ){
       checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0);
     }

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.175 2004/11/08 07:13:14 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.176 2004/11/08 09:26:10 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -34,7 +34,7 @@
 #define TRACE2(X,Y)     sqlite3DebugPrintf(X,Y)
 #define TRACE3(X,Y,Z)   sqlite3DebugPrintf(X,Y,Z)
 #define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W)
-#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W, V)
+#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W,V)
 #else
 #define TRACE1(X)
 #define TRACE2(X,Y)
@@ -3303,6 +3303,7 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
   PgHdr *pPg = DATA_TO_PGHDR(pData);
   PgHdr *pPgOld; 
   int h;
+  Pgno needSyncPgno = 0;
 
   assert( !pPager->stmtInUse );
   assert( pPg->nRef>0 );
@@ -3310,6 +3311,12 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
   TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
       PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
 
+  if( pPg->needSync ){
+    needSyncPgno = pPg->pgno;
+    assert( pPg->inJournal );
+    assert( pPg->dirty );
+  }
+
   /* Unlink pPg from it's hash-chain */
   unlinkHashChain(pPager, pPg);
 
@@ -3342,6 +3349,23 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
   pPg->dirty = 1;
   pPager->dirtyCache = 1;
 
+  if( needSyncPgno ){
+    /* If needSyncPgno is non-zero, then the journal file needs to be 
+    ** sync()ed before any data is written to database file page needSyncPgno.
+    ** Currently, no such page exists in the page-cache and the 
+    ** Pager.aInJournal bit has been set. This needs to be remedied by loading
+    ** the page into the pager-cache and setting the PgHdr.needSync flag.
+    */
+    int rc;
+    void *pNeedSync;
+    rc = sqlite3pager_get(pPager, needSyncPgno, &pNeedSync);
+    if( rc!=SQLITE_OK ) return rc;
+    DATA_TO_PGHDR(pNeedSync)->needSync = 1;
+    DATA_TO_PGHDR(pNeedSync)->inJournal = 1;
+    DATA_TO_PGHDR(pNeedSync)->dirty = 1;
+    sqlite3pager_unref(pNeedSync);
+  }
+
   return SQLITE_OK;
 }
 #endif

test/crash.test

@@ -20,7 +20,7 @@
 # The special crash-test module with its os_test.c backend only works
 # on Unix.
 #
-# $Id: crash.test,v 1.9 2004/08/21 19:20:42 drh Exp $
+# $Id: crash.test,v 1.10 2004/11/08 09:26:10 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -68,6 +68,11 @@ proc signature2 {} {
   return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc2}]
 }
 
+# This variable is set to 1 if the databases being used support auto-vacuum.
+# This is because some of the tests in this file verify file-size, which is
+# slightly larger for auto-vacuum databases.
+set AUTOVACUUM [db eval {pragma auto_vacuum}]
+
 #--------------------------------------------------------------------------
 # Simple crash test:
 #
@@ -170,7 +175,7 @@ do_test crash-2.1 {
 } {499500.0 999000.0 1498500.0}
 do_test crash-2.2 {
   expr [file size test.db] / 1024
-} {19}
+} [expr $AUTOVACUUM ? 20 : 19]
 do_test crash-2.3 {
   crashsql 2 test.db-journal {
     DELETE FROM abc WHERE a < 800;
@@ -210,7 +215,7 @@ do_test crash-3.0 {
     INSERT INTO abc SELECT * FROM abc;
   }
   expr [file size test.db] / 1024
-} {554}
+} [expr $AUTOVACUUM ? 557 : 554]
 for {set i 1} {$i < $repeats} {incr i} {
   set sig [signature]
   do_test crash-3.$i.1 {
@@ -324,3 +329,64 @@ for {set i 1} {$i < 5} {incr i} {
     signature2
   } $sig2
 }
+
+#--------------------------------------------------------------------------
+# The following test cases - crash-5.* - exposes a bug that existed in the
+# sqlite3pager_movepage() API used by auto-vacuum databases.
+# database when a crash occurs during a multi-file transaction. See comments
+# in test crash-5.3 for details.
+#
+db close
+file delete -force test.db
+sqlite3 db test.db
+do_test crash-5.1 {
+  execsql {
+    CREATE TABLE abc(a, b, c);                          -- Root page 3
+    INSERT INTO abc VALUES(randstr(1500,1500), 0, 0);   -- Overflow page 4
+    INSERT INTO abc SELECT * FROM abc;
+    INSERT INTO abc SELECT * FROM abc;
+    INSERT INTO abc SELECT * FROM abc;
+  }
+} {}
+do_test crash-5.2 {
+  expr [file size test.db] / 1024
+} [expr $AUTOVACUUM ? 11 : 10]
+set sig [signature]
+do_test crash-5.3 {
+# The SQL below is used to expose a bug that existed in
+# sqlite3pager_movepage() during development of the auto-vacuum feature. It
+# functions as follows:
+# 
+# 1: Begin a transaction.
+# 2: Put page 4 on the free-list (was the overflow page for the row deleted).
+# 3: Write data to page 4 (it becomes the overflow page for the row inserted).
+#    The old page 4 data has been written to the journal file, but the
+#    journal file has not been sync()hronized.
+# 4: Create a table, which calls sqlite3pager_movepage() to move page 4
+#    to the end of the database (page 12) to make room for the new root-page.
+# 5: Put pressure on the pager-cache. This results in page 4 being written
+#    to the database file to make space in the cache to load a new page. The
+#    bug was that page 4 was written to the database file before the journal
+#    is sync()hronized.
+# 6: Commit. A crash occurs during the sync of the journal file.
+#
+# End result: Before the bug was fixed, data has been written to page 4 of the
+# database file and the journal file does not contain trustworthy rollback
+# data for this page.
+#
+  crashsql 1 test.db-journal {
+    BEGIN;                                             -- 1
+    DELETE FROM abc WHERE oid = 1;                     -- 2
+    INSERT INTO abc VALUES(randstr(1500,1500), 0, 0);  -- 3
+    CREATE TABLE abc2(a, b, c);                        -- 4
+    SELECT * FROM abc;                                 -- 5
+    COMMIT;                                            -- 6
+  }
+} {1 {child process exited abnormally}}
+integrity_check crash-5.4
+do_test crash-5.5 {
+  signature
+} $sig
+
+
+

test/quick.test

@@ -10,7 +10,7 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: quick.test,v 1.31 2004/11/02 12:56:41 danielk1977 Exp $
+# $Id: quick.test,v 1.32 2004/11/08 09:26:10 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -32,8 +32,7 @@ set EXCLUDE {
   misuse.test
   quick.test
   utf16.test
-
+  autovacuum_crash.test
-  autovacuum.test
 }
 
 if {[sqlite3 -has-codec]} {