More rollback problems: Fix two more errors introduced by checking (410) that

can cause database corruption after a ROLLBACK.  Also add new tests to make
sure everything is working this time. (CVS 663)

FossilOrigin-Name: f6e24d5ccbcfcf5863ffbd65860dafa2f5663e99

VERSION

@@ -1 +1 @@
-2.5.5
+2.5.6

manifest

@@ -1,9 +1,9 @@
-C Version\s2.5.5\s(CVS\s662)
+C More\srollback\sproblems:\s\sFix\stwo\smore\serrors\sintroduced\sby\schecking\s(410)\sthat\ncan\scause\sdatabase\scorruption\safter\sa\sROLLBACK.\s\sAlso\sadd\snew\stests\sto\smake\nsure\severything\sis\sworking\sthis\stime.\s(CVS\s663)
-D 2002-07-06T16:34:21
+D 2002-07-07T16:52:47
 F Makefile.in 6291a33b87d2a395aafd7646ee1ed562c6f2c28c
 F Makefile.template 4e11752e0b5c7a043ca50af4296ec562857ba495
 F README a4c0ba11354ef6ba0776b400d057c59da47a4cc0
-F VERSION 913f3b0d995e090a9f126ca63215e49ef22904e5
+F VERSION 0afb45a36f2b97c8455247659f1155967b8bb883
 F aclocal.m4 11faa843caa38fd451bc6aeb43e248d1723a269d
 F config.guess f38b1e93d1e0fa6f5a6913e9e7b12774b9232588
 F config.sub f14b07d544ca26b5d698259045136b783e18fc7f
@@ -18,21 +18,21 @@ F publish.sh 1a04b9aa0d9c9661e338268343476ed0851c5778
 F spec.template 238f7db425a78dc1bb7682e56e3834c7270a3f5e
 F sqlite.1 83f4a9d37bdf2b7ef079a82d54eaf2e3509ee6ea
 F src/TODO af7f3cab0228e34149cf98e073aa83d45878e7e6
-F src/btree.c f722f9ff5cca79cc765d06116a60a106a0ddfdf1
+F src/btree.c 73212bda34c491ce9165464c3e1ada737acf5e06
 F src/btree.h 8abeabfe6e0b1a990b64fa457592a6482f6674f3
 F src/build.c ea4a3bc15d6338294e68100f642edf48e4082403
 F src/delete.c 215492ffcea4262a993e55f3c4a67dc9fea4da9c
 F src/encode.c 346b12b46148506c32038524b95c4631ab46d760
 F src/expr.c 4b25ee5e65f351d40dea8575b998605762556d76
-F src/func.c 5eae8227a8b0d276a64d51a3880a6e86f238fedf
+F src/func.c e45cd908b9b723d9b91473d09e12c23f786b3fc2
 F src/hash.c 6a6236b89c8c060c65dabd300a1c8ce7c10edb72
 F src/hash.h cd0433998bc1a3759d244e1637fe5a3c13b53bf8
 F src/insert.c 4511e06abce1688664ce90cbf09fa13433b82c43
 F src/main.c 6ac32ca71ab4728212c5b44aed25e26dc6cfe73c
 F src/md5.c 0ae1f3e2cac92d06fc6246d1b4b8f61a2fe66d3b
-F src/os.c 9cc40c5384baba4a85e160e67807645ca98ba3cc
+F src/os.c 6902568313e39f7c942b7ba8e316887a5b1aa40e
 F src/os.h 4a361fccfbc4e7609b3e1557f604f94c1e96ad10
-F src/pager.c 103341600b4044e41b5bd95cd3850f046b976ff2
+F src/pager.c b8415b02a2a5c4e9d8bfd45311a5f15e785c7f1f
 F src/pager.h 6fddfddd3b73aa8abc081b973886320e3c614f0e
 F src/parse.y 35437ac29441ce2d34904d8e93f40b7d112147a9
 F src/printf.c 236ed7a79386feed4456fa728fff8be793f1547c
@@ -55,7 +55,7 @@ F src/util.c 7a99e754c44dd220e881122e30581c08b6d6adef
 F src/vdbe.c 0169270bb73e8dec4174b90dffc7070c4cabe039
 F src/vdbe.h a9292f2b5fcecef924fa255fb74609e9cbc776c2
 F src/where.c 6a43aa6c80eab12221eeca754cba852a9ecd1e13
-F test/all.test e4d3821eeba751829b419cd47814bd20af4286d1
+F test/all.test f296d27fff6aca72348af15092154f879d1fc7d4
 F test/bigrow.test 8ab252dba108f12ad64e337b0f2ff31a807ac578
 F test/btree.test bf326f546a666617367a7033fa2c07451bd4f8e1
 F test/btree2.test e3b81ec33dc2f89b3e6087436dfe605b870c9080
@@ -84,7 +84,7 @@ F test/null.test 5c2b57307e4b6178aae825eb65ddbee01e76b0fd
 F test/pager.test b0c0d00cd5dce0ce21f16926956b195c0ab5044c
 F test/pragma.test 0b9675ef1f5ba5b43abfa337744445fc5b01a34a
 F test/printf.test a29b8afa24edb4411adfe473b12ac32c84098fce
-F test/quick.test 6f023c7a73fc413e6d65b7a1879c79764038dc05
+F test/quick.test 21f710471a6eb9b2513d8fe4abd07bdb874f23bb
 F test/quote.test 08f23385c685d3dc7914ec760d492cacea7f6e3d
 F test/rowid.test 4c55943300cddf73dd0f88d40a268cab14c83274
 F test/select1.test 0d708cec567104653ec9aa49fecf3444a2e7d150
@@ -100,7 +100,7 @@ F test/tableapi.test 3c80421a889e1d106df16e5800fa787f0d2914a6
 F test/tclsqlite.test 6f4b9760681c7dbca52a18d0ab46a1679cdc79b9
 F test/temptable.test 9ed7ec0288f887e132de66d90c428ad109105f67
 F test/tester.tcl 6f603d90881bd835ea27c568a7fecaa57dce91cc
-F test/trans.test 4eeb19975bb9607dce4fd4c0ef6723c2185c7d84
+F test/trans.test 1fd1ecdece1eca22abe631674c3bd24046df4150
 F test/trigger1.test bb63749fa8a395a60541100607d86381604b7194
 F test/trigger2.test c12759a0d7ba6488d9d24c96a1352ddee995c1ab
 F test/trigger3.test 7dfe798d7e72c13720394685fe353112e3f31adf
@@ -125,7 +125,7 @@ F www/arch.fig d5f9752a4dbf242e9cfffffd3f5762b6c63b3bcf
 F www/arch.png 82ef36db1143828a7abc88b1e308a5f55d4336f4
 F www/arch.tcl 72a0c80e9054cc7025a50928d28d9c75c02c2b8b
 F www/c_interface.tcl 58cf4d128dcae08d91d0011c6d4d11de323f470f
-F www/changes.tcl 8b15a85a0a338234ddff5804ecad636521efcab5
+F www/changes.tcl a6d732a78b451eab29a66a068dc07b359f32c5a8
 F www/conflict.tcl 81dd21f9a679e60aae049e9dd8ab53d59570cda2
 F www/crosscompile.tcl 3622ebbe518927a3854a12de51344673eb2dd060
 F www/download.tcl 29aa6679ca29621d10613f60ebbbda18f4b91c49
@@ -140,7 +140,7 @@ F www/speed.tcl da8afcc1d3ccc5696cfb388a68982bc3d9f7f00f
 F www/sqlite.tcl ae3dcfb077e53833b59d4fcc94d8a12c50a44098
 F www/tclsqlite.tcl 1db15abeb446aad0caf0b95b8b9579720e4ea331
 F www/vdbe.tcl 2013852c27a02a091d39a766bc87cff329f21218
-P 00f83c2576f4a9689720d344788f66219c1f6827
+P 6284c65c174bb2cd049fd1db81de52be3abf4000
-R 5a2631b9a3e9098f0e7b3bbcc4662784
+R 66486ccd900763207e459fadea926940
 U drh
-Z 4b1afc40fad39701659a14b71a9eda50
+Z 6cbbb10ace5dd6c600cdbe2673a30799

manifest.uuid

@@ -1 +1 @@
-6284c65c174bb2cd049fd1db81de52be3abf4000
+f6e24d5ccbcfcf5863ffbd65860dafa2f5663e99

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.64 2002/07/06 16:32:15 drh Exp $
+** $Id: btree.c,v 1.65 2002/07/07 16:52:47 drh Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -1569,6 +1569,12 @@ static int freePage(Btree *pBt, void *pPage, Pgno pgno){
     pgno = sqlitepager_pagenumber(pOvfl);
   }
   assert( pgno>2 );
+  pMemPage = (MemPage*)pPage;
+  pMemPage->isInit = 0;
+  if( pMemPage->pParent ){
+    sqlitepager_unref(pMemPage->pParent);
+    pMemPage->pParent = 0;
+  }
   rc = sqlitepager_write(pPage1);
   if( rc ){
     return rc;
@@ -1606,12 +1612,6 @@ static int freePage(Btree *pBt, void *pPage, Pgno pgno){
   pOvfl->iNext = pPage1->freeList;
   pPage1->freeList = pgno;
   memset(pOvfl->aPayload, 0, OVERFLOW_SIZE);
-  pMemPage = (MemPage*)pPage;
-  pMemPage->isInit = 0;
-  if( pMemPage->pParent ){
-    sqlitepager_unref(pMemPage->pParent);
-    pMemPage->pParent = 0;
-  }
   if( needUnref ) rc = sqlitepager_unref(pOvfl);
   return rc;
 }

src/func.c

@@ -16,7 +16,7 @@
 ** sqliteRegisterBuildinFunctions() found at the bottom of the file.
 ** All other code has file scope.
 **
-** $Id: func.c,v 1.21 2002/06/20 11:36:49 drh Exp $
+** $Id: func.c,v 1.22 2002/07/07 16:52:47 drh Exp $
 */
 #include <ctype.h>
 #include <math.h>
@@ -247,6 +247,49 @@ static void nullifFunc(sqlite_func *context, int argc, const char **argv){
   }
 }
 
+#ifdef SQLITE_TEST
+/*
+** This function generates a string of random characters.  Used for
+** generating test data.
+*/
+static void randStr(sqlite_func *context, int argc, const char **argv){
+  static const char zSrc[] = 
+     "abcdefghijklmnopqrstuvwxyz"
+     "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+     "0123456789"
+     ".-!,:*^+=_|?/<> ";
+  int iMin, iMax, n, r, i;
+  char zBuf[1000];
+  if( argc>=1 ){
+    iMin = atoi(argv[0]);
+    if( iMin<0 ) iMin = 0;
+    if( iMin>=sizeof(zBuf) ) iMin = sizeof(zBuf)-1;
+  }else{
+    iMin = 1;
+  }
+  if( argc>=2 ){
+    iMax = atoi(argv[1]);
+    if( iMax<iMin ) iMax = iMin;
+    if( iMax>=sizeof(zBuf) ) iMax = sizeof(zBuf);
+  }else{
+    iMax = 50;
+  }
+  n = iMin;
+  if( iMax>iMin ){
+    r = sqliteRandomInteger();
+    if( r<0 ) r = -r;
+    n += r%(iMax + 1 - iMin);
+  }
+  r = 0;
+  for(i=0; i<n; i++){
+    r = (r + sqliteRandomByte())% (sizeof(zSrc)-1);
+    zBuf[i] = zSrc[r];
+  }
+  zBuf[n] = 0;
+  sqlite_set_result_string(context, zBuf, n);
+}
+#endif
+
 /*
 ** An instance of the following structure holds the context of a
 ** sum() or avg() aggregate computation.
@@ -438,6 +481,9 @@ void sqliteRegisterBuiltinFunctions(sqlite *db){
     { "like",       2, SQLITE_NUMERIC, likeFunc   },
     { "glob",       2, SQLITE_NUMERIC, globFunc   },
     { "nullif",     2, SQLITE_ARGS,    nullifFunc },
+#ifdef SQLITE_TEST
+    { "randstr",    2, SQLITE_TEXT,    randStr    },
+#endif
   };
   static struct {
     char *zName;

src/os.c

@@ -717,13 +717,19 @@ int sqliteOsUnlock(OsFile *id){
 */
 int sqliteOsRandomSeed(char *zBuf){
   static int once = 1;
-#if OS_UNIX
+#ifdef SQLITE_TEST
+  /* When testing, always use the same random number sequence.
+  ** This makes the tests repeatable.
+  */
+  memset(zBuf, 0, 256);
+#endif
+#if OS_UNIX && !defined(SQLITE_TEST)
   int pid;
   time((time_t*)zBuf);
   pid = getpid();
   memcpy(&zBuf[sizeof(time_t)], &pid, sizeof(pid));
 #endif
-#if OS_WIN
+#if OS_WIN && !defined(SQLITE_TEST)
   GetSystemTime((LPSYSTEMTIME)zBuf);
 #endif
   if( once ){

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.48 2002/07/06 16:28:48 drh Exp $
+** @(#) $Id: pager.c,v 1.49 2002/07/07 16:52:47 drh Exp $
 */
 #include "sqliteInt.h"
 #include "pager.h"
@@ -74,10 +74,10 @@ struct PgHdr {
   int nRef;                      /* Number of users of this page */
   PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
   PgHdr *pNextAll, *pPrevAll;    /* A list of all pages */
-  char inJournal;                /* TRUE if has been written to journal */
+  u8 inJournal;                  /* TRUE if has been written to journal */
-  char inCkpt;                   /* TRUE if written to the checkpoint journal */
+  u8 inCkpt;                     /* TRUE if written to the checkpoint journal */
-  char dirty;                    /* TRUE if we need to write back changes */
+  u8 dirty;                      /* TRUE if we need to write back changes */
-  char alwaysRollback;           /* Ignore dont_rollback() calls if true */
+  u8 alwaysRollback;             /* Disable dont_rollback() for this page */
   /* SQLITE_PAGE_SIZE bytes of page data follow this header */
   /* Pager.nExtra bytes of local data follow the page data */
 };
@@ -123,6 +123,7 @@ struct Pager {
   u8 readOnly;                /* True for a read-only database */
   u8 needSync;                /* True if an fsync() is needed on the journal */
   u8 dirtyFile;               /* True if database file has changed in any way */
+  u8 alwaysRollback;          /* Disable dont_rollback() for all pages */
   u8 *aInJournal;             /* One bit for each page in the database file */
   u8 *aInCkpt;                /* One bit for each page in the database */
   PgHdr *pFirst, *pLast;      /* List of free pages */
@@ -861,6 +862,18 @@ int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){
       assert( pPg->nRef==0 );
       assert( pPg->dirty==0 );
 
+      /* If the page we are recyclying is marked as alwaysRollback, then
+      ** set the global alwaysRollback flag, thus disabling the
+      ** sqlite_dont_rollback() optimization for the rest of this transaction.
+      ** It is necessary to do this because the page marked alwaysRollback
+      ** might be reloaded at a later time but at that point we won't remember
+      ** that is was marked alwaysRollback.  This means that all pages must
+      ** be marked as alwaysRollback from here on out.
+      */
+      if( pPg->alwaysRollback ){
+        pPager->alwaysRollback = 1;
+      }
+
       /* Unlink the old page from the free list and the hash table
       */
       if( pPg->pPrevFree ){
@@ -1056,6 +1069,7 @@ int sqlitepager_begin(void *pData){
     pPager->journalOpen = 1;
     pPager->needSync = 0;
     pPager->dirtyFile = 0;
+    pPager->alwaysRollback = 0;
     pPager->state = SQLITE_WRITELOCK;
     sqlitepager_pagecount(pPager);
     pPager->origDbSize = pPager->dbSize;
@@ -1242,7 +1256,7 @@ void sqlitepager_dont_rollback(void *pData){
   Pager *pPager = pPg->pPager;
 
   if( pPager->state!=SQLITE_WRITELOCK || pPager->journalOpen==0 ) return;
-  if( pPg->alwaysRollback ) return;
+  if( pPg->alwaysRollback || pPager->alwaysRollback ) return;
   if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
     assert( pPager->aInJournal!=0 );
     pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);

test/all.test

@@ -10,7 +10,7 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: all.test,v 1.14 2002/05/10 13:14:08 drh Exp $
+# $Id: all.test,v 1.15 2002/07/07 16:52:47 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -34,8 +34,8 @@ set EXCLUDE {
   quick.test
   malloc.test
   misuse.test
-  btree2.test
 }
+#  btree2.test
 
 for {set Counter 0} {$Counter<$COUNT && $nErr==0} {incr Counter} {
   if {$Counter%2} {

test/quick.test

@@ -10,12 +10,13 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: quick.test,v 1.2 2001/10/22 02:58:11 drh Exp $
+# $Id: quick.test,v 1.3 2002/07/07 16:52:47 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 rename finish_test really_finish_test
 proc finish_test {} {}
+set ISQUICK 1
 
 set EXCLUDE {
   all.test

test/trans.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is database locks.
 #
-# $Id: trans.test,v 1.12 2002/06/25 13:16:04 drh Exp $
+# $Id: trans.test,v 1.13 2002/07/07 16:52:47 drh Exp $
 
 
 set testdir [file dirname $argv0]
@@ -805,5 +805,86 @@ do_test trans-8.2 {
 } $checksum2
 integrity_check trans-8.3
 
+# In the following sequence of tests, compute the MD5 sum of the content
+# of a table, make lots of modifications to that table, then do a rollback.
+# Verify that after the rollback, the MD5 checksum is unchanged.
+#
+do_test trans-9.1 {
+  execsql {
+    PRAGMA cache_size=10;
+    BEGIN;
+    CREATE TABLE t3(x TEXT);
+    INSERT INTO t3 VALUES(randstr(10,400));
+    INSERT INTO t3 VALUES(randstr(10,400));
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    INSERT INTO t3 SELECT randstr(10,400) FROM t3;
+    COMMIT;
+    SELECT count(*) FROM t3;
+  }
+} {1024}
+
+# The following procedure computes a "signature" for table "t3".  If
+# T3 changes in any way, the signature should change.  
+#
+# This is used to test ROLLBACK.  We gather a signature for t3, then
+# make lots of changes to t3, then rollback and take another signature.
+# The two signatures should be the same.
+#
+proc signature {} {
+  return [db eval {SELECT count(*), md5sum(x) FROM t3}]
+}
+
+# Repeat the following group of tests 20 times for quick testing and
+# 40 times for full testing.  Each iteration of the test makes table
+# t3 a little larger, and thus takes a little longer, so doing 40 tests
+# is more than 2.0 times slower than doing 20 tests.  Considerably more.
+#
+if {[info exists ISQUICK]} {
+  set limit 20
+} else {
+  set limit 40
+}
+
+# Do rollbacks.  Make sure the signature does not change.
+#
+for {set i 2} {$i<=$limit} {incr i} {
+  set ::sig [signature]
+  set cnt [lindex $::sig 0]
+  do_test trans-9.$i.1-$cnt {
+     execsql {
+       BEGIN;
+       DELETE FROM t3 WHERE random()%10!=0;
+       INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
+       INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
+       ROLLBACK;
+     }
+     signature
+  } $sig
+  do_test trans-9.$i.2-$cnt {
+     execsql {
+       BEGIN;
+       DELETE FROM t3 WHERE random()%10!=0;
+       INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
+       DELETE FROM t3 WHERE random()%10!=0;
+       INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
+       ROLLBACK;
+     }
+     signature
+  } $sig
+  if {$i<$limit} {
+    do_test trans-9.$i.9-$cnt {
+       execsql {
+         INSERT INTO t3 SELECT randstr(10,400) FROM t3 WHERE random()%10==0;
+       }
+    } {}
+  }
+}
    
 finish_test

www/changes.tcl

@@ -25,6 +25,12 @@ proc chng {date desc} {
   puts "<DD><P><UL>$desc</UL></P></DD>"
 }
 
+chng {2002 Jly 7 (2.5.6)} {
+<li>Fix more problems with rollback.  Enhance the test suite to exercise
+    the rollback logic extensively in order to prevent any future problems.
+    </li>
+}
+
 chng {2002 Jly 6 (2.5.5)} {
 <li>Fix a bug which could cause database corruption during a rollback.
     This bugs was introduced in version 2.4.0 by the freelist