Fixed several more crashes due to corrupt db files. Added corruptC.test to soak.test. (CVS 5905)

FossilOrigin-Name: 9b7a52e952c81e50611e04d2d79003b0ddc57ee5

manifest

@@ -1,5 +1,5 @@
-C Added\ssupport\sfor\s-DSQLITE_NO_SYNC\sto\sos_win.c.\s(CVS\s5904)
+C Fixed\sseveral\smore\scrashes\sdue\sto\scorrupt\sdb\sfiles.\s\sAdded\scorruptC.test\sto\ssoak.test.\s(CVS\s5905)
-D 2008-11-13T18:20:43
+D 2008-11-13T18:29:51
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 6cbc7db84c23804c368bc7ffe51367412212d7b2
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -99,10 +99,10 @@ F src/attach.c 208881c87160d9e2c73a46cf86116c5a6d66f9d7
 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627
 F src/bitvec.c 9e922b2577b7e46d8f95349bca6a52f7674d7582
 F src/btmutex.c 3a90096c3080b9057dc570b8e16e46511e1c788a
-F src/btree.c be3e0aa63755b094941f9c4298a987fe93df0f22
+F src/btree.c 81c229650b57e1ca129279f4ca7f49a5111903e6
 F src/btree.h 179c3ea813780df78a289a8f5130db18e6d4616e
 F src/btreeInt.h e38e9b2b285f40f5bc0a6664f630d4a141622f16
-F src/build.c 98a6884d47c3cc12faeb2e9a926018d3a7382133
+F src/build.c 7723123a571fcf9b0c3362dcfffeb1b64ec4f043
 F src/callback.c e970e5beddbdb23f89a6d05cb1a6419d9f755624
 F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c
 F src/date.c 6f4277fa56d8c1b8e70c0bde838c9e99609f5ec0
@@ -191,7 +191,7 @@ F src/update.c f22a6f4507f9a0ef082418919382f83b90fd2e63
 F src/utf.c 86dc0f8076f606432a01f1498ae054c32de1f9d2
 F src/util.c afe659ccc05d1f8af9e8631dabfec3ee3a7144af
 F src/vacuum.c fd77433d0c26d3ff1eb96eab017a1787ac5aa642
-F src/vdbe.c b6b989bbd0e306581695f8914c4246905a5c0d14
+F src/vdbe.c c15dc80bd1a5dcb83a91792976eacf434d691ef6
 F src/vdbe.h 03516f28bf5aca00a53c4dccd6c313f96adb94f6
 F src/vdbeInt.h c9400778d6f801c2cb8ebe6151c909e19dd2d793
 F src/vdbeapi.c ea22e171704906632cd971668359b8c0c5053001
@@ -266,7 +266,7 @@ F test/corrupt8.test 9992ef7f67cefc576b92373f6bf5ab8775280f51
 F test/corrupt9.test 794d284109c65c8f10a2b275479045e02d163bae
 F test/corruptA.test 99e95620b980161cb3e79f06a884a4bb8ae265ff
 F test/corruptB.test 505331779fe7a96fe38ecbb817f19c63bc27d171
-F test/corruptC.test bcedf37afa205aff7cf1729a32b862c6a037fb5f
+F test/corruptC.test b8030bbeb90ae7b05c63bb4dc5fdd44810810963
 F test/crash.test 1b6ac8410689ff78028887f445062dc897c9ac89
 F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651
 F test/crash3.test 776f9363554c029fcce71d9e6600fa0ba6359ce7
@@ -496,7 +496,7 @@ F test/shared4.test d0fadacb50bb6981b2fb9dc6d1da30fa1edddf83
 F test/shared_err.test 91e26ec4f3fbe07951967955585137e2f18993de
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329
-F test/soak.test 3c317b3e55e1160731030c8e865d1858fab66fea
+F test/soak.test d9d0a5e5c0157115c9a17f526f12691fe146768d
 F test/softheap1.test 73ebd6e020d2954d965da2072baba5922fc8fb6a
 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5
 F test/speed1.test c74564fea46e094d6b518bf464c355991905eea2
@@ -657,7 +657,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P 428a5479200dc24e2ee9b4a85ef6caadacbdbbd7
+P 2649337937077d2dba7cdc7473fcd176aa252a52
-R 0862238cedee5bfa8e1cf288700d9240
+R eda786b9ca4d828a37954e12c4d14e7a
 U shane
-Z 83a561c6084b21462d245b6589e2fbad
+Z df50ccbe255d3c847dcb2b81abf98520

manifest.uuid

@@ -1 +1 @@
-2649337937077d2dba7cdc7473fcd176aa252a52
+9b7a52e952c81e50611e04d2d79003b0ddc57ee5

src/btree.c

@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.535 2008/11/13 14:28:29 danielk1977 Exp $
+** $Id: btree.c,v 1.536 2008/11/13 18:29:51 shane Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** See the header comment on "btreeInt.h" for additional information.
@@ -734,7 +734,7 @@ static int defragmentPage(MemPage *pPage){
     u8 *pAddr;     /* The i-th cell pointer */
     pAddr = &data[cellOffset + i*2];
     pc = get2byte(pAddr);
-    if( pc>=pPage->pBt->usableSize ){
+    if( pc>=usableSize ){
       return SQLITE_CORRUPT_BKPT;
     }
     size = cellSizePtr(pPage, &temp[pc]);
@@ -835,7 +835,7 @@ static int allocateSpace(MemPage *pPage, int nByte){
 ** Most of the effort here is involved in coalesing adjacent
 ** free blocks into a single big free block.
 */
-static void freeSpace(MemPage *pPage, int start, int size){
+static int freeSpace(MemPage *pPage, int start, int size){
   int addr, pbegin, hdr;
   unsigned char *data = pPage->aData;
 
@@ -857,10 +857,14 @@ static void freeSpace(MemPage *pPage, int start, int size){
   addr = hdr + 1;
   while( (pbegin = get2byte(&data[addr]))<start && pbegin>0 ){
     assert( pbegin<=pPage->pBt->usableSize-4 );
-    assert( pbegin>addr );
+    if( pbegin<=addr ) {
+      return SQLITE_CORRUPT_BKPT;
+    }
     addr = pbegin;
   }
-  assert( pbegin<=pPage->pBt->usableSize-4 );
+  if ( pbegin>pPage->pBt->usableSize-4 ) {
+    return SQLITE_CORRUPT_BKPT;
+  }
   assert( pbegin>addr || pbegin==0 );
   put2byte(&data[addr], start);
   put2byte(&data[start], pbegin);
@@ -877,7 +881,9 @@ static void freeSpace(MemPage *pPage, int start, int size){
     psize = get2byte(&data[pbegin+2]);
     if( pbegin + psize + 3 >= pnext && pnext>0 ){
       int frag = pnext - (pbegin+psize);
-      assert( frag<=data[pPage->hdrOffset+7] );
+      if( (frag<0) || (frag>data[pPage->hdrOffset+7]) ){
+        return SQLITE_CORRUPT_BKPT;
+      }
       data[pPage->hdrOffset+7] -= frag;
       put2byte(&data[pbegin], get2byte(&data[pnext]));
       put2byte(&data[pbegin+2], pnext+get2byte(&data[pnext+2])-pbegin);
@@ -894,6 +900,7 @@ static void freeSpace(MemPage *pPage, int start, int size){
     top = get2byte(&data[hdr+5]);
     put2byte(&data[hdr+5], top + get2byte(&data[pbegin+2]));
   }
+  return SQLITE_OK;
 }
 
 /*
@@ -4579,6 +4586,7 @@ static int dropCell(MemPage *pPage, int idx, int sz){
   int pc;         /* Offset to cell content of cell being deleted */
   u8 *data;       /* pPage->aData */
   u8 *ptr;        /* Used to move bytes around within data[] */
+  int rc;         /* The return code */
 
   assert( idx>=0 && idx<pPage->nCell );
   assert( sz==cellSize(pPage, idx) );
@@ -4587,10 +4595,13 @@ static int dropCell(MemPage *pPage, int idx, int sz){
   data = pPage->aData;
   ptr = &data[pPage->cellOffset + 2*idx];
   pc = get2byte(ptr);
-  if ( pc<=10 || pc+sz>pPage->pBt->usableSize ) {
+  if ( (pc<pPage->hdrOffset+6+(pPage->leaf?0:4)) || (pc+sz>pPage->pBt->usableSize) ) {
     return SQLITE_CORRUPT_BKPT;
   }
-  freeSpace(pPage, pc, sz);
+  rc = freeSpace(pPage, pc, sz);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
   for(i=idx+1; i<pPage->nCell; i++, ptr+=2){
     ptr[0] = ptr[2];
     ptr[1] = ptr[3];
@@ -6051,8 +6062,10 @@ int sqlite3BtreeDelete(BtCursor *pCur){
   }else{
     TRACE(("DELETE: table=%d delete from leaf %d\n",
        pCur->pgnoRoot, pPage->pgno));
-    dropCell(pPage, idx, cellSizePtr(pPage, pCell));
+    rc = dropCell(pPage, idx, cellSizePtr(pPage, pCell));
-    rc = balance(pCur, 0);
+    if( rc==SQLITE_OK ){
+      rc = balance(pCur, 0);
+    }
   }
   if( rc==SQLITE_OK ){
     moveToRoot(pCur);

src/build.c

@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.501 2008/11/11 18:28:59 drh Exp $
+** $Id: build.c,v 1.502 2008/11/13 18:29:51 shane Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -670,7 +670,11 @@ int sqlite3TwoPartName(
   sqlite3 *db = pParse->db;
 
   if( pName2 && pName2->n>0 ){
-    assert( !db->init.busy );
+    if( db->init.busy ) {
+      sqlite3ErrorMsg(pParse, "corrupt database");
+      pParse->nErr++;
+      return -1;
+    }
     *pUnqual = pName2;
     iDb = sqlite3FindDb(db, pName1);
     if( iDb<0 ){

src/vdbe.c

@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.786 2008/11/05 16:37:35 drh Exp $
+** $Id: vdbe.c,v 1.787 2008/11/13 18:29:51 shane Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -2698,7 +2698,10 @@ case OP_OpenWrite: {
     pIn2 = &p->aMem[p2];
     sqlite3VdbeMemIntegerify(pIn2);
     p2 = pIn2->u.i;
-    assert( p2>=2 );
+    if( p2<2 ) {
+      rc = SQLITE_CORRUPT_BKPT;
+      goto abort_due_to_error;
+    }
   }
   assert( i>=0 );
   pCur = allocateCursor(p, i, &pOp[-1], iDb, 1);

test/corruptC.test

@@ -15,16 +15,13 @@
 # data base file, then tests that single byte corruptions in 
 # increasingly larger quantities are handled gracefully.
 #
-# $Id: corruptC.test,v 1.8 2008/11/12 18:21:36 danielk1977 Exp $
+# $Id: corruptC.test,v 1.9 2008/11/13 18:29:51 shane Exp $
 
 catch {file delete -force test.db test.db-journal test.bu}
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-# Set a uniform random seed
-expr srand(0)
-
 # Construct a compact, dense database for testing.
 #
 do_test corruptC-1.1 {
@@ -52,19 +49,13 @@ ifcapable {integrityck} {
 # Generate random integer
 #
 proc random {range} {
-    return [expr {round(rand()*$range)}]
+  return [expr {round(rand()*$range)}]
 }
 
 # Copy file $from into $to
 #
 proc copy_file {from to} {
-  set f [open $from]
+  file copy -force $from $to
-  fconfigure $f -translation binary
-  set t [open $to w]
-  fconfigure $t -translation binary
-  puts -nonewline $t [read $f [file size $from]]
-  close $t
-  close $f
 }
 
 # Setup for the tests.  Make a backup copy of the good database in test.bu.
@@ -74,8 +65,22 @@ copy_file test.db test.bu
 sqlite3 db test.db
 set fsize [file size test.db]
 
+# Set a quasi-random random seed. 
+if {[info exists SOAKTEST]} {
+  # If we are doing SOAK tests, we want a different
+  # random seed for each run.  Ideally we would like 
+  # to use [clock clicks] or something like that here.
+  set qseed [file mtime test.db]
+} else {
+  # If we are not doing soak tests,
+  # make it repeatable.
+  set qseed 0
+}
+expr srand($qseed)
+
 #
-# first test some specific corruption tests found from earlier runs
+# First test some specific corruption tests found from earlier runs
+# with specific seeds.
 #
 
 # test that a corrupt content offset size is handled (seed 5577)
@@ -166,13 +171,98 @@ do_test corruptC-2.6 {
   catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;}
 } {1 {database disk image is malformed}}
 
+# corruption (seed 178692)
+do_test corruptC-2.7 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 3074 [format %02x 0xa0]
+
+  sqlite3 db test.db
+  catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+# corruption (seed 179069)
+do_test corruptC-2.8 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 1393 [format %02x 0x7d]
+  hexio_write test.db 84 [format %02x 0x19]
+  hexio_write test.db 3287 [format %02x 0x3b]
+  hexio_write test.db 2564 [format %02x 0xed]
+  hexio_write test.db 2139 [format %02x 0x55]
+
+  sqlite3 db test.db
+  catchsql {BEGIN; DELETE FROM t1 WHERE x>13; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+# corruption (seed 170434)
+do_test corruptC-2.9 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 2095 [format %02x 0xd6]
+
+  sqlite3 db test.db
+  catchsql {BEGIN; DELETE FROM t1 WHERE x>13; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+# corruption (seed 186504)
+do_test corruptC-2.10 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 3130 [format %02x 0x02]
+  
+  sqlite3 db test.db
+  catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+# corruption (seed 1589)
+do_test corruptC-2.11 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 55 [format %02x 0xa7]
+  
+  sqlite3 db test.db
+  catchsql {BEGIN; CREATE TABLE t3 AS SELECT x,3 as y FROM t2 WHERE rowid%5!=0; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+# corruption (seed 14166)
+do_test corruptC-2.12 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 974 [format %02x 0x2e]
+  
+  sqlite3 db test.db
+  catchsql {SELECT count(*) FROM sqlite_master;}
+} {1 {malformed database schema (t1i1) - corrupt database}}
+
+# corruption (seed 218803)
+do_test corruptC-2.13 {
+  db close
+  copy_file test.bu test.db
+
+  # insert corrupt byte(s)
+  hexio_write test.db 102 [format %02x 0x12]
+  
+  sqlite3 db test.db
+  catchsql {BEGIN; CREATE TABLE t3 AS SELECT x,3 as y FROM t2 WHERE rowid%5!=0; ROLLBACK;}
+} {1 {database disk image is malformed}}
+
+
 #
 # now test for a series of quasi-random seeds
-#
+for {set tn 0} {$tn<1024} {incr tn 1} {
-for {set tn 0} {$tn<=1024} {incr tn 1} {
-
-  # Set a quasi-random random seed
-  expr srand($tn)
 
   # setup for test
   db close
@@ -184,43 +274,65 @@ for {set tn 0} {$tn<=1024} {incr tn 1} {
   # the database engine can handle the corruption gracefully.
   #
   set last 0
-  for {set i 1} {$i<=1024 && !$last} {incr i 1} {
+  for {set i 1} {$i<=512 && !$last} {incr i 1} {
 
     # insert random byte at random location
     db close
-    hexio_write test.db [random $fsize] [format %02x [random 255]]
+    set roffset [random $fsize]
+    set rbyte [format %02x [random 255]]
+
+    # You can uncomment the following to have it trace
+    # exactly how it's corrupting the file.  This is 
+    # useful for generating the "seed specific" tests
+    # above.
+    # set rline "$roffset $rbyte"
+    # puts stdout $rline
+
+    hexio_write test.db $roffset $rbyte
     sqlite3 db test.db
 
     # do a few random operations to make sure that if 
     # they error, they error gracefully instead of crashing.
-    do_test corruptC-3.$tn.$i.1 {
+    do_test corruptC-3.$tn.($qseed).$i.1 {
       catchsql {SELECT count(*) FROM sqlite_master}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.2 {
+    do_test corruptC-3.$tn.($qseed).$i.2 {
       catchsql {SELECT count(*) FROM t1}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.3 {
+    do_test corruptC-3.$tn.($qseed).$i.3 {
       catchsql {SELECT count(*) FROM t1 WHERE x>13}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.4 {
+    do_test corruptC-3.$tn.($qseed).$i.4 {
       catchsql {SELECT count(*) FROM t2}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.5 {
+    do_test corruptC-3.$tn.($qseed).$i.5 {
       catchsql {SELECT count(*) FROM t2 WHERE x<13}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.6 {
+    do_test corruptC-3.$tn.($qseed).$i.6 {
       catchsql {BEGIN; UPDATE t1 SET y=1; ROLLBACK;}
       set x {}
     } {}
-    do_test corruptC-3.$tn.$i.7 {
+    do_test corruptC-3.$tn.($qseed).$i.7 {
       catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;}
       set x {}
     } {}
+    do_test corruptC-3.$tn.($qseed).$i.8 {
+      catchsql {BEGIN; DELETE FROM t1 WHERE x>13; ROLLBACK;}
+      set x {}
+    } {}
+    do_test corruptC-3.$tn.($qseed).$i.9 {
+      catchsql {BEGIN; DELETE FROM t2 WHERE x<13; ROLLBACK;}
+      set x {}
+    } {}
+    do_test corruptC-3.$tn.($qseed).$i.10 {
+      catchsql {BEGIN; CREATE TABLE t3 AS SELECT x,3 as y FROM t2 WHERE rowid%5!=0; ROLLBACK;}
+      set x {}
+    } {}
 
     # check the integrity of the database.
     # once the corruption is detected, we can stop.
@@ -243,7 +355,7 @@ for {set tn 0} {$tn<=1024} {incr tn 1} {
     # TBD:  need to figure out why this doesn't work
     # work with ROLLBACKs...
     if {0} {
-      do_test corruptC-3.$tn.$i.8 {
+      do_test corruptC-3.$tn.($qseed).$i.11 {
         set bt [btree_from_db db]
         db_enter db
         array set stats [btree_pager_stats $bt]

test/soak.test

@@ -11,7 +11,7 @@
 # This file is the driver for the "soak" tests. It is a peer of the
 # quick.test and all.test scripts.
 #
-# $Id: soak.test,v 1.3 2008/07/12 14:52:20 drh Exp $
+# $Id: soak.test,v 1.4 2008/11/13 18:29:51 shane Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -47,10 +47,10 @@ set argv [list]
 # global variable $TIMEOUT - tests are run for at least $TIMEOUT 
 # seconds.
 #
-#   fuzz.test   (pseudo-random SQL statements)
+#   fuzz.test     (pseudo-random SQL statements)
-#   trans.test  (pseudo-random changes to a database followed by rollbacks)
+#   trans.test    (pseudo-random changes to a database followed by rollbacks)
-#
+#   fuzz_malloc.test
-# fuzzy malloc?
+#   corruptC.test (pseudo-random corruption to a database)
 #
 # Many database changes maintaining some kind of invariant. 
 # Storing checksums etc.
@@ -62,6 +62,7 @@ set SOAKTESTS {
   fuzz.test
   fuzz_malloc.test
   trans.test
+  corruptC.test
 }
 
 set ISQUICK 1