diff --git a/manifest b/manifest index 95e347bb41..74f035f05d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Make\ssure\sa\sWAL\sframe\sof\sall\szeros\sis\sdetected\sas\san\sinvalid\sframe. -D 2010-05-24T13:28:36 +C Change\sthe\schecksum\sused\sin\sWAL\sfiles\sso\sthat\seach\sframes\schecksum\sdepends\son\sthe\scontent\sof\sthe\sWAL\sheader\sand\sall\sframe\sheaders\sand\scontent\sup\sto\sand\sincluding\sthe\sframe\sto\swhich\sthe\schecksum\sis\sattached. +D 2010-05-24T13:57:43 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -157,7 +154,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19 F src/os_unix.c 35ace483789db8ede92acc46134930c2c4267645 F src/os_win.c 1e44ee84210b59db1e098bbbc66f6dee68e20d5f -F src/pager.c d3284a6bbedeaa4ef3f5668af309d4381df97618 +F src/pager.c 0fbfe2ccd98cd893f3b5254a9297e153440e5e37 F src/pager.h 76466c3a5af56943537f68b1f16567101a0cd1d0 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -227,8 +224,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c e8c58e529bcc50c0fb3797bc39750e802cbace78 -F src/wal.h 434f76f51225bb614e43ccb6bd2341541ba6a06e +F src/wal.c c09f4e33aad4ec3822ef3e9626f8bd7c273542af +F src/wal.h 111c6f3efd83fe2fc707b29e26431e8eff4c6f28 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -764,10 +761,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 3b8ad018c1faf89d3f5bb23704775f5d20e486de -F test/wal2.test 053c9ea94194c5bce5b742429be75ff2432794ab +F test/wal.test be8ef043253ca735ffcabb92a7dac2d79ebfe8c1 +F test/wal2.test d9a50d1b2e0f0735b8a21538631100eaf845364f F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432 -F test/walcksum.test cc41a85d8b6f1471ebdf847f82f39dd0003a37bc +F test/walcksum.test a0712107b6a73397fc7e3f92d5b16e206caa7d3d F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f F test/walfault.test f71d4c9a13d4e27086aef55f1e0e94734ffa2f6a F test/walhook.test 67e675127f4acb72f061a12667ce6e5460b06b78 @@ -817,14 +814,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 51fd38152b92db637d1d346fca35ec2d3e4d4f57 -R 3ae8439c7818113b3e55b5c33ec15fc6 -U drh -Z 157804aa9c723c5bf0d5892d1297d942 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL+n8IoxKgR168RlERAjqiAJ93cfWX1aiOOxPNMQnGl5zdjLHO3ACfQbUq -4XUL8oGpoSF35Q/+0pQNofc= -=ktcp ------END PGP SIGNATURE----- +P 02d99ad4b51065c67cc7689916130774be1c4c87 +R 0b567cb4cfa023970e21a7a28609c3b9 +U dan +Z fafe9597e653734ba29a55af1088e0f3 diff --git a/manifest.uuid b/manifest.uuid index ab6b56274d..9c65286320 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02d99ad4b51065c67cc7689916130774be1c4c87 \ No newline at end of file +8a53f12c83a107684b99f4a9de371b5ea3ca810a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 43ac8eb322..beb2b2dfc0 100644 --- a/src/pager.c +++ b/src/pager.c @@ -221,7 +221,7 @@ struct PagerSavepoint { Bitvec *pInSavepoint; /* Set of pages in this savepoint */ Pgno nOrig; /* Original number of pages in file */ Pgno iSubRec; /* Index of first record in sub-journal */ - u32 iFrame; /* Last frame in WAL when savepoint opened */ + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ }; /* @@ -2567,7 +2567,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); if( pagerUseWal(pPager) ){ - rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->iFrame); + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); } for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ assert( offset==ii*(4+pPager->pageSize) ); @@ -5448,7 +5448,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ return SQLITE_NOMEM; } if( pagerUseWal(pPager) ){ - aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pWal); + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); } } diff --git a/src/wal.c b/src/wal.c index 0d1cb96249..09b3517505 100644 --- a/src/wal.c +++ b/src/wal.c @@ -214,13 +214,14 @@ typedef struct WalIterator WalIterator; ** object. */ struct WalIndexHdr { - u32 iChange; /* Counter incremented each transaction */ - u16 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes */ - u32 mxFrame; /* Index of last valid frame in the WAL */ - u32 nPage; /* Size of database in pages */ - u32 aSalt[2]; /* Salt-1 and salt-2 values copied from WAL header */ - u32 aCksum[2]; /* Checksum over all prior fields */ + u32 iChange; /* Counter incremented each transaction */ + u16 bigEndCksum; /* True if checksums in WAL are big-endian */ + u16 szPage; /* Database page size in bytes */ + u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 nPage; /* Size of database in pages */ + u32 aFrameCksum[2]; /* Checksum of last frame in log */ + u32 aSalt[2]; /* Two salt values copied from WAL header */ + u32 aCksum[2]; /* Checksum over all prior fields */ }; /* A block of WALINDEX_LOCK_RESERVED bytes beginning at @@ -424,14 +425,14 @@ static void walEncodeFrame( u8 *aFrame /* OUT: Write encoded frame here */ ){ int nativeCksum; /* True for native byte-order checksums */ - u32 aCksum[2]; + u32 *aCksum = pWal->hdr.aFrameCksum; assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); memcpy(&aFrame[8], pWal->hdr.aSalt, 8); nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); sqlite3Put4byte(&aFrame[16], aCksum[0]); @@ -451,8 +452,8 @@ static int walDecodeFrame( u8 *aFrame /* Frame data */ ){ int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; u32 pgno; /* Page number of the frame */ - u32 aCksum[2]; assert( WAL_FRAME_HDRSIZE==24 ); /* A frame is only valid if the salt values in the frame-header @@ -474,7 +475,7 @@ static int walDecodeFrame( ** the checksum in the last 8 bytes of the frame-header. */ nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) @@ -739,10 +740,10 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ static int walIndexRecover(Wal *pWal){ int rc; /* Return Code */ i64 nSize; /* Size of log file */ - WalIndexHdr hdr; /* Recovered wal-index header */ + u32 aFrameCksum[2] = {0, 0}; assert( pWal->lockState>SQLITE_SHM_READ ); - memset(&hdr, 0, sizeof(hdr)); + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); if( rc!=SQLITE_OK ){ @@ -779,10 +780,13 @@ static int walIndexRecover(Wal *pWal){ ){ goto finished; } - hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001); + pWal->hdr.bigEndCksum = (magic&0x00000001); pWal->szPage = szPage; pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE, 0, pWal->hdr.aFrameCksum + ); /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; @@ -809,23 +813,24 @@ static int walIndexRecover(Wal *pWal){ /* If nTruncate is non-zero, this is a commit record. */ if( nTruncate ){ - hdr.mxFrame = iFrame; - hdr.nPage = nTruncate; - hdr.szPage = szPage; + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = szPage; + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } } sqlite3_free(aFrame); - }else{ - memset(&hdr, 0, sizeof(hdr)); } finished: - if( rc==SQLITE_OK && hdr.mxFrame==0 ){ + if( rc==SQLITE_OK && pWal->hdr.mxFrame==0 ){ rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); } if( rc==SQLITE_OK ){ - memcpy(&pWal->hdr, &hdr, sizeof(hdr)); + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; walIndexWriteHdr(pWal); } return rc; @@ -1626,25 +1631,35 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ return rc; } -/* Return an integer that records the current (uncommitted) write -** position in the WAL +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current +** point in the event of a savepoint rollback (via WalSavepointUndo()). */ -u32 sqlite3WalSavepoint(Wal *pWal){ +void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ assert( pWal->lockState==SQLITE_SHM_WRITE ); - return pWal->hdr.mxFrame; + aWalData[0] = pWal->hdr.mxFrame; + aWalData[1] = pWal->hdr.aFrameCksum[0]; + aWalData[2] = pWal->hdr.aFrameCksum[1]; } -/* Move the write position of the WAL back to iFrame. Called in -** response to a ROLLBACK TO command. +/* +** Move the write position of the WAL back to the point identified by +** the values in the aWalData[] array. aWalData must point to an array +** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated +** by a call to WalSavepoint(). */ -int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){ +int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; assert( pWal->lockState==SQLITE_SHM_WRITE ); - assert( iFrame<=pWal->hdr.mxFrame ); - if( iFramehdr.mxFrame ){ + assert( aWalData[0]<=pWal->hdr.mxFrame ); + if( aWalData[0]hdr.mxFrame ){ rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); - pWal->hdr.mxFrame = iFrame; + pWal->hdr.mxFrame = aWalData[0]; + pWal->hdr.aFrameCksum[0] = aWalData[1]; + pWal->hdr.aFrameCksum[1] = aWalData[2]; if( rc==SQLITE_OK ){ walCleanupHash(pWal); walIndexUnmap(pWal); @@ -1694,6 +1709,7 @@ int sqlite3WalFrames( if( rc!=SQLITE_OK ){ return rc; } + walChecksumBytes(1, aWalHdr, sizeof(aWalHdr), 0, pWal->hdr.aFrameCksum); } assert( pWal->szPage==szPage ); diff --git a/src/wal.h b/src/wal.h index 1e25fc8116..bf40c3bd9f 100644 --- a/src/wal.h +++ b/src/wal.h @@ -28,13 +28,15 @@ # define sqlite3WalDbsize(y,z) # define sqlite3WalWriteLock(y,z) 0 # define sqlite3WalUndo(x,y,z) 0 -# define sqlite3WalSavepoint(z) 0 +# define sqlite3WalSavepoint(y,z) # define sqlite3WalSavepointUndo(y,z) 0 # define sqlite3WalFrames(u,v,w,x,y,z) 0 # define sqlite3WalCheckpoint(u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 #else +#define WAL_SAVEPOINT_NDATA 3 + /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ @@ -69,11 +71,11 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); /* Return an integer that records the current (uncommitted) write ** position in the WAL */ -u32 sqlite3WalSavepoint(Wal *pWal); +void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); /* Move the write position of the WAL back to iFrame. Called in ** response to a ROLLBACK TO command. */ -int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame); +int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); /* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); diff --git a/test/wal.test b/test/wal.test index 7005bce68f..fbe0bd3ee7 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1348,7 +1348,8 @@ foreach {tn pgsz works} { set framehdr [binary format IIII $pg 5 22 23] set c1 0 set c2 0 - logcksum c1 c2 $framehdr + logcksum c1 c2 $walhdr + logcksum c1 c2 [string range $framehdr 0 7] logcksum c1 c2 $framebody set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2] diff --git a/test/wal2.test b/test/wal2.test index 1e93818046..80fe9f3549 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -19,17 +19,24 @@ source $testdir/lock_common.tcl ifcapable !wal {finish_test ; return } proc set_tvfs_hdr {file args} { + + # Set $nHdr to the number of bytes in the wal-index header: + set nHdr 80 + set nInt [expr {$nHdr/4}] + if {[llength $args]>1} { return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"} } set blob [tvfs shm $file] if {[llength $args]} { - set blob [binary format i16a* [lindex $args 0] [string range $blob 64 end]] + set blob [ + binary format i${nInt}a* [lindex $args 0] [string range $blob $nHdr end] + ] tvfs shm $file $blob } - binary scan $blob i16 ints + binary scan $blob i${nInt} ints return $ints } diff --git a/test/walcksum.test b/test/walcksum.test index d2339c74db..f21cf9fea9 100644 --- a/test/walcksum.test +++ b/test/walcksum.test @@ -62,18 +62,13 @@ proc readfile {filename} { # proc log_checksum_verify {filename iFrame endian} { set data [readfile $filename] - set c1 0 - set c2 0 - - binary scan [string range $data 8 11] I pgsz - set n [log_file_size [expr $iFrame-1] $pgsz] - binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2 - log_cksum $endian c1 c2 [string range $data $n [expr $n+15]] - log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} + binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2 set expect1 [expr $expect1&0xFFFFFFFF] set expect2 [expr $expect2&0xFFFFFFFF] + expr {$c1==$expect1 && $c2==$expect2} } @@ -85,24 +80,37 @@ proc log_checksum_verify {filename iFrame endian} { # proc log_checksum_write {filename iFrame endian} { set data [readfile $filename] - set c1 0 - set c2 0 - - binary scan [string range $data 8 11] I pgsz - set n [log_file_size [expr $iFrame-1] $pgsz] - log_cksum $endian c1 c2 [string range $data $n [expr $n+15]] - log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} set bin [binary format II $c1 $c2] set fd [open $filename r+] fconfigure $fd -encoding binary fconfigure $fd -translation binary - seek $fd [expr $n+16] + seek $fd $offset puts -nonewline $fd $bin close $fd } +proc log_checksum_calc {data iFrame endian} { + + binary scan [string range $data 8 11] I pgsz + if {$iFrame > 1} { + set n [log_file_size [expr $iFrame-2] $pgsz] + binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2 + } else { + set c1 0 + set c2 0 + log_cksum $endian c1 c2 [string range $data 0 23] + } + + set n [log_file_size [expr $iFrame-1] $pgsz] + log_cksum $endian c1 c2 [string range $data $n [expr $n+7]] + log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + + list [expr $n+16] $c1 $c2 +} + # # File $filename must be a WAL file on disk. Set the 'magic' field of the # WAL header to indicate that checksums are $endian-endian ($endian must be @@ -180,6 +188,7 @@ foreach endian {big little} { # Replace all checksums in the current WAL file with $endian versions. # Then check that it is still possible to recover and read the database. # + log_checksum_writemagic test2.db-wal $endian for {set f 1} {$f <= 6} {incr f} { do_test walcksum-1.$endian.3.$f { log_checksum_write test2.db-wal $f $endian @@ -187,7 +196,6 @@ foreach endian {big little} { } {1} } do_test walcksum-1.$endian.4.1 { - log_checksum_writemagic test2.db-wal $endian file copy -force test2.db test.db file copy -force test2.db-wal test.db-wal sqlite3 db test.db @@ -263,7 +271,7 @@ foreach endian {big little} { } {1} do_test walcksum-1.$endian.8.3 { log_checksum_verify test.db-wal 3 $native - } [expr {$native == $endian}] + } {0} do_test walcksum-1.$endian.9 { execsql { @@ -276,4 +284,45 @@ foreach endian {big little} { catch { db2 close } } +do_test walcksum-2.1 { + file delete -force test.db test.db-wal test.db-journal + sqlite3 db test.db + execsql { + PRAGMA synchronous = NORMAL; + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA cache_size = 10; + CREATE TABLE t1(x PRIMARY KEY); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(randomblob(800)); + BEGIN; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 16 */ + SAVEPOINT one; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ + ROLLBACK TO one; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ + COMMIT; + } + + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + + sqlite3 db2 test2.db + execsql { + PRAGMA integrity_check; + SELECT count(*) FROM t1; + } db2 +} {ok 256} +catch { db close } +catch { db2 close } + finish_test