database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-07 02:42:48 +03:00
Code Activity

Fix problems with recovering wal files that use a page-size other than the default.

Browse Source 
FossilOrigin-Name: 1a391f3c55dc9d4266552fa26d2a9839c6bafce4
This commit is contained in:
dan
2010-05-04 14:47:39 +00:00
parent ef37802545
commit b6e099a973
7 changed files with 220 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
manifest
View File

@@ -1,5 +1,5 @@
C Fix\sa\stypo\sin\swalfault.test.
D 2010-05-04T11:06:03
C Fix\sproblems\swith\srecovering\swal\sfiles\sthat\suse\sa\spage-size\sother\sthan\sthe\sdefault.
D 2010-05-04T14:47:40
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -152,7 +152,7 @@ F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d
F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0
F src/os_unix.c 7ab8d2fe59954dca619af3e1bbd4789f5032fece
F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1
F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8
F src/pager.c f030536592582d29a761b75988f409c894c37d5e
F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c
F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf
@@ -221,8 +221,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
F src/wal.c 1d0abe01ae9836f207ecfe2d3f1ece97b744fc3e
F src/wal.h d6d4a6809151e30bed5b01dd05cf27858f5a7bc8
F src/wal.c 558700d2d596af389305c193fe98e79406a50982
F src/wal.h b4c42014b5fa3b4e6244ac8c65de7ff67adeb27c
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -758,9 +758,9 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
F test/wal.test 511c630993d5adc170f56e581909b6eb580b0f53
F test/wal.test 2b3626a6c839ef281e2a51f6223578e5fa7616c2
F test/walbak.test a0e45187c7d8928df035dfea29b99b016b21ca3c
F test/walcrash.test 63edc6a9e05f645b54d649186a5818fc82953e2e
F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f
F test/walfault.test 2504c5c50d8f9a9e48969de381eb98e2a8b89195
F test/walhook.test 5f18e0fc8787f1f8889d7a9971af18f334f83786
F test/walmode.test bac6f06544a8554588a1543def996bbe2fc41792
@@ -809,7 +809,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P a60104aa7e38e7d9f2ff2eae02687dc9c5dd5d77
R 9692cfa3edc87873be9342b2b7fc3777
P 232dbe8ecec16485be5c5995fdf7a0ed951c2097
R 8e6184a7c7d1eb3d58e7d51edd48bf22
U dan
Z 6f47b056126e5089750d5cf38e878789
Z 520c27e9585f17af9827fbfa6794dca0

2
manifest.uuid
View File

@@ -1 +1 @@
232dbe8ecec16485be5c5995fdf7a0ed951c2097
1a391f3c55dc9d4266552fa26d2a9839c6bafce4

25
src/pager.c
View File

@@ -2196,8 +2196,8 @@ static int readDbPage(PgHdr *pPg){
Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
Pgno pgno = pPg->pgno; /* Page number to read */
int rc = SQLITE_OK; /* Return code */
i64 iOffset; /* Byte offset of file to read from */
int isInWal = 0; /* True if page is in log file */
int pgsz = pPager->pageSize; /* Number of bytes to read */
assert( pPager->state>=PAGER_SHARED && !MEMDB );
assert( isOpen(pPager->fd) );
@@ -2210,11 +2210,11 @@ static int readDbPage(PgHdr *pPg){
if( pagerUseWal(pPager) ){
/* Try to pull the page from the write-ahead log. */
rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pPg->pData);
rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData);
}
if( rc==SQLITE_OK && !isInWal ){
iOffset = (pgno-1)*(i64)pPager->pageSize;
rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset);
i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
if( rc==SQLITE_IOERR_SHORT_READ ){
rc = SQLITE_OK;
}
@@ -2832,6 +2832,15 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){
int rc = SQLITE_OK;
memset(pDest, 0, N);
assert( isOpen(pPager->fd) || pPager->tempFile );
if( pagerUseWal(pPager) ){
int isInWal = 0;
rc = sqlite3WalRead(pPager->pWal, 1, &isInWal, N, pDest);
if( rc!=SQLITE_OK || isInWal ){
return rc;
}
}
if( isOpen(pPager->fd) ){
IOTRACE(("DBHDR %p 0 %d\n", pPager, N))
rc = sqlite3OsRead(pPager->fd, pDest, N, 0);
@@ -3055,7 +3064,8 @@ int sqlite3PagerClose(Pager *pPager){
pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
sqlite3WalClose(pPager->pWal, pPager->fd,
(pPager->noSync ? 0 : pPager->sync_flags), pTmp
(pPager->noSync ? 0 : pPager->sync_flags),
pPager->pageSize, pTmp
);
pPager->pWal = 0;
#endif
@@ -5833,7 +5843,8 @@ int sqlite3PagerCheckpoint(Pager *pPager){
u8 *zBuf = (u8 *)pPager->pTmpSpace;
rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd,
(pPager->noSync ? 0 : pPager->sync_flags),
zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg
pPager->pageSize, zBuf,
pPager->xBusyHandler, pPager->pBusyHandlerArg
);
}
return rc;
@@ -5908,7 +5919,7 @@ int sqlite3PagerCloseWal(Pager *pPager){
if( rc==SQLITE_OK ){
rc = sqlite3WalClose(pPager->pWal, pPager->fd,
(pPager->noSync ? 0 : pPager->sync_flags),
(u8*)pPager->pTmpSpace
pPager->pageSize, (u8*)pPager->pTmpSpace
);
pPager->pWal = 0;
}

33
src/wal.c
View File

@@ -710,6 +710,7 @@ static int walCheckpoint(
Wal *pWal, /* Wal connection */
sqlite3_file *pFd, /* File descriptor open on db file */
int sync_flags, /* Flags for OsSync() (or 0) */
int nBuf, /* Size of zBuf in bytes */
u8 *zBuf /* Temporary buffer to use */
){
int rc; /* Return code */
@@ -718,14 +719,20 @@ static int walCheckpoint(
u32 iDbpage = 0; /* Next database page to write */
u32 iFrame = 0; /* Wal frame containing data for iDbpage */
if( pWal->hdr.iLastPg==0 ){
return SQLITE_OK;
}
/* Allocate the iterator */
pIter = walIteratorInit(pWal);
if( !pIter ) return SQLITE_NOMEM;
if( pWal->hdr.iLastPg==0 ){
rc = SQLITE_OK;
goto out;
}
if( pWal->hdr.pgsz!=nBuf ){
rc = SQLITE_CORRUPT_BKPT;
goto out;
}
/* Sync the log file to disk */
if( sync_flags ){
rc = sqlite3OsSync(pWal->pFd, sync_flags);
@@ -788,7 +795,8 @@ int sqlite3WalClose(
Wal *pWal, /* Wal to close */
sqlite3_file *pFd, /* Database file */
int sync_flags, /* Flags to pass to OsSync() (or 0) */
u8 *zBuf /* Buffer of at least page-size bytes */
int nBuf,
u8 *zBuf /* Buffer of at least nBuf bytes */
){
int rc = SQLITE_OK;
if( pWal ){
@@ -804,7 +812,7 @@ int sqlite3WalClose(
*/
rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE);
if( rc==SQLITE_OK ){
rc = walCheckpoint(pWal, pFd, sync_flags, zBuf);
rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf);
if( rc==SQLITE_OK ){
isDelete = 1;
}
@@ -953,7 +961,13 @@ void sqlite3WalCloseSnapshot(Wal *pWal){
/*
** Read a page from the log, if it is present.
*/
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){
int sqlite3WalRead(
Wal *pWal,
Pgno pgno,
int *pInWal,
int nOut,
u8 *pOut
){
u32 iRead = 0;
u32 *aData;
int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00);
@@ -1011,7 +1025,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){
if( iRead ){
i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE;
*pInWal = 1;
return sqlite3OsRead(pWal->pFd, pOut, pWal->hdr.pgsz, iOffset);
return sqlite3OsRead(pWal->pFd, pOut, nOut, iOffset);
}
*pInWal = 0;
@@ -1266,6 +1280,7 @@ int sqlite3WalCheckpoint(
Wal *pWal, /* Wal connection */
sqlite3_file *pFd, /* File descriptor open on db file */
int sync_flags, /* Flags to sync db file with (or 0) */
int nBuf, /* Size of temporary buffer */
u8 *zBuf, /* Temporary buffer to use */
int (*xBusyHandler)(void *), /* Pointer to busy-handler function */
void *pBusyHandlerArg /* Argument to pass to xBusyHandler */
@@ -1288,7 +1303,7 @@ int sqlite3WalCheckpoint(
/* Copy data from the log to the database file. */
rc = walIndexReadHdr(pWal, &isChanged);
if( rc==SQLITE_OK ){
rc = walCheckpoint(pWal, pFd, sync_flags, zBuf);
rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf);
}
if( isChanged ){
/* If a new wal-index header was loaded before the checkpoint was

7
src/wal.h
View File

@@ -24,7 +24,7 @@
# define sqlite3WalClose(w,x,y,z) 0
# define sqlite3WalOpenSnapshot(y,z) 0
# define sqlite3WalCloseSnapshot(z)
# define sqlite3WalRead(w,x,y,z) 0
# define sqlite3WalRead(v,w,x,y,z) 0
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z) 0
# define sqlite3WalUndo(x,y,z) 0
@@ -42,7 +42,7 @@ typedef struct Wal Wal;
/* Open and close a connection to a write-ahead log. */
int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal);
int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, u8 *zBuf);
int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, int, u8 *);
/* Used by readers to open (lock) and close (unlock) a snapshot. A
** snapshot is like a read-transaction. It is the state of the database
@@ -55,7 +55,7 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *);
void sqlite3WalCloseSnapshot(Wal *pWal);
/* Read a page from the write-ahead log, if it is present. */
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut);
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut);
/* Return the size of the database as it existed at the beginning
** of the snapshot */
@@ -83,6 +83,7 @@ int sqlite3WalCheckpoint(
Wal *pWal, /* Write-ahead log connection */
sqlite3_file *pFd, /* File descriptor open on db file */
int sync_flags, /* Flags to sync db file with (or 0) */
int nBuf, /* Size of buffer nBuf */
u8 *zBuf, /* Temporary buffer to use */
int (*xBusyHandler)(void *), /* Pointer to busy-handler function */
void *pBusyHandlerArg /* Argument to pass to xBusyHandler */

146
test/wal.test
View File

@@ -1193,8 +1193,152 @@ foreach {tn sectorsize logsize} {
}
sqlite3_test_control_pending_byte $old_pending_byte
#-------------------------------------------------------------------------
# This test - wal-18.* - verifies a couple of specific conditions that
# may be encountered while recovering a log file are handled correctly:
#
# wal-18.1.* When the first 32-bits of a frame checksum is correct but
# the second 32-bits are false, and
#
# wal-18.2.* When the page-size field that occurs at the start of a log
# file is a power of 2 greater than 16384 or smaller than 512.
#
file delete -force test.db test.db-wal test.db-journal
do_test wal-18.0 {
sqlite3 db test.db
execsql {
PRAGMA page_size = 1024;
PRAGMA auto_vacuum = 0;
PRAGMA journal_mode = WAL;
PRAGMA synchronous = OFF;
CREATE TABLE t1(a, b, UNIQUE(a, b));
INSERT INTO t1 VALUES(0, 0);
PRAGMA wal_checkpoint;
INSERT INTO t1 VALUES(1, 2); -- frames 1 and 2
INSERT INTO t1 VALUES(3, 4); -- frames 3 and 4
INSERT INTO t1 VALUES(5, 6); -- frames 5 and 6
}
file copy -force test.db testX.db
file copy -force test.db-wal testX.db-wal
db close
list [file size testX.db] [file size testX.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]
foreach {nFrame result} {
0 {0 0}
1 {0 0}
2 {0 0 1 2}
3 {0 0 1 2}
4 {0 0 1 2 3 4}
5 {0 0 1 2 3 4}
6 {0 0 1 2 3 4 5 6}
} {
do_test wal-18.1.$nFrame {
file copy -force testX.db test.db
file copy -force testX.db-wal test.db-wal
hexio_write test.db-wal [expr 12 + $nFrame*(16+1024) + 12] 00000000
sqlite3 db test.db
execsql {
SELECT * FROM t1;
PRAGMA integrity_check;
}
} [concat $result ok]
db close
}
proc randomblob {pgsz} {
sqlite3 rbdb :memory:
set blob [rbdb one {SELECT randomblob($pgsz)}]
rbdb close
set blob
}
proc logcksum {ckv1 ckv2 blob} {
upvar $ckv1 c1
upvar $ckv2 c2
binary scan $blob iu* values
foreach v $values {
incr c1 $v
incr c2 $c1
}
set c1 [expr ($c1 + ($c1>>24))&0xFFFFFFFF]
set c2 [expr ($c2 + ($c2>>24))&0xFFFFFFFF]
}
file copy -force test.db testX.db
foreach {tn pgsz works} {
1 128 0
2 256 0
3 512 1
4 1024 1
5 2048 1
6 4096 1
7 8192 1
8 16384 1
9 32768 1
10 65536 0
} {
for {set pg 1} {$pg <= 3} {incr pg} {
file copy -force testX.db test.db
file delete -force test.db-wal
# Check that the database now exists and consists of three pages. And
# that there is no associated wal file.
#
do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0
do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1
do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3]
do_test wal-18.2.$tn.$pg.4 {
# Create a wal file that contains a single frame (database page
# number $pg) with the commit flag set. The frame checksum is
# correct, but the contents of the database page are corrupt.
#
# The page-size in the log file header is set to $pgsz. If the
# WAL code considers $pgsz to be a valid SQLite database file page-size,
# the database will be corrupt (because the garbage frame contents
# will be treated as valid content). If $pgsz is invalid (too small
# or too large), the db will not be corrupt as the log file will
# be ignored.
#
set c1 22
set c2 23
set walhdr [binary format III $pgsz $c1 $c2]
set framebody [randomblob $pgsz]
set framehdr [binary format II $pg 5]
logcksum c1 c2 $framehdr
logcksum c1 c2 $framebody
set framehdr [binary format IIII $pg 5 $c1 $c2]
set fd [open test.db-wal w]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd $walhdr
puts -nonewline $fd $framehdr
puts -nonewline $fd $framebody
close $fd
file size test.db-wal
} [log_file_size 1 $pgsz]
do_test wal-18.2.$tn.$pg.5 {
sqlite3 db test.db
set rc [catch { db one {PRAGMA integrity_check} } msg]
expr { $rc!=0 || $msg!="ok" }
} $works
db close
}
}
catch { db2 close }
catch { db close }
finish_test

20
test/walcrash.test
View File

@@ -250,11 +250,27 @@ for {set i 1} {$i < $REPEATS} {incr i} {
db close
}
#-------------------------------------------------------------------------
# This test case simulates a crash while checkpointing the database. Page
# 1 is one of the pages overwritten by the checkpoint. This is a special
# case because it means the content of page 1 may be damaged. SQLite will
# have to determine:
#
# (a) that the database is a WAL database, and
# (b) the database page-size
#
# based on the log file.
#
for {set i 1} {$i < $REPEATS} {incr i} {
file delete -force test.db test.db-wal
# Select a page-size for this test.
#
set pgsz [lindex {512 1024 2048 4096 8192 16384} [expr $i%6]]
do_test walcrash-7.$i.1 {
crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 {
crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 "
PRAGMA page_size = $pgsz;
PRAGMA journal_mode = wal;
BEGIN;
CREATE TABLE t1(a, b);
@@ -263,7 +279,7 @@ for {set i 1} {$i < $REPEATS} {incr i} {
PRAGMA wal_checkpoint;
CREATE INDEX i1 ON t1(a);
PRAGMA wal_checkpoint;
}
"
} {1 {child process exited abnormally}}
do_test walcrash-7.$i.2 {