database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Smaller and faster PRAGMA integrity_check that also does a better job of

Browse Source 
detecting errors.  Some output text describing discovered file corruption
has changed for clarity.

FossilOrigin-Name: 251a7590ff4f65f59a1c871892533e4e2c544515
This commit is contained in:
drh
2015-07-02 16:17:30 +00:00
parent 3f09beda45
commit cbc6b71f39
6 changed files with 168 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
manifest
View File

@ -1,5 +1,5 @@
C Remove\s"#ifdef\sSQLITE_ENABLE_FTS5"\sfrom\sindividual\sfts5\ssource\sfiles.\sAdd\sa\ssingle\s"#if\s!defined(SQLITE_CORE)\s||\sdefined(SQLITE_ENABLE_FTS5)"\sto\sfts5.c.
D 2015-07-02T15:52:21.801
C Smaller\sand\sfaster\sPRAGMA\sintegrity_check\sthat\salso\sdoes\sa\sbetter\sjob\sof\ndetecting\serrors.\s\sSome\soutput\stext\sdescribing\sdiscovered\sfile\scorruption\nhas\schanged\sfor\sclarity.
D 2015-07-02T16:17:30.545
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 1f525f24e2d3a4defd0ce819c10980caeec967fe
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -269,7 +269,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452
F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
F src/btree.c e283de2f9da7ec5e47bedcd442a66a671aa2e118
F src/btree.c 98ef3db8f90e2258eae6428cbe9cf47802b81958
F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1
F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e
F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70
@ -518,19 +518,19 @@ F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09
F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b
F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4
F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97
F test/corrupt2.test f2064e0bf934124cc38868fd8badb8f0dd67b552
F test/corrupt2.test 08cec1e5ffa68a3610306d6068f112d08bc9f090
F test/corrupt3.test 4b548d0bbe2933bc81d3f54099a05fc4d28aff18
F test/corrupt4.test b99652079d542b21f4965f6248703b983e40fe80
F test/corrupt5.test 8ead52af76006f3286e9396cb41898018ccea107
F test/corrupt6.test 269548d19427ac554c830763b1c5ea54a0252f80
F test/corrupt7.test 22cc644c2e76c9804bc844121267aa6f8f7c0ded
F test/corrupt7.test e4fa6d6584276679cc1d20c4e58beb9559a4eb85
F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516
F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85
F test/corruptA.test 53e56dafd180addcdadb402244b8cb9771d2ba26
F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec
F test/corruptC.test 3fcc0f73d2cf2d69befe2d96332b942426a6aae2
F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040
F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee
F test/corruptE.test be8e5088c369fc7979c662cd644efdaafc0f7f6d
F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4
F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804
F test/corruptH.test 5dd4fa98c6c1ed33b178f9e8a48c4fdd3cfc9067
@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P c9ddbd88998d9523e72ad910ea67eb55024b3a88
R 6e7391ae89eb1933cb67f3ce933af577
U dan
Z 75f643c411d6a670d86998374f15370f
P 7819002ed85497bbd0f9cf4d39df641573324436
R f272e35570685d031c8340df24a1680f
U drh
Z 39a394fff4020992662b6a21d701df1d

2
manifest.uuid
View File

@ -1 +1 @@
7819002ed85497bbd0f9cf4d39df641573324436
251a7590ff4f65f59a1c871892533e4e2c544515

242
src/btree.c
View File

@ -8932,20 +8932,30 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){
static int checkTreePage(
IntegrityCk *pCheck, /* Context for the sanity check */
int iPage, /* Page number of the page to check */
i64 *pnParentMinKey,
i64 *pnParentMaxKey
i64 *piMinKey, /* Write minimum integer primary key here */
i64 maxKey /* Error if integer primary key greater than this */
){
MemPage *pPage = 0;
int i, rc, depth, d2, pgno, cnt;
int hdr, cellStart;
int nCell;
u8 *data;
BtShared *pBt;
int usableSize;
u32 *heap = 0;
MemPage *pPage = 0; /* The page being analyzed */
int i; /* Loop counter */
int rc; /* Result code from subroutine call */
int depth = -1, d2; /* Depth of a subtree */
int pgno; /* Page number */
int nFrag; /* Number of fragmented bytes on the page */
int hdr; /* Offset to the page header */
int cellStart; /* Offset to the start of the cell pointer array */
int nCell; /* Number of cells */
int doCoverageCheck = 1; /* True if cell coverage checking should be done */
int keyCanBeEqual = 1; /* True if IPK can be equal to maxKey
** False if IPK must be strictly less than maxKey */
u8 *data; /* Page content */
u8 *pCell; /* Cell content */
u8 *pCellIdx; /* Next element of the cell pointer array */
BtShared *pBt; /* The BtShared object that owns pPage */
u32 pc; /* Address of a cell */
u32 usableSize; /* Usable size of the page */
u32 contentOffset; /* Offset to the start of the cell content area */
u32 *heap = 0; /* Min-heap used for checking cell coverage */
u32 x, prev = 0;
i64 nMinKey = 0;
i64 nMaxKey = 0;
const char *saved_zPfx = pCheck->zPfx;
int saved_v1 = pCheck->v1;
int saved_v2 = pCheck->v2;
@ -8961,7 +8971,6 @@ static int checkTreePage(
if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){
checkAppendMsg(pCheck,
"unable to get the page. error code=%d", rc);
depth = -1;
goto end_of_check;
}
@ -8972,41 +8981,85 @@ static int checkTreePage(
assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */
checkAppendMsg(pCheck,
"btreeInitPage() returns error code %d", rc);
depth = -1;
goto end_of_check;
}
data = pPage->aData;
hdr = pPage->hdrOffset;
/* Check out all the cells.
*/
depth = 0;
/* Set up for cell analysis */
pCheck->zPfx = "On tree page %d cell %d: ";
for(i=0; i<pPage->nCell && pCheck->mxErr; i++){
u8 *pCell;
u32 sz;
contentOffset = get2byteNotZero(&data[hdr+5]);
assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */
/* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
** number of cells on the page. */
nCell = get2byte(&data[hdr+3]);
assert( pPage->nCell==nCell );
/* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page
** immediately follows the b-tree page header. */
cellStart = hdr + 12 - 4*pPage->leaf;
assert( pPage->aCellIdx==&data[cellStart] );
pCellIdx = &data[cellStart + 2*(nCell-1)];
if( !pPage->leaf ){
/* Analyze the right-child page of internal pages */
pgno = get4byte(&data[hdr+8]);
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
pCheck->zPfx = "On page %d at right child: ";
checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
}
#endif
depth = checkTreePage(pCheck, pgno, &maxKey, maxKey);
keyCanBeEqual = 0;
}else{
/* For leaf pages, the coverage check will occur in the same loop
** as the other cell checks, so initialize the heap. */
heap = pCheck->heap;
heap[0] = 0;
btreeHeapInsert(heap, contentOffset-1);
}
/* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
** integer offsets to the cell contents. */
for(i=nCell-1; i>=0 && pCheck->mxErr; i--){
CellInfo info;
/* Check payload overflow pages
*/
/* Check cell size */
pCheck->v2 = i;
pCell = findCell(pPage,i);
assert( pCellIdx==&data[cellStart + i*2] );
pc = get2byteAligned(pCellIdx);
pCellIdx -= 2;
if( pc<contentOffset || pc>usableSize-4 ){
checkAppendMsg(pCheck, "Offset %d out of range %d..%d",
pc, contentOffset, usableSize-4);
doCoverageCheck = 0;
continue;
}
pCell = &data[pc];
pPage->xParseCell(pPage, pCell, &info);
sz = info.nPayload;
/* For intKey pages, check that the keys are in order.
*/
if( pc+info.nSize>usableSize ){
checkAppendMsg(pCheck, "Extends off end of page");
doCoverageCheck = 0;
continue;
}
/* Check for integer primary key out of range */
if( pPage->intKey ){
if( i==0 ){
nMinKey = nMaxKey = info.nKey;
}else if( info.nKey <= nMaxKey ){
checkAppendMsg(pCheck,
"Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey);
if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){
checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey);
}
nMaxKey = info.nKey;
maxKey = info.nKey;
}
if( (sz>info.nLocal)
&& (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize])
){
int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4);
Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
/* Check the content overflow list */
if( info.nPayload>info.nLocal ){
int nPage; /* Number of pages on the overflow chain */
Pgno pgnoOvfl; /* First page of the overflow chain */
assert( pc + info.iOverflow <= usableSize );
nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4);
pgnoOvfl = get4byte(&pCell[info.iOverflow]);
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage);
@ -9015,107 +9068,50 @@ static int checkTreePage(
checkList(pCheck, 0, pgnoOvfl, nPage);
}
/* Check sanity of left child page.
*/
if( !pPage->leaf ){
/* Check sanity of left child page for internal pages */
pgno = get4byte(pCell);
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
}
#endif
d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey);
if( i>0 && d2!=depth ){
d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey);
keyCanBeEqual = 0;
if( d2!=depth ){
checkAppendMsg(pCheck, "Child page depth differs");
}
depth = d2;
}
}
if( !pPage->leaf ){
pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
pCheck->zPfx = "On page %d at right child: ";
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
}
#endif
d2 = checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey);
if( d2!=depth && iPage!=1 ){
checkAppendMsg(pCheck, "Child page depth differs");
}
}
/* For intKey leaf pages, check that the min/max keys are in order
** with any left/parent/right pages.
*/
pCheck->zPfx = "Page %d: ";
if( pPage->leaf && pPage->intKey ){
/* if we are a left child page */
if( pnParentMinKey ){
/* if we are the left most child page */
if( !pnParentMaxKey ){
if( nMaxKey > *pnParentMinKey ){
checkAppendMsg(pCheck,
"Rowid %lld out of order (max larger than parent min of %lld)",
nMaxKey, *pnParentMinKey);
}
}else{
if( nMinKey <= *pnParentMinKey ){
checkAppendMsg(pCheck,
"Rowid %lld out of order (min less than parent min of %lld)",
nMinKey, *pnParentMinKey);
}
if( nMaxKey > *pnParentMaxKey ){
checkAppendMsg(pCheck,
"Rowid %lld out of order (max larger than parent max of %lld)",
nMaxKey, *pnParentMaxKey);
}
*pnParentMinKey = nMaxKey;
}
/* else if we're a right child page */
} else if( pnParentMaxKey ){
if( nMinKey <= *pnParentMaxKey ){
checkAppendMsg(pCheck,
"Rowid %lld out of order (min less than parent max of %lld)",
nMinKey, *pnParentMaxKey);
}
/* Populate the coverage-checking heap for leaf pages */
btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1));
}
}
*piMinKey = maxKey;
/* Check for complete coverage of the page
*/
data = pPage->aData;
hdr = pPage->hdrOffset;
pCheck->zPfx = 0;
if( doCoverageCheck && pCheck->mxErr>0 ){
/* For leaf pages, the min-heap has already been initialized and the
** cells have already been inserted. But for internal pages, that has
** not yet been done, so do it now */
if( !pPage->leaf ){
heap = pCheck->heap;
heap[0] = 0;
pCheck->zPfx = 0;
{
int contentOffset = get2byteNotZero(&data[hdr+5]);
assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */
btreeHeapInsert(heap, contentOffset-1);
/* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
** number of cells on the page. */
nCell = get2byte(&data[hdr+3]);
/* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page
** immediately follows the b-tree page header. */
cellStart = hdr + 12 - 4*pPage->leaf;
/* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
** integer offsets to the cell contents. */
for(i=nCell-1; i>=0; i--){
u32 pc = get2byteAligned(&data[cellStart+i*2]);
u32 size = pPage->xCellSize(pPage, &data[pc]);
if( (int)(pc+size-1)>=usableSize ){
pCheck->zPfx = 0;
checkAppendMsg(pCheck,
"Corruption detected in cell %d on page %d",i,iPage);
}else{
btreeHeapInsert(heap, (pc<<16)|(pc+size-1));
}
}
/* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header
/* Add the freeblocks to the min-heap
**
** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header
** is the offset of the first freeblock, or zero if there are no
** freeblocks on the page. */
** freeblocks on the page.
*/
i = get2byte(&data[hdr+1]);
while( i>0 ){
int size, j;
@ -9134,7 +9130,10 @@ static int checkTreePage(
assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */
i = j;
}
cnt = 0;
/* Analyze the min-heap looking for overlap between cells and/or
** freeblocks, and counting the number of untracked bytes in nFrag.
*/
nFrag = 0;
assert( heap[0]>0 );
assert( (heap[1]>>16)==0 );
btreeHeapPull(heap,&prev);
@ -9144,20 +9143,20 @@ static int checkTreePage(
"Multiple uses for byte %u of page %d", x>>16, iPage);
break;
}else{
cnt += (x>>16) - (prev&0xffff) - 1;
nFrag += (x>>16) - (prev&0xffff) - 1;
prev = x;
}
}
cnt += usableSize - (prev&0xffff) - 1;
nFrag += usableSize - (prev&0xffff) - 1;
/* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments
** is stored in the fifth field of the b-tree page header.
** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the
** number of fragmented free bytes within the cell content area.
*/
if( heap[0]==0 && cnt!=data[hdr+7] ){
if( heap[0]==0 && nFrag!=data[hdr+7] ){
checkAppendMsg(pCheck,
"Fragmentation of %d bytes reported as %d on page %d",
cnt, data[hdr+7], iPage);
nFrag, data[hdr+7], iPage);
}
}
@ -9192,10 +9191,11 @@ char *sqlite3BtreeIntegrityCheck(
int *pnErr /* Write number of errors seen to this variable */
){
Pgno i;
VVA_ONLY( int nRef );
IntegrityCk sCheck;
BtShared *pBt = p->pBt;
int savedDbFlags = pBt->db->flags;
char zErr[100];
VVA_ONLY( int nRef );
sqlite3BtreeEnter(p);
assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE );
@ -9239,15 +9239,19 @@ char *sqlite3BtreeIntegrityCheck(
/* Check all the tables.
*/
testcase( pBt->db->flags & SQLITE_CellSizeCk );
pBt->db->flags &= ~SQLITE_CellSizeCk;
for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
i64 notUsed;
if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum && aRoot[i]>1 ){
checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
}
#endif
checkTreePage(&sCheck, aRoot[i], NULL, NULL);
checkTreePage(&sCheck, aRoot[i], &notUsed, LARGEST_INT64);
}
pBt->db->flags = savedDbFlags;
/* Make sure every page in the file is referenced
*/

1
test/corrupt2.test
View File

@ -249,7 +249,6 @@ do_test corrupt2-5.1 {
set result
} {{*** in database main ***
On tree page 2 cell 0: 2nd reference to page 10
On tree page 2 cell 1: Child page depth differs
Page 4 is never used}}
db2 close

25
test/corrupt7.test
View File

@ -65,41 +65,20 @@ integrity_check corrupt7-1.4
# Deliberately corrupt some of the cell offsets in the btree page
# on page 2 of the database.
#
# The error message is different depending on whether or not the
# SQLITE_ENABLE_OVERSIZE_CELL_CHECK compile-time option is engaged.
#
ifcapable oversize_cell_check {
do_test corrupt7-2.1 {
db close
hexio_write test.db 1062 FF
sqlite3 db test.db
db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***
Page 2: btreeInitPage() returns error code 11}}
On tree page 2 cell 15: Offset 65457 out of range 945..1020}}
do_test corrupt7-2.2 {
db close
hexio_write test.db 1062 04
sqlite3 db test.db
db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***
Page 2: btreeInitPage() returns error code 11}}
} else {
do_test corrupt7-2.1 {
db close
hexio_write test.db 1062 FF
sqlite3 db test.db
db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***
Corruption detected in cell 15 on page 2}}
do_test corrupt7-2.2 {
db close
hexio_write test.db 1062 04
sqlite3 db test.db
db eval {PRAGMA integrity_check(1)}
} {{*** in database main ***
On tree page 2 cell 15: Rowid 0 out of order (previous was 15)}}
}
On tree page 2 cell 15: Offset 1201 out of range 945..1020}}
# The code path that was causing the buffer overrun that this test
# case was checking for was removed.

46
test/corruptE.test
View File

@ -14,7 +14,6 @@
# segfault if it sees a corrupt database file. It specifcally
# focuses on rowid order corruption.
#
# $Id: corruptE.test,v 1.14 2009/07/11 06:55:34 danielk1977 Exp $
set testdir [file dirname $argv0]
source $testdir/tester.tcl
@ -79,12 +78,8 @@ do_test corruptE-2.1 {
sqlite3 db test.db
set res [ catchsql {PRAGMA integrity_check} ]
set ans [lindex $res 1]
list [regexp {out of order.*previous was} $ans] \
[regexp {out of order.*max larger than parent max} $ans]
} {1 1}
catchsql {PRAGMA integrity_check}
} {/ out of order/}
do_test corruptE-2.2 {
db close
@ -95,12 +90,8 @@ do_test corruptE-2.2 {
sqlite3 db test.db
set res [ catchsql {PRAGMA integrity_check} ]
set ans [lindex $res 1]
list [regexp {out of order.*previous was} $ans] \
[regexp {out of order.*min less than parent min} $ans]
} {1 1}
catchsql {PRAGMA integrity_check}
} {/ Extends off end of page/}
do_test corruptE-2.3 {
db close
@ -112,11 +103,8 @@ do_test corruptE-2.3 {
sqlite3 db test.db
set res [ catchsql {PRAGMA integrity_check} ]
set ans [lindex $res 1]
list [regexp {out of order.*max larger than parent min} $ans]
} {1}
catchsql {PRAGMA integrity_check}
} {/out of order/}
do_test corruptE-2.4 {
db close
@ -127,33 +115,22 @@ do_test corruptE-2.4 {
sqlite3 db test.db
set res [ catchsql {PRAGMA integrity_check} ]
set ans [lindex $res 1]
list [regexp {out of order.*min less than parent max} $ans]
} {1}
catchsql {PRAGMA integrity_check}
} {/out of order/}
set tests [list {10233 0xd0} \
{941 0x42} \
{1028 0x53} \
{2041 0xd0} \
{2042 0x1f} \
{2047 0xaa} \
{2263 0x29} \
{2274 0x75} \
{3267 0xf2} \
{4104 0x2c} \
{5113 0x36} \
{10233 0x84} \
{10234 0x74} \
{10239 0x41} \
{10453 0x11} \
{11273 0x28} \
{11455 0x11} \
{11461 0xe6} \
{12281 0x99} \
{12296 0x9e} \
{12297 0xd7} \
{13303 0x53} ]
@ -168,11 +145,8 @@ foreach test $tests {
sqlite3 db test.db
set res [ catchsql {PRAGMA integrity_check} ]
set ans [lindex $res 1]
list [regexp {out of order} $ans]
} {1}
catchsql {PRAGMA integrity_check}
} {/out of order/}
incr tc 1
}