database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-01-06 08:01:16 +03:00
Code Activity

Add extra tests for corrupt database handling in fts5.

Browse Source 
FossilOrigin-Name: 41449f7a0b5da6332eef48386c91ef63382c4783
This commit is contained in:
dan
2015-04-24 15:56:09 +00:00
parent 9686db7770
commit def90aae18
8 changed files with 230 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ext/fts5/fts5Int.h
View File

@@ -39,6 +39,18 @@ int sqlite3Fts5Corrupt(void);
# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
#endif
/*
** The assert_nc() macro is similar to the assert() macro, except that it
** is used for assert() conditions that are true only if it can be
** guranteed that the database is not corrupt.
*/
#ifdef SQLITE_TEST
extern int sqlite3_fts5_may_be_corrupt;
# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
#else
# define assert_nc(x) assert(x)
#endif
/**************************************************************************
** Interface to code in fts5.c.
*/

61
ext/fts5/fts5_index.c
View File

@@ -752,6 +752,46 @@ static void fts5CloseReader(Fts5Index *p){
}
}
/*
** Check if row iRowid exists in the %_data table, and that it contains
** a blob value. If so, return SQLITE_ERROR (yes - SQLITE_ERROR, not
** SQLITE_OK). If not, return SQLITE_CORRUPT_VTAB.
**
** If an error occurs (e.g. OOM or IOERR), return the relevant error code.
**
** This function does not need to be efficient. It is part of vary rarely
** invoked error handling code only.
*/
#if 0
static int fts5CheckMissingRowid(Fts5Index *p, i64 iRowid){
const char *zFmt = "SELECT typeof(block)=='blob' FROM '%q'.%Q WHERE id=%lld";
int bOk = 0;
int rc;
char *zSql;
zSql = sqlite3_mprintf(zFmt, p->pConfig->zDb, p->zDataTbl, iRowid);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
sqlite3_stmt *pStmt;
rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, &pStmt, 0);
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
bOk = sqlite3_column_int(pStmt, 0);
}
rc = sqlite3_finalize(pStmt);
}
sqlite3_free(zSql);
}
if( rc==SQLITE_OK ){
rc = bOk ? SQLITE_ERROR : FTS5_CORRUPT;
}
return rc;
}
#endif
static Fts5Data *fts5DataReadOrBuffer(
Fts5Index *p,
Fts5Buffer *pBuf,
@@ -761,13 +801,6 @@ static Fts5Data *fts5DataReadOrBuffer(
if( p->rc==SQLITE_OK ){
int rc = SQLITE_OK;
#if 0
Fts5Buffer buf = {0,0,0};
fts5DebugRowid(&rc, &buf, iRowid);
fprintf(stdout, "read: %s\n", buf.p);
fflush(stdout);
sqlite3_free(buf.p);
#endif
if( p->pReader ){
/* This call may return SQLITE_ABORT if there has been a savepoint
** rollback since it was last used. In this case a new blob handle
@@ -788,6 +821,13 @@ sqlite3_free(buf.p);
);
}
/* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls
** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead.
** All the reasons those functions might return SQLITE_ERROR - missing
** table, missing row, non-blob/text in block column - indicate
** backing store corruption. */
if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT;
if( rc==SQLITE_OK ){
u8 *aOut; /* Read blob data into this buffer */
int nByte = sqlite3_blob_bytes(p->pReader);
@@ -1563,7 +1603,7 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
if( p->rc==SQLITE_OK ){
const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
int iOff = pIter->iLeafOffset; /* Offset to read at */
pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos,&pIter->bDel);
pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
}
}
@@ -1577,8 +1617,6 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
**
** Fts5SegIter.term
** Fts5SegIter.rowid
** Fts5SegIter.nPos
** Fts5SegIter.bDel
**
** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of
** the first position list. The position list belonging to document
@@ -3912,7 +3950,6 @@ static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
pIter->iLeaf = pIter->aLvl[0].s.iChild;
assert( p->rc==SQLITE_OK || pIter->bEof );
}
static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
@@ -3985,7 +4022,7 @@ static void fts5IndexIntegrityCheckSegment(
/* Iterate through the b-tree hierarchy. */
for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
iter.bEof==0;
p->rc==SQLITE_OK && iter.bEof==0;
fts5BtreeIterNext(&iter)
){
i64 iRow; /* Rowid for this leaf */

38
ext/fts5/fts5_tcl.c
View File

@@ -22,6 +22,14 @@
#include <string.h>
#include <assert.h>
/*
** This variable is set to true when running corruption tests. Otherwise
** false. If it is false, extra assert() conditions in the fts5 code are
** activated - conditions that are only true if it is guaranteed that the
** fts5 database is not corrupt.
*/
int sqlite3_fts5_may_be_corrupt = 0;
/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in
** tclsqlite.c. We need it here so that the get_sqlite_pointer routine
@@ -830,6 +838,33 @@ static void xF5tFree(ClientData clientData){
ckfree(clientData);
}
/*
** sqlite3_fts5_may_be_corrupt BOOLEAN
**
** Set or clear the global "may-be-corrupt" flag. Return the old value.
*/
static int f5tMayBeCorrupt(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
int bOld = sqlite3_fts5_may_be_corrupt;
if( objc!=2 && objc!=1 ){
Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
return TCL_ERROR;
}
if( objc==2 ){
int bNew;
if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
sqlite3_fts5_may_be_corrupt = bNew;
}
Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
return TCL_OK;
}
/*
** Entry point.
*/
@@ -842,7 +877,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
{ "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
{ "sqlite3_fts5_token", f5tTokenizerReturn, 1 },
{ "sqlite3_fts5_tokenize", f5tTokenize, 0 },
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 }
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 },
{ "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }
};
int i;
F5tTokenizerContext *pContext;

4
ext/fts5/test/fts5corrupt.test
View File

@@ -9,6 +9,8 @@
#
#***********************************************************************
#
# This file tests that the FTS5 'integrity-check' command detects
# inconsistencies (corruption) in the on-disk backing tables.
#
source [file join [file dirname [info script]] fts5_common.tcl]
@@ -38,7 +40,7 @@ do_test 1.3 {
DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4);
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {SQL logic error or missing database}}
} {1 {database disk image is malformed}}
do_test 1.4 {
db_restore_and_reopen

116
ext/fts5/test/fts5corrupt2.test Normal file
View File

@@ -0,0 +1,116 @@
# 2015 Apr 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that FTS5 handles corrupt databases (i.e. internal
# inconsistencies in the backing tables) correctly. In this case
# "correctly" means without crashing.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt2
# Create a simple FTS5 table containing 100 documents. Each document
# contains 10 terms, each of which start with the character "x".
#
expr srand(0)
db func rnddoc fts5_rnddoc
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
set mask [expr 31 << 31]
# Test 1:
#
# For each page in the t1_data table, open a transaction and DELETE
# the t1_data entry. Then run:
#
# * an integrity-check, and
# * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'"
#
# and check that the corruption is detected in both cases. The
# rollback the transaction.
#
# Test 2:
#
# Same thing, except instead of deleting a row from t1_data, replace its
# blob content with integer value 14.
#
foreach {tno stmt} {
1 { DELETE FROM t1_data WHERE rowid=$rowid }
2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
} {
break
set tn 0
foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
incr tn
#if {$tn!=224} continue
do_test 1.$tno.$tn.1.$rowid {
execsql { BEGIN }
execsql $stmt
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
if {($rowid & $mask)==0} {
# Node is a leaf node, not a b-tree node.
do_catchsql_test 1.$tno.$tn.2.$rowid {
SELECT rowid FROM t1 WHERE t1 MATCH 'x*'
} {1 {database disk image is malformed}}
}
do_execsql_test 1.$tno.$tn.3.$rowid {
ROLLBACK;
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
}
}
# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
# of the fts index. For test $i, truncate the rightmost leafpage to $i
# bytes. Then test both the integrity-check detects the corruption.
#
# Also tested is that "MATCH 'x*'" does not crash and sometimes reports
# corruption. It may not report the db as corrupt because truncating the
# final leaf to some sizes may create a valid leaf page.
#
set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
set all [db eval {SELECT rowid FROM t1}]
for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
do_execsql_test 2.$i.1 {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
}
do_catchsql_test 2.$i.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_test 2.$i.3 {
set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}]
expr {
$res=="1 {database disk image is malformed}"
|| $res=="0 {$all}"
}
} 1
do_execsql_test 2.$i.4 {
ROLLBACK;
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
}
finish_test

2
ext/fts5/test/fts5rebuild.test
View File

@@ -39,7 +39,7 @@ do_execsql_test 1.5 {
do_catchsql_test 1.6 {
INSERT INTO f1(f1) VALUES('integrity-check');
} {1 {SQL logic error or missing database}}
} {1 {database disk image is malformed}}
do_execsql_test 1.7 {
INSERT INTO f1(f1) VALUES('rebuild');

21
manifest
View File

@@ -1,5 +1,5 @@
C Fix\san\sfts5\sbuild\sproblem\sin\smain.mk.
D 2015-04-24T06:02:29.587
C Add\sextra\stests\sfor\scorrupt\sdatabase\shandling\sin\sfts5.
D 2015-04-24T15:56:09.379
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in faaf75b89840659d74501bea269c7e33414761c1
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,15 +106,15 @@ F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
F ext/fts5/fts5.c 1eb8ca073be5222c43e4eee5408764c2cbb4200b
F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a
F ext/fts5/fts5Int.h 1b537736f8838df7fca10245c0f70a23cfddc7f5
F ext/fts5/fts5Int.h 1309320cb233e1c5b38d7f1e2cab2138bbf497d8
F ext/fts5/fts5_aux.c fcea18b1a2a3f95a498b52aba2983557d7678a22
F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
F ext/fts5/fts5_config.c 0847facc8914f57ea4452c43ce109200dc65e894
F ext/fts5/fts5_expr.c 05da381ab26031243266069302c6eb4094b2c5dd
F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
F ext/fts5/fts5_index.c 7c9615a83e0ca928817e81be65e266f639f45532
F ext/fts5/fts5_index.c 1663ad6a9ae221f14f27442b9b1a9d5088a2c5fe
F ext/fts5/fts5_storage.c ac0f0937059c8d4f38a1f13aa5f2c2cd7edf3e0d
F ext/fts5/fts5_tcl.c 617b6bb96545be8d9045de6967c688cd9cd15541
F ext/fts5/fts5_tcl.c 10bf0eb678d34c1bfdcfaf653d2e6dd92afa8b38
F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
@@ -135,7 +135,8 @@ F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede
F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
F ext/fts5/test/fts5corrupt.test dbdcfe75749ed2f2eb3915cf68fd55d3dc3b058d
F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
F ext/fts5/test/fts5corrupt2.test 3be48d8a30d30e3ae819f04e957c45d091bfbb85
F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
@@ -145,7 +146,7 @@ F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
F ext/fts5/test/fts5prefix.test 4610dfba4460d92f23a8014874a46493f1be77b5
F ext/fts5/test/fts5rebuild.test 2a5e98205393487b4a732c8290999af7c0b907b4
F ext/fts5/test/fts5rebuild.test ee6792715c6c528cc188e7869d67c3c655889ddb
F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6
F ext/fts5/test/fts5tokenizer.test b34ae592db66f6e89546d791ce1f905ba0b3395c
F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d
@@ -1301,7 +1302,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 2dd59b5762c944b3bdd270e95c6739cd4f530bfa
R 16931392bbfaadd753438e7b377b1156
P 60045cedef109f03317dc878fe6bb3d03867ae69
R 9c5c238cdd6f30dc8d0223c36173d961
U dan
Z b1c66eea22fb17fe989c58ccfd8fe427
Z ec7cf237df9e7bd8116f5a496704530c

2
manifest.uuid
View File

@@ -1 +1 @@
60045cedef109f03317dc878fe6bb3d03867ae69
41449f7a0b5da6332eef48386c91ef63382c4783