database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00
Code Activity

Add tests to improve coverage of fts3. Associated bugfixes.

Browse Source 
FossilOrigin-Name: f0eac4175aee6c50ee68acc253f76fbe44574250
This commit is contained in:
dan
2009-12-08 19:05:53 +00:00
parent 4800b2ee95
commit 84db21ec6a
9 changed files with 210 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
ext/fts3/fts3.c
View File

@@ -1244,9 +1244,10 @@ static int fts3PoslistPhraseMerge(
if( (*p1&0xFE)==0 ) break;
fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
}
}
if( pSave && pp ){
if( pSave ){
assert( pp && p );
p = pSave;
}
@@ -1387,10 +1388,6 @@ static int fts3DoclistMerge(
if( !aBuffer ){
return SQLITE_NOMEM;
}
if( n1==0 && n2==0 ){
*pnBuffer = 0;
return SQLITE_OK;
}
/* Read the first docid from each doclist */
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
@@ -1475,7 +1472,7 @@ static int fts3DoclistMerge(
char **ppPos = 0;
if( mergetype==MERGE_POS_NEAR ){
ppPos = &p;
aTmp = sqlite3_malloc(2*(n1+n2));
aTmp = sqlite3_malloc(2*(n1+n2+1));
if( !aTmp ){
return SQLITE_NOMEM;
}
@@ -1908,6 +1905,7 @@ static int fts3FilterMethod(
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr->aDoclist);
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
/* Compile a SELECT statement for this cursor. For a full-table-scan, the
@@ -2307,13 +2305,14 @@ int sqlite3Fts3Init(sqlite3 *db){
Fts3Hash *pHash = 0;
const sqlite3_tokenizer_module *pSimple = 0;
const sqlite3_tokenizer_module *pPorter = 0;
#ifdef SQLITE_ENABLE_ICU
const sqlite3_tokenizer_module *pIcu = 0;
sqlite3Fts3IcuTokenizerModule(&pIcu);
#endif
sqlite3Fts3SimpleTokenizerModule(&pSimple);
sqlite3Fts3PorterTokenizerModule(&pPorter);
#ifdef SQLITE_ENABLE_ICU
sqlite3Fts3IcuTokenizerModule(&pIcu);
#endif
/* Allocate and initialise the hash-table used to store tokenizers. */
pHash = sqlite3_malloc(sizeof(Fts3Hash));
@@ -2327,14 +2326,18 @@ int sqlite3Fts3Init(sqlite3 *db){
if( rc==SQLITE_OK ){
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
#ifdef SQLITE_ENABLE_ICU
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
#endif
){
rc = SQLITE_NOMEM;
}
}
#ifdef SQLITE_TEST
sqlite3Fts3ExprInitTestInterface(db);
if( rc==SQLITE_OK ){
rc = sqlite3Fts3ExprInitTestInterface(db);
}
#endif
/* Create the virtual table wrapper around the hash-table and overload

2
ext/fts3/fts3Int.h
View File

@@ -273,7 +273,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif
#endif /* _FTSINT_H */

4
ext/fts3/fts3_expr.c
View File

@@ -913,8 +913,8 @@ exprtest_out:
** Register the query expression parser test function fts3_exprtest()
** with database connection db.
*/
void sqlite3Fts3ExprInitTestInterface(sqlite3* db){
sqlite3_create_function(
int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
return sqlite3_create_function(
db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
);
}

8
ext/fts3/fts3_tokenizer.c
View File

@@ -119,14 +119,14 @@ const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
/* Find the start of the next token. */
z1 = zStr;
while( z2==0 ){
switch( *z1 ){
char c = *z1;
switch( c ){
case '\0': return 0; /* No more tokens here */
case '\'':
case '"':
case '`': {
z2 = &z1[1];
while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++;
if( *z2 ) z2++;
z2 = z1;
while( *++z2 && (*z2!=c || *++z2==c) );
break;
}
case '[':

3
ext/fts3/fts3_write.c
View File

@@ -2162,7 +2162,8 @@ int sqlite3Fts3Optimize(Fts3Table *p){
if( rc==SQLITE_OK ){
rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}else{
sqlite3_exec(p->db, "ROLLBACK TO fts3 ; RELEASE fts3", 0, 0, 0);
sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}
}
return rc;

36
manifest
View File

@@ -1,8 +1,5 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
C Avoid\spointer\saliasing\sin\sthe\sallocSpace()\sroutine\sin\svdbeaux.c.
D 2009-12-08T15:35:23
C Add\stests\sto\simprove\scoverage\sof\sfts3.\sAssociated\sbugfixes.
D 2009-12-08T19:05:54
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -59,19 +56,19 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 684a55d603f11c8432323171082ff8a9437b4681
F ext/fts3/fts3.c ece6d5b17b4be847ff139a8103c71443c699371f
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h cc716c74afa7da8e0f8ef39404f33ea62a823eb3
F ext/fts3/fts3_expr.c c18794a62c257d3456d3314c5a18e348ae0d84bd
F ext/fts3/fts3Int.h 36f8e6d6fafa6d71a48e1810095e1e58fd55b199
F ext/fts3/fts3_expr.c 09a0e1bec4c780fb94661e146b93efbea04f7416
F ext/fts3/fts3_hash.c 18feef38fca216992725e9eae775a0c7735e6724
F ext/fts3/fts3_hash.h d410ff2c93c81a56b927fcf07b2099ccbfa7a479
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c a651e287e02b49b565a6ccf9441959d434489156
F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
F ext/fts3/fts3_tokenizer.c 52112e7b50fbf24eb68e356081c5969917406cc6
F ext/fts3/fts3_tokenizer.c 3dc76eaea6b58ecfbe50135b8473aa668d712dcd
F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
F ext/fts3/fts3_write.c ec51fb6886f910e78ae32158ec0301aa675f52d8
F ext/fts3/fts3_write.c 643be9c7b097f5ea8189cf6ed840edd59485c45f
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@@ -220,7 +217,7 @@ F src/vdbeaux.c 86d43cdf9615235e1def993a945dfaabc20ab079
F src/vdbeblob.c 84f924700a7a889152aeebef77ca5f4e3875ffb4
F src/vdbemem.c 1e16e3a16e55f4c3452834f0e041726021aa66e0
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c 456fc226614569f0e46f216e33265bea268bd917
F src/vtab.c 4e73b2692080821aed36d62b20576e409f910883
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c 11b5b00c49d53e767a7eb855bc60790edeca6185
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -329,7 +326,7 @@ F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f
F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb
F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17
F test/e_fts3.test 8907e25b2c7d6bda9f7077356f64bc5e26c251a7
F test/e_fts3.test 3ad57e7b920c7dcfa373dc2ed2105f83e1cf69a6
F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea
F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398
F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041
@@ -780,14 +777,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P 973c5c86eed31dcce54d14e71938f2e255f5f1c7
R c570a97b0ffcfd39f4ed66f9f0662c03
U drh
Z ee7e56c2d879987294a2e31ad1fde920
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.6 (GNU/Linux)
iD8DBQFLHnI+oxKgR168RlERAn/LAJ9meRPYFG8kZK/fIyNChkghTUHQWgCdGqla
K42FnTiSLRLIME8ykgShkdk=
=3fbN
-----END PGP SIGNATURE-----
P d6ae27512229d95502c584b17bb2cbdba401f80a
R 9a53ddf5917c05e76f26075e2be743be
U dan
Z d714a86f89f40b4ba38c62abeeffc91f

2
manifest.uuid
View File

@@ -1 +1 @@
d6ae27512229d95502c584b17bb2cbdba401f80a
f0eac4175aee6c50ee68acc253f76fbe44574250

10
src/vtab.c
View File

@@ -669,11 +669,11 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
pParse->declareVtab = 1;
pParse->db = db;
if(
SQLITE_OK == sqlite3RunParser(pParse, zCreateTable, &zErr) &&
pParse->pNewTable &&
!pParse->pNewTable->pSelect &&
(pParse->pNewTable->tabFlags & TF_Virtual)==0
if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr)
&& !db->mallocFailed
&& pParse->pNewTable
&& !pParse->pNewTable->pSelect
&& (pParse->pNewTable->tabFlags & TF_Virtual)==0
){
if( !pTab->aCol ){
pTab->aCol = pParse->pNewTable->aCol;

169
test/e_fts3.test
View File

@@ -20,6 +20,7 @@ source $testdir/tester.tcl
#
ifcapable !fts3 { finish_test ; return }
source $testdir/fts3_common.tcl
source $testdir/malloc_common.tcl
# Procs used to make the tests in this file easier to read.
#
@@ -36,7 +37,6 @@ proc error_test {tn sql result} {
uplevel [list do_error_test e_fts3-$tn $sql $result]
}
#-------------------------------------------------------------------------
# The body of the following [foreach] block contains test cases to verify
# that the example code in fts3.html works as expected. The tests run three
@@ -46,7 +46,14 @@ proc error_test {tn sql result} {
# DO_MALLOC_TEST=1: Run tests with transient OOM errors.
# DO_MALLOC_TEST=2: Run tests with persistent OOM errors.
#
foreach DO_MALLOC_TEST [lrange {0 1 2} 0 end] {
foreach {DO_MALLOC_TEST enc} {
0 utf-8
1 utf-8
2 utf-8
1 utf-16
} {
#if {$DO_MALLOC_TEST} break
# Reset the database and database connection. If this iteration of the
# [foreach] loop is testing with OOM errors, disable the lookaside buffer.
@@ -55,6 +62,7 @@ db close
file delete -force test.db test.db-journal
sqlite3 db test.db
if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 }
db eval "PRAGMA encoding = '$enc'"
##########################################################################
# Test the example CREATE VIRTUAL TABLE statements in section 1.1
@@ -465,5 +473,162 @@ error_test 2.1.7 {
error_test 2.1.8 {
SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
#
do_malloc_test e_fts3-3 -tclbody {
if {[catch {sqlite3 db test.db}]} { error "out of memory" }
}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Verify the return values of the optimize() function. If no error occurs,
# the returned value should be "Index optimized" if the data structure
# was modified, or "Index already optimal" if it were not.
#
set DO_MALLOC_TEST 0
ddl_test 4.1 { CREATE VIRTUAL TABLE t4 USING fts3(a, b) }
write_test 4.2 t4_content {
INSERT INTO t4 VALUES('In Xanadu', 'did Kubla Khan');
}
write_test 4.3 t4_content {
INSERT INTO t4 VALUES('a stately pleasure', 'dome decree');
}
do_test e_fts3-4.4 {
execsql { SELECT optimize(t4) FROM t4 LIMIT 1 }
} {{Index optimized}}
do_test e_fts3-4.5 {
execsql { SELECT optimize(t4) FROM t4 LIMIT 1 }
} {{Index already optimal}}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test that the snippet function appears to work correctly with 1, 2, 3
# or 4 arguments passed to it.
#
set DO_MALLOC_TEST 0
ddl_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea. So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 {
SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <b>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.4 {
SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <i>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.5 {
SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.6 {
SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXX Down to a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towers were XXX}}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0
ddl_test 6.1 { CREATE VIRTUAL TABLE t6 USING fts3(x) }
write_test 6.2 t5_content { INSERT INTO t6 VALUES('a'); }
write_test 6.3 t5_content { INSERT INTO t6 VALUES('b'); }
write_test 6.4 t5_content { INSERT INTO t6 VALUES('c'); }
read_test 6.5 { SELECT * FROM t6 WHERE t6 MATCH '' } {}
read_test 6.6 { SELECT * FROM t6 WHERE x MATCH '' } {}
read_test 6.7 { SELECT * FROM t6 WHERE t6 MATCH NULL } {}
read_test 6.8 { SELECT * FROM t6 WHERE x MATCH NULL } {}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test a few facets of the FTS3 xFilter() callback implementation:
#
# 1. That the sqlite3_index_constraint.usable flag is respected.
#
# 2. That it is an error to use the "docid" or "rowid" column of
# an FTS3 table as the LHS of a MATCH operator.
#
# 3. That it is an error to AND together two MATCH expressions in
# that refer to a single FTS3 table in a WHERE clause.
#
#
ddl_test 7.1.1 { CREATE VIRTUAL TABLE t7 USING fts3(a) }
ddl_test 7.1.2 { CREATE VIRTUAL TABLE t8 USING fts3(b) }
write_test 7.1.3 t7_content { INSERT INTO t7(docid, a) VALUES(4,'number four') }
write_test 7.1.4 t7_content { INSERT INTO t7(docid, a) VALUES(5,'number five') }
write_test 7.1.5 t8_content { INSERT INTO t8(docid, b) VALUES(4,'letter D') }
write_test 7.1.6 t8_content { INSERT INTO t8(docid, b) VALUES(5,'letter E') }
read_test 7.1.7 {
SELECT a || ':' || b FROM t7 JOIN t8 USING(docid)
} {{number four:letter D} {number five:letter E}}
error_test 7.2.1 {
SELECT * FROM t7 WHERE docid MATCH 'number'
} {unable to use function MATCH in the requested context}
error_test 7.2.2 {
SELECT * FROM t7 WHERE rowid MATCH 'number'
} {unable to use function MATCH in the requested context}
error_test 7.3.1 {
SELECT * FROM t7 WHERE a MATCH 'number' AND a MATCH 'four'
} {unable to use function MATCH in the requested context}
error_test 7.3.2 {
SELECT * FROM t7, t8 WHERE a MATCH 'number' AND a MATCH 'four'
} {unable to use function MATCH in the requested context}
error_test 7.3.3 {
SELECT * FROM t7, t8 WHERE b MATCH 'letter' AND b MATCH 'd'
} {unable to use function MATCH in the requested context}
read_test 7.3.4 {
SELECT * FROM t7, t8 WHERE a MATCH 'number' AND b MATCH 'letter'
} {{number four} {letter D} {number four} {letter E} {number five} {letter D} {number five} {letter E}}
#-------------------------------------------------------------------------
# Test the quoting of FTS3 table column names. Names may be quoted using
# any of "", '', ``` or [].
#
ddl_test 8.1.1 { CREATE VIRTUAL TABLE t9a USING fts3("c1", [c2]) }
ddl_test 8.1.2 { CREATE VIRTUAL TABLE t9b USING fts3('c1', `c2`) }
read_test 8.1.3 { PRAGMA table_info(t9a) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0}
read_test 8.1.4 { PRAGMA table_info(t9b) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0}
ddl_test 8.2.1 { CREATE VIRTUAL TABLE t9c USING fts3("c""1", 'c''2') }
read_test 8.2.2 { PRAGMA table_info(t9c) } {0 c\"1 {} 0 {} 0 1 c'2 {} 0 {} 0}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test that FTS3 tables can be renamed using the ALTER RENAME command.
# OOM errors are tested during ALTER RENAME commands also.
#
foreach DO_MALLOC_TEST {0 1 2} {
db close
file delete -force test.db test.db-journal
sqlite3 db test.db
if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 }
ddl_test 9.1.1 { CREATE VIRTUAL TABLE t10 USING fts3(x) }
write_test 9.1.2 t10_content { INSERT INTO t10 VALUES('fts3 tables') }
write_test 9.1.3 t10_content { INSERT INTO t10 VALUES('are renameable') }
read_test 9.1.4 {
SELECT * FROM t10 WHERE t10 MATCH 'table*'
} {{fts3 tables}}
read_test 9.1.5 {
SELECT * FROM t10 WHERE x MATCH 'rename*'
} {{are renameable}}
ddl_test 9.1.6 { ALTER TABLE t10 RENAME TO t11 }
read_test 9.1.7 {
SELECT * FROM t11 WHERE t11 MATCH 'table*'
} {{fts3 tables}}
read_test 9.1.8 {
SELECT * FROM t11 WHERE x MATCH 'rename*'
} {{are renameable}}
}
finish_test