1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-12-24 14:17:58 +03:00

Further tests for the new code on this branch.

FossilOrigin-Name: 59d008b6c23ab900377bc696ee19381feb7614bac80546eae361e401c3620c4e
This commit is contained in:
dan
2023-12-05 18:36:23 +00:00
parent 49bfbc1ef3
commit fb923fc4cc
8 changed files with 257 additions and 82 deletions

View File

@@ -1768,7 +1768,9 @@ static int fts5ParseTokenize(
memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
if( pCtx->pConfig->bTokendata ) pTerm->nQueryTerm = strlen(pTerm->pTerm);
if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){
pTerm->nQueryTerm = strlen(pTerm->pTerm);
}
}
}
@@ -3027,7 +3029,7 @@ static int fts5ExprPopulatePoslistsCb(
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){
if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
int iCol = p->iOff>>32;
int iTokOff = p->iOff & 0x7FFFFFFF;
rc = sqlite3Fts5IndexIterWriteTokendata(

View File

@@ -6661,57 +6661,6 @@ static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
}
}
/*
** The iterator passed as the first argument must be a tokendata=1 iterator
** (pIter->pTokenDataIter!=0). This function is used to access the token
** instance located at offset iOff of column iCol of row iRowid. It is
** returned via output pointers *ppOut and *pnOut. This is used by the
** xInstToken() API.
*/
static int fts5TokendataIterToken(
Fts5Iter *pIter,
i64 iRowid,
int iCol, int iOff,
const char **ppOut, int *pnOut
){
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
Fts5TokenDataMap *aMap = pT->aMap;
i64 iPos = (((i64)iCol)<<32) + iOff;
int i1 = 0;
int i2 = pT->nMap;
int iTest = 0;
while( i2>i1 ){
iTest = (i1 + i2) / 2;
if( aMap[iTest].iRowid<iRowid ){
i1 = iTest+1;
}else if( aMap[iTest].iRowid>iRowid ){
i2 = iTest;
}else{
if( aMap[iTest].iPos<iPos ){
if( aMap[iTest].iPos<0 ){
break;
}
i1 = iTest+1;
}else if( aMap[iTest].iPos>iPos ){
i2 = iTest;
}else{
break;
}
}
}
if( i2>i1 ){
Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
*ppOut = (const char*)pMap->aSeg[0].term.p+1;
*pnOut = pMap->aSeg[0].term.n-1;
}
return SQLITE_OK;
}
/*
** Append a mapping to the token-map belonging to object pT.
*/
@@ -6960,9 +6909,7 @@ static Fts5Iter *fts5SetupTokendataIter(
memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter));
memset(pPrevIter, 0, sizeof(Fts5SegIter));
bDone = 1;
}else if( pPrevIter->pLeaf
&& pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf
){
}else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){
fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter);
bDone = 1;
}
@@ -7071,7 +7018,7 @@ int sqlite3Fts5IndexQuery(
if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
int iIdx = 0; /* Index to search */
int iPrefixIdx = 0; /* +1 prefix index */
int bTokendata = (flags&FTS5INDEX_QUERY_NOTOKENDATA)?0:pConfig->bTokendata;
int bTokendata = pConfig->bTokendata;
if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
/* Figure out which index to search and set iIdx accordingly. If this
@@ -7085,6 +7032,7 @@ int sqlite3Fts5IndexQuery(
** for internal sanity checking by the integrity-check in debug
** mode only. */
#ifdef SQLITE_DEBUG
if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0;
if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
assert( flags & FTS5INDEX_QUERY_PREFIX );
iIdx = 1+pConfig->nPrefix;
@@ -7208,7 +7156,8 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
/*
** This is used by xInstToken() to access the token at offset iOff, column
** iCol of row iRowid. The token is returned via output variables *ppOut
** and *pnOut.
** and *pnOut. The iterator passed as the first argument must be a tokendata=1
** iterator (pIter->pTokenDataIter!=0).
*/
int sqlite3Fts5IterToken(
Fts5IndexIter *pIndexIter,
@@ -7218,9 +7167,39 @@ int sqlite3Fts5IterToken(
const char **ppOut, int *pnOut
){
Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
Fts5TokenDataMap *aMap = pT->aMap;
i64 iPos = (((i64)iCol)<<32) + iOff;
if( pIter->pTokenDataIter ){
return fts5TokendataIterToken(pIter, iRowid, iCol, iOff, ppOut, pnOut);
int i1 = 0;
int i2 = pT->nMap;
int iTest = 0;
while( i2>i1 ){
iTest = (i1 + i2) / 2;
if( aMap[iTest].iRowid<iRowid ){
i1 = iTest+1;
}else if( aMap[iTest].iRowid>iRowid ){
i2 = iTest;
}else{
if( aMap[iTest].iPos<iPos ){
if( aMap[iTest].iPos<0 ){
break;
}
i1 = iTest+1;
}else if( aMap[iTest].iPos>iPos ){
i2 = iTest;
}else{
break;
}
}
}
if( i2>i1 ){
Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
*ppOut = (const char*)pMap->aSeg[0].term.p+1;
*pnOut = pMap->aSeg[0].term.n-1;
}
return SQLITE_OK;
@@ -7252,17 +7231,17 @@ int sqlite3Fts5IndexIterWriteTokendata(
Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
Fts5Index *p = pIter->pIndex;
int ii;
assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
if( pT ){
int ii;
for(ii=0; ii<pT->nIter; ii++){
Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
}
if( ii<pT->nIter ){
fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
}
assert( pIter->pTokenDataIter );
for(ii=0; ii<pT->nIter; ii++){
Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
}
if( ii<pT->nIter ){
fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
}
return fts5IndexReturn(p);
}

View File

@@ -0,0 +1,93 @@
# 2010 June 15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultG
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set ::testprefix fts5faultH
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
BEGIN;
INSERT INTO t1 VALUES('oNe tWo thRee');
INSERT INTO t1 VALUES('One Two Three');
INSERT INTO t1 VALUES('onE twO threE');
COMMIT;
BEGIN;
INSERT INTO t1 VALUES('one two three');
INSERT INTO t1 VALUES('one two three');
INSERT INTO t1 VALUES('one two three');
COMMIT;
}
do_faultsim_test 1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('three');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {1 2 3 4 5 6}}
}
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
BEGIN;
INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
COMMIT;
}
do_faultsim_test 2 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('BBB AND AAA');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {10 24}}
}
finish_test

View File

@@ -103,5 +103,44 @@ do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
#------------------------------------------------------------------------
reset_db
sqlite3_fts5_register_origintext db
proc tokens {cmd} {
set ret [list]
for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
set txt [$cmd xInstToken $iTok 0]
set txt [string map [list "\0" "."] $txt]
lappend ret $txt
}
set ret
}
sqlite3_fts5_create_function db tokens tokens
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE x1 USING fts5(
v, tokenize="origintext unicode61", tokendata=1, detail=none
);
INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
}
do_execsql_test 2.1 {
SELECT tokens(x1) FROM x1('xxx');
} {
{xxx xxx.Xxx xxx.XXX} {xxx xxx}
}
do_execsql_test 2.2 {
UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
}
do_execsql_test 2.3 {
SELECT tokens(x1) FROM x1('xxx');
} {
{xxx {} xxx} {xxx xxx}
}
finish_test

View File

@@ -74,6 +74,27 @@ foreach_detail_mode $testprefix {
do_execsql_test 1.6 {
SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
} {three.THREE 3 three 1 three 2}
do_execsql_test 1.7 {
INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
}
do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
}
finish_test

View File

@@ -121,7 +121,7 @@ proc ctrl_tokens {term args} {
db func ctrl_tokens ctrl_tokens
proc do_all_vocab_test {tn} {
foreach ::v [vocab] {
foreach ::v [concat [vocab] nnn] {
set answer [execsql {
SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
}]
@@ -134,7 +134,6 @@ proc do_all_vocab_test {tn} {
}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", content=, contentless_delete=1,
@@ -229,5 +228,46 @@ for {set ii 0} {$ii < $NLOOP} {incr ii} {
do_all_vocab_test 2.3.$ii
}
#-------------------------------------------------------------------------
unset -nocomplain ::expanded_vocab
proc vocab {} {
list abcde fghij klmno
}
proc do_all_vocab_test3 {tn} {
foreach ::v [concat [vocab] nnn] {
set answer [execsql {
SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
}]
do_execsql_test $tn.$::v.1 {
SELECT rowid, tokens(ft3) FROM ft3($::v)
} $answer
do_execsql_test $tn.$::v.2 {
SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
} $answer
}
}
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE ft3 USING fts5(
w, tokenize="origintext unicode61", content=, contentless_delete=1,
tokendata=1
);
INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
CREATE TABLE ctrl3(w);
}
do_execsql_test 3.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
)
INSERT INTO ctrl3 SELECT document() FROM s;
INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
}
do_all_vocab_test3 3.2
finish_test

View File

@@ -1,5 +1,5 @@
C Add\sfurther\stests\sfor\sxInstToken().
D 2023-12-04T19:48:08.530
C Further\stests\sfor\sthe\snew\scode\son\sthis\sbranch.
D 2023-12-05T18:36:23.618
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,9 +94,9 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880
F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
F ext/fts5/fts5_expr.c 920516be4aac44eccdd9e747fe26f367442848b9a58971e4b36edb0c8284b6b8
F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba
F ext/fts5/fts5_index.c be39b44ff8773cff56bcbc01f74701a83e068c20d773cafd01e8bb2fa0fc1bc5
F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a
F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
@@ -170,6 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860
F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
F ext/fts5/test/fts5faultH.test d845f45dac3e1a3f20c7e0a2be95280c95d3204c06802f86ab2c110e52ed3d14
F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -191,10 +192,10 @@ F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696
F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
F ext/fts5/test/fts5origintext5.test 067bfb3008323585df640ab29e8ef7c4ca6dec62c597be07a9f896d88f98cd10
F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2150,8 +2151,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1
R ced1cc2c59284a9ef3026043e080f745
P 8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4
R c18539c880ee946e73119264163283e7
U dan
Z e2cce7acff968ef3c9472723192ae452
Z 86bd4754d1465fbe5cee9e784de6e900
# Remove this line to create a well-formed Fossil manifest.

View File

@@ -1 +1 @@
8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4
59d008b6c23ab900377bc696ee19381feb7614bac80546eae361e401c3620c4e