database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Fix another problem with fts3/4 auxiliary functions and NEAR expressions that consist entirely of deferred tokens.

Browse Source 
FossilOrigin-Name: a8c91c132f6157b7e3649f57a799984b1d7f8a18fd434515c875617d4195db29
This commit is contained in:
dan
2023-01-25 13:42:55 +00:00
parent f36fa4e902
commit 0fa2bbf9e1
6 changed files with 85 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
ext/fts3/fts3.c
View File

@ -5765,6 +5765,21 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
}
}
/*
** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array
** has not yet been allocated, allocate and zero it. Otherwise, just zero
** it.
*/
static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){
Fts3Table *pTab = (Fts3Table*)pCtx;
if( pExpr->aMI==0 ){
pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
if( pExpr->aMI==0 ) return SQLITE_NOMEM;
}
memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
return SQLITE_OK;
}
/*
** Expression pExpr must be of type FTSQUERY_PHRASE.
**
@ -5786,7 +5801,6 @@ static int fts3EvalGatherStats(
if( pExpr->aMI==0 ){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
Fts3Expr *pRoot; /* Root of NEAR expression */
Fts3Expr *p; /* Iterator used for several purposes */
sqlite3_int64 iPrevId = pCsr->iPrevId;
sqlite3_int64 iDocid;
@ -5794,7 +5808,9 @@ static int fts3EvalGatherStats(
/* Find the root of the NEAR expression */
pRoot = pExpr;
while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
while( pRoot->pParent
&& (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred)
){
pRoot = pRoot->pParent;
}
iDocid = pRoot->iDocid;
@ -5802,14 +5818,8 @@ static int fts3EvalGatherStats(
assert( pRoot->bStart );
/* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
for(p=pRoot; p; p=p->pLeft){
Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
assert( pE->aMI==0 );
pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
if( !pE->aMI ) return SQLITE_NOMEM;
memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
}
rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab);
if( rc!=SQLITE_OK ) return rc;
fts3EvalRestart(pCsr, pRoot, &rc);
while( pCsr->isEof==0 && rc==SQLITE_OK ){
@ -5965,6 +5975,7 @@ int sqlite3Fts3EvalPhrasePoslist(
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */
int bMatch;
/* Check if this phrase descends from an OR expression node. If not,
@ -5979,25 +5990,30 @@ int sqlite3Fts3EvalPhrasePoslist(
if( p->bEof ) bTreeEof = 1;
}
if( bOr==0 ) return SQLITE_OK;
pRun = pNear;
while( pRun->bDeferred ){
assert( pRun->pParent );
pRun = pRun->pParent;
}
/* This is the descendent of an OR node. In this case we cannot use
** an incremental phrase. Load the entire doclist for the phrase
** into memory in this case. */
if( pPhrase->bIncr ){
int bEofSave = pNear->bEof;
fts3EvalRestart(pCsr, pNear, &rc);
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
if( bEofSave==0 && pNear->iDocid==iDocid ) break;
int bEofSave = pRun->bEof;
fts3EvalRestart(pCsr, pRun, &rc);
while( rc==SQLITE_OK && !pRun->bEof ){
fts3EvalNextRow(pCsr, pRun, &rc);
if( bEofSave==0 && pRun->iDocid==iDocid ) break;
}
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){
if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){
rc = FTS_CORRUPT_VTAB;
}
}
if( bTreeEof ){
while( rc==SQLITE_OK && !pNear->bEof && !pNear->bDeferred ){
fts3EvalNextRow(pCsr, pNear, &rc);
while( rc==SQLITE_OK && !pRun->bEof ){
fts3EvalNextRow(pCsr, pRun, &rc);
}
}
if( rc!=SQLITE_OK ) return rc;

2
ext/fts3/fts3Int.h
View File

@ -650,5 +650,7 @@ int sqlite3FtsUnicodeIsalnum(int);
int sqlite3FtsUnicodeIsdiacritic(int);
#endif
int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */

48
ext/fts3/fts3_snippet.c
View File

@ -41,7 +41,7 @@ typedef sqlite3_int64 i64;
/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
@ -85,7 +85,7 @@ struct SnippetFragment {
};
/*
** This type is used as an fts3ExprIterate() context object while
** This type is used as an sqlite3Fts3ExprIterate() context object while
** accumulating the data returned by the matchinfo() function.
*/
typedef struct MatchInfo MatchInfo;
@ -244,7 +244,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){
}
/*
** Helper function for fts3ExprIterate() (see below).
** Helper function for sqlite3Fts3ExprIterate() (see below).
*/
static int fts3ExprIterate2(
Fts3Expr *pExpr, /* Expression to iterate phrases of */
@ -278,7 +278,7 @@ static int fts3ExprIterate2(
** Otherwise, SQLITE_OK is returned after a callback has been made for
** all eligible phrase nodes.
*/
static int fts3ExprIterate(
int sqlite3Fts3ExprIterate(
Fts3Expr *pExpr, /* Expression to iterate phrases of */
int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
@ -287,10 +287,9 @@ static int fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** This is an sqlite3Fts3ExprIterate() callback used while loading the
** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** fts3ExprLoadDoclists().
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
@ -322,9 +321,9 @@ static int fts3ExprLoadDoclists(
int *pnToken /* OUT: Number of tokens in query */
){
int rc; /* Return Code */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */
sCtx.pCsr = pCsr;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx);
if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
if( pnToken ) *pnToken = sCtx.nToken;
return rc;
@ -337,7 +336,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
int nPhrase = 0;
(void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
(void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
return nPhrase;
}
@ -465,8 +464,9 @@ static void fts3SnippetDetails(
}
/*
** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
** Each invocation populates an element of the SnippetIter.aPhrase[] array.
** This function is an sqlite3Fts3ExprIterate() callback used by
** fts3BestSnippet(). Each invocation populates an element of the
** SnippetIter.aPhrase[] array.
*/
static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
SnippetIter *p = (SnippetIter *)ctx;
@ -556,7 +556,9 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
rc = sqlite3Fts3ExprIterate(
pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter
);
if( rc==SQLITE_OK ){
/* Set the *pmSeen output variable. */
@ -917,10 +919,10 @@ static int fts3ExprLHitGather(
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo
** stats for a single query.
**
** fts3ExprIterate() callback to load the 'global' elements of a
** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
** of the matchinfo array that are constant for all rows returned by the
** current query.
@ -955,7 +957,7 @@ static int fts3ExprGlobalHitsCb(
}
/*
** fts3ExprIterate() callback used to collect the "local" part of the
** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the
** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
** array that are different for each row returned by the query.
*/
@ -1151,7 +1153,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
**/
aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase);
if( !aIter ) return SQLITE_NOMEM;
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
(void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
@ -1328,11 +1330,11 @@ static int fts3MatchinfoValues(
rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
if( rc!=SQLITE_OK ) break;
}
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
sqlite3Fts3EvalTestDeferred(pCsr, &rc);
if( rc!=SQLITE_OK ) break;
}
(void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
(void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
break;
}
}
@ -1555,7 +1557,7 @@ struct TermOffsetCtx {
};
/*
** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets().
*/
static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
TermOffsetCtx *p = (TermOffsetCtx *)ctx;
@ -1637,7 +1639,9 @@ void sqlite3Fts3Offsets(
*/
sCtx.iCol = iCol;
sCtx.iTerm = 0;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
rc = sqlite3Fts3ExprIterate(
pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx
);
if( rc!=SQLITE_OK ) goto offsets_out;
/* Retreive the text stored in column iCol. If an SQL NULL is stored

19
manifest
View File

@ -1,5 +1,5 @@
C Add\sscalar\sSQL\sfunction\sunhex().
D 2023-01-24T20:17:43.328
C Fix\sanother\sproblem\swith\sfts3/4\sauxiliary\sfunctions\sand\sNEAR\sexpressions\sthat\sconsist\sentirely\sof\sdeferred\stokens.
D 2023-01-25T13:42:55.473
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -58,16 +58,16 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 36e4555d96d6d322d46d8bbf5a2fe30422be2071d160ebeb3e2156b0ce085b91
F ext/fts3/fts3.c a613b81b208a330ce155b5f41ee038358aa4ddbae8b12fe9d07d7b399bf57dc8
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h ae2a44b04cddb5fb35cac4ea5f7f819b2894fd258186465777a19f7acfdf84ed
F ext/fts3/fts3Int.h e573c6d881f7238d77cc3fd2396cbb9b2fe13efef7d2ad295a155151c4e7efbd
F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f
F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a
F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8
F ext/fts3/fts3_snippet.c f9a8149173553113f3c495a503843e30028b5dc3723d0ca798c5ad6142e130e6
F ext/fts3/fts3_snippet.c 4d6523e3eddeb7b46e7a82b3476a0a86a0c04821e0e2b8dd40f45ee28057cb13
F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1
F ext/fts3/fts3_test.c d8d7b2734f894e8a489987447658e374cdd3a3bc8575c401decf1911cb7c6454
F ext/fts3/fts3_tokenize_vtab.c a95feda3590f3c3e17672fe35b67ea6112471aeea4c07ef7744a6606b66549aa
@ -1067,7 +1067,7 @@ F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05ee
F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf
F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f
F test/fts3defer.test f4c20e4c7153d20a98ee49ee5f3faef624fefc9a067f8d8d629db380c4d9f1de
F test/fts3defer2.test 0c0448ec227f470250173dd41b94fa4776d7dcaee07a7b8c0f7e6c1aba97f024
F test/fts3defer2.test 356c980bc28cdf4b14dbd3d7599929d3e54ecf9a37760c037bb895f614918459
F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
F test/fts3dropmod.test 7de242ea1c8a713a8b143ea54468f4b1c4953fa068349e23ac178e2c90c59889
@ -2044,9 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 39bfae4c4698a13e07c4a0725f2790955e03b601fe64e17a000c691def1bdcb8 66c8562690b19f17972589611810e1dccad3a48777acb05208289c1f77076f71
R e4f755d732b8075e7847f3615f7f161a
T +closed 66c8562690b19f17972589611810e1dccad3a48777acb05208289c1f77076f71
P 890e9629a7480138c9c1d3acc2d1e7b3c05e0d156e5c5fba428bc1aeb790fbfb
R 1da822a94c4b832d6a6cd41c425e2e73
U dan
Z b2c1f41f5ac4263ca444db1f00dc6050
Z 20acb0606f0068559b373b7ba0fbeacf
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
890e9629a7480138c9c1d3acc2d1e7b3c05e0d156e5c5fba428bc1aeb790fbfb
a8c91c132f6157b7e3649f57a799984b1d7f8a18fd434515c875617d4195db29

16
test/fts3defer2.test
View File

@ -164,15 +164,25 @@ foreach {tn sql} {
do_execsql_test 2.5 {
INSERT INTO t3(t3) VALUES('rebuild');
}
do_execsql_test 2.5 {
do_execsql_test 2.6 {
SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR a)';
} {11 228929}
do_execsql_test 2.6 {
do_execsql_test 2.7 {
SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b NEAR a)';
} {1 23 3 23 11 205}
do_execsql_test 2.7 {
do_execsql_test 2.8 {
SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b)';
} {1 15 3 15 11 106}
do_execsql_test 2.9 {
SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR a)';
} {11 32}
do_execsql_test 2.10 {
SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b NEAR a)'
} {1 44 3 44 11 44}
do_execsql_test 2.11 {
SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b)';
} {1 32 3 32 11 32}
finish_test