database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Have the FTS auxiliary functions correctly handle terms that appear in non-matching branches of the FTS expression. Fix for [bdc6bbbb38].

Browse Source 
FossilOrigin-Name: 4d3e1673b309a3d6a6cac01d67dfd79a8007f991
This commit is contained in:
dan
2012-05-10 17:43:14 +00:00
parent aeb281c2ea
commit 5cfed52dd1
8 changed files with 393 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
ext/fts3/fts3.c
View File

@ -3945,7 +3945,7 @@ void sqlite3Fts3DoclistPrev(
int nDoclist, /* Length of aDoclist in bytes */
char **ppIter, /* IN/OUT: Iterator pointer */
sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
int *pnList, /* IN/OUT: List length pointer */
int *pnList, /* OUT: List length pointer */
u8 *pbEof /* OUT: End-of-file flag */
){
char *p = *ppIter;
@ -3992,6 +3992,41 @@ void sqlite3Fts3DoclistPrev(
}
}
/*
** Iterate forwards through a doclist.
*/
void sqlite3Fts3DoclistNext(
int bDescIdx, /* True if the doclist is desc */
char *aDoclist, /* Pointer to entire doclist */
int nDoclist, /* Length of aDoclist in bytes */
char **ppIter, /* IN/OUT: Iterator pointer */
sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
u8 *pbEof /* OUT: End-of-file flag */
){
char *p = *ppIter;
assert( nDoclist>0 );
assert( *pbEof==0 );
assert( p || *piDocid==0 );
assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
if( p==0 ){
p = aDoclist;
p += sqlite3Fts3GetVarint(p, piDocid);
}else{
fts3PoslistCopy(0, &p);
if( p>=&aDoclist[nDoclist] ){
*pbEof = 1;
}else{
sqlite3_int64 iVar;
p += sqlite3Fts3GetVarint(p, &iVar);
*piDocid += ((bDescIdx ? -1 : 1) * iVar);
}
}
*ppIter = p;
}
/*
** Attempt to move the phrase iterator to point to the next matching docid.
** If an error occurs, return an SQLite error code. Otherwise, return
@ -5147,26 +5182,87 @@ int sqlite3Fts3EvalPhraseStats(
** This function works regardless of whether or not the phrase is deferred,
** incremental, or neither.
*/
char *sqlite3Fts3EvalPhrasePoslist(
int sqlite3Fts3EvalPhrasePoslist(
Fts3Cursor *pCsr, /* FTS3 cursor object */
Fts3Expr *pExpr, /* Phrase to return doclist for */
int iCol /* Column to return position list for */
int iCol, /* Column to return position list for */
char **ppOut /* OUT: Pointer to position list */
){
Fts3Phrase *pPhrase = pExpr->pPhrase;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
char *pIter = pPhrase->doclist.pList;
char *pIter;
int iThis;
sqlite3_int64 iDocid;
/* If this phrase is applies specifically to some column other than
** column iCol, return a NULL pointer. */
*ppOut = 0;
assert( iCol>=0 && iCol<pTab->nColumn );
if( !pIter
|| pExpr->bEof
|| pExpr->iDocid!=pCsr->iPrevId
|| (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
){
return 0;
if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){
return SQLITE_OK;
}
assert( pPhrase->doclist.nList>0 );
iDocid = pExpr->iDocid;
pIter = pPhrase->doclist.pList;
if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int bOr = 0;
u8 bEof = 0;
Fts3Expr *p;
/* Check if this phrase descends from an OR expression node. If not,
** return NULL. Otherwise, the entry that corresponds to docid
** pCsr->iPrevId may lie earlier in the doclist buffer. */
for(p=pExpr->pParent; p; p=p->pParent){
if( p->eType==FTSQUERY_OR ) bOr = 1;
}
if( bOr==0 ) return SQLITE_OK;
/* This is the descendent of an OR node. In this case we cannot use
** an incremental phrase. Load the entire doclist for the phrase
** into memory in this case. */
if( pPhrase->bIncr ){
int rc = SQLITE_OK;
int bEofSave = pExpr->bEof;
fts3EvalRestart(pCsr, pExpr, &rc);
while( rc==SQLITE_OK && !pExpr->bEof ){
fts3EvalNextRow(pCsr, pExpr, &rc);
if( bEofSave==0 && pExpr->iDocid==iDocid ) break;
}
pIter = pPhrase->doclist.pList;
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
if( rc!=SQLITE_OK ) return rc;
}
if( pExpr->bEof ){
pIter = 0;
iDocid = 0;
}
bEof = (pPhrase->doclist.nAll==0);
assert( bDescDoclist==0 || bDescDoclist==1 );
assert( pCsr->bDesc==0 || pCsr->bDesc==1 );
if( pCsr->bDesc==bDescDoclist ){
int dummy;
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
sqlite3Fts3DoclistPrev(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}else{
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}
if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
}
if( pIter==0 ) return SQLITE_OK;
if( *pIter==0x01 ){
pIter++;
pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
@ -5180,7 +5276,8 @@ char *sqlite3Fts3EvalPhrasePoslist(
pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
}
return ((iCol==iThis)?pIter:0);
*ppOut = ((iCol==iThis)?pIter:0);
return SQLITE_OK;
}
/*

2
ext/fts3/fts3Int.h
View File

@ -535,7 +535,7 @@ int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);

27
ext/fts3/fts3_snippet.c
View File

@ -360,10 +360,11 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
SnippetIter *p = (SnippetIter *)ctx;
SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
char *pCsr;
int rc;
pPhrase->nToken = pExpr->pPhrase->nToken;
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr);
assert( rc==SQLITE_OK || pCsr==0 );
if( pCsr ){
int iFirst = 0;
pPhrase->pList = pCsr;
@ -374,10 +375,12 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
pPhrase->iHead = iFirst;
pPhrase->iTail = iFirst;
}else{
assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
assert( rc!=SQLITE_OK || (
pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0
));
}
return SQLITE_OK;
return rc;
}
/*
@ -770,13 +773,14 @@ static int fts3ExprLocalHitsCb(
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
){
int rc = SQLITE_OK;
MatchInfo *p = (MatchInfo *)pCtx;
int iStart = iPhrase * p->nCol * 3;
int i;
for(i=0; i<p->nCol; i++){
for(i=0; i<p->nCol && rc==SQLITE_OK; i++){
char *pCsr;
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr);
if( pCsr ){
p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
}else{
@ -784,7 +788,7 @@ static int fts3ExprLocalHitsCb(
}
}
return SQLITE_OK;
return rc;
}
static int fts3MatchinfoCheck(
@ -945,8 +949,10 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
int nLive = 0; /* Number of iterators in aIter not at EOF */
for(i=0; i<pInfo->nPhrase; i++){
int rc;
LcsIterator *pIt = &aIter[i];
pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
if( rc!=SQLITE_OK ) return rc;
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
@ -1298,9 +1304,10 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iTerm; /* For looping through nTerm phrase terms */
char *pList; /* Pointer to position list for phrase */
int iPos = 0; /* First position in position-list */
int rc;
UNUSED_PARAMETER(iPhrase);
pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList);
nTerm = pExpr->pPhrase->nToken;
if( pList ){
fts3GetDeltaPosition(&pList, &iPos);
@ -1314,7 +1321,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
pT->iPos = iPos;
}
return SQLITE_OK;
return rc;
}
/*

23
manifest
View File

@ -1,5 +1,5 @@
C Further\schanges\sto\sconstraint\sparsing\sto\ssupport\slegacy\ssyntax.
D 2012-05-08T11:17:33.934
C Have\sthe\sFTS\sauxiliary\sfunctions\scorrectly\shandle\sterms\sthat\sappear\sin\snon-matching\sbranches\sof\sthe\sFTS\sexpression.\sFix\sfor\s[bdc6bbbb38].
D 2012-05-10T17:43:14.961
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -55,16 +55,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 111626ce72b0df93f509ebd14ce31804fed24be0
F ext/fts3/fts3.c a7adf6747d1fdd627ecd421c1709996741ca6693
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 5fd2ec4e47faf17bf4a508d6b8ec5fc0f2c80bff
F ext/fts3/fts3Int.h aca752b99c15ee738f5bcf0910eafb9e4aeb1b97
F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551
F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c 62ec177c55f6a5c6e994dd3e5fd3194b4045c347
F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3
F ext/fts3/fts3_snippet.c 51a3a34c217e24678a133782c1dfb6f2f70fe559
F ext/fts3/fts3_snippet.c bf67520ae9d2352a65368ed101729ff701c08808
F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763
F ext/fts3/fts3_test.c 348f7d08cae05285794e23dc4fe8b8fdf66e264a
F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
@ -282,7 +282,7 @@ F test/autoindex1.test 058d0b331ae6840a61bbee910d8cbae27bfd5991
F test/autovacuum.test fcaf4616ae5bb18098db1cb36262565e5c841c3c
F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85
F test/backcompat.test 94778872ed9f6158ba1b8534264d3413ae7e27f8
F test/backcompat.test bccbc64769d9c755ad65ee7c2f7336b86e3cc0c8
F test/backup.test 717346db953e9e435c2a94916e4af177330d60d3
F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf
F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0
@ -460,7 +460,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
F test/fts3auto.test 868a2afea308d7d8b45ef29fcf022644a9e6d662
F test/fts3auto.test b39f3f51227aea145eae6638690355dbdf9abf18
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@ -768,6 +768,7 @@ F test/tkt-b1d3a2e531.test 610ef582413171b379652663111b1f996d9f8f78
F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0
F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3
F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898
F test/tkt-bdc6bbbb38.test 1ec1c8e0fd0d294dc5d6de1a65eab8bb1b565ebf
F test/tkt-c48d99d690.test bed446e3513ae10eec1b86fdd186ef750226c408
F test/tkt-cbd054fa6b.test bd9fb546f63bc0c79d1776978d059fa51c5b1c63
F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7
@ -996,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
P a1c014d8a87c8940b3a037d8d8cc4d5678809802
R b6207703cdccfd3ac416f9da168138e7
U drh
Z c70e51594d9a44ff78a8a89d0fd856a7
P 38bf90af1ede6ee64ef7be66392e895e60c9126e
R 8aa768ab15356f78201e8a08039f9508
U dan
Z 18c4ee9f4eaf33e7622486489a2f279a

2
manifest.uuid
View File

@ -1 +1 @@
38bf90af1ede6ee64ef7be66392e895e60c9126e
4d3e1673b309a3d6a6cac01d67dfd79a8007f991

4
test/backcompat.test
View File

@ -251,6 +251,7 @@ do_allbackcompat_test {
# Test that FTS3 tables may be read/written by different versions of
# SQLite.
#
ifcapable fts3 {
set contents {
CREATE VIRTUAL TABLE t1 USING fts3(a, b);
}
@ -344,11 +345,13 @@ do_allbackcompat_test {
}
}
}
}
#-------------------------------------------------------------------------
# Test that Rtree tables may be read/written by different versions of
# SQLite.
#
ifcapable rtree {
set contents {
CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2, y1, y2);
}
@ -417,5 +420,6 @@ do_allbackcompat_test {
}
}
}
finish_test

14
test/fts3auto.test
View File

@ -75,26 +75,27 @@ proc do_fts3query_test {tn args} {
}
}
get_near_results $tbl $match $deferred aMatchinfo
get_near_results $tbl $match $deferred aHit
get_near_results $tbl [string map {AND OR} $match] $deferred aMatchinfo
set matchinfo_asc [list]
foreach docid [lsort -integer -incr [array names aMatchinfo]] {
foreach docid [lsort -integer -incr [array names aHit]] {
lappend matchinfo_asc $docid $aMatchinfo($docid)
}
set matchinfo_desc [list]
foreach docid [lsort -integer -decr [array names aMatchinfo]] {
foreach docid [lsort -integer -decr [array names aHit]] {
lappend matchinfo_desc $docid $aMatchinfo($docid)
}
set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)"
set title "(\"$match\" -> [llength [array names aHit]] rows)"
do_execsql_test $tn$title.1 "
SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
" [lsort -integer -incr [array names aMatchinfo]]
" [lsort -integer -incr [array names aHit]]
do_execsql_test $tn$title.2 "
SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
" [lsort -integer -decr [array names aMatchinfo]]
" [lsort -integer -decr [array names aHit]]
do_execsql_test $tn$title.3 "
SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl
@ -650,6 +651,7 @@ foreach {tn pending create} {
do_fts3query_test 6.$tn.2 t1 {b:G AND c:I}
do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I}
do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C}
do_fts3query_test 6.$tn.5 t1 {a:G OR b:G}
catchsql { COMMIT }

89
test/tkt-bdc6bbbb38.test Normal file
View File

@ -0,0 +1,89 @@
# 2012 May 11
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to verify that ticket [bdc6bbbb38] has been
# fixed.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix tkt-bdc6bbbb38
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 { finish_test ; return }
set sqlite_fts3_enable_parentheses 1
foreach {tn idxdir} {1 ASC 2 DESC} {
execsql { DROP TABLE IF EXISTS t2 }
do_execsql_test $tn.1.1 "CREATE VIRTUAL TABLE t2 USING fts4(x, order=$idxdir)"
do_execsql_test $tn.1.2 { INSERT INTO t2 VALUES('a b c') }
do_execsql_test $tn.1.3 {
SELECT offsets(t2) FROM t2 WHERE t2 MATCH 'a AND d OR b' ORDER BY docid ASC
} {
{0 0 0 1 0 2 2 1}
}
do_execsql_test $tn.1.4 {
SELECT snippet(t2,'[',']') FROM t2 WHERE t2 MATCH 'a AND d OR b'
ORDER BY docid ASC
} {
{[a] [b] c}
}
do_execsql_test $tn.1.5 { INSERT INTO t2 VALUES('a c d') }
do_execsql_test $tn.1.6 {
SELECT offsets(t2) FROM t2 WHERE t2 MATCH 'a AND d OR b' ORDER BY docid ASC
} {
{0 0 0 1 0 2 2 1}
{0 0 0 1 0 1 4 1}
}
do_execsql_test $tn.1.7 {
SELECT snippet(t2,'[',']') FROM t2 WHERE t2 MATCH 'a AND d OR b'
ORDER BY docid ASC
} {
{[a] [b] c}
{[a] c [d]}
}
execsql { DROP TABLE IF EXISTS t3 }
do_execsql_test $tn.2.1 "CREATE VIRTUAL TABLE t3 USING fts4(x, order=$idxdir)"
do_execsql_test $tn.2.2 { INSERT INTO t3 VALUES('a c d') }
do_execsql_test $tn.2.3 {
SELECT offsets(t3) FROM t3 WHERE t3 MATCH 'a AND d OR b' ORDER BY docid DESC
} {
{0 0 0 1 0 1 4 1}
}
do_execsql_test $tn.2.4 {
SELECT snippet(t3,'[',']') FROM t3 WHERE t3 MATCH 'a AND d OR b'
ORDER BY docid DESC
} {
{[a] c [d]}
}
do_execsql_test $tn.2.5 {
INSERT INTO t3 VALUES('a b c');
}
do_execsql_test $tn.2.6 {
SELECT offsets(t3) FROM t3 WHERE t3 MATCH 'a AND d OR b' ORDER BY docid DESC
} {
{0 0 0 1 0 2 2 1}
{0 0 0 1 0 1 4 1}
}
do_execsql_test $tn.2.7 {
SELECT snippet(t3,'[',']') FROM t3 WHERE t3 MATCH 'a AND d OR b'
ORDER BY docid DESC
} {
{[a] [b] c}
{[a] c [d]}
}
}
finish_test