diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 1eceafa050..a0286bbb43 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -28,6 +28,7 @@ #define FTS3_MATCHINFO_LCS 's' /* nCol values */ #define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ #define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ /* ** The default value for the second argument to matchinfo(). @@ -89,6 +90,7 @@ struct MatchInfo { int nCol; /* Number of columns in table */ int nPhrase; /* Number of matchable phrases in query */ sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; u32 *aMatchinfo; /* Pre-allocated buffer */ }; @@ -236,28 +238,27 @@ static void fts3GetDeltaPosition(char **pp, int *piPos){ *piPos += (iVal-2); } -static int fts3ExprLHitsCb(Fts3Expr*,int,void*); - /* ** Helper function for fts3ExprIterate() (see below). */ static int fts3ExprIterate2( Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int bExcludeEof, int *piPhrase, /* Pointer to phrase counter */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ ){ int rc; /* Return code */ - if( x==fts3ExprLHitsCb && pExpr->bEof ){ + if( bExcludeEof && pExpr->bEof ){ rc = SQLITE_OK; }else{ int eType = pExpr->eType; /* Type of expression node pExpr */ if( eType!=FTSQUERY_PHRASE ){ assert( pExpr->pLeft && pExpr->pRight ); - rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + rc = fts3ExprIterate2(pExpr->pLeft, bExcludeEof, piPhrase, x, pCtx); if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ - rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + rc = fts3ExprIterate2(pExpr->pRight, bExcludeEof, piPhrase, x, pCtx); } }else{ rc = x(pExpr, *piPhrase, pCtx); @@ -279,11 +280,12 @@ static int fts3ExprIterate2( */ static int fts3ExprIterate( Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int bExcludeEof, /* Include nodes already at EOF */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ ){ int iPhrase = 0; /* Variable used as the phrase counter */ - return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); + return fts3ExprIterate2(pExpr, bExcludeEof, &iPhrase, x, pCtx); } /* @@ -322,7 +324,7 @@ static int fts3ExprLoadDoclists( int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + rc = fts3ExprIterate(pCsr->pExpr, 0, fts3ExprLoadDoclistsCb, (void *)&sCtx); if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; @@ -336,7 +338,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + (void)fts3ExprIterate(pExpr, 0, fts3ExprPhraseCountCb, (void *)&nPhrase); return nPhrase; } @@ -552,7 +554,7 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); + rc = fts3ExprIterate(pCsr->pExpr, 0, fts3SnippetFindPositions, (void*)&sIter); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ @@ -936,15 +938,26 @@ static int fts3ExprLHitsCb( if( pExpr->iDocid==p->pCursor->iPrevId ){ Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; - int iStart = iPhrase * p->nCol; + int iStart; Fts3Phrase *pPhrase = pExpr->pPhrase; char *pIter = pPhrase->doclist.pList; int iCol = 0; + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = iPhrase * p->nCol; + }else{ + iStart = iPhrase * ((p->nCol + 31) / 32); + } + while( 1 ){ int nHit = fts3ColumnlistCount(&pIter); if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ - p->aMatchinfo[iStart + iCol] = (u32)nHit; + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } } assert( *pIter==0x00 || *pIter==0x01 ); if( *pIter!=0x01 ) break; @@ -969,6 +982,7 @@ static int fts3MatchinfoCheck( || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) ){ return SQLITE_OK; } @@ -996,6 +1010,10 @@ static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ nVal = pInfo->nCol * pInfo->nPhrase; break; + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + default: assert( cArg==FTS3_MATCHINFO_HITS ); nVal = pInfo->nCol * pInfo->nPhrase * 3; @@ -1106,7 +1124,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + (void)fts3ExprIterate(pCsr->pExpr, 0, fts3MatchinfoLcsCb, (void*)aIter); for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; @@ -1190,7 +1208,7 @@ static int fts3MatchinfoValues( sqlite3_stmt *pSelect = 0; for(i=0; rc==SQLITE_OK && zArg[i]; i++){ - + pInfo->flag = zArg[i]; switch( zArg[i] ){ case FTS3_MATCHINFO_NPHRASE: if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; @@ -1250,10 +1268,11 @@ static int fts3MatchinfoValues( } break; + case FTS3_MATCHINFO_LHITS_BM: case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, FTS3_MATCHINFO_LHITS)*sizeof(u32); + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); memset(pInfo->aMatchinfo, 0, nZero); - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo); + (void)fts3ExprIterate(pCsr->pExpr, 1, fts3ExprLHitsCb, (void*)pInfo); break; } @@ -1268,10 +1287,10 @@ static int fts3MatchinfoValues( rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); if( rc!=SQLITE_OK ) break; } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + rc = fts3ExprIterate(pExpr, 0, fts3ExprGlobalHitsCb,(void*)pInfo); if( rc!=SQLITE_OK ) break; } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + (void)fts3ExprIterate(pExpr, 0, fts3ExprLocalHitsCb,(void*)pInfo); break; } } @@ -1570,7 +1589,7 @@ void sqlite3Fts3Offsets( */ sCtx.iCol = iCol; sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); + (void)fts3ExprIterate(pCsr->pExpr, 0, fts3ExprTermOffsetInit, (void*)&sCtx); /* Retreive the text stored in column iCol. If an SQL NULL is stored ** in column iCol, jump immediately to the next iteration of the loop. diff --git a/manifest b/manifest index 738c7e07a1..cfd87670bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\sfor\sthe\smatchinfo()\sfunction,\sparticularly\sthe\s'y'\sflag. -D 2015-05-05T19:37:07.819 +C Add\sthe\sfts3\smatchinfo\s'b'\sflag. +D 2015-05-05T20:39:53.730 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 08728ecbeddca339c77bfd564d3484b523dffdb1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -87,7 +87,7 @@ F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 -F ext/fts3/fts3_snippet.c a1c62f1b7c55d14a13385689ce11aa0e1ada55b8 +F ext/fts3/fts3_snippet.c 2224cdfddd4825c449bab0420549b76963f5e444 F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 @@ -593,11 +593,11 @@ F test/fts3fault2.test f953bb3cf903988172270a9a0aafd5a890b0f98f F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 -F test/fts3matchinfo.test 3e5f5ac2e0a8ba42eafd4c685f803ca48b4c3a83 +F test/fts3matchinfo.test 07009313ad6c082f94d8c9c3228eb8940c93ac71 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce -F test/fts3query.test c838b18f2b859e15fd31c64be3d79ef1556803ca +F test/fts3query.test f33eb71a1fe1084ea585eeb7ee76b390729f5170 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e F test/fts3snippet.test 63dbd687d5bf5191f1b8e6a0977aa9c1e28a7004 @@ -1256,10 +1256,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b8f090e65d010c62df335d0520a36a24904e8bc6 -R db4b4f916c43503b3d5880cc35083941 -T *branch * fts3-matchinfo-y -T *sym-fts3-matchinfo-y * -T -sym-trunk * +P dddd7e182943a1d3a9d32830e819a63f1a228d6d +R 163d76f4b269e89cdb470ec09f3dac69 U dan -Z 2cd9b208c5989e901293be093105f297 +Z 5b29ad8a6cfef6f4d3b7a8b71870984c diff --git a/manifest.uuid b/manifest.uuid index c76ab92b50..9f54e8112d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dddd7e182943a1d3a9d32830e819a63f1a228d6d \ No newline at end of file +b9b77972d88171e4239b8194f308eb5d60b5d172 \ No newline at end of file diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 2681d0068f..b17d7da0a5 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -507,10 +507,49 @@ foreach {tn expr res} { } } { - do_execsql_test 11.1.$tn { + do_execsql_test 11.1.$tn.1 { SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr } $res + + set r2 [list] + foreach {rowid L} $res { + lappend r2 $rowid + set M [list] + foreach {a b} $L { + lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)] + } + lappend r2 $M + } + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 + breakpoint + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 } set sqlite_fts3_enable_parentheses 0 +#--------------------------------------------------------------------------- +# Test the 'b' matchinfo flag +# +set sqlite_fts3_enable_parentheses 1 +reset_db +db func mit mit + +do_test 12.0 { + set cols [list] + for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } + execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" +} {} + +do_execsql_test 12.1 { + INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); + SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; +} [list [list [expr 1<<4] [expr 1<<(45-32)]]] + +set sqlite_fts3_enable_parentheses 0 finish_test + diff --git a/test/fts3query.test b/test/fts3query.test index 06019d14e6..7d5ae991f7 100644 --- a/test/fts3query.test +++ b/test/fts3query.test @@ -173,8 +173,8 @@ do_select_tests 5.4 -errorformat { 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize } do_catchsql_test 5.5.1 { - SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history' -} {1 {unrecognized matchinfo request: b}} + SELECT matchinfo(t2, 'abcd') FROM t2 WHERE t2 MATCH 'history' +} {1 {unrecognized matchinfo request: d}} do_execsql_test 5.5 { DROP TABLE t2 }