database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Add experimental implementation of FTS3 functions matchinfo() and snippet() (not enabled by default).

Browse Source 
FossilOrigin-Name: 51f7ee844057086789dcfcdcba7daf45343cae62
This commit is contained in:
dan
2010-01-02 19:02:02 +00:00
parent 855787a1b3
commit a98af17af6
8 changed files with 739 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

610
ext/fts3/fts3_snippet.c
View File

@ -731,4 +731,614 @@ void sqlite3Fts3Snippet(
fts3SnippetFree(p);
}
/*************************************************************************
** Below this point is the alternative, experimental snippet() implementation.
*/
#define SNIPPET_BUFFER_CHUNK 64
#define SNIPPET_BUFFER_SIZE SNIPPET_BUFFER_CHUNK*4
#define SNIPPET_BUFFER_MASK (SNIPPET_BUFFER_SIZE-1)
static void fts3GetDeltaPosition(char **pp, int *piPos){
int iVal;
*pp += sqlite3Fts3GetVarint32(*pp, &iVal);
*piPos += (iVal-2);
}
/*
** Iterate through all phrase nodes in an FTS3 query, except those that
** are part of a sub-tree that is the right-hand-side of a NOT operator.
** For each phrase node found, the supplied callback function is invoked.
**
** If the callback function returns anything other than SQLITE_OK,
** the iteration is abandoned and the error code returned immediately.
** Otherwise, SQLITE_OK is returned after a callback has been made for
** all eligible phrase nodes.
*/
static int fts3ExprIterate(
Fts3Expr *pExpr, /* Expression to iterate phrases of */
int (*x)(Fts3Expr *, void *), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
){
int rc;
int eType = pExpr->eType;
if( eType==FTSQUERY_NOT ){
rc = SQLITE_OK;
}else if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
rc = fts3ExprIterate(pExpr->pLeft, x, pCtx);
if( rc==SQLITE_OK ){
rc = fts3ExprIterate(pExpr->pRight, x, pCtx);
}
}else{
rc = x(pExpr, pCtx);
}
return rc;
}
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {
Fts3Table *pTab; /* FTS3 Table */
int nPhrase; /* Number of phrases so far */
};
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, void *ctx){
int rc = SQLITE_OK;
LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
p->nPhrase++;
if( pExpr->isLoaded==0 ){
rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
pExpr->isLoaded = 1;
if( rc==SQLITE_OK && pExpr->aDoclist ){
pExpr->pCurrent = pExpr->aDoclist;
pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent);
}
}
return rc;
}
static int fts3ExprLoadDoclists(Fts3Cursor *pCsr, int *pnPhrase){
int rc;
LoadDoclistCtx sCtx = {0, 0};
sCtx.pTab = (Fts3Table *)pCsr->base.pVtab;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
*pnPhrase = sCtx.nPhrase;
return rc;
}
/*
** Each call to this function populates a chunk of a snippet-buffer
** SNIPPET_BUFFER_CHUNK bytes in size.
**
** Return true if the end of the data has been reached (and all subsequent
** calls to fts3LoadSnippetBuffer() with the same arguments will be no-ops),
** or false otherwise.
*/
static int fts3LoadSnippetBuffer(
int iPos, /* Document token offset to load data for */
u8 *aBuffer, /* Circular snippet buffer to populate */
int nList, /* Number of position lists in appList */
char **apList, /* IN/OUT: nList position list pointers */
int *aiPrev /* IN/OUT: Previous positions read */
){
int i;
int nFin = 0;
assert( (iPos&(SNIPPET_BUFFER_CHUNK-1))==0 );
memset(&aBuffer[iPos&SNIPPET_BUFFER_MASK], 0, SNIPPET_BUFFER_CHUNK);
for(i=0; i<nList; i++){
int iPrev = aiPrev[i];
char *pList = apList[i];
if( !pList ){
nFin++;
continue;
}
while( iPrev<(iPos+SNIPPET_BUFFER_CHUNK) ){
if( iPrev>=iPos ){
aBuffer[iPrev&SNIPPET_BUFFER_MASK] = i+1;
}
if( 0==((*pList)&0xFE) ){
nFin++;
break;
}
fts3GetDeltaPosition(&pList, &iPrev);
}
aiPrev[i] = iPrev;
apList[i] = pList;
}
return (nFin==nList);
}
typedef struct SnippetCtx SnippetCtx;
struct SnippetCtx {
Fts3Cursor *pCsr;
int iCol;
int iPhrase;
int *aiPrev;
int *anToken;
char **apList;
};
static int fts3SnippetFindPositions(Fts3Expr *pExpr, void *ctx){
SnippetCtx *p = (SnippetCtx *)ctx;
int iPhrase = p->iPhrase++;
char *pCsr;
p->anToken[iPhrase] = pExpr->pPhrase->nToken;
pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol);
if( pCsr ){
int iVal;
pCsr += sqlite3Fts3GetVarint32(pCsr, &iVal);
p->apList[iPhrase] = pCsr;
p->aiPrev[iPhrase] = iVal-2;
}
return SQLITE_OK;
}
static void fts3SnippetCnt(
int iIdx,
int nSnippet,
int *anCnt,
u8 *aBuffer,
int *anToken,
u64 *pHlmask
){
int iSub = (iIdx-1)&SNIPPET_BUFFER_MASK;
int iAdd = (iIdx+nSnippet-1)&SNIPPET_BUFFER_MASK;
int iSub2 = (iIdx+(nSnippet/3)-1)&SNIPPET_BUFFER_MASK;
int iAdd2 = (iIdx+(nSnippet*2/3)-1)&SNIPPET_BUFFER_MASK;
u64 h = *pHlmask;
anCnt[ aBuffer[iSub] ]--;
anCnt[ aBuffer[iSub2] ]--;
anCnt[ aBuffer[iAdd] ]++;
anCnt[ aBuffer[iAdd2] ]++;
h = h >> 1;
if( aBuffer[iAdd] ){
int j;
for(j=anToken[aBuffer[iAdd]-1]; j>=1; j--){
h |= (u64)1 << (nSnippet-j);
}
}
*pHlmask = h;
}
static int fts3SnippetScore(int n, int *anCnt){
int j;
int iScore = 0;
for(j=1; j<=n; j++){
int nCnt = anCnt[j];
iScore += nCnt + (nCnt ? 1000 : 0);
}
return iScore;
}
static int fts3BestSnippet(
int nSnippet, /* Desired snippet length */
Fts3Cursor *pCsr, /* Cursor to create snippet for */
int iCol, /* Index of column to create snippet from */
int *piPos, /* OUT: Starting token for best snippet */
u64 *pHlmask /* OUT: Highlight mask for best snippet */
){
int rc; /* Return Code */
u8 aBuffer[SNIPPET_BUFFER_SIZE];/* Circular snippet buffer */
int *aiPrev; /* Used by fts3LoadSnippetBuffer() */
int *anToken; /* Number of tokens in each phrase */
char **apList; /* Array of position lists */
int *anCnt; /* Running totals of phrase occurences */
int nList;
int i;
u64 hlmask = 0; /* Current mask of highlighted terms */
u64 besthlmask = 0; /* Mask of highlighted terms for iBestPos */
int iBestPos = 0; /* Starting position of 'best' snippet */
int iBestScore = 0; /* Score of best snippet higher->better */
SnippetCtx sCtx;
/* Iterate through the phrases in the expression to count them. The same
** callback makes sure the doclists are loaded for each phrase.
*/
rc = fts3ExprLoadDoclists(pCsr, &nList);
if( rc!=SQLITE_OK ){
return rc;
}
/* Now that it is known how many phrases there are, allocate and zero
** the required arrays using malloc().
*/
apList = sqlite3_malloc(
sizeof(u8*)*nList + /* apList */
sizeof(int)*(nList) + /* anToken */
sizeof(int)*nList + /* aiPrev */
sizeof(int)*(nList+1) /* anCnt */
);
if( !apList ){
return SQLITE_NOMEM;
}
memset(apList, 0, sizeof(u8*)*nList+sizeof(int)*nList+sizeof(int)*nList);
anToken = (int *)&apList[nList];
aiPrev = &anToken[nList];
anCnt = &aiPrev[nList];
/* Initialize the contents of the aiPrev and aiList arrays. */
sCtx.pCsr = pCsr;
sCtx.iCol = iCol;
sCtx.apList = apList;
sCtx.aiPrev = aiPrev;
sCtx.anToken = anToken;
sCtx.iPhrase = 0;
(void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sCtx);
/* Load the first two chunks of data into the buffer. */
memset(aBuffer, 0, SNIPPET_BUFFER_SIZE);
fts3LoadSnippetBuffer(0, aBuffer, nList, apList, aiPrev);
fts3LoadSnippetBuffer(SNIPPET_BUFFER_CHUNK, aBuffer, nList, apList, aiPrev);
/* Set the initial contents of the highlight-mask and anCnt[] array. */
for(i=1-nSnippet; i<=0; i++){
fts3SnippetCnt(i, nSnippet, anCnt, aBuffer, anToken, &hlmask);
}
iBestScore = fts3SnippetScore(nList, anCnt);
besthlmask = hlmask;
iBestPos = 0;
for(i=1; 1; i++){
int iScore;
if( 0==(i&(SNIPPET_BUFFER_CHUNK-1)) ){
int iLoad = i + SNIPPET_BUFFER_CHUNK;
if( fts3LoadSnippetBuffer(iLoad, aBuffer, nList, apList, aiPrev) ) break;
}
/* Figure out how highly a snippet starting at token offset i scores
** according to fts3SnippetScore(). If it is higher than any previously
** considered position, save the current position, score and hlmask as
** the best snippet candidate found so far.
*/
fts3SnippetCnt(i, nSnippet, anCnt, aBuffer, anToken, &hlmask);
iScore = fts3SnippetScore(nList, anCnt);
if( iScore>iBestScore ){
iBestPos = i;
iBestScore = iScore;
besthlmask = hlmask;
}
}
sqlite3_free(apList);
*piPos = iBestPos;
*pHlmask = besthlmask;
return SQLITE_OK;
}
typedef struct StrBuffer StrBuffer;
struct StrBuffer {
char *z;
int n;
int nAlloc;
};
static int fts3StringAppend(
StrBuffer *pStr,
const char *zAppend,
int nAppend
){
if( nAppend<0 ){
nAppend = strlen(zAppend);
}
if( pStr->n+nAppend+1>=pStr->nAlloc ){
int nAlloc = pStr->nAlloc+nAppend+100;
char *zNew = sqlite3_realloc(pStr->z, nAlloc);
if( !zNew ){
return SQLITE_NOMEM;
}
pStr->z = zNew;
pStr->nAlloc = nAlloc;
}
memcpy(&pStr->z[pStr->n], zAppend, nAppend);
pStr->n += nAppend;
pStr->z[pStr->n] = '\0';
return SQLITE_OK;
}
static int fts3SnippetText(
Fts3Cursor *pCsr, /* FTS3 Cursor */
const char *zDoc, /* Document to extract snippet from */
int nDoc, /* Size of zDoc in bytes */
int nSnippet, /* Number of tokens in extracted snippet */
int iPos, /* Index of first document token in snippet */
u64 hlmask, /* Bitmask of terms to highlight in snippet */
const char *zOpen, /* String inserted before highlighted term */
const char *zClose, /* String inserted after highlighted term */
const char *zEllipsis,
char **pzSnippet /* OUT: Snippet text */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc; /* Return code */
int iCurrent = 0;
int iStart = 0;
int iEnd;
sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */
const char *ZDUMMY; /* Dummy arguments used with tokenizer */
int DUMMY1, DUMMY2, DUMMY3; /* Dummy arguments used with tokenizer */
StrBuffer res = {0, 0, 0}; /* Result string */
/* Open a token cursor on the document. Read all tokens up to and
** including token iPos (the first token of the snippet). Set variable
** iStart to the byte offset in zDoc of the start of token iPos.
*/
pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
while( rc==SQLITE_OK && iCurrent<iPos ){
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iStart, &DUMMY2, &iCurrent);
}
iEnd = iStart;
if( rc==SQLITE_OK && iStart>0 ){
rc = fts3StringAppend(&res, zEllipsis, -1);
}
while( rc==SQLITE_OK ){
int iBegin;
int iFin;
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
if( rc==SQLITE_OK ){
if( iCurrent>=(iPos+nSnippet) ){
rc = SQLITE_DONE;
}else{
iEnd = iFin;
if( hlmask & ((u64)1 << (iCurrent-iPos)) ){
if( fts3StringAppend(&res, &zDoc[iStart], iBegin-iStart)
|| fts3StringAppend(&res, zOpen, -1)
|| fts3StringAppend(&res, &zDoc[iBegin], iEnd-iBegin)
|| fts3StringAppend(&res, zClose, -1)
){
rc = SQLITE_NOMEM;
}
iStart = iEnd;
}
}
}
}
assert( rc!=SQLITE_OK );
if( rc==SQLITE_DONE ){
rc = fts3StringAppend(&res, &zDoc[iStart], iEnd-iStart);
if( rc==SQLITE_OK ){
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
if( rc==SQLITE_OK ){
rc = fts3StringAppend(&res, zEllipsis, -1);
}else if( rc==SQLITE_DONE ){
rc = fts3StringAppend(&res, &zDoc[iEnd], -1);
}
}
}
pMod->xClose(pC);
if( rc!=SQLITE_OK ){
sqlite3_free(res.z);
}else{
*pzSnippet = res.z;
}
return rc;
}
/*
** An instance of this structure is used to collect the 'global' part of
** the matchinfo statistics. The 'global' part consists of the following:
**
** 1. The number of phrases in the query (nPhrase).
**
** 2. The number of columns in the FTS3 table (nCol).
**
** 3. A matrix of (nPhrase*nCol) integers containing the sum of the
** number of hits for each phrase in each column across all rows
** of the table.
**
** The total size of the global matchinfo array, assuming the number of
** columns is N and the number of phrases is P is:
**
** 2 + P*(N+1)
**
** The number of hits for the 3rd phrase in the second column is found
** using the expression:
**
** aGlobal[2 + P*(1+2) + 1]
*/
typedef struct MatchInfo MatchInfo;
struct MatchInfo {
Fts3Table *pTab; /* FTS3 Table */
Fts3Cursor *pCursor; /* FTS3 Cursor */
int iPhrase; /* Number of phrases so far */
int nCol; /* Number of columns in table */
u32 *aGlobal; /* Pre-allocated buffer */
};
/*
** This function is used to count the entries in a column-list (delta-encoded
** list of term offsets within a single column of a single row).
*/
static int fts3ColumnlistCount(char **ppCollist){
char *pEnd = *ppCollist;
char c = 0;
int nEntry = 0;
/* A column-list is terminated by either a 0x01 or 0x00. */
while( 0xFE & (*pEnd | c) ){
c = *pEnd++ & 0x80;
if( !c ) nEntry++;
}
*ppCollist = pEnd;
return nEntry;
}
static void fts3LoadColumnlistCounts(char **pp, u32 *aOut){
char *pCsr = *pp;
while( *pCsr ){
sqlite3_int64 iCol = 0;
if( *pCsr==0x01 ){
pCsr++;
pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
}
aOut[iCol] += fts3ColumnlistCount(&pCsr);
}
pCsr++;
*pp = pCsr;
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
*/
static int fts3ExprGlobalMatchinfoCb(
Fts3Expr *pExpr, /* Phrase expression node */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
char *pCsr;
char *pEnd;
const int iStart = 2 + p->nCol*p->iPhrase;
assert( pExpr->isLoaded );
/* Fill in the global hit count matrix row for this phrase. */
pCsr = pExpr->aDoclist;
pEnd = &pExpr->aDoclist[pExpr->nDoclist];
while( pCsr<pEnd ){
while( *pCsr++ & 0x80 );
fts3LoadColumnlistCounts(&pCsr, &p->aGlobal[iStart]);
}
p->iPhrase++;
return SQLITE_OK;
}
static int fts3ExprLocalMatchinfoCb(
Fts3Expr *pExpr, /* Phrase expression node */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
int iPhrase = p->iPhrase++;
if( pExpr->aDoclist ){
char *pCsr;
int iOffset = 2 + p->nCol*(p->aGlobal[0]+iPhrase);
memset(&p->aGlobal[iOffset], 0, p->nCol*sizeof(u32));
pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
if( pCsr ) fts3LoadColumnlistCounts(&pCsr, &p->aGlobal[iOffset]);
}
return SQLITE_OK;
}
/*
** Populate pCsr->aMatchinfo[] with data for the current row. The 'matchinfo'
** data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(Fts3Cursor *pCsr){
MatchInfo g;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
if( pCsr->aMatchinfo==0 ){
int rc;
int nPhrase;
int nMatchinfo;
g.pTab = pTab;
g.nCol = pTab->nColumn;
g.iPhrase = 0;
rc = fts3ExprLoadDoclists(pCsr, &nPhrase);
if( rc!=SQLITE_OK ){
return rc;
}
nMatchinfo = 2 + 2*g.nCol*nPhrase;
g.iPhrase = 0;
g.aGlobal = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
if( !g.aGlobal ){
return SQLITE_NOMEM;
}
memset(g.aGlobal, 0, sizeof(u32)*nMatchinfo);
g.aGlobal[0] = nPhrase;
g.aGlobal[1] = g.nCol;
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb, (void *)&g);
pCsr->aMatchinfo = g.aGlobal;
}
g.pTab = pTab;
g.pCursor = pCsr;
g.nCol = pTab->nColumn;
g.iPhrase = 0;
g.aGlobal = pCsr->aMatchinfo;
if( pCsr->isMatchinfoOk ){
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void *)&g);
pCsr->isMatchinfoOk = 0;
}
return SQLITE_OK;
}
void sqlite3Fts3Snippet2(
sqlite3_context *pCtx, /* SQLite function call context */
Fts3Cursor *pCsr, /* Cursor object */
const char *zStart, /* Snippet start text - "<b>" */
const char *zEnd, /* Snippet end text - "</b>" */
const char *zEllipsis, /* Snippet ellipsis text - "<b>...</b>" */
int iCol, /* Extract snippet from this column */
int nToken /* Approximate number of tokens in snippet */
){
int rc;
int iPos = 0;
u64 hlmask = 0;
char *z = 0;
int nDoc;
const char *zDoc;
rc = fts3BestSnippet(nToken, pCsr, iCol, &iPos, &hlmask);
nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
if( rc==SQLITE_OK ){
rc = fts3SnippetText(
pCsr, zDoc, nDoc, nToken, iPos, hlmask, zStart, zEnd, zEllipsis, &z);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}else{
sqlite3_result_text(pCtx, z, -1, sqlite3_free);
}
}
void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
int rc = fts3GetMatchinfo(pCsr);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*2);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
}
}
#endif