database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Changes to improve performance and support LIMIT clauses on fts3 tables. This branch is unstable for now.

Browse Source 
FossilOrigin-Name: 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9
This commit is contained in:
dan
2011-06-02 19:57:24 +00:00
parent 382874fc5c
commit e414854800
11 changed files with 1274 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1120
ext/fts3/fts3.c
View File

File diff suppressed because it is too large Load Diff

86
ext/fts3/fts3Int.h
View File

@ -47,6 +47,11 @@
*/ */
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
#ifndef MIN
# define MIN(x,y) ((x)<(y)?(x):(y))
#endif
/* /*
** Maximum length of a varint encoded integer. The varint format is different ** Maximum length of a varint encoded integer. The varint format is different
** from that used by SQLite, so the maximum length is 10, not 9. ** from that used by SQLite, so the maximum length is 10, not 9.
@ -142,10 +147,11 @@ typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3PhraseToken Fts3PhraseToken;
typedef struct Fts3Doclist Fts3Doclist;
typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3SegFilter Fts3SegFilter;
typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; typedef struct Fts3MultiSegReader Fts3MultiSegReader;
/* /*
** A connection to a fulltext index is an instance of the following ** A connection to a fulltext index is an instance of the following
@ -224,6 +230,7 @@ struct Fts3Cursor {
u8 isRequireSeek; /* True if must seek pStmt to %_content row */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */ Fts3Expr *pExpr; /* Parsed MATCH query string */
int bIncremental; /* True to use incremental querying */
int nPhrase; /* Number of matchable phrases in query */ int nPhrase; /* Number of matchable phrases in query */
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
@ -263,6 +270,17 @@ struct Fts3Cursor {
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
struct Fts3Doclist {
char *aAll; /* Array containing doclist (or NULL) */
int nAll; /* Size of a[] in bytes */
sqlite3_int64 iDocid; /* Current docid (if p!=0) */
char *pNextDocid; /* Pointer to next docid */
char *pList; /* Pointer to position list following iDocid */
int nList; /* Length of position list */
} doclist;
/* /*
** A "phrase" is a sequence of one or more tokens that must match in ** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case. ** sequence. A single token is the base case and the most common case.
@ -277,23 +295,29 @@ struct Fts3PhraseToken {
/* Variables above this point are populated when the expression is /* Variables above this point are populated when the expression is
** parsed (by code in fts3_expr.c). Below this point the variables are ** parsed (by code in fts3_expr.c). Below this point the variables are
** used when evaluating the expression. */ ** used when evaluating the expression. */
int bFulltext; /* True if full-text index was used */ int bFulltext; /* True if full-text index was used */
Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
}; };
struct Fts3Phrase { struct Fts3Phrase {
/* Variables populated by fts3_expr.c when parsing a MATCH expression */ /* Cache of doclist for this phrase. */
int nToken; /* Number of tokens in the phrase */ Fts3Doclist doclist;
int iColumn; /* Index of column this phrase must match */ int bIncr; /* True if doclist is loaded incrementally */
#if 1
int isLoaded; /* True if aDoclist/nDoclist are initialized. */ int isLoaded; /* True if aDoclist/nDoclist are initialized. */
char *aDoclist; /* Buffer containing doclist */ char *aDoclist; /* Buffer containing doclist */
int nDoclist; /* Size of aDoclist in bytes */ int nDoclist; /* Size of aDoclist in bytes */
sqlite3_int64 iCurrent; sqlite3_int64 iCurrent;
char *pCurrent; char *pCurrent;
#endif
/* Variables below this point are populated by fts3_expr.c when parsing
** a MATCH expression. Everything above is part of the evaluation phase.
*/
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
}; };
@ -317,6 +341,12 @@ struct Fts3Expr {
Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pLeft; /* Left operand */
Fts3Expr *pRight; /* Right operand */ Fts3Expr *pRight; /* Right operand */
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
/* The following are used by the fts3_eval.c module. */
sqlite3_int64 iDocid; /* Current docid */
u8 bEof; /* True this expression is at EOF already */
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
}; };
/* /*
@ -366,11 +396,12 @@ void sqlite3Fts3SegmentsClose(Fts3Table *);
#define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_PENDING -1
#define FTS3_SEGCURSOR_ALL -2 #define FTS3_SEGCURSOR_ALL -2
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
int sqlite3Fts3SegReaderCursor( int sqlite3Fts3SegReaderCursor(
Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@ -387,7 +418,7 @@ struct Fts3SegFilter {
int flags; int flags;
}; };
struct Fts3SegReaderCursor { struct Fts3MultiSegReader {
/* Used internally by sqlite3Fts3SegReaderXXX() calls */ /* Used internally by sqlite3Fts3SegReaderXXX() calls */
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
int nSegment; /* Size of apSegment array */ int nSegment; /* Size of apSegment array */
@ -396,8 +427,11 @@ struct Fts3SegReaderCursor {
char *aBuffer; /* Buffer to merge doclists in */ char *aBuffer; /* Buffer to merge doclists in */
int nBuffer; /* Allocated size of aBuffer[] in bytes */ int nBuffer; /* Allocated size of aBuffer[] in bytes */
/* Cost of running this iterator. Used by fts3.c only. */ int iColFilter; /* If >=0, filter for this column */
int nCost;
/* Used by fts3.c only. */
int nCost; /* Cost of running iterator */
int bLookup; /* True if a lookup of a single entry. */
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
char *zTerm; /* Pointer to term buffer */ char *zTerm; /* Pointer to term buffer */
@ -413,7 +447,6 @@ int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64); int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *); void sqlite3Fts3Dequote(char *);
char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
@ -446,4 +479,29 @@ int sqlite3Fts3InitTerm(sqlite3 *db);
/* fts3_aux.c */ /* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db); int sqlite3Fts3InitAux(sqlite3 *db);
int sqlite3Fts3TermSegReaderCursor(
Fts3Cursor *pCsr, /* Virtual table cursor handle */
const char *zTerm, /* Term to query for */
int nTerm, /* Size of zTerm in bytes */
int isPrefix, /* True for a prefix search */
Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
);
int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *);
sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*);
void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr);
int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(
Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
#endif /* _FTSINT_H */ #endif /* _FTSINT_H */

2
ext/fts3/fts3_aux.c
View File

@ -28,7 +28,7 @@ struct Fts3auxTable {
struct Fts3auxCursor { struct Fts3auxCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */ Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter; Fts3SegFilter filter;
char *zStop; char *zStop;
int nStop; /* Byte-length of string zStop */ int nStop; /* Byte-length of string zStop */

5
ext/fts3/fts3_expr.c
View File

@ -768,7 +768,10 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){
assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight); sqlite3Fts3ExprFree(p->pRight);
if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist); if( p->pPhrase ){
sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
sqlite3_free(p->pPhrase->aDoclist);
}
sqlite3_free(p); sqlite3_free(p);
} }
} }

110
ext/fts3/fts3_snippet.c
View File

@ -416,7 +416,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
pPhrase->nToken = pExpr->pPhrase->nToken; pPhrase->nToken = pExpr->pPhrase->nToken;
pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol);
if( pCsr ){ if( pCsr ){
int iFirst = 0; int iFirst = 0;
pPhrase->pList = pCsr; pPhrase->pList = pCsr;
@ -826,46 +826,31 @@ static int fts3ExprGlobalHitsCb(
void *pCtx /* Pointer to MatchInfo structure */ void *pCtx /* Pointer to MatchInfo structure */
){ ){
MatchInfo *p = (MatchInfo *)pCtx; MatchInfo *p = (MatchInfo *)pCtx;
Fts3Cursor *pCsr = p->pCursor;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter;
char *pEnd;
char *pFree = 0;
u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
assert( pPhrase->isLoaded ); if( pExpr->bDeferred ){
int iCol; /* Column index */
if( pCsr->pDeferred ){ for(iCol=0; iCol<p->nCol; iCol++){
int ii; aOut[iCol*3 + 1] = (u32)p->nDoc;
for(ii=0; ii<pPhrase->nToken; ii++){ aOut[iCol*3 + 2] = (u32)p->nDoc;
if( pPhrase->aToken[ii].bFulltext ) break;
}
if( ii<pPhrase->nToken ){
int nFree = 0;
int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
if( rc!=SQLITE_OK ) return rc;
pIter = pFree;
pEnd = &pFree[nFree];
}else{
int iCol; /* Column index */
for(iCol=0; iCol<p->nCol; iCol++){
aOut[iCol*3 + 1] = (u32)p->nDoc;
aOut[iCol*3 + 2] = (u32)p->nDoc;
}
return SQLITE_OK;
} }
}else{ }else{
pIter = pPhrase->aDoclist; char *pIter;
pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; char *pEnd;
int n;
int rc = sqlite3Fts3EvalPhraseDoclist(
p->pCursor, pExpr, (const char **)&pIter, &n
);
if( rc!=SQLITE_OK ) return rc;
pEnd = &pIter[n];
/* Fill in the global hit count matrix row for this phrase. */
while( pIter<pEnd ){
while( *pIter++ & 0x80 ); /* Skip past docid. */
fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
}
} }
/* Fill in the global hit count matrix row for this phrase. */
while( pIter<pEnd ){
while( *pIter++ & 0x80 ); /* Skip past docid. */
fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
}
sqlite3_free(pFree);
return SQLITE_OK; return SQLITE_OK;
} }
@ -882,15 +867,15 @@ static int fts3ExprLocalHitsCb(
MatchInfo *p = (MatchInfo *)pCtx; MatchInfo *p = (MatchInfo *)pCtx;
int iStart = iPhrase * p->nCol * 3; int iStart = iPhrase * p->nCol * 3;
int i; int i;
sqlite3_int64 iDocid = p->pCursor->iPrevId;
for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0; for(i=0; i<p->nCol; i++){
if( pExpr->pPhrase->aDoclist ){
char *pCsr; char *pCsr;
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i);
pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
if( pCsr ){ if( pCsr ){
fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
}else{
p->aMatchinfo[iStart+i*3] = 0;
} }
} }
@ -976,9 +961,8 @@ static int fts3MatchinfoSelectDoctotal(
typedef struct LcsIterator LcsIterator; typedef struct LcsIterator LcsIterator;
struct LcsIterator { struct LcsIterator {
Fts3Expr *pExpr; /* Pointer to phrase expression */ Fts3Expr *pExpr; /* Pointer to phrase expression */
char *pRead; /* Cursor used to iterate through aDoclist */
int iPosOffset; /* Tokens count up to end of this phrase */ int iPosOffset; /* Tokens count up to end of this phrase */
int iCol; /* Current column number */ char *pRead; /* Cursor used to iterate through aDoclist */
int iPos; /* Current position */ int iPos; /* Current position */
}; };
@ -1009,17 +993,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){
int rc = 0; int rc = 0;
pRead += sqlite3Fts3GetVarint(pRead, &iRead); pRead += sqlite3Fts3GetVarint(pRead, &iRead);
if( iRead==0 ){ if( iRead==0 || iRead==1 ){
pIter->iCol = LCS_ITERATOR_FINISHED; pRead = 0;
rc = 1; rc = 1;
}else{ }else{
if( iRead==1 ){
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
pIter->iCol = (int)iRead;
pIter->iPos = pIter->iPosOffset;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
rc = 1;
}
pIter->iPos += (int)(iRead-2); pIter->iPos += (int)(iRead-2);
} }
@ -1043,6 +1020,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
int i; int i;
int iCol; int iCol;
int nToken = 0; int nToken = 0;
sqlite3_int64 iDocid = pCsr->iPrevId;
/* Allocate and populate the array of LcsIterator objects. The array /* Allocate and populate the array of LcsIterator objects. The array
** contains one element for each matchable phrase in the query. ** contains one element for each matchable phrase in the query.
@ -1051,42 +1029,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
if( !aIter ) return SQLITE_NOMEM; if( !aIter ) return SQLITE_NOMEM;
memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
for(i=0; i<pInfo->nPhrase; i++){ for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i]; LcsIterator *pIter = &aIter[i];
nToken -= pIter->pExpr->pPhrase->nToken; nToken -= pIter->pExpr->pPhrase->nToken;
pIter->iPosOffset = nToken; pIter->iPosOffset = nToken;
pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
if( pIter->pRead ){
pIter->iPos = pIter->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
}else{
pIter->iCol = LCS_ITERATOR_FINISHED;
}
} }
for(iCol=0; iCol<pInfo->nCol; iCol++){ for(iCol=0; iCol<pInfo->nCol; iCol++){
int nLcs = 0; /* LCS value for this column */ int nLcs = 0; /* LCS value for this column */
int nLive = 0; /* Number of iterators in aIter not at EOF */ int nLive = 0; /* Number of iterators in aIter not at EOF */
/* Loop through the iterators in aIter[]. Set nLive to the number of
** iterators that point to a position-list corresponding to column iCol.
*/
for(i=0; i<pInfo->nPhrase; i++){ for(i=0; i<pInfo->nPhrase; i++){
assert( aIter[i].iCol>=iCol ); LcsIterator *pIt = &aIter[i];
if( aIter[i].iCol==iCol ) nLive++; pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol);
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
nLive++;
}
} }
/* The following loop runs until all iterators in aIter[] have finished
** iterating through positions in column iCol. Exactly one of the
** iterators is advanced each time the body of the loop is run.
*/
while( nLive>0 ){ while( nLive>0 ){
LcsIterator *pAdv = 0; /* The iterator to advance by one position */ LcsIterator *pAdv = 0; /* The iterator to advance by one position */
int nThisLcs = 0; /* LCS for the current iterator positions */ int nThisLcs = 0; /* LCS for the current iterator positions */
for(i=0; i<pInfo->nPhrase; i++){ for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i]; LcsIterator *pIter = &aIter[i];
if( iCol!=pIter->iCol ){ if( pIter->pRead==0 ){
/* This iterator is already at EOF for this column. */ /* This iterator is already at EOF for this column. */
nThisLcs = 0; nThisLcs = 0;
}else{ }else{
@ -1426,7 +1396,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iPos = 0; /* First position in position-list */ int iPos = 0; /* First position in position-list */
UNUSED_PARAMETER(iPhrase); UNUSED_PARAMETER(iPhrase);
pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol);
nTerm = pExpr->pPhrase->nToken; nTerm = pExpr->pPhrase->nToken;
if( pList ){ if( pList ){
fts3GetDeltaPosition(&pList, &iPos); fts3GetDeltaPosition(&pList, &iPos);

2
ext/fts3/fts3_term.c
View File

@ -33,7 +33,7 @@ struct Fts3termTable {
struct Fts3termCursor { struct Fts3termCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */ Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter; Fts3SegFilter filter;
int isEof; /* True if cursor is at EOF */ int isEof; /* True if cursor is at EOF */

170
ext/fts3/fts3_write.c
View File

@ -1155,6 +1155,8 @@ int sqlite3Fts3SegReaderCost(
int nCost = 0; /* Cost in bytes to return */ int nCost = 0; /* Cost in bytes to return */
int pgsz = p->nPgsz; /* Database page size */ int pgsz = p->nPgsz; /* Database page size */
assert( pgsz>0 );
/* If this seg-reader is reading the pending-terms table, or if all data /* If this seg-reader is reading the pending-terms table, or if all data
** for the segment is stored on the root page of the b-tree, then the cost ** for the segment is stored on the root page of the b-tree, then the cost
** is zero. In this case all required data is already in main memory. ** is zero. In this case all required data is already in main memory.
@ -1223,6 +1225,40 @@ int sqlite3Fts3SegReaderCost(
return rc; return rc;
} }
int sqlite3Fts3MsrOvfl(
Fts3Cursor *pCsr,
Fts3MultiSegReader *pMsr,
int *pnOvfl
){
Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
int nOvfl = 0;
int ii;
int rc = SQLITE_OK;
int pgsz = p->nPgsz;
assert( p->bHasStat );
assert( pgsz>0 );
for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
Fts3SegReader *pReader = pMsr->apSegment[ii];
if( !fts3SegReaderIsPending(pReader)
&& !fts3SegReaderIsRootOnly(pReader)
){
int jj;
for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
int nBlob;
rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob);
if( rc!=SQLITE_OK ) break;
if( (nBlob+35)>pgsz ){
nOvfl += (nBlob + 34)/pgsz;
}
}
}
}
*pnOvfl = nOvfl;
return rc;
}
/* /*
** Free all allocations associated with the iterator passed as the ** Free all allocations associated with the iterator passed as the
** second argument. ** second argument.
@ -2140,9 +2176,107 @@ static void fts3ColumnFilter(
*pnList = nList; *pnList = nList;
} }
int sqlite3Fts3MsrIncrStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
int iCol, /* Column to match on. */
const char *zTerm, /* Term to iterate through a doclist for */
int nTerm /* Number of bytes in zTerm */
){
int i;
int nSegment = pCsr->nSegment;
assert( pCsr->pFilter==0 );
assert( zTerm && nTerm>0 );
/* Advance each segment iterator until it points to the term zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg);
if( rc!=SQLITE_OK ) return rc;
}while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
}
fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
/* Determine how many of the segments actually point to zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
break;
}
}
pCsr->nAdvance = i;
/* Advance each of the segments to point to the first docid. */
for(i=0; i<pCsr->nAdvance; i++){
fts3SegReaderFirstDocid(pCsr->apSegment[i]);
}
assert( iCol<0 || iCol<p->nColumn );
pCsr->iColFilter = iCol;
return SQLITE_OK;
}
int sqlite3Fts3MsrIncrNext(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
sqlite3_int64 *piDocid, /* OUT: Docid value */
char **paPoslist, /* OUT: Pointer to position list */
int *pnPoslist /* OUT: Size of position list in bytes */
){
int rc = SQLITE_OK;
int nMerge = pMsr->nAdvance;
Fts3SegReader **apSegment = pMsr->apSegment;
if( nMerge==0 ){
*paPoslist = 0;
return SQLITE_OK;
}
while( 1 ){
Fts3SegReader *pSeg;
fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
pSeg = pMsr->apSegment[0];
if( pSeg->pOffsetList==0 ){
*paPoslist = 0;
break;
}else{
char *pList;
int nList;
int j;
sqlite3_int64 iDocid = apSegment[0]->iDocid;
fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
j = 1;
while( j<nMerge
&& apSegment[j]->pOffsetList
&& apSegment[j]->iDocid==iDocid
){
fts3SegReaderNextDocid(apSegment[j], 0, 0);
}
if( pMsr->iColFilter>=0 ){
fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
}
if( nList>0 ){
*piDocid = iDocid;
*paPoslist = pList;
*pnPoslist = nList;
break;
}
}
}
return rc;
}
int sqlite3Fts3SegReaderStart( int sqlite3Fts3SegReaderStart(
Fts3Table *p, /* Virtual table handle */ Fts3Table *p, /* Virtual table handle */
Fts3SegReaderCursor *pCsr, /* Cursor object */ Fts3MultiSegReader *pCsr, /* Cursor object */
Fts3SegFilter *pFilter /* Restrictions on range of iteration */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */
){ ){
int i; int i;
@ -2173,7 +2307,7 @@ int sqlite3Fts3SegReaderStart(
int sqlite3Fts3SegReaderStep( int sqlite3Fts3SegReaderStep(
Fts3Table *p, /* Virtual table handle */ Fts3Table *p, /* Virtual table handle */
Fts3SegReaderCursor *pCsr /* Cursor object */ Fts3MultiSegReader *pCsr /* Cursor object */
){ ){
int rc = SQLITE_OK; int rc = SQLITE_OK;
@ -2308,8 +2442,9 @@ int sqlite3Fts3SegReaderStep(
return rc; return rc;
} }
void sqlite3Fts3SegReaderFinish( void sqlite3Fts3SegReaderFinish(
Fts3SegReaderCursor *pCsr /* Cursor object */ Fts3MultiSegReader *pCsr /* Cursor object */
){ ){
if( pCsr ){ if( pCsr ){
int i; int i;
@ -2342,7 +2477,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
int iNewLevel = 0; /* Level/index to create new segment at */ int iNewLevel = 0; /* Level/index to create new segment at */
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegFilter filter; /* Segment term filter condition */
Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */ int bIgnoreEmpty = 0; /* True to ignore empty segments */
assert( iLevel==FTS3_SEGCURSOR_ALL assert( iLevel==FTS3_SEGCURSOR_ALL
@ -2746,6 +2881,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
return rc; return rc;
} }
int sqlite3Fts3DeferredTokenList(
Fts3DeferredToken *p,
char **ppData,
int *pnData
){
char *pRet;
int nSkip;
sqlite3_int64 dummy;
*ppData = 0;
*pnData = 0;
if( p->pList==0 ){
return SQLITE_OK;
}
pRet = (char *)sqlite3_malloc(p->pList->nData);
if( !pRet ) return SQLITE_NOMEM;
nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
*pnData = p->pList->nData - nSkip;
*ppData = pRet;
memcpy(pRet, &p->pList->aData[nSkip], *pnData);
return SQLITE_OK;
}
/* /*
** Add an entry for token pToken to the pCsr->pDeferred list. ** Add an entry for token pToken to the pCsr->pDeferred list.
*/ */

28
manifest
View File

@ -1,5 +1,5 @@
C Minor\schanges\smade\swhile\splanning\sa\slarger\schange. C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow.
D 2011-05-28T15:57:40.694 D 2011-06-02T19:57:24.733
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319 F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62 F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50
F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f
F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903 F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2
F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7 F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274
F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1
F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4 F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -463,7 +463,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35
F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
@ -612,7 +612,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6 F test/permutations.test d27eac16dae111ff7cec331dab4bca08625ba65a
F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850
F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P cc83991caae7c7d647432d5711b6cd80228c3002 P 84097a4c759b1d65890af885f137d3cb16eef584
R eef52aec1faead4921ebaf39c1605d2b R e38e0cc84edbfdb5eae395b3be2f7a86
U dan U dan
Z 01d8f7aea2ce8ebb35c05ccc519b614f Z 53d59cf3dcd8991c66b0afd5fb898b1a

2
manifest.uuid
View File

@ -1 +1 @@
84097a4c759b1d65890af885f137d3cb16eef584 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9

29
test/fts3defer.test
View File

@ -20,6 +20,8 @@ ifcapable !fts3 {
set sqlite_fts3_enable_parentheses 1 set sqlite_fts3_enable_parentheses 1
set fts3_simple_deferred_tokens_only 1
set ::testprefix fts3defer set ::testprefix fts3defer
#-------------------------------------------------------------------------- #--------------------------------------------------------------------------
@ -257,7 +259,6 @@ foreach {tn setup} {
do_select_test 1.2 { do_select_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
} {100} } {100}
if {$tn==3} breakpoint
do_select_test 1.3 { do_select_test 1.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
} {7 70 98} } {7 70 98}
@ -282,13 +283,16 @@ if {$tn==3} breakpoint
do_select_test 1.10 { do_select_test 1.10 {
SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100} } {10 13 17 31 35 51 58 88 89 90 93 100}
do_select_test 1.11 {
SELECT rowid FROM t1 if { $fts3_simple_deferred_tokens_only==0 } {
WHERE t1 MATCH '( do_select_test 1.11 {
zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR SELECT rowid FROM t1
zk OR zkhdvkw OR zm OR zsmhnf WHERE t1 MATCH '(
) vgsld' zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
} {10 13 17 31 35 51 58 88 89 90 93 100} zk OR zkhdvkw OR zm OR zsmhnf
) vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
}
do_select_test 2.1 { do_select_test 2.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
@ -364,6 +368,7 @@ if {$tn==3} breakpoint
foreach DO_MALLOC_TEST $dmt_modes { foreach DO_MALLOC_TEST $dmt_modes {
# Phrase search. # Phrase search.
#
do_select_test 5.$DO_MALLOC_TEST.1 { do_select_test 5.$DO_MALLOC_TEST.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
} {8 15 36 64 67 72} } {8 15 36 64 67 72}
@ -416,9 +421,11 @@ if {$tn==3} breakpoint
do_select_test 6.2.2 { do_select_test 6.2.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
} {15 26 92 96} } {15 26 92 96}
do_select_test 6.2.3 { if {$fts3_simple_deferred_tokens_only==0} {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' do_select_test 6.2.3 {
} {8 15 26 92 96} SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
} }
set testprefix fts3defer set testprefix fts3defer

1
test/permutations.test
View File

@ -179,6 +179,7 @@ test_suite "fts3" -prefix "" -description {
fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test
fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3near.test fts3query.test fts3shared.test fts3snippet.test
fts3sort.test
fts3fault.test fts3malloc.test fts3matchinfo.test fts3fault.test fts3malloc.test fts3matchinfo.test