database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Improvements to the way fts3 reads the full-text index.

Browse Source 
FossilOrigin-Name: 45c051e78651d8204c17cecdda2bde705698881f
This commit is contained in:
dan
2009-11-17 12:52:10 +00:00
parent 948a5f88ea
commit f13b704ee6
6 changed files with 237 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
ext/fts3/fts3.c
View File

@ -905,7 +905,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
static int fts3ReadBlock( static int fts3ReadBlock(
Fts3Table *p, Fts3Table *p,
sqlite3_int64 iBlock, sqlite3_int64 iBlock,
char **pzBlock, char const **pzBlock,
int *pnBlock int *pnBlock
){ ){
sqlite3_stmt *pStmt; sqlite3_stmt *pStmt;
@ -928,27 +928,28 @@ static int fts3ReadBlock(
} }
/* /*
** The buffer pointed to by argument zNode (size nNode bytes) contains a ** The buffer pointed to by argument zNode (size nNode bytes) contains the
** b-tree segment interior node. This function inspects the sub-tree headed ** root node of a b-tree segment. The segment is guaranteed to be at least
** by the node to determine the range of leaf-nodes (if any) that may ** one level high (i.e. the root node is not also a leaf). If successful,
** contain a term that matches the contents of buffer zTerm (size nTerm ** this function locates the leaf node of the segment that may contain the
** bytes). If the isPrefix parameter is true, then the range of leaves ** term specified by arguments zTerm and nTerm and writes its block number
** returned are those that may contain any term for which zTerm/nTerm is ** to *piLeaf.
** a prefix.
** **
** If successful, SQLITE_OK is returned. The blockid of the first leaf in the ** It is possible that the returned leaf node does not contain the specified
** selected range is written to piStart before returning. The blockid of the ** term. However, if the segment does contain said term, it is stored on
** final leaf in the selected range is written to *piEnd. ** the identified leaf node. Because this function only inspects interior
** segment nodes (and never loads leaf nodes into memory), it is not possible
** to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.
*/ */
static int fts3SelectLeaves( static int fts3SelectLeaf(
Fts3Table *p, /* Virtual table handle */ Fts3Table *p, /* Virtual table handle */
const char *zTerm, /* Term to select leaves for */ const char *zTerm, /* Term to select leaves for */
int nTerm, /* Size of term zTerm in bytes */ int nTerm, /* Size of term zTerm in bytes */
int isPrefix, /* True for a prefix search */
const char *zNode, /* Buffer containing segment interior node */ const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */ int nNode, /* Size of buffer at zNode */
sqlite3_int64 *piStart, /* First selected leaf */ sqlite3_int64 *piLeaf /* Selected leaf node */
sqlite3_int64 *piEnd /* Second selected leaf */
){ ){
int rc = SQLITE_OK; /* Return code */ int rc = SQLITE_OK; /* Return code */
const char *zCsr = zNode; /* Cursor to iterate through node */ const char *zCsr = zNode; /* Cursor to iterate through node */
@ -956,79 +957,67 @@ static int fts3SelectLeaves(
char *zBuffer = 0; /* Buffer to load terms into */ char *zBuffer = 0; /* Buffer to load terms into */
int nAlloc = 0; /* Size of allocated buffer */ int nAlloc = 0; /* Size of allocated buffer */
int iHeight; /* Height of this node in tree */ while( 1 ){
sqlite3_int64 iChild; int iHeight; /* Height of this node in tree */
sqlite3_int64 iStart = 0; sqlite3_int64 iChild; /* Block id of child node to descend to */
sqlite3_int64 iEnd; int nBlock; /* Size of child node in bytes */
zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
while( zCsr<zEnd ){ while( zCsr<zEnd ){
int nSuffix; /* Size of term suffix */ int nSuffix; /* Size of term suffix */
int nPrefix = 0; /* Size of term prefix */ int nPrefix = 0; /* Size of term prefix */
int nBuffer; /* Total term size */ int nBuffer; /* Total term size */
int nMin; /* Minimum of nBuffer and nTerm */ int nMin; /* Minimum of nBuffer and nTerm */
int cmp; /* Result of comparing term and buffer */
/* Load the next term on the node into zBuffer */
/* Load the next term on the node into zBuffer */ if( zBuffer ){
if( zBuffer ){ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
}
zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
if( nPrefix+nSuffix>nAlloc ){
char *zNew;
nAlloc = (nPrefix+nSuffix) * 2;
zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
if( !zNew ){
sqlite3_free(zBuffer);
return SQLITE_NOMEM;
} }
zBuffer = zNew; zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
} if( nPrefix+nSuffix>nAlloc ){
memcpy(&zBuffer[nPrefix], zCsr, nSuffix); char *zNew;
nBuffer = nPrefix + nSuffix; nAlloc = (nPrefix+nSuffix) * 2;
zCsr += nSuffix; zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
if( !zNew ){
sqlite3_free(zBuffer);
return SQLITE_NOMEM;
}
zBuffer = zNew;
}
memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
nBuffer = nPrefix + nSuffix;
zCsr += nSuffix;
/* Compare the term we are searching for with the term just loaded from
** the interior node. If the specified term is greater than or equal
** to the term from the interior node, then all terms on the sub-tree
** headed by node iChild are smaller than zTerm. No need to search
** iChild.
**
** If the interior node term is larger than the specified term, then
** the tree headed by iChild may contain the specified term.
*/
nMin = (nBuffer>nTerm ? nTerm : nBuffer);
if( memcmp(zTerm, zBuffer, nMin)<0 ) break;
iChild++;
};
/* Compare the term we are searching for with the term just loaded from /* If (iHeight==1), the children of this interior node are leaves. The
** the interior node. If variable cmp is greater than or equal to zero, ** specified term may be present on leaf node iChild.
** then all terms on the sub-tree headed by node iChild are smaller than
** zTerm. No need to search iChild.
**
** If variable cmp is less than zero, then the sub-tree headed by
*/ */
nMin = (nBuffer>nTerm ? nTerm : nBuffer); if( iHeight==1 ){
cmp = memcmp(zTerm, zBuffer, nMin); *piLeaf = iChild;
if( isPrefix && cmp==0 && iStart==0 ){ break;
iStart = iChild;
} }
if( cmp<0 ) break;
iChild++;
};
iEnd = iChild;
if( iStart==0 ) iStart = iChild;
sqlite3_free(zBuffer);
if( iHeight==1 ){ /* Descend to interior node iChild. */
if( piEnd ) *piEnd = iEnd; rc = fts3ReadBlock(p, iChild, &zCsr, &nBlock);
if( piStart ) *piStart = iStart; if( rc!=SQLITE_OK ) break;
}else{ zEnd = &zCsr[nBlock];
char *zBlock;
int nBlock;
if( piEnd ){
rc = fts3ReadBlock(p, iEnd, &zBlock, &nBlock);
if( rc==SQLITE_OK ){
rc = fts3SelectLeaves(p,zTerm,nTerm,isPrefix,zBlock,nBlock,0,piEnd);
}
}
if( piStart && rc==SQLITE_OK ){
rc = fts3ReadBlock(p, iStart, &zBlock, &nBlock);
if( rc==SQLITE_OK ){
rc = fts3SelectLeaves(p,zTerm,nTerm,isPrefix,zBlock,nBlock,piStart,0);
}
}
} }
sqlite3_free(zBuffer);
return rc; return rc;
} }
@ -1409,7 +1398,6 @@ static int fts3DoclistMerge(
return SQLITE_NOMEM; return SQLITE_NOMEM;
} }
} }
(mergetype==MERGE_NEAR ? 0 : &p);
while( p1 && p2 ){ while( p1 && p2 ){
if( i1==i2 ){ if( i1==i2 ){
@ -1444,58 +1432,54 @@ static int fts3DoclistMerge(
return SQLITE_OK; return SQLITE_OK;
} }
/*
** A pointer to an instance of this structure is used as the context
** argument to sqlite3Fts3SegReaderIterate()
*/
typedef struct TermSelect TermSelect; typedef struct TermSelect TermSelect;
struct TermSelect { struct TermSelect {
char const *zTerm;
int nTerm;
int isPrefix;
int isReqPos; int isReqPos;
char *aOutput; /* Malloc'd output buffer */ char *aOutput; /* Malloc'd output buffer */
int nOutput; /* Size of output in bytes */ int nOutput; /* Size of output in bytes */
}; };
static int fts3TermSelectCb( static int fts3TermSelectCb(
Fts3Table *p, Fts3Table *p, /* Virtual table object */
void *pContext, void *pContext, /* Pointer to TermSelect structure */
char *zTerm, char *zTerm,
int nTerm, int nTerm,
char *aDoclist, char *aDoclist,
int nDoclist int nDoclist
){ ){
TermSelect *pTS = (TermSelect *)pContext; TermSelect *pTS = (TermSelect *)pContext;
int nNew = pTS->nOutput + nDoclist;
if( (pTS->nTerm==nTerm || (pTS->isPrefix && pTS->nTerm<nTerm)) char *aNew = sqlite3_malloc(nNew);
&& 0==memcmp(zTerm, pTS->zTerm, pTS->nTerm) if( !aNew ){
){ return SQLITE_NOMEM;
int nNew = pTS->nOutput + nDoclist;
char *aNew = sqlite3_malloc(nNew);
if( !aNew ){
return SQLITE_NOMEM;
}
if( pTS->nOutput==0 ){
/* If this is the first term selected, copy the doclist to the output
** buffer using memcpy(). TODO: Add a way to transfer control of the
** aDoclist buffer from the caller so as to avoid the memcpy().
*/
memcpy(aNew, aDoclist, nDoclist);
}else{
/* The output buffer is not empty. Merge doclist aDoclist with the
** existing output. This can only happen with prefix-searches (as
** searches for exact terms return exactly one doclist).
*/
int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
assert( pTS->isPrefix );
fts3DoclistMerge(mergetype, 0, 0,
aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
);
}
sqlite3_free(pTS->aOutput);
pTS->aOutput = aNew;
pTS->nOutput = nNew;
} }
if( pTS->nOutput==0 ){
/* If this is the first term selected, copy the doclist to the output
** buffer using memcpy(). TODO: Add a way to transfer control of the
** aDoclist buffer from the caller so as to avoid the memcpy().
*/
memcpy(aNew, aDoclist, nDoclist);
}else{
/* The output buffer is not empty. Merge doclist aDoclist with the
** existing output. This can only happen with prefix-searches (as
** searches for exact terms return exactly one doclist).
*/
int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
fts3DoclistMerge(mergetype, 0, 0,
aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
);
}
sqlite3_free(pTS->aOutput);
pTS->aOutput = aNew;
pTS->nOutput = nNew;
return SQLITE_OK; return SQLITE_OK;
} }
@ -1522,13 +1506,13 @@ static int fts3TermSelect(
){ ){
int i; int i;
TermSelect tsc; TermSelect tsc;
Fts3SegFilter filter; /* Segment term filter configuration */
Fts3SegReader **apSegment = 0; /* Array of segments to read data from */ Fts3SegReader **apSegment = 0; /* Array of segments to read data from */
int nSegment = 0; /* Size of apSegment array */ int nSegment = 0; /* Size of apSegment array */
int nAlloc = 0; /* Allocated size of segment array */ int nAlloc = 0; /* Allocated size of segment array */
int rc; /* Return code */ int rc; /* Return code */
sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */ sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */
int iAge = 0; /* Used to assign ages to segments */ int iAge = 0; /* Used to assign ages to segments */
int flags;
/* Loop through the entire %_segdir table. For each segment, create a /* Loop through the entire %_segdir table. For each segment, create a
** Fts3SegReader to iterate through the subset of the segment leaves ** Fts3SegReader to iterate through the subset of the segment leaves
@ -1552,10 +1536,10 @@ static int fts3TermSelect(
*/ */
rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
}else{ }else{
sqlite3_int64 i1, i2; sqlite3_int64 i1;
rc = fts3SelectLeaves(p, zTerm, nTerm, isPrefix, zRoot, nRoot, &i1, &i2); rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
if( rc==SQLITE_OK ){ if( rc==SQLITE_OK ){
assert( i1 && i2 ); sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 3);
rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
} }
} }
@ -1585,16 +1569,17 @@ static int fts3TermSelect(
} }
memset(&tsc, 0, sizeof(TermSelect)); memset(&tsc, 0, sizeof(TermSelect));
tsc.zTerm = zTerm;
tsc.nTerm = nTerm;
tsc.isPrefix = isPrefix;
tsc.isReqPos = isReqPos; tsc.isReqPos = isReqPos;
flags = FTS3_SEGMENT_IGNORE_EMPTY filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
| (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, flags, filter.iCol = iColumn;
iColumn, fts3TermSelectCb, (void *)&tsc filter.zTerm = zTerm;
filter.nTerm = nTerm;
rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
fts3TermSelectCb, (void *)&tsc
); );
if( rc==SQLITE_OK ){ if( rc==SQLITE_OK ){

26
ext/fts3/fts3Int.h
View File

@ -58,6 +58,7 @@ typedef struct Fts3Cursor Fts3Cursor;
typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegFilter Fts3SegFilter;
/* /*
** A connection to a fulltext index is an instance of the following ** A connection to a fulltext index is an instance of the following
@ -123,14 +124,14 @@ struct Fts3Cursor {
** of tokens in the string. ** of tokens in the string.
*/ */
struct Fts3Phrase { struct Fts3Phrase {
int nToken; /* Number of tokens in the phrase */ int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */ int iColumn; /* Index of column this phrase must match */
int isNot; /* Phrase prefixed by unary not (-) operator */ int isNot; /* Phrase prefixed by unary not (-) operator */
struct PhraseToken { struct PhraseToken {
char *z; /* Text of the token */ char *z; /* Text of the token */
int n; /* Number of bytes in buffer pointed to by z */ int n; /* Number of bytes in buffer pointed to by z */
int isPrefix; /* True if token ends in with a "*" character */ int isPrefix; /* True if token ends in with a "*" character */
} aToken[1]; /* One entry for each token in the phrase */ } aToken[1]; /* One entry for each token in the phrase */
}; };
/* /*
@ -178,12 +179,21 @@ int sqlite3Fts3Optimize(Fts3Table *);
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
#define FTS3_SEGMENT_PREFIX 0x00000008
struct Fts3SegFilter {
const char *zTerm;
int nTerm;
int iCol;
int flags;
};
int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *); void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3SegReaderIterate( int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, int, int, Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void * int (*)(Fts3Table *, void *, char *, int, char *, int), void *
); );

95
ext/fts3/fts3_write.c
View File

@ -873,6 +873,33 @@ static int fts3SegReaderCmp2(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
return rc; return rc;
} }
/*
** Compare the term that the Fts3SegReader object passed as the first argument
** points to with the term specified by arguments zTerm and nTerm.
**
** If the pSeg iterator is already at EOF, return 0. Otherwise, return
** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
*/
static int fts3SegReaderTermCmp(
Fts3SegReader *pSeg, /* Segment reader object */
const char *zTerm, /* Term to compare to */
int nTerm /* Size of term zTerm in bytes */
){
int res = 0;
if( pSeg->aNode ){
if( pSeg->nTerm>nTerm ){
res = memcmp(pSeg->zTerm, zTerm, nTerm);
}else{
res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
}
if( res==0 ){
res = pSeg->nTerm-nTerm;
}
}
return res;
}
/* /*
** Argument apSegment is an array of nSegment elements. It is known that ** Argument apSegment is an array of nSegment elements. It is known that
** the final (nSegment-nSuspect) members are already in sorted order ** the final (nSegment-nSuspect) members are already in sorted order
@ -1427,8 +1454,9 @@ static void fts3ColumnFilter(int iCol, char **ppList, int *pnList){
int nList = *pnList; int nList = *pnList;
char *pEnd = &pList[nList]; char *pEnd = &pList[nList];
int iCurrent = 0; int iCurrent = 0;
char *p = pList; char *p = pList;
assert( iCol>=0 );
while( 1 ){ while( 1 ){
char c = 0; char c = 0;
while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80; while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
@ -1467,8 +1495,7 @@ int sqlite3Fts3SegReaderIterate(
Fts3Table *p, /* Virtual table handle */ Fts3Table *p, /* Virtual table handle */
Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */ Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */
int nSegment, /* Size of apSegment array */ int nSegment, /* Size of apSegment array */
int flags, /* Flags mask */ Fts3SegFilter *pFilter, /* Restrictions on range of iteration */
int iCol, /* Column to filter for */
int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */ int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */
void *pContext /* Callback context (2nd argument) */ void *pContext /* Callback context (2nd argument) */
){ ){
@ -1477,9 +1504,28 @@ int sqlite3Fts3SegReaderIterate(
int nAlloc = 0; /* Allocated size of aBuffer buffer */ int nAlloc = 0; /* Allocated size of aBuffer buffer */
int rc = SQLITE_OK; /* Return code */ int rc = SQLITE_OK; /* Return code */
int isIgnoreEmpty = (flags&FTS3_SEGMENT_IGNORE_EMPTY); int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
int isRequirePos = (flags&FTS3_SEGMENT_REQUIRE_POS); int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
int isColFilter = (flags&FTS3_SEGMENT_COLUMN_FILTER); int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX);
/* If the Fts3SegFilter defines a specific term (or term prefix) to search
** for, then advance each segment iterator until it points to a term of
** equal or greater value than the specified term. This prevents many
** unnecessary merge/sort operations for the case where single segment
** b-tree leaf nodes contain more than one term.
*/
if( pFilter->zTerm ){
int nTerm = pFilter->nTerm;
char *zTerm = pFilter->zTerm;
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = apSegment[i];
while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
rc = fts3SegReaderNext(pSeg);
if( rc!=SQLITE_OK ) goto finished;
}
}
}
fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
while( apSegment[0]->aNode ){ while( apSegment[0]->aNode ){
@ -1487,6 +1533,22 @@ int sqlite3Fts3SegReaderIterate(
char *zTerm = apSegment[0]->zTerm; char *zTerm = apSegment[0]->zTerm;
int nMerge = 1; int nMerge = 1;
/* If this is a prefix-search, and if the term that apSegment[0] points
** to does not share a suffix with pFilter->zTerm/nTerm, then all
** required callbacks have been made. In this case exit early.
**
** Similarly, if this is a search for an exact match, and the first term
** of segment apSegment[0] is not a match, exit early.
*/
if( pFilter->zTerm ){
if( nTerm<pFilter->nTerm
|| (!isPrefix && nTerm>pFilter->nTerm)
|| memcmp(zTerm, pFilter->zTerm, pFilter->nTerm)
){
goto finished;
}
}
while( nMerge<nSegment while( nMerge<nSegment
&& apSegment[nMerge]->aNode && apSegment[nMerge]->aNode
&& apSegment[nMerge]->nTerm==nTerm && apSegment[nMerge]->nTerm==nTerm
@ -1527,9 +1589,8 @@ int sqlite3Fts3SegReaderIterate(
j++; j++;
} }
assert( iCol>=0 || isColFilter==0 );
if( isColFilter ){ if( isColFilter ){
fts3ColumnFilter(iCol, &pList, &nList); fts3ColumnFilter(pFilter->iCol, &pList, &nList);
} }
if( !isIgnoreEmpty || nList>0 ){ if( !isIgnoreEmpty || nList>0 ){
@ -1562,6 +1623,14 @@ int sqlite3Fts3SegReaderIterate(
} }
} }
/* If there is a term specified to filter on, and this is not a prefix
** search, return now. The callback that corresponds to the required
** term (if such a term exists in the index) has already been made.
*/
if( pFilter->zTerm && !isPrefix ){
goto finished;
}
for(i=0; i<nMerge; i++){ for(i=0; i<nMerge; i++){
rc = fts3SegReaderNext(apSegment[i]); rc = fts3SegReaderNext(apSegment[i]);
if( rc!=SQLITE_OK ) goto finished; if( rc!=SQLITE_OK ) goto finished;
@ -1594,6 +1663,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
SegmentWriter *pWriter = 0; SegmentWriter *pWriter = 0;
int nSegment = 0; /* Number of segments being merged */ int nSegment = 0; /* Number of segments being merged */
Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */
Fts3SegFilter filter; /* Segment term filter condition */
if( iLevel<0 ){ if( iLevel<0 ){
/* This call is to merge all segments in the database to a single /* This call is to merge all segments in the database to a single
@ -1646,10 +1716,11 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
pStmt = 0; pStmt = 0;
if( rc!=SQLITE_OK ) goto finished; if( rc!=SQLITE_OK ) goto finished;
rc = sqlite3Fts3SegReaderIterate( memset(&filter, 0, sizeof(Fts3SegFilter));
p, apSegment, nSegment, filter.flags = FTS3_SEGMENT_REQUIRE_POS;
(iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0)|FTS3_SEGMENT_REQUIRE_POS, filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
0, fts3MergeCallback, (void *)&pWriter rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
&filter, fts3MergeCallback, (void *)&pWriter
); );
if( rc!=SQLITE_OK ) goto finished; if( rc!=SQLITE_OK ) goto finished;

18
manifest
View File

@ -1,5 +1,5 @@
C Add\sa\sfew\sextra\scoverage\stest\scases\sfor\sfts3. C Improvements\sto\sthe\sway\sfts3\sreads\sthe\sfull-text\sindex.
D 2009-11-16T16:36:23 D 2009-11-17T12:52:10
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in 53f3dfa49f28ab5b80cb083fb7c9051e596bcfa1 F Makefile.in 53f3dfa49f28ab5b80cb083fb7c9051e596bcfa1
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@ -56,9 +56,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 4d701e31cb32cfb9b535edcc33271d8bcb2fa76f F ext/fts3/fts3.c a72c19fa6270b5f88ad9b1215c821f7082164655
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h f8419da445790c0666d4b4d72dc15a07dd7ae93e F ext/fts3/fts3Int.h 5c040c0fb47ed81aaba589a55a7455c980592bea
F ext/fts3/fts3_expr.c bdf11f3602f62f36f0e42823680bf22033dae0de F ext/fts3/fts3_expr.c bdf11f3602f62f36f0e42823680bf22033dae0de
F ext/fts3/fts3_hash.c 1af1833a4d581ee8d668bb71f5a500f7a0104982 F ext/fts3/fts3_hash.c 1af1833a4d581ee8d668bb71f5a500f7a0104982
F ext/fts3/fts3_hash.h 39524725425078bf9e814e9569c74a8e5a21b9fb F ext/fts3/fts3_hash.h 39524725425078bf9e814e9569c74a8e5a21b9fb
@ -68,7 +68,7 @@ F ext/fts3/fts3_snippet.c 8ea9619247ac61c79aca650fc3307b8b4097b5f3
F ext/fts3/fts3_tokenizer.c 185a212670a9bbdeb5cad6942305e681bce5c87b F ext/fts3/fts3_tokenizer.c 185a212670a9bbdeb5cad6942305e681bce5c87b
F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
F ext/fts3/fts3_tokenizer1.c 0a5bcc579f35de5d24a9345d7908dc25ae403ee7 F ext/fts3/fts3_tokenizer1.c 0a5bcc579f35de5d24a9345d7908dc25ae403ee7
F ext/fts3/fts3_write.c 4285a2804ef308ed2eef946dae20d9d0361554d0 F ext/fts3/fts3_write.c edf123f978fca3d26707452a380fa169849eb655
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@ -398,7 +398,7 @@ F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F test/fts3malloc.test ed11f188d03560d7f44bd5c126ac004f011cc719 F test/fts3malloc.test cda1b22d8c86c3e434d93b63f2fc7a0191fb6d30
F test/fts3near.test dc196dd17b4606f440c580d45b3d23aa975fd077 F test/fts3near.test dc196dd17b4606f440c580d45b3d23aa975fd077
F test/func.test af106ed834001738246d276659406823e35cde7b F test/func.test af106ed834001738246d276659406823e35cde7b
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@ -772,7 +772,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P d3cae986ee1a176b1b015c3cebcd58ff0c3bdf92 P f29c8fcade4aadeae3824975cf59f306c11c906b
R 4deae89fec5a2f223cc1fc606b6691c8 R a419bf1bcdd810be07b904f21b0a1ef0
U dan U dan
Z 34f2a8a40095974f35e0328293748f2b Z 4e40dbade582f444c0b93ffcad88c385

2
manifest.uuid
View File

@ -1 +1 @@
f29c8fcade4aadeae3824975cf59f306c11c906b 45c051e78651d8204c17cecdda2bde705698881f

21
test/fts3malloc.test
View File

@ -181,17 +181,24 @@ do_write_test fts3_malloc-1.6 sqlite_master {
} }
# Test the xConnect/xDisconnect methods: # Test the xConnect/xDisconnect methods:
db eval { ATTACH 'test2.db' AS aux } #db eval { ATTACH 'test2.db' AS aux }
do_write_test fts3_malloc-1.6 aux.sqlite_master { #do_write_test fts3_malloc-1.6 aux.sqlite_master {
CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); # CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);
} #}
do_write_test fts3_malloc-1.6 aux.sqlite_master { #do_write_test fts3_malloc-1.6 aux.sqlite_master {
CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); # CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);
} #}
do_test fts3_malloc-2.0 { do_test fts3_malloc-2.0 {
execsql {
DROP TABLE ft1;
DROP TABLE ft2;
DROP TABLE ft3;
DROP TABLE ft4;
DROP TABLE ft6;
}
execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) } execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
for {set ii 1} {$ii < 32} {incr ii} { for {set ii 1} {$ii < 32} {incr ii} {
set a [list] set a [list]