From 4e76cc3650d51fb37915449e6f7b438d056aeced Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Oct 2010 18:56:04 +0000 Subject: [PATCH] Updates to FTS4 to improve performance and make more accurate cost estimates for prefix terms. FossilOrigin-Name: d0a450ce78e99f55c862f26f9332786660007a0a --- ext/fts3/fts3.c | 245 +++++++++++++++++++++++++--------------- ext/fts3/fts3Int.h | 4 + ext/fts3/fts3_snippet.c | 1 + ext/fts3/fts3_write.c | 100 +++++++++------- ext/fts3/fts3speed.tcl | 5 +- manifest | 36 +++--- manifest.uuid | 2 +- src/btree.c | 3 +- src/sqlite.h.in | 5 + src/test1.c | 46 ++++++++ src/vdbeblob.c | 180 +++++++++++++++++++---------- test/fts3ah.test | 6 +- test/fts3defer.test | 33 +++++- 13 files changed, 442 insertions(+), 224 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index bfd0f57ee7..752d0e9bcf 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -926,8 +926,9 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ -static int fulltextClose(sqlite3_vtab_cursor *pCursor){ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); @@ -969,6 +970,84 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ } +static int fts3ScanInteriorNode( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + + int isFirstTerm = 1; /* True when processing first term on page */ + int dummy; + sqlite3_int64 iChild; /* Block id of child node to descend to */ + int nBlock; /* Size of child node in bytes */ + + zCsr += sqlite3Fts3GetVarint32(zCsr, &dummy); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + + while( zCsrnAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + sqlite3_free(zBuffer); + return SQLITE_NOMEM; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + sqlite3_free(zBuffer); + return rc; +} /* @@ -993,77 +1072,33 @@ static int fts3SelectLeaf( int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf /* Selected leaf node */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ ){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ + int rc; /* Return code */ + int iHeight; /* Height of this node in tree */ - while( 1 ){ - int isFirstTerm = 1; /* True when processing first term on page */ - int iHeight; /* Height of this node in tree */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ - int nBlock; /* Size of child node in bytes */ + sqlite3Fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + + if( rc==SQLITE_OK && iHeight>1 ){ + const char *zBlob; + int nBlob; - zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - - while( zCsrnAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - sqlite3_free(zBuffer); - return SQLITE_NOMEM; - } - zBuffer = zNew; - } - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; - iChild++; - }; - - /* If (iHeight==1), the children of this interior node are leaves. The - ** specified term may be present on leaf node iChild. - */ - if( iHeight==1 ){ - *piLeaf = iChild; - break; + piLeaf = 0; } - /* Descend to interior node iChild. */ - rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); - if( rc!=SQLITE_OK ) break; - zEnd = &zCsr[nBlock]; + rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } } - sqlite3_free(zBuffer); + return rc; } @@ -1506,7 +1541,8 @@ static int fts3DoclistMerge( char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ - int n2 /* Size of buffer a2 */ + int n2, /* Size of buffer a2 */ + int *pnDoc /* OUT: Number of docids in output */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -1517,6 +1553,7 @@ static int fts3DoclistMerge( char *p2 = a2; char *pEnd1 = &a1[n1]; char *pEnd2 = &a2[n2]; + int nDoc = 0; assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT @@ -1560,6 +1597,7 @@ static int fts3DoclistMerge( fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); + nDoc++; }else if( i1aaOutput[i], pTS->anOutput[i], aOut, nOut + aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 ); sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); @@ -1763,8 +1804,8 @@ static int fts3TermSelectCb( } return SQLITE_NOMEM; } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge + fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, + pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 ); if( iOut>0 ) sqlite3_free(aMerge); @@ -1905,10 +1946,11 @@ static int fts3TermSegReaderArray( rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); }else{ int rc2; /* Return value of sqlite3Fts3ReadBlock() */ - sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */ - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); + sqlite3_int64 i1; /* First leaf that may contain zTerm */ + sqlite3_int64 i2; /* Last leaf that may contain zTerm */ + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); + if( isPrefix==0 ) i2 = i1; if( rc==SQLITE_OK ){ - sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); } @@ -2008,17 +2050,34 @@ static int fts3TermSelect( return rc; } +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ int nDoc = 0; /* Return value */ + if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ - sqlite3_int64 dummy; /* For Fts3GetVarint() */ - - while( pdoDeferred==0 && pPhrase->nToken>1 ){ + nDoc = fts3DoclistCountDocids(1, pOut, nOut); + } }else{ /* Merge the new term list and the current output. */ char *aLeft, *aRight; @@ -2189,14 +2251,14 @@ static int fts3PhraseSelect( nDist = iPrevTok-iTok; } pOut = aRight; - - fts3DoclistMerge(mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight); + fts3DoclistMerge( + mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc + ); sqlite3_free(aLeft); } assert( nOut==0 || pOut!=0 ); iPrevTok = iTok; - nDoc = fts3DoclistCountDocids(ii<(pPhrase->nToken-1), pOut, nOut); } if( rc==SQLITE_OK ){ @@ -2234,7 +2296,7 @@ static int fts3NearMerge( rc = SQLITE_NOMEM; }else{ rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight + aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 ); if( rc!=SQLITE_OK ){ sqlite3_free(aOut); @@ -2442,13 +2504,15 @@ static int fts3EvalExpr( if( ii==0 ){ aRet = aNew; nRet = nNew; + if( nExpr>1 ){ + nDoc = fts3DoclistCountDocids(0, aRet, nRet); + } }else{ fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew + MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc ); sqlite3_free(aNew); } - nDoc = fts3DoclistCountDocids(0, aRet, nRet); } } } @@ -2507,7 +2571,7 @@ static int fts3EvalExpr( */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aBuffer; sqlite3_free(aLeft); @@ -2517,7 +2581,7 @@ static int fts3EvalExpr( default: { assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aLeft; break; @@ -2597,7 +2661,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ return rc; } - /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -2641,6 +2704,7 @@ static int fts3FilterMethod( assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + assert( p->pSegments==0 ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); @@ -2671,6 +2735,7 @@ static int fts3FilterMethod( if( rc!=SQLITE_OK ) return rc; rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; @@ -2782,7 +2847,9 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + return rc; } /* @@ -3116,7 +3183,7 @@ static const sqlite3_module fts3Module = { /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, - /* xClose */ fulltextClose, + /* xClose */ fts3CloseMethod, /* xFilter */ fts3FilterMethod, /* xNext */ fts3NextMethod, /* xEof */ fts3EofMethod, diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index ef00377a7f..e905dfc716 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -130,6 +130,8 @@ struct Fts3Table { u8 bHasContent; /* True if %_content table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. @@ -287,6 +289,8 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); +void sqlite3Fts3SegmentsClose(Fts3Table *); + /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 4f3d9ec3e8..21f858545d 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -268,6 +268,7 @@ static int fts3ExprLoadDoclists( } if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; + sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab); return rc; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 92eaf4dc93..b54ece3365 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -807,29 +807,51 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** The %_segments table is declared as follows: ** ** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) -** -** This function opens a read-only blob handle on the "block" column of -** row iSegment of the %_segments table associated with FTS3 table p. -** -** If all goes well, SQLITE_OK is returned and *ppBlob set to the -** read-only blob handle. It is the responsibility of the caller to call -** sqlite3_blob_close() on the blob handle. Or, if an error occurs, an -** SQLite error code is returned and *ppBlob is either not modified or -** set to 0. */ -static int fts3OpenSegmentsBlob( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iSegment, /* Rowid in %_segments table */ - sqlite3_blob **ppBlob /* OUT: Read-only blob handle */ +static int fts3SegmentsBlob( + Fts3Table *p, + sqlite3_int64 iSegment, + char **paBlob, + int *pnBlob ){ - if( 0==p->zSegmentsTbl - && 0==(p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName)) - ) { - return SQLITE_NOMEM; + int rc; + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iSegment); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, &p->pSegments + ); } - return sqlite3_blob_open( - p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, ppBlob - ); + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + *pnBlob = nByte; + } + + return rc; +} + +void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; } @@ -880,17 +902,9 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ return SQLITE_OK; } - rc = fts3OpenSegmentsBlob(p, ++pReader->iCurrentBlock, &pBlob); - if( rc==SQLITE_OK ){ - pReader->nNode = sqlite3_blob_bytes(pBlob); - pReader->aNode = (char *)sqlite3_malloc(pReader->nNode); - if( pReader->aNode ){ - rc = sqlite3_blob_read(pBlob, pReader->aNode, pReader->nNode, 0); - }else{ - rc = SQLITE_NOMEM; - } - sqlite3_blob_close(pBlob); - } + rc = fts3SegmentsBlob( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + ); if( rc!=SQLITE_OK ){ return rc; @@ -1008,7 +1022,8 @@ int sqlite3Fts3SegReaderCost( && !fts3SegReaderIsPending(pReader) && !fts3SegReaderIsRootOnly(pReader) ){ - sqlite3_blob *pBlob = 0; + int nBlob = 0; + sqlite3_int64 iBlock; if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost @@ -1045,20 +1060,18 @@ int sqlite3Fts3SegReaderCost( if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; } - rc = fts3OpenSegmentsBlob(p, pReader->iStartBlock, &pBlob); - if( rc==SQLITE_OK ){ - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - int nBlob = sqlite3_blob_bytes(pBlob); + /* Assume that a blob flows over onto overflow pages if it is larger + ** than (pgsz-35) bytes in size (the file-format documentation + ** confirms this). + */ + for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ + rc = fts3SegmentsBlob(p, iBlock, 0, &nBlob); + if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ int nOvfl = (nBlob + 34)/pgsz; nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); } } - assert( rc==SQLITE_OK || pBlob==0 ); - sqlite3_blob_close(pBlob); } *pnCost += nCost; @@ -1096,6 +1109,7 @@ int sqlite3Fts3SegReaderNew( Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ + assert( iStartLeaf<=iEndLeaf ); if( iStartLeaf==0 ){ nExtra = nRoot; } @@ -2619,6 +2633,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -2787,6 +2802,7 @@ int sqlite3Fts3UpdateMethod( u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + assert( p->pSegments==0 ); /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 ); @@ -2842,6 +2858,7 @@ int sqlite3Fts3UpdateMethod( } sqlite3_free(aSzIns); + sqlite3Fts3SegmentsClose(p); return rc; } @@ -2865,6 +2882,7 @@ int sqlite3Fts3Optimize(Fts3Table *p){ sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } + sqlite3Fts3SegmentsClose(p); return rc; } diff --git a/ext/fts3/fts3speed.tcl b/ext/fts3/fts3speed.tcl index bf420aacb9..377cb19606 100644 --- a/ext/fts3/fts3speed.tcl +++ b/ext/fts3/fts3speed.tcl @@ -18,7 +18,7 @@ set VOCAB_SIZE 2000 set DOC_SIZE 100 -set NUM_INSERTS 1000 +set NUM_INSERTS 100000 set NUM_SELECTS 1000 # Force everything in this script to be deterministic. @@ -74,8 +74,7 @@ proc test_1 {nInsert} { sql "CREATE VIRTUAL TABLE t1 USING fts4;" for {set i 0} {$i < $nInsert} {incr i} { set doc [select_doc $::DOC_SIZE] - #sql "INSERT INTO t1 VALUES('$doc');" - sql "\"$doc\"" + sql "INSERT INTO t1 VALUES('$doc');" } } diff --git a/manifest b/manifest index 3a7cae981d..2d3d7d30ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\sto\sfts4\sto\stry\sto\sselectively\savoid\sloading\svery\slarge\sdoclists. -D 2010-10-19T14:08:00 +C Updates\sto\sFTS4\sto\simprove\sperformance\sand\smake\smore\saccurate\scost\sestimates\sfor\sprefix\sterms. +D 2010-10-20T18:56:04 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b01fdfcfecf8a0716c29867a67959f6148b79961 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,20 +61,20 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 9d4ccf3b7bbfbeeef03dba91377c4d72b757dcb9 +F ext/fts3/fts3.c ce7bcd1f42e74912149fe6201fc63a6ac0db42a8 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h a640e4fbdb2fcab1457f87993ca3f4ceaa31e776 +F ext/fts3/fts3Int.h 9fbe422f7d0e005371702acaa3cd44283a67c389 F ext/fts3/fts3_expr.c a5aee50edde20e5c9116199bd58be869a3a22c9f F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba -F ext/fts3/fts3_snippet.c 474c11e718610cade73e6009f75ffc173d4c42c5 +F ext/fts3/fts3_snippet.c ca60a2a47de5e7abb22a804ccd1a743f81c2fe3e F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 29b63a98de55d4eb34b7fc6fd90b3224d6cdc7ff -F ext/fts3/fts3speed.tcl 71b9cdc8f499822124a9eef42003e31a88f26f16 +F ext/fts3/fts3_write.c be47d30cf80bc91e050ece18e2de7e207432be1a +F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c 850e9a36567bbcce6bd85a4b68243cad8e3c2de2 @@ -119,7 +119,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c d5b0137bc20327af08c14772227cc35134839c30 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 8a1b0267a4f1914aedbaa93d3fcf4f2e42141ea8 +F src/btree.c 3edab36d03d86c200cb9551467410f975d510aa9 F src/btree.h 2d1a83ad509047e8cc314fda7e054f99ff52414d F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0 F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d @@ -176,14 +176,14 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 6a5c72fb0e8dc7f6133f5a9d7a747130ef0a00ea F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056 -F src/sqlite.h.in 13f219b9ab78f22603019fd193f09d5c8913795a +F src/sqlite.h.in 460599b35c035deb339d1c9933089ef32187ecc6 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 F src/sqliteInt.h c63b0340dfdfde18ff255ddccf004edd2d073288 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e91019fb6787166abca23a81b16c07fecc2ed751 -F src/test1.c cbedc6ea7905b1361db054fbf7fcd0dafb6d844e +F src/test1.c f6e39615c8315e03798217a360810e4c59595627 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 056093cfef69ff4227a6bdb9108564dc7f45e4bc F src/test4.c 0528360b5025688002a5feb6be906ddce52eaaee @@ -231,7 +231,7 @@ F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2 F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4 F src/vdbeapi.c 5368714fa750270cf6430160287c21adff44582d F src/vdbeaux.c de0b06b11a25293e820a49159eca9f1c51a64716 -F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 +F src/vdbeblob.c c8cbe6ce28cc8bf806ea0818b5167dd9a27c48a3 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 6c90e3e65b2f026fc54703a8f3c917155f419d87 @@ -420,7 +420,7 @@ F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49 F test/fts3ae.test ce32a13b34b0260928e4213b4481acf801533bda F test/fts3af.test d394978c534eabf22dd0837e718b913fd66b499c F test/fts3ag.test 0b7d303f61ae5d620c4efb5e825713ea34ff9441 -F test/fts3ah.test 3c5a1bd49979d7b5b5ed9fdbcdd14a7bfe5a5ff9 +F test/fts3ah.test dc9f66c32c296f1bc8bcc4535126bddfeca62894 F test/fts3ai.test d29cee6ed653e30de478066881cec8aa766531b2 F test/fts3aj.test 584facbc9ac4381a7ec624bfde677340ffc2a5a4 F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d @@ -433,7 +433,7 @@ F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3cov.test 6f1ff88ff6b5abcfff6979098cb9d0c68a69202e F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test a9f81bba6e1132dd6a2ad3cf11e4628733975c8c +F test/fts3defer.test cf66bf69afcc2fb8373d3aed31c55399409e83f2 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a @@ -875,11 +875,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c0ee614fd988f445c4884a37f494479bdd669185 -R 0c4aebbf44d624104504e37bec992917 -T *bgcolor * #c0ffc0 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P 5ae0ba447a561e3b6637b52f9b83a9fc683d2572 +R acd75bd5094c2beea7f56f4633b47ea2 U dan -Z c2b99a58ccad27405ff8b0fcedef5c33 +Z 910cba31e6572b93e93d39c91f91b9da diff --git a/manifest.uuid b/manifest.uuid index ee8a49e8ae..ab22dd85a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ae0ba447a561e3b6637b52f9b83a9fc683d2572 \ No newline at end of file +d0a450ce78e99f55c862f26f9332786660007a0a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f9c368a93d..7e8c39feb6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8096,8 +8096,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ void sqlite3BtreeCacheOverflow(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert(!pCur->isIncrblobHandle); - assert(!pCur->aOverflow); + invalidateOverflowCache(pCur); pCur->isIncrblobHandle = 1; } #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e827e828bc..5e32c10c7c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4790,6 +4790,11 @@ int sqlite3_blob_open( sqlite3_blob **ppBlob ); +/* +** CAPI3REF: Move a BLOB Handle +*/ +SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + /* ** CAPI3REF: Close A BLOB Handle ** diff --git a/src/test1.c b/src/test1.c index 2cf8c9764a..6ea6e82975 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1703,6 +1703,51 @@ static int test_blob_write( return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); } + +static int test_blob_reopen( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + Tcl_WideInt iRowid; + Tcl_Channel channel; + ClientData instanceData; + sqlite3_blob *pBlob; + int notUsed; + int rc; + + unsigned char *zBuf; + int nBuf; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID"); + return TCL_ERROR; + } + + channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), ¬Used); + if( !channel || TCL_OK!=Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ){ + return TCL_ERROR; + } + + if( TCL_OK!=(rc = Tcl_Flush(channel)) ){ + return rc; + } + if( TCL_OK!=(rc = Tcl_Seek(channel, 0, SEEK_SET)) ){ + return rc; + } + + instanceData = Tcl_GetChannelInstanceData(channel); + pBlob = *((sqlite3_blob **)instanceData); + + rc = sqlite3_blob_reopen(pBlob, iRowid); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + } + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + #endif /* @@ -5328,6 +5373,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifndef SQLITE_OMIT_INCRBLOB { "sqlite3_blob_read", test_blob_read, 0 }, { "sqlite3_blob_write", test_blob_write, 0 }, + { "sqlite3_blob_reopen", test_blob_reopen, 0 }, #endif { "pcache_stats", test_pcache_stats, 0 }, #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY diff --git a/src/vdbeblob.c b/src/vdbeblob.c index b2b9f0ed00..f7ee670f26 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -26,11 +26,61 @@ struct Incrblob { int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ + int iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ }; + +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + v->aVar[0].u.i = iRow; + rc = sqlite3_step(p->pStmt); + + if( rc==SQLITE_ROW ){ + Vdbe *v = (Vdbe *)p->pStmt; + u32 type = v->apCsr[0]->aType[p->iCol]; + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = v->apCsr[0]->aOffset[p->iCol]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = v->apCsr[0]->pCursor; + sqlite3BtreeEnterCursor(p->pCsr); + sqlite3BtreeCacheOverflow(p->pCsr); + sqlite3BtreeLeaveCursor(p->pCsr); + } + } + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; +} + /* ** Open a blob handle. */ @@ -71,11 +121,12 @@ int sqlite3_blob_open( {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ - {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */ + {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ {OP_Column, 0, 0, 1}, /* 7 */ {OP_ResultRow, 1, 0, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ + {OP_Goto, 0, 5, 0}, /* 9 */ + {OP_Close, 0, 0, 0}, /* 10 */ + {OP_Halt, 0, 0, 0}, /* 11 */ }; Vdbe *v = 0; @@ -83,14 +134,20 @@ int sqlite3_blob_open( char *zErr = 0; Table *pTab; Parse *pParse; + Incrblob *pBlob; + flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; + if( pParse==0 || pBlob==0 ){ + assert( db->mallocFailed ); goto blob_open_out; } + do { memset(pParse, 0, sizeof(Parse)); pParse->db = db; @@ -177,7 +234,6 @@ int sqlite3_blob_open( if( v ){ int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); - flags = !!flags; /* flags = (flags ? 1 : 0); */ /* Configure the OP_Transaction */ sqlite3VdbeChangeP1(v, 0, iDb); @@ -220,65 +276,30 @@ int sqlite3_blob_open( } } - sqlite3BtreeLeaveAll(db); - if( db->mallocFailed ){ - goto blob_open_out; - } - - sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow); - rc = sqlite3_step((sqlite3_stmt *)v); - if( rc!=SQLITE_ROW ){ - nAttempt++; - rc = sqlite3_finalize((sqlite3_stmt *)v); - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, sqlite3_errmsg(db)); - v = 0; - } - } while( nAttempt<5 && rc==SQLITE_SCHEMA ); - - if( rc==SQLITE_ROW ){ - /* The row-record has been opened successfully. Check that the - ** column in question contains text or a blob. If it contains - ** text, it is up to the caller to get the encoding right. - */ - Incrblob *pBlob; - u32 type = v->apCsr[0]->aType[iCol]; - - if( type<12 ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "cannot open value of type %s", - type==0?"null": type==7?"real": "integer" - ); - rc = SQLITE_ERROR; - goto blob_open_out; - } - pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( db->mallocFailed ){ - sqlite3DbFree(db, pBlob); - goto blob_open_out; - } pBlob->flags = flags; - pBlob->pCsr = v->apCsr[0]->pCursor; - sqlite3BtreeEnterCursor(pBlob->pCsr); - sqlite3BtreeCacheOverflow(pBlob->pCsr); - sqlite3BtreeLeaveCursor(pBlob->pCsr); pBlob->pStmt = (sqlite3_stmt *)v; - pBlob->iOffset = v->apCsr[0]->aOffset[iCol]; - pBlob->nByte = sqlite3VdbeSerialTypeLen(type); + pBlob->iCol = iCol; pBlob->db = db; - *ppBlob = (sqlite3_blob *)pBlob; - rc = SQLITE_OK; - }else if( rc==SQLITE_OK ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow); - rc = SQLITE_ERROR; - } + sqlite3BtreeLeaveAll(db); + v = 0; + if( db->mallocFailed ){ + goto blob_open_out; + } + + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); blob_open_out: - if( v && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(v); + if( rc==SQLITE_OK && db->mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( v ) sqlite3VdbeFinalize(v); + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, zErr); + + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); @@ -331,7 +352,7 @@ static int blobReadWrite( /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, 0); - } else if( v==0 ){ + }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. */ @@ -382,4 +403,43 @@ int sqlite3_blob_bytes(sqlite3_blob *pBlob){ return p ? p->nByte : 0; } +/* +** Move an existing blob handle to point to a different row of the same +** database table. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. +*/ +int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); + } + + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ diff --git a/test/fts3ah.test b/test/fts3ah.test index 6e8f2d541a..3810ec37b5 100644 --- a/test/fts3ah.test +++ b/test/fts3ah.test @@ -56,15 +56,15 @@ do_test fts3ah-1.2 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm} } {1 3} -do_test fts3ah-1.2 { +do_test fts3ah-1.3 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm} } {} -do_test fts3ah-1.3 { +do_test fts3ah-1.4 { execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'" } {1 3} -do_test fts3ah-1.4 { +do_test fts3ah-1.5 { execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'" } {1} diff --git a/test/fts3defer.test b/test/fts3defer.test index dd35be8e1e..a6de8ac546 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -18,6 +18,8 @@ ifcapable !fts3 { return } +set sqlite_fts3_enable_parentheses 1 + set ::testprefix fts3defer #-------------------------------------------------------------------------- @@ -187,7 +189,14 @@ lappend data {*}{ "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki" } +#set e [list] +#foreach d $data {set e [concat $e $d]} +#puts [lsort -unique $e] +#exit +set zero_long_doclists { + UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000 +} foreach {tn setup} { 1 { @@ -204,11 +213,14 @@ foreach {tn setup} { set dmt_modes {0 1 2} execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } - execsql { - UPDATE t1_segments - SET block = zeroblob(length(block)) - WHERE length(block)>10000; - } + execsql $zero_long_doclists + } + 4 { + set dmt_modes 0 + execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } + foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } + execsql "INSERT INTO t1(t1) VALUES('optimize')" + execsql $zero_long_doclists } } { @@ -271,6 +283,17 @@ foreach {tn setup} { SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"' } {13 17} + do_select_test 2.8 { + SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + do_select_test 2.9 { + SELECT rowid FROM t1 + WHERE t1 MATCH '( + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR + zk OR zkhdvkw OR zm OR zsmhnf + ) vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + do_select_test 3.1 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"' } {