FTS changes: Remove unreachable code. Fix bugs. When processing a large doclist incrementally, read from disk incrementally too.

FossilOrigin-Name: a4c7e2820824e82580730c36f85aede2efa66754

ext/fts3/fts3Int.h

@@ -230,14 +230,13 @@ struct Fts3Cursor {
   u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
   sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
   Fts3Expr *pExpr;                /* Parsed MATCH query string */
-  int bIncremental;               /* True to use incremental querying */
   int nPhrase;                    /* Number of matchable phrases in query */
   Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   char *pNextId;                  /* Pointer into the body of aDoclist */
   char *aDoclist;                 /* List of docids for full-text queries */
   int nDoclist;                   /* Size of buffer at aDoclist */
-  int desc;                       /* True to sort in descending order */
+  int bDesc;                      /* True to sort in descending order */
   int eEvalmode;                  /* An FTS3_EVAL_XX constant */
   int nRowAvg;                    /* Average size of database rows, in pages */
 
@@ -274,9 +273,10 @@ struct Fts3Cursor {
 struct Fts3Doclist {
   char *aAll;                    /* Array containing doclist (or NULL) */
   int nAll;                      /* Size of a[] in bytes */
-
-  sqlite3_int64 iDocid;          /* Current docid (if p!=0) */
   char *pNextDocid;              /* Pointer to next docid */
+
+  sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
+  int bFreeList;                 /* True if pList should be sqlite3_free()d */
   char *pList;                   /* Pointer to position list following iDocid */
   int nList;                     /* Length of position list */
 } doclist;
@@ -305,14 +305,6 @@ struct Fts3Phrase {
   Fts3Doclist doclist;
   int bIncr;                 /* True if doclist is loaded incrementally */
 
-#if 1
-  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
-  char *aDoclist;            /* Buffer containing doclist */
-  int nDoclist;              /* Size of aDoclist in bytes */
-  sqlite3_int64 iCurrent;
-  char *pCurrent;
-#endif
-
   /* Variables below this point are populated by fts3_expr.c when parsing 
   ** a MATCH expression. Everything above is part of the evaluation phase. 
   */
@@ -380,7 +372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *);
 int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
 int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
 int sqlite3Fts3ReadLock(Fts3Table *);
-int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
+int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
 
 int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
 int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
@@ -448,7 +440,6 @@ int sqlite3Fts3VarintLen(sqlite3_uint64);
 void sqlite3Fts3Dequote(char *);
 
 int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
-int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
 int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
 
 /* fts3_tokenizer.c */
@@ -493,13 +484,13 @@ int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*);
 void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
 
 int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
-int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr);
+int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
+
 int sqlite3Fts3MsrIncrStart(
     Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
 int sqlite3Fts3MsrIncrNext(
     Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
-char *sqlite3Fts3EvalPhrasePoslist(
+char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
-  Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol); 
 int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
 
 int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);

ext/fts3/fts3_expr.c

@@ -768,10 +768,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){
     assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
     sqlite3Fts3ExprFree(p->pLeft);
     sqlite3Fts3ExprFree(p->pRight);
-    if( p->pPhrase ){
+    sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
-      sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
-      sqlite3_free(p->pPhrase->aDoclist);
-    }
     sqlite3_free(p);
   }
 }

ext/fts3/fts3_snippet.c

@@ -176,51 +176,6 @@ static int fts3ExprIterate(
   return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
 }
 
-/*
-** The argument to this function is always a phrase node. Its doclist 
-** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
-** to the left of this one in the query tree have already been loaded.
-**
-** If this phrase node is part of a series of phrase nodes joined by 
-** NEAR operators (and is not the left-most of said series), then elements are
-** removed from the phrases doclist consistent with the NEAR restriction. If
-** required, elements may be removed from the doclists of phrases to the
-** left of this one that are part of the same series of NEAR operator 
-** connected phrases.
-**
-** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
-*/
-static int fts3ExprNearTrim(Fts3Expr *pExpr){
-  int rc = SQLITE_OK;
-  Fts3Expr *pParent = pExpr->pParent;
-
-  assert( pExpr->eType==FTSQUERY_PHRASE );
-  while( rc==SQLITE_OK
-   && pParent 
-   && pParent->eType==FTSQUERY_NEAR 
-   && pParent->pRight==pExpr 
-  ){
-    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
-    ** Find the expression to the left of the same operator.
-    */
-    int nNear = pParent->nNear;
-    Fts3Expr *pLeft = pParent->pLeft;
-
-    if( pLeft->eType!=FTSQUERY_PHRASE ){
-      assert( pLeft->eType==FTSQUERY_NEAR );
-      assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
-      pLeft = pLeft->pRight;
-    }
-
-    rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
-
-    pExpr = pLeft;
-    pParent = pExpr->pParent;
-  }
-
-  return rc;
-}
-
 /*
 ** This is an fts3ExprIterate() callback used while loading the doclists
 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
@@ -236,14 +191,6 @@ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   p->nPhrase++;
   p->nToken += pPhrase->nToken;
 
-  if( pPhrase->isLoaded==0 ){
-    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
-    pPhrase->isLoaded = 1;
-    if( rc==SQLITE_OK ){
-      rc = fts3ExprNearTrim(pExpr);
-    }
-  }
-
   return rc;
 }
 
@@ -416,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
 
   pPhrase->nToken = pExpr->pPhrase->nToken;
 
-  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol);
+  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
   if( pCsr ){
     int iFirst = 0;
     pPhrase->pList = pCsr;
@@ -867,11 +814,10 @@ static int fts3ExprLocalHitsCb(
   MatchInfo *p = (MatchInfo *)pCtx;
   int iStart = iPhrase * p->nCol * 3;
   int i;
-  sqlite3_int64 iDocid = p->pCursor->iPrevId;
 
   for(i=0; i<p->nCol; i++){
     char *pCsr;
-    pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i);
+    pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
     if( pCsr ){
       p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
     }else{
@@ -1020,7 +966,6 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
   int i;
   int iCol;
   int nToken = 0;
-  sqlite3_int64 iDocid = pCsr->iPrevId;
 
   /* Allocate and populate the array of LcsIterator objects. The array
   ** contains one element for each matchable phrase in the query.
@@ -1042,7 +987,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
 
     for(i=0; i<pInfo->nPhrase; i++){
       LcsIterator *pIt = &aIter[i];
-      pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol);
+      pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
       if( pIt->pRead ){
         pIt->iPos = pIt->iPosOffset;
         fts3LcsIteratorAdvance(&aIter[i]);
@@ -1396,7 +1341,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
   int iPos = 0;                   /* First position in position-list */
 
   UNUSED_PARAMETER(iPhrase);
-  pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol);
+  pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
   nTerm = pExpr->pPhrase->nToken;
   if( pList ){
     fts3GetDeltaPosition(&pList, &iPos);

ext/fts3/fts3_write.c

@@ -36,6 +36,27 @@
 */
 #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
 
+/*
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into
+** memory incrementally instead of all at once. This can be a big performance
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
+** method before retrieving all query results (as may happen, for example,
+** if a query has a LIMIT clause).
+**
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD 
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
+** The code is written so that the hard lower-limit for each of these values 
+** is 1. Clearly such small values would be inefficient, but can be useful 
+** for testing purposes.
+**
+** TODO: Add a test interface to modify these "constants" from a script for
+** this purpose.
+*/
+#define FTS3_NODE_CHUNKSIZE (4*1024) 
+#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
+/* #define FTS3_NODE_CHUNKSIZE 1 */
+/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */
+
 typedef struct PendingList PendingList;
 typedef struct SegmentNode SegmentNode;
 typedef struct SegmentWriter SegmentWriter;
@@ -93,6 +114,9 @@ struct Fts3SegReader {
 
   char *aNode;                    /* Pointer to node data (or NULL) */
   int nNode;                      /* Size of buffer at aNode (or 0) */
+  int nPopulate;                  /* If >0, bytes of buffer aNode[] loaded */
+  sqlite3_blob *pBlob;            /* If not NULL, blob handle to read node */
+
   Fts3HashElem **ppNextElem;
 
   /* Variables set by fts3SegReaderNext(). These may be read directly
@@ -933,7 +957,8 @@ int sqlite3Fts3ReadBlock(
   Fts3Table *p,                   /* FTS3 table handle */
   sqlite3_int64 iBlockid,         /* Access the row with blockid=$iBlockid */
   char **paBlob,                  /* OUT: Blob data in malloc'd buffer */
-  int *pnBlob                     /* OUT: Size of blob data */
+  int *pnBlob,                    /* OUT: Size of blob data */
+  int *pnLoad                     /* OUT: Bytes actually loaded */
 ){
   int rc;                         /* Return code */
 
@@ -954,11 +979,16 @@ int sqlite3Fts3ReadBlock(
 
   if( rc==SQLITE_OK ){
     int nByte = sqlite3_blob_bytes(p->pSegments);
+    *pnBlob = nByte;
     if( paBlob ){
       char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
       if( !aByte ){
         rc = SQLITE_NOMEM;
       }else{
+        if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
+          nByte = FTS3_NODE_CHUNKSIZE;
+          *pnLoad = nByte;
+        }
         rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
         memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
         if( rc!=SQLITE_OK ){
@@ -968,7 +998,6 @@ int sqlite3Fts3ReadBlock(
       }
       *paBlob = aByte;
     }
-    *pnBlob = nByte;
   }
 
   return rc;
@@ -982,13 +1011,55 @@ void sqlite3Fts3SegmentsClose(Fts3Table *p){
   sqlite3_blob_close(p->pSegments);
   p->pSegments = 0;
 }
+    
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
+  int nRead;                      /* Number of bytes to read */
+  int rc;                         /* Return code */
+
+  nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
+  rc = sqlite3_blob_read(
+      pReader->pBlob, 
+      &pReader->aNode[pReader->nPopulate],
+      nRead,
+      pReader->nPopulate
+  );
+
+  if( rc==SQLITE_OK ){
+    pReader->nPopulate += nRead;
+    memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
+    if( pReader->nPopulate==pReader->nNode ){
+      sqlite3_blob_close(pReader->pBlob);
+      pReader->pBlob = 0;
+      pReader->nPopulate = 0;
+    }
+  }
+  return rc;
+}
+
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
+  int rc = SQLITE_OK;
+  assert( !pReader->pBlob 
+       || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
+  );
+  while( pReader->pBlob && rc==SQLITE_OK 
+     &&  (pFrom - pReader->aNode + nByte)>pReader->nPopulate
+  ){
+    rc = fts3SegReaderIncrRead(pReader);
+  }
+  return rc;
+}
 
 /*
 ** Move the iterator passed as the first argument to the next term in the
 ** segment. If successful, SQLITE_OK is returned. If there is no next term,
 ** SQLITE_DONE. Otherwise, an SQLite error code.
 */
-static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
+static int fts3SegReaderNext(
+  Fts3Table *p, 
+  Fts3SegReader *pReader,
+  int bIncr
+){
+  int rc;                         /* Return code of various sub-routines */
   char *pNext;                    /* Cursor variable */
   int nPrefix;                    /* Number of bytes in term prefix */
   int nSuffix;                    /* Number of bytes in term suffix */
@@ -1000,7 +1071,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
   }
 
   if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
-    int rc;                       /* Return code from Fts3ReadBlock() */
 
     if( fts3SegReaderIsPending(pReader) ){
       Fts3HashElem *pElem = *(pReader->ppNextElem);
@@ -1020,6 +1090,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
 
     if( !fts3SegReaderIsRootOnly(pReader) ){
       sqlite3_free(pReader->aNode);
+      sqlite3_blob_close(pReader->pBlob);
+      pReader->pBlob = 0;
     }
     pReader->aNode = 0;
 
@@ -1031,11 +1103,20 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
     }
 
     rc = sqlite3Fts3ReadBlock(
-        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
+        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, 
+        (bIncr ? &pReader->nPopulate : 0)
     );
     if( rc!=SQLITE_OK ) return rc;
+    assert( pReader->pBlob==0 );
+    if( bIncr && pReader->nPopulate<pReader->nNode ){
+      pReader->pBlob = p->pSegments;
+      p->pSegments = 0;
+    }
     pNext = pReader->aNode;
   }
+
+  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
+  if( rc!=SQLITE_OK ) return rc;
   
   /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
   ** safe (no risk of overread) even if the node data is corrupted.  
@@ -1057,6 +1138,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
     pReader->zTerm = zNew;
     pReader->nTermAlloc = nNew;
   }
+
+  rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
+  if( rc!=SQLITE_OK ) return rc;
+
   memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
   pReader->nTerm = nPrefix+nSuffix;
   pNext += nSuffix;
@@ -1069,7 +1154,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
   ** of these statements is untrue, then the data structure is corrupt.
   */
   if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
-   || pReader->aDoclist[pReader->nDoclist-1]
+   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
   ){
     return SQLITE_CORRUPT_VTAB;
   }
@@ -1080,12 +1165,16 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
 ** Set the SegReader to point to the first docid in the doclist associated
 ** with the current term.
 */
-static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
+static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){
-  int n;
+  int rc;
   assert( pReader->aDoclist );
   assert( !pReader->pOffsetList );
-  n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+  rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
-  pReader->pOffsetList = &pReader->aDoclist[n];
+  if( rc==SQLITE_OK ){
+    int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+    pReader->pOffsetList = &pReader->aDoclist[n];
+  }
+  return rc;
 }
 
 /*
@@ -1098,11 +1187,12 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
 ** *pnOffsetList is set to the length of the set of column-offset
 ** lists, not including the nul-terminator byte. For example:
 */
-static void fts3SegReaderNextDocid(
+static int fts3SegReaderNextDocid(
   Fts3SegReader *pReader,
   char **ppOffsetList,
   int *pnOffsetList
 ){
+  int rc = SQLITE_OK;
   char *p = pReader->pOffsetList;
   char c = 0;
 
@@ -1110,7 +1200,16 @@ static void fts3SegReaderNextDocid(
   ** following two lines advance it to point one byte past the end of
   ** the same offset list.
   */
-  while( *p | c ) c = *p++ & 0x80;
+  while( 1 ){
+    int nRead;
+    int rc;
+
+    while( *p | c ) c = *p++ & 0x80;
+    assert( *p==0 );
+    if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break;
+    rc = fts3SegReaderIncrRead(pReader);
+    if( rc!=SQLITE_OK ) return rc;
+  }
   p++;
 
   /* If required, populate the output variables with a pointer to and the
@@ -1129,10 +1228,15 @@ static void fts3SegReaderNextDocid(
   if( p>=&pReader->aDoclist[pReader->nDoclist] ){
     pReader->pOffsetList = 0;
   }else{
-    sqlite3_int64 iDelta;
+    rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
-    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+    if( rc==SQLITE_OK ){
-    pReader->iDocid += iDelta;
+      sqlite3_int64 iDelta;
+      pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+      pReader->iDocid += iDelta;
+    }
   }
+
+  return SQLITE_OK;
 }
 
 /*
@@ -1212,7 +1316,7 @@ int sqlite3Fts3SegReaderCost(
     ** confirms this).
     */
     for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
-      rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
+      rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob, 0);
       if( rc!=SQLITE_OK ) break;
       if( (nBlob+35)>pgsz ){
         int nOvfl = (nBlob + 34)/pgsz;
@@ -1247,7 +1351,7 @@ int sqlite3Fts3MsrOvfl(
       int jj;
       for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
         int nBlob;
-        rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob);
+        rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
         if( rc!=SQLITE_OK ) break;
         if( (nBlob+35)>pgsz ){
           nOvfl += (nBlob + 34)/pgsz;
@@ -1268,6 +1372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
     sqlite3_free(pReader->zTerm);
     if( !fts3SegReaderIsRootOnly(pReader) ){
       sqlite3_free(pReader->aNode);
+      sqlite3_blob_close(pReader->pBlob);
     }
   }
   sqlite3_free(pReader);
@@ -2193,7 +2298,7 @@ int sqlite3Fts3MsrIncrStart(
   for(i=0; i<nSegment; i++){
     Fts3SegReader *pSeg = pCsr->apSegment[i];
     do {
-      int rc = fts3SegReaderNext(p, pSeg);
+      int rc = fts3SegReaderNext(p, pSeg, 1);
       if( rc!=SQLITE_OK ) return rc;
     }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
   }
@@ -2210,8 +2315,10 @@ int sqlite3Fts3MsrIncrStart(
 
   /* Advance each of the segments to point to the first docid. */
   for(i=0; i<pCsr->nAdvance; i++){
-    fts3SegReaderFirstDocid(pCsr->apSegment[i]);
+    int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]);
+    if( rc!=SQLITE_OK ) return rc;
   }
+  fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp);
 
   assert( iCol<0 || iCol<p->nColumn );
   pCsr->iColFilter = iCol;
@@ -2226,7 +2333,6 @@ int sqlite3Fts3MsrIncrNext(
   char **paPoslist,               /* OUT: Pointer to position list */
   int *pnPoslist                  /* OUT: Size of position list in bytes */
 ){
-  int rc = SQLITE_OK;
   int nMerge = pMsr->nAdvance;
   Fts3SegReader **apSegment = pMsr->apSegment;
 
@@ -2236,27 +2342,33 @@ int sqlite3Fts3MsrIncrNext(
   }
 
   while( 1 ){
+    int nSort;
     Fts3SegReader *pSeg;
-    fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
     pSeg = pMsr->apSegment[0];
 
     if( pSeg->pOffsetList==0 ){
       *paPoslist = 0;
       break;
     }else{
+      int rc;
       char *pList;
       int nList;
       int j;
       sqlite3_int64 iDocid = apSegment[0]->iDocid;
 
-      fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
+      rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
       j = 1;
-      while( j<nMerge
+      while( rc==SQLITE_OK 
+        && j<nMerge
         && apSegment[j]->pOffsetList
         && apSegment[j]->iDocid==iDocid
       ){
         fts3SegReaderNextDocid(apSegment[j], 0, 0);
+        j++;
       }
+      if( rc!=SQLITE_OK ) return rc;
+
+      fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp);
 
       if( pMsr->iColFilter>=0 ){
         fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
@@ -2269,9 +2381,10 @@ int sqlite3Fts3MsrIncrNext(
         break;
       }
     }
+    
   }
 
-  return rc;
+  return SQLITE_OK;
 }
 
 int sqlite3Fts3SegReaderStart(
@@ -2295,7 +2408,7 @@ int sqlite3Fts3SegReaderStart(
     const char *zTerm = pFilter->zTerm;
     Fts3SegReader *pSeg = pCsr->apSegment[i];
     do {
-      int rc = fts3SegReaderNext(p, pSeg);
+      int rc = fts3SegReaderNext(p, pSeg, 0);
       if( rc!=SQLITE_OK ) return rc;
     }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
   }
@@ -2331,7 +2444,7 @@ int sqlite3Fts3SegReaderStep(
     ** forward. Then sort the list in order of current term again.  
     */
     for(i=0; i<pCsr->nAdvance; i++){
-      rc = fts3SegReaderNext(p, apSegment[i]);
+      rc = fts3SegReaderNext(p, apSegment[i], 0);
       if( rc!=SQLITE_OK ) return rc;
     }
     fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
@@ -2780,15 +2893,6 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
     Fts3Phrase *pPhrase = pExpr->pPhrase;
     fts3DeferredDoclistClear(pExpr->pLeft);
     fts3DeferredDoclistClear(pExpr->pRight);
-    if( pPhrase ){
-      assert( pExpr->eType==FTSQUERY_PHRASE );
-      sqlite3_free(pPhrase->aDoclist);
-      pPhrase->isLoaded = 0;
-      pPhrase->aDoclist = 0;
-      pPhrase->nDoclist = 0;
-      pPhrase->pCurrent = 0;
-      pPhrase->iCurrent = 0;
-    }
   }
 }
 

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow.
+C FTS\schanges:\sRemove\sunreachable\scode.\sFix\sbugs.\sWhen\sprocessing\sa\slarge\sdoclist\sincrementally,\sread\sfrom\sdisk\sincrementally\stoo.
-D 2011-06-02T19:57:24.733
+D 2011-06-03T18:00:19.691
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4
+F ext/fts3/fts3.c 3183cf6aa7bfb00f227be1950b326feb8294da7d
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50
+F ext/fts3/fts3Int.h ba6f831fcde80ed5f0ccb112bb593b117cf11538
 F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f
-F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517
+F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b
 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
 F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2
-F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274
+F ext/fts3/fts3_snippet.c 0485969cce410760b50d587a77186f9c7f7e96be
 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1
 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
-F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870
+F ext/fts3/fts3_write.c c2f041d74f38e40c7487440f78d20f2aaa1f5d3c
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 84097a4c759b1d65890af885f137d3cb16eef584
+P 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9
-R e38e0cc84edbfdb5eae395b3be2f7a86
+R b735e6f33e9ec766163a902e084c30a6
 U dan
-Z 53d59cf3dcd8991c66b0afd5fb898b1a
+Z f3b0ba5ae8b73d0176f1d82c224a912a

manifest.uuid

@@ -1 +1 @@
-28149a7882a1e9dfe4a75ec5b91d176ebe6284e9
+a4c7e2820824e82580730c36f85aede2efa66754