Updates to FTS4 to improve performance and make more accurate cost estimates for prefix terms.

FossilOrigin-Name: d0a450ce78e99f55c862f26f9332786660007a0a

ext/fts3/fts3.c

@@ -926,8 +926,9 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
 ** Close the cursor.  For additional information see the documentation
 ** on the xClose method of the virtual table interface.
 */
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
   Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
   sqlite3_finalize(pCsr->pStmt);
   sqlite3Fts3ExprFree(pCsr->pExpr);
   sqlite3Fts3FreeDeferredTokens(pCsr);
@@ -969,6 +970,84 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
 }
 
 
+static int fts3ScanInteriorNode(
+  Fts3Table *p,                   /* Virtual table handle */
+  const char *zTerm,              /* Term to select leaves for */
+  int nTerm,                      /* Size of term zTerm in bytes */
+  const char *zNode,              /* Buffer containing segment interior node */
+  int nNode,                      /* Size of buffer at zNode */
+  sqlite3_int64 *piFirst,         /* OUT: Selected child node */
+  sqlite3_int64 *piLast           /* OUT: Selected child node */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  const char *zCsr = zNode;       /* Cursor to iterate through node */
+  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+  char *zBuffer = 0;              /* Buffer to load terms into */
+  int nAlloc = 0;                 /* Size of allocated buffer */
+
+  int isFirstTerm = 1;          /* True when processing first term on page */
+  int dummy;
+  sqlite3_int64 iChild;         /* Block id of child node to descend to */
+  int nBlock;                   /* Size of child node in bytes */
+
+  zCsr += sqlite3Fts3GetVarint32(zCsr, &dummy);
+  zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+  
+  while( zCsr<zEnd && (piFirst || piLast) ){
+    int cmp;                    /* memcmp() result */
+    int nSuffix;                /* Size of term suffix */
+    int nPrefix = 0;            /* Size of term prefix */
+    int nBuffer;                /* Total term size */
+  
+    /* Load the next term on the node into zBuffer */
+    if( !isFirstTerm ){
+      zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+    }
+    isFirstTerm = 0;
+    zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+    if( nPrefix+nSuffix>nAlloc ){
+      char *zNew;
+      nAlloc = (nPrefix+nSuffix) * 2;
+      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      if( !zNew ){
+        sqlite3_free(zBuffer);
+        return SQLITE_NOMEM;
+      }
+      zBuffer = zNew;
+    }
+    memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+    nBuffer = nPrefix + nSuffix;
+    zCsr += nSuffix;
+
+    /* Compare the term we are searching for with the term just loaded from
+    ** the interior node. If the specified term is greater than or equal
+    ** to the term from the interior node, then all terms on the sub-tree 
+    ** headed by node iChild are smaller than zTerm. No need to search 
+    ** iChild.
+    **
+    ** If the interior node term is larger than the specified term, then
+    ** the tree headed by iChild may contain the specified term.
+    */
+    cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+    if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+      *piFirst = iChild;
+      piFirst = 0;
+    }
+
+    if( piLast && cmp<0 ){
+      *piLast = iChild;
+      piLast = 0;
+    }
+
+    iChild++;
+  };
+
+  if( piFirst ) *piFirst = iChild;
+  if( piLast ) *piLast = iChild;
+
+  sqlite3_free(zBuffer);
+  return rc;
+}
 
 
 /*
@@ -993,77 +1072,33 @@ static int fts3SelectLeaf(
   int nTerm,                      /* Size of term zTerm in bytes */
   const char *zNode,              /* Buffer containing segment interior node */
   int nNode,                      /* Size of buffer at zNode */
-  sqlite3_int64 *piLeaf           /* Selected leaf node */
+  sqlite3_int64 *piLeaf,          /* Selected leaf node */
+  sqlite3_int64 *piLeaf2          /* Selected leaf node */
 ){
-  int rc = SQLITE_OK;             /* Return code */
+  int rc;                         /* Return code */
-  const char *zCsr = zNode;       /* Cursor to iterate through node */
+  int iHeight;                    /* Height of this node in tree */
-  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
-  char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
 
-  while( 1 ){
+  sqlite3Fts3GetVarint32(zNode, &iHeight);
-    int isFirstTerm = 1;          /* True when processing first term on page */
+  rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
-    int iHeight;                  /* Height of this node in tree */
+  
-    sqlite3_int64 iChild;         /* Block id of child node to descend to */
+  if( rc==SQLITE_OK && iHeight>1 ){
-    int nBlock;                   /* Size of child node in bytes */
+    const char *zBlob;
+    int nBlob;
 
-    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
+    if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
-    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
-  
+      if( rc==SQLITE_OK ){
-    while( zCsr<zEnd ){
+        rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
-      int cmp;                    /* memcmp() result */
-      int nSuffix;                /* Size of term suffix */
-      int nPrefix = 0;            /* Size of term prefix */
-      int nBuffer;                /* Total term size */
-  
-      /* Load the next term on the node into zBuffer */
-      if( !isFirstTerm ){
-        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
       }
-      isFirstTerm = 0;
+      piLeaf = 0;
-      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
-      if( nPrefix+nSuffix>nAlloc ){
-        char *zNew;
-        nAlloc = (nPrefix+nSuffix) * 2;
-        zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
-        if( !zNew ){
-          sqlite3_free(zBuffer);
-          return SQLITE_NOMEM;
-        }
-        zBuffer = zNew;
-      }
-      memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
-      nBuffer = nPrefix + nSuffix;
-      zCsr += nSuffix;
-  
-      /* Compare the term we are searching for with the term just loaded from
-      ** the interior node. If the specified term is greater than or equal
-      ** to the term from the interior node, then all terms on the sub-tree 
-      ** headed by node iChild are smaller than zTerm. No need to search 
-      ** iChild.
-      **
-      ** If the interior node term is larger than the specified term, then
-      ** the tree headed by iChild may contain the specified term.
-      */
-      cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
-      if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
-      iChild++;
-    };
-
-    /* If (iHeight==1), the children of this interior node are leaves. The
-    ** specified term may be present on leaf node iChild.
-    */
-    if( iHeight==1 ){
-      *piLeaf = iChild;
-      break;
     }
 
-    /* Descend to interior node iChild. */
+    rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
-    rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
+    if( rc==SQLITE_OK ){
-    if( rc!=SQLITE_OK ) break;
+      rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
-    zEnd = &zCsr[nBlock];
+    }
   }
-  sqlite3_free(zBuffer);
+
   return rc;
 }
 
@@ -1506,7 +1541,8 @@ static int fts3DoclistMerge(
   char *a1,                       /* Buffer containing first doclist */
   int n1,                         /* Size of buffer a1 */
   char *a2,                       /* Buffer containing second doclist */
-  int n2                          /* Size of buffer a2 */
+  int n2,                         /* Size of buffer a2 */
+  int *pnDoc                      /* OUT: Number of docids in output */
 ){
   sqlite3_int64 i1 = 0;
   sqlite3_int64 i2 = 0;
@@ -1517,6 +1553,7 @@ static int fts3DoclistMerge(
   char *p2 = a2;
   char *pEnd1 = &a1[n1];
   char *pEnd2 = &a2[n2];
+  int nDoc = 0;
 
   assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
        || mergetype==MERGE_AND    || mergetype==MERGE_NOT
@@ -1560,6 +1597,7 @@ static int fts3DoclistMerge(
           fts3PutDeltaVarint(&p, &iPrev, i1);
           fts3GetDeltaVarint2(&p1, pEnd1, &i1);
           fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+          nDoc++;
         }else if( i1<i2 ){
           fts3GetDeltaVarint2(&p1, pEnd1, &i1);
         }else{
@@ -1593,6 +1631,8 @@ static int fts3DoclistMerge(
           if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
             p = pSave;
             iPrev = iPrevSave;
+          }else{
+            nDoc++;
           }
           fts3GetDeltaVarint2(&p1, pEnd1, &i1);
           fts3GetDeltaVarint2(&p2, pEnd2, &i2);
@@ -1645,6 +1685,7 @@ static int fts3DoclistMerge(
     }
   }
 
+  if( pnDoc ) *pnDoc = nDoc;
   *pnBuffer = (int)(p-aBuffer);
   return SQLITE_OK;
 }
@@ -1692,7 +1733,7 @@ static int fts3TermSelectMerge(TermSelect *pTS){
           return SQLITE_NOMEM;
         }
         fts3DoclistMerge(mergetype, 0, 0,
-            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
         );
         sqlite3_free(pTS->aaOutput[i]);
         sqlite3_free(aOut);
@@ -1763,8 +1804,8 @@ static int fts3TermSelectCb(
         }
         return SQLITE_NOMEM;
       }
-      fts3DoclistMerge(mergetype, 0, 0,
+      fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 
-          aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+          pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
       );
 
       if( iOut>0 ) sqlite3_free(aMerge);
@@ -1905,10 +1946,11 @@ static int fts3TermSegReaderArray(
       rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
     }else{
       int rc2;                    /* Return value of sqlite3Fts3ReadBlock() */
-      sqlite3_int64 i1;           /* Blockid of leaf that may contain zTerm */
+      sqlite3_int64 i1;           /* First leaf that may contain zTerm */
-      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
+      sqlite3_int64 i2;           /* Last leaf that may contain zTerm */
+      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
+      if( isPrefix==0 ) i2 = i1;
       if( rc==SQLITE_OK ){
-        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
         rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
       }
 
@@ -2008,17 +2050,34 @@ static int fts3TermSelect(
   return rc;
 }
 
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded 
+** varints.
+*/
 static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
   int nDoc = 0;                   /* Return value */
+
   if( aList ){
     char *aEnd = &aList[nList];   /* Pointer to one byte after EOF */
     char *p = aList;              /* Cursor */
-    sqlite3_int64 dummy;          /* For Fts3GetVarint() */
+    if( !isPoslist ){
-  
+      /* The number of docids in the list is the same as the number of 
-    while( p<aEnd ){
+      ** varints. In FTS3 a varint consists of a single byte with the 0x80 
-      nDoc++;
+      ** bit cleared and zero or more bytes with the 0x80 bit set. So to
-      p += sqlite3Fts3GetVarint(p, &dummy);
+      ** count the varints in the buffer, just count the number of bytes
-      if( isPoslist ) fts3PoslistCopy(0, &p);
+      ** with the 0x80 bit clear.  */
+      while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
+    }else{
+      while( p<aEnd ){
+        nDoc++;
+        while( (*p++)&0x80 );     /* Skip docid varint */
+        fts3PoslistCopy(0, &p);   /* Skip over position list */
+      }
     }
   }
 
@@ -2160,6 +2219,9 @@ static int fts3PhraseSelect(
     if( ii==0 ){
       pOut = pList;
       nOut = nList;
+      if( pCsr->doDeferred==0 && pPhrase->nToken>1 ){
+        nDoc = fts3DoclistCountDocids(1, pOut, nOut);
+      }
     }else{
       /* Merge the new term list and the current output. */
       char *aLeft, *aRight;
@@ -2189,14 +2251,14 @@ static int fts3PhraseSelect(
         nDist = iPrevTok-iTok;
       }
       pOut = aRight;
-     
+      fts3DoclistMerge(
-      fts3DoclistMerge(mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight);
+          mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
+      );
       sqlite3_free(aLeft);
     }
     assert( nOut==0 || pOut!=0 );
 
     iPrevTok = iTok;
-    nDoc = fts3DoclistCountDocids(ii<(pPhrase->nToken-1), pOut, nOut);
   }
 
   if( rc==SQLITE_OK ){
@@ -2234,7 +2296,7 @@ static int fts3NearMerge(
     rc = SQLITE_NOMEM;
   }else{
     rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
-      aOut, pnOut, aLeft, nLeft, aRight, nRight
+      aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
     );
     if( rc!=SQLITE_OK ){
       sqlite3_free(aOut);
@@ -2442,13 +2504,15 @@ static int fts3EvalExpr(
             if( ii==0 ){
               aRet = aNew;
               nRet = nNew;
+              if( nExpr>1 ){
+                nDoc = fts3DoclistCountDocids(0, aRet, nRet);
+              }
             }else{
               fts3DoclistMerge(
-                  MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew
+                  MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
               );
               sqlite3_free(aNew);
             }
-            nDoc = fts3DoclistCountDocids(0, aRet, nRet);
           }
         }
       }
@@ -2507,7 +2571,7 @@ static int fts3EvalExpr(
             */
             char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
             rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
-                aLeft, nLeft, aRight, nRight
+                aLeft, nLeft, aRight, nRight, 0
             );
             *paOut = aBuffer;
             sqlite3_free(aLeft);
@@ -2517,7 +2581,7 @@ static int fts3EvalExpr(
           default: {
             assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
             fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
-                aLeft, nLeft, aRight, nRight
+                aLeft, nLeft, aRight, nRight, 0
             );
             *paOut = aLeft;
             break;
@@ -2597,7 +2661,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
   return rc;
 }
 
-
 /*
 ** This is the xFilter interface for the virtual table.  See
 ** the virtual table xFilter method documentation for additional
@@ -2641,6 +2704,7 @@ static int fts3FilterMethod(
   assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
   assert( nVal==0 || nVal==1 );
   assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+  assert( p->pSegments==0 );
 
   /* In case the cursor has been used before, clear it now. */
   sqlite3_finalize(pCsr->pStmt);
@@ -2671,6 +2735,7 @@ static int fts3FilterMethod(
     if( rc!=SQLITE_OK ) return rc;
 
     rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+    sqlite3Fts3SegmentsClose(p);
     if( rc!=SQLITE_OK ) return rc;
     pCsr->pNextId = pCsr->aDoclist;
     pCsr->iPrevId = 0;
@@ -2782,7 +2847,9 @@ static int fts3UpdateMethod(
 ** hash-table to the database.
 */
 static int fts3SyncMethod(sqlite3_vtab *pVtab){
-  return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+  int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+  sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+  return rc;
 }
 
 /*
@@ -3116,7 +3183,7 @@ static const sqlite3_module fts3Module = {
   /* xDisconnect   */ fts3DisconnectMethod,
   /* xDestroy      */ fts3DestroyMethod,
   /* xOpen         */ fts3OpenMethod,
-  /* xClose        */ fulltextClose,
+  /* xClose        */ fts3CloseMethod,
   /* xFilter       */ fts3FilterMethod,
   /* xNext         */ fts3NextMethod,
   /* xEof          */ fts3EofMethod,

ext/fts3/fts3Int.h

@@ -130,6 +130,8 @@ struct Fts3Table {
   u8 bHasContent;                 /* True if %_content table exists */
   u8 bHasDocsize;                 /* True if %_docsize table exists */
 
+  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
+
   /* The following hash table is used to buffer pending index updates during
   ** transactions. Variable nPendingData estimates the memory size of the 
   ** pending data, including hash table overhead, but not malloc overhead. 
@@ -287,6 +289,8 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
 void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
 char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
 
+void sqlite3Fts3SegmentsClose(Fts3Table *);
+
 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
 #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
 #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002

ext/fts3/fts3_snippet.c

@@ -268,6 +268,7 @@ static int fts3ExprLoadDoclists(
   }
   if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   if( pnToken ) *pnToken = sCtx.nToken;
+  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab);
   return rc;
 }
 

ext/fts3/fts3_write.c

@@ -807,29 +807,51 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
 ** The %_segments table is declared as follows:
 **
 **   CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
-**
-** This function opens a read-only blob handle on the "block" column of
-** row iSegment of the %_segments table associated with FTS3 table p.
-**
-** If all goes well, SQLITE_OK is returned and *ppBlob set to the 
-** read-only blob handle. It is the responsibility of the caller to call
-** sqlite3_blob_close() on the blob handle. Or, if an error occurs, an
-** SQLite error code is returned and *ppBlob is either not modified or
-** set to 0.
 */
-static int fts3OpenSegmentsBlob(
+static int fts3SegmentsBlob(
-  Fts3Table *p,                   /* FTS3 table handle */
+  Fts3Table *p,
-  sqlite3_int64 iSegment,         /* Rowid in %_segments table */
+  sqlite3_int64 iSegment,
-  sqlite3_blob **ppBlob           /* OUT: Read-only blob handle */
+  char **paBlob,
+  int *pnBlob
 ){
-  if( 0==p->zSegmentsTbl
+  int rc;
-   && 0==(p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName))
+
-  ) {
+  if( p->pSegments ){
-    return SQLITE_NOMEM;
+    rc = sqlite3_blob_reopen(p->pSegments, iSegment);
+  }else{
+    if( 0==p->zSegmentsTbl ){
+      p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+      if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+    }
+    rc = sqlite3_blob_open(
+       p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, &p->pSegments
+    );
   }
-  return sqlite3_blob_open(
+
-     p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, ppBlob
+  if( rc==SQLITE_OK ){
-  );
+    int nByte = sqlite3_blob_bytes(p->pSegments);
+    if( paBlob ){
+      char *aByte = sqlite3_malloc(nByte);
+      if( !aByte ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+        if( rc!=SQLITE_OK ){
+          sqlite3_free(aByte);
+          aByte = 0;
+        }
+      }
+      *paBlob = aByte;
+    }
+    *pnBlob = nByte;
+  }
+
+  return rc;
+}
+
+void sqlite3Fts3SegmentsClose(Fts3Table *p){
+  sqlite3_blob_close(p->pSegments);
+  p->pSegments = 0;
 }
 
 
@@ -880,17 +902,9 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
       return SQLITE_OK;
     }
 
-    rc = fts3OpenSegmentsBlob(p, ++pReader->iCurrentBlock, &pBlob);
+    rc = fts3SegmentsBlob(
-    if( rc==SQLITE_OK ){
+        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
-      pReader->nNode = sqlite3_blob_bytes(pBlob);
+    );
-      pReader->aNode = (char *)sqlite3_malloc(pReader->nNode);
-      if( pReader->aNode ){
-        rc = sqlite3_blob_read(pBlob, pReader->aNode, pReader->nNode, 0);
-      }else{
-        rc = SQLITE_NOMEM;
-      }
-      sqlite3_blob_close(pBlob);
-    }
 
     if( rc!=SQLITE_OK ){
       return rc;
@@ -1008,7 +1022,8 @@ int sqlite3Fts3SegReaderCost(
    && !fts3SegReaderIsPending(pReader) 
    && !fts3SegReaderIsRootOnly(pReader) 
   ){
-    sqlite3_blob *pBlob = 0;
+    int nBlob = 0;
+    sqlite3_int64 iBlock;
 
     if( pCsr->nRowAvg==0 ){
       /* The average document size, which is required to calculate the cost
@@ -1045,20 +1060,18 @@ int sqlite3Fts3SegReaderCost(
       if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
     }
 
-    rc = fts3OpenSegmentsBlob(p, pReader->iStartBlock, &pBlob);
+    /* Assume that a blob flows over onto overflow pages if it is larger
-    if( rc==SQLITE_OK ){
+    ** than (pgsz-35) bytes in size (the file-format documentation
-      /* Assume that a blob flows over onto overflow pages if it is larger
+    ** confirms this).
-      ** than (pgsz-35) bytes in size (the file-format documentation
+    */
-      ** confirms this).
+    for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
-      */
+      rc = fts3SegmentsBlob(p, iBlock, 0, &nBlob);
-      int nBlob = sqlite3_blob_bytes(pBlob);
+      if( rc!=SQLITE_OK ) break;
       if( (nBlob+35)>pgsz ){
         int nOvfl = (nBlob + 34)/pgsz;
         nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
       }
     }
-    assert( rc==SQLITE_OK || pBlob==0 );
-    sqlite3_blob_close(pBlob);
   }
 
   *pnCost += nCost;
@@ -1096,6 +1109,7 @@ int sqlite3Fts3SegReaderNew(
   Fts3SegReader *pReader;         /* Newly allocated SegReader object */
   int nExtra = 0;                 /* Bytes to allocate segment root node */
 
+  assert( iStartLeaf<=iEndLeaf );
   if( iStartLeaf==0 ){
     nExtra = nRoot;
   }
@@ -2619,6 +2633,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     rc = SQLITE_ERROR;
   }
 
+  sqlite3Fts3SegmentsClose(p);
   return rc;
 }
 
@@ -2787,6 +2802,7 @@ int sqlite3Fts3UpdateMethod(
   u32 *aSzDel;                    /* Sizes of deleted documents */
   int nChng = 0;                  /* Net change in number of documents */
 
+  assert( p->pSegments==0 );
 
   /* Allocate space to hold the change in document sizes */
   aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
@@ -2842,6 +2858,7 @@ int sqlite3Fts3UpdateMethod(
   }
 
   sqlite3_free(aSzIns);
+  sqlite3Fts3SegmentsClose(p);
   return rc;
 }
 
@@ -2865,6 +2882,7 @@ int sqlite3Fts3Optimize(Fts3Table *p){
       sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
     }
   }
+  sqlite3Fts3SegmentsClose(p);
   return rc;
 }
 

ext/fts3/fts3speed.tcl

@@ -18,7 +18,7 @@
 set VOCAB_SIZE  2000
 set DOC_SIZE     100
 
-set NUM_INSERTS 1000
+set NUM_INSERTS 100000
 set NUM_SELECTS 1000
 
 # Force everything in this script to be deterministic.
@@ -74,8 +74,7 @@ proc test_1 {nInsert} {
   sql "CREATE VIRTUAL TABLE t1 USING fts4;"
   for {set i 0} {$i < $nInsert} {incr i} {
     set doc [select_doc $::DOC_SIZE]
-    #sql "INSERT INTO t1 VALUES('$doc');"
+    sql "INSERT INTO t1 VALUES('$doc');"
-    sql "\"$doc\""
   }
 }
 

manifest

@@ -1,5 +1,5 @@
-C Experimental\schanges\sto\sfts4\sto\stry\sto\sselectively\savoid\sloading\svery\slarge\sdoclists.
+C Updates\sto\sFTS4\sto\simprove\sperformance\sand\smake\smore\saccurate\scost\sestimates\sfor\sprefix\sterms.
-D 2010-10-19T14:08:00
+D 2010-10-20T18:56:04
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b01fdfcfecf8a0716c29867a67959f6148b79961
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -61,20 +61,20 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 9d4ccf3b7bbfbeeef03dba91377c4d72b757dcb9
+F ext/fts3/fts3.c ce7bcd1f42e74912149fe6201fc63a6ac0db42a8
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h a640e4fbdb2fcab1457f87993ca3f4ceaa31e776
+F ext/fts3/fts3Int.h 9fbe422f7d0e005371702acaa3cd44283a67c389
 F ext/fts3/fts3_expr.c a5aee50edde20e5c9116199bd58be869a3a22c9f
 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
 F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
 F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba
-F ext/fts3/fts3_snippet.c 474c11e718610cade73e6009f75ffc173d4c42c5
+F ext/fts3/fts3_snippet.c ca60a2a47de5e7abb22a804ccd1a743f81c2fe3e
 F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
-F ext/fts3/fts3_write.c 29b63a98de55d4eb34b7fc6fd90b3224d6cdc7ff
+F ext/fts3/fts3_write.c be47d30cf80bc91e050ece18e2de7e207432be1a
-F ext/fts3/fts3speed.tcl 71b9cdc8f499822124a9eef42003e31a88f26f16
+F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
 F ext/icu/icu.c 850e9a36567bbcce6bd85a4b68243cad8e3c2de2
@@ -119,7 +119,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c d5b0137bc20327af08c14772227cc35134839c30
 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff
-F src/btree.c 8a1b0267a4f1914aedbaa93d3fcf4f2e42141ea8
+F src/btree.c 3edab36d03d86c200cb9551467410f975d510aa9
 F src/btree.h 2d1a83ad509047e8cc314fda7e054f99ff52414d
 F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0
 F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d
@@ -176,14 +176,14 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706
 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
 F src/select.c 6a5c72fb0e8dc7f6133f5a9d7a747130ef0a00ea
 F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056
-F src/sqlite.h.in 13f219b9ab78f22603019fd193f09d5c8913795a
+F src/sqlite.h.in 460599b35c035deb339d1c9933089ef32187ecc6
 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754
 F src/sqliteInt.h c63b0340dfdfde18ff255ddccf004edd2d073288
 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44
 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
 F src/tclsqlite.c e91019fb6787166abca23a81b16c07fecc2ed751
-F src/test1.c cbedc6ea7905b1361db054fbf7fcd0dafb6d844e
+F src/test1.c f6e39615c8315e03798217a360810e4c59595627
 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31
 F src/test3.c 056093cfef69ff4227a6bdb9108564dc7f45e4bc
 F src/test4.c 0528360b5025688002a5feb6be906ddce52eaaee
@@ -231,7 +231,7 @@ F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2
 F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4
 F src/vdbeapi.c 5368714fa750270cf6430160287c21adff44582d
 F src/vdbeaux.c de0b06b11a25293e820a49159eca9f1c51a64716
-F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256
+F src/vdbeblob.c c8cbe6ce28cc8bf806ea0818b5167dd9a27c48a3
 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9
 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
 F src/vtab.c 6c90e3e65b2f026fc54703a8f3c917155f419d87
@@ -420,7 +420,7 @@ F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49
 F test/fts3ae.test ce32a13b34b0260928e4213b4481acf801533bda
 F test/fts3af.test d394978c534eabf22dd0837e718b913fd66b499c
 F test/fts3ag.test 0b7d303f61ae5d620c4efb5e825713ea34ff9441
-F test/fts3ah.test 3c5a1bd49979d7b5b5ed9fdbcdd14a7bfe5a5ff9
+F test/fts3ah.test dc9f66c32c296f1bc8bcc4535126bddfeca62894
 F test/fts3ai.test d29cee6ed653e30de478066881cec8aa766531b2
 F test/fts3aj.test 584facbc9ac4381a7ec624bfde677340ffc2a5a4
 F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d
@@ -433,7 +433,7 @@ F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
 F test/fts3cov.test 6f1ff88ff6b5abcfff6979098cb9d0c68a69202e
 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
-F test/fts3defer.test a9f81bba6e1132dd6a2ad3cf11e4628733975c8c
+F test/fts3defer.test cf66bf69afcc2fb8373d3aed31c55399409e83f2
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
@@ -875,11 +875,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P c0ee614fd988f445c4884a37f494479bdd669185
+P 5ae0ba447a561e3b6637b52f9b83a9fc683d2572
-R 0c4aebbf44d624104504e37bec992917
+R acd75bd5094c2beea7f56f4633b47ea2
-T *bgcolor * #c0ffc0
-T *branch * experimental
-T *sym-experimental *
-T -sym-trunk *
 U dan
-Z c2b99a58ccad27405ff8b0fcedef5c33
+Z 910cba31e6572b93e93d39c91f91b9da

manifest.uuid

@@ -1 +1 @@
-5ae0ba447a561e3b6637b52f9b83a9fc683d2572
+d0a450ce78e99f55c862f26f9332786660007a0a

src/btree.c

@@ -8096,8 +8096,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){
 void sqlite3BtreeCacheOverflow(BtCursor *pCur){
   assert( cursorHoldsMutex(pCur) );
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
-  assert(!pCur->isIncrblobHandle);
+  invalidateOverflowCache(pCur);
-  assert(!pCur->aOverflow);
   pCur->isIncrblobHandle = 1;
 }
 #endif

src/sqlite.h.in

@@ -4790,6 +4790,11 @@ int sqlite3_blob_open(
   sqlite3_blob **ppBlob
 );
 
+/*
+** CAPI3REF: Move a BLOB Handle
+*/
+SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
 /*
 ** CAPI3REF: Close A BLOB Handle
 **

src/test1.c

@@ -1703,6 +1703,51 @@ static int test_blob_write(
 
   return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
 }
+
+static int test_blob_reopen(
+  ClientData clientData, /* Not used */
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int objc,              /* Number of arguments */
+  Tcl_Obj *CONST objv[]  /* Command arguments */
+){
+  Tcl_WideInt iRowid;
+  Tcl_Channel channel;
+  ClientData instanceData;
+  sqlite3_blob *pBlob;
+  int notUsed;
+  int rc;
+
+  unsigned char *zBuf;
+  int nBuf;
+  
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID");
+    return TCL_ERROR;
+  }
+
+  channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), &notUsed);
+  if( !channel || TCL_OK!=Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ){ 
+    return TCL_ERROR;
+  }
+
+  if( TCL_OK!=(rc = Tcl_Flush(channel)) ){
+    return rc;
+  }
+  if( TCL_OK!=(rc = Tcl_Seek(channel, 0, SEEK_SET)) ){
+    return rc;
+  }
+
+  instanceData = Tcl_GetChannelInstanceData(channel);
+  pBlob = *((sqlite3_blob **)instanceData);
+
+  rc = sqlite3_blob_reopen(pBlob, iRowid);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE);
+  }
+
+  return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
+}
+
 #endif
 
 /*
@@ -5328,6 +5373,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_INCRBLOB
      { "sqlite3_blob_read",  test_blob_read, 0  },
      { "sqlite3_blob_write", test_blob_write, 0  },
+     { "sqlite3_blob_reopen", test_blob_reopen, 0  },
 #endif
      { "pcache_stats",       test_pcache_stats, 0  },
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY

src/vdbeblob.c

@@ -26,11 +26,61 @@ struct Incrblob {
   int flags;              /* Copy of "flags" passed to sqlite3_blob_open() */
   int nByte;              /* Size of open blob, in bytes */
   int iOffset;            /* Byte offset of blob in cursor data */
+  int iCol;               /* Table column this handle is open on */
   BtCursor *pCsr;         /* Cursor pointing at blob row */
   sqlite3_stmt *pStmt;    /* Statement holding cursor open */
   sqlite3 *db;            /* The associated database */
 };
 
+
+static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
+  int rc;                         /* Error code */
+  char *zErr = 0;                 /* Error message */
+  Vdbe *v = (Vdbe *)p->pStmt;
+
+  v->aVar[0].u.i = iRow;
+  rc = sqlite3_step(p->pStmt);
+
+  if( rc==SQLITE_ROW ){
+    Vdbe *v = (Vdbe *)p->pStmt;
+    u32 type = v->apCsr[0]->aType[p->iCol];
+    if( type<12 ){
+      zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
+          type==0?"null": type==7?"real": "integer"
+      );
+      rc = SQLITE_ERROR;
+      sqlite3_finalize(p->pStmt);
+      p->pStmt = 0;
+    }else{
+      p->iOffset = v->apCsr[0]->aOffset[p->iCol];
+      p->nByte = sqlite3VdbeSerialTypeLen(type);
+      p->pCsr =  v->apCsr[0]->pCursor;
+      sqlite3BtreeEnterCursor(p->pCsr);
+      sqlite3BtreeCacheOverflow(p->pCsr);
+      sqlite3BtreeLeaveCursor(p->pCsr);
+    }
+  }
+
+  if( rc==SQLITE_ROW ){
+    rc = SQLITE_OK;
+  }else if( p->pStmt ){
+    rc = sqlite3_finalize(p->pStmt);
+    p->pStmt = 0;
+    if( rc==SQLITE_OK ){
+      zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow);
+      rc = SQLITE_ERROR;
+    }else{
+      zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db));
+    }
+  }
+
+  assert( rc!=SQLITE_OK || zErr==0 );
+  assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE );
+
+  *pzErr = zErr;
+  return rc;
+}
+
 /*
 ** Open a blob handle.
 */
@@ -71,11 +121,12 @@ int sqlite3_blob_open(
     {OP_OpenWrite, 0, 0, 0},       /* 4: Open cursor 0 for read/write */
 
     {OP_Variable, 1, 1, 1},        /* 5: Push the rowid to the stack */
-    {OP_NotExists, 0, 9, 1},       /* 6: Seek the cursor */
+    {OP_NotExists, 0, 10, 1},      /* 6: Seek the cursor */
     {OP_Column, 0, 0, 1},          /* 7  */
     {OP_ResultRow, 1, 0, 0},       /* 8  */
-    {OP_Close, 0, 0, 0},           /* 9  */
+    {OP_Goto, 0, 5, 0},            /* 9  */
-    {OP_Halt, 0, 0, 0},            /* 10 */
+    {OP_Close, 0, 0, 0},           /* 10 */
+    {OP_Halt, 0, 0, 0},            /* 11 */
   };
 
   Vdbe *v = 0;
@@ -83,14 +134,20 @@ int sqlite3_blob_open(
   char *zErr = 0;
   Table *pTab;
   Parse *pParse;
+  Incrblob *pBlob;
 
+  flags = !!flags;                /* flags = (flags ? 1 : 0); */
   *ppBlob = 0;
+
   sqlite3_mutex_enter(db->mutex);
+
+  pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
   pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
-  if( pParse==0 ){
+  if( pParse==0 || pBlob==0 ){
-    rc = SQLITE_NOMEM;
+    assert( db->mallocFailed );
     goto blob_open_out;
   }
+
   do {
     memset(pParse, 0, sizeof(Parse));
     pParse->db = db;
@@ -177,7 +234,6 @@ int sqlite3_blob_open(
     if( v ){
       int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
       sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob);
-      flags = !!flags;                 /* flags = (flags ? 1 : 0); */
 
       /* Configure the OP_Transaction */
       sqlite3VdbeChangeP1(v, 0, iDb);
@@ -220,65 +276,30 @@ int sqlite3_blob_open(
       }
     }
    
-    sqlite3BtreeLeaveAll(db);
-    if( db->mallocFailed ){
-      goto blob_open_out;
-    }
-
-    sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow);
-    rc = sqlite3_step((sqlite3_stmt *)v);
-    if( rc!=SQLITE_ROW ){
-      nAttempt++;
-      rc = sqlite3_finalize((sqlite3_stmt *)v);
-      sqlite3DbFree(db, zErr);
-      zErr = sqlite3MPrintf(db, sqlite3_errmsg(db));
-      v = 0;
-    }
-  } while( nAttempt<5 && rc==SQLITE_SCHEMA );
-
-  if( rc==SQLITE_ROW ){
-    /* The row-record has been opened successfully. Check that the
-    ** column in question contains text or a blob. If it contains
-    ** text, it is up to the caller to get the encoding right.
-    */
-    Incrblob *pBlob;
-    u32 type = v->apCsr[0]->aType[iCol];
-
-    if( type<12 ){
-      sqlite3DbFree(db, zErr);
-      zErr = sqlite3MPrintf(db, "cannot open value of type %s",
-          type==0?"null": type==7?"real": "integer"
-      );
-      rc = SQLITE_ERROR;
-      goto blob_open_out;
-    }
-    pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
-    if( db->mallocFailed ){
-      sqlite3DbFree(db, pBlob);
-      goto blob_open_out;
-    }
     pBlob->flags = flags;
-    pBlob->pCsr =  v->apCsr[0]->pCursor;
-    sqlite3BtreeEnterCursor(pBlob->pCsr);
-    sqlite3BtreeCacheOverflow(pBlob->pCsr);
-    sqlite3BtreeLeaveCursor(pBlob->pCsr);
     pBlob->pStmt = (sqlite3_stmt *)v;
-    pBlob->iOffset = v->apCsr[0]->aOffset[iCol];
+    pBlob->iCol = iCol;
-    pBlob->nByte = sqlite3VdbeSerialTypeLen(type);
     pBlob->db = db;
-    *ppBlob = (sqlite3_blob *)pBlob;
+    sqlite3BtreeLeaveAll(db);
-    rc = SQLITE_OK;
+    v = 0;
-  }else if( rc==SQLITE_OK ){
+    if( db->mallocFailed ){
-    sqlite3DbFree(db, zErr);
+      goto blob_open_out;
-    zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow);
+    }
-    rc = SQLITE_ERROR;
+
-  }
+    sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
+    rc = blobSeekToRow(pBlob, iRow, &zErr);
+  } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA );
 
 blob_open_out:
-  if( v && (rc!=SQLITE_OK || db->mallocFailed) ){
+  if( rc==SQLITE_OK && db->mallocFailed==0 ){
-    sqlite3VdbeFinalize(v);
+    *ppBlob = (sqlite3_blob *)pBlob;
+  }else{
+    if( v ) sqlite3VdbeFinalize(v);
+    if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
+    sqlite3DbFree(db, pBlob);
   }
-  sqlite3Error(db, rc, zErr);
+
+  sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
   sqlite3DbFree(db, zErr);
   sqlite3StackFree(db, pParse);
   rc = sqlite3ApiExit(db, rc);
@@ -331,7 +352,7 @@ static int blobReadWrite(
     /* Request is out of range. Return a transient error. */
     rc = SQLITE_ERROR;
     sqlite3Error(db, SQLITE_ERROR, 0);
-  } else if( v==0 ){
+  }else if( v==0 ){
     /* If there is no statement handle, then the blob-handle has
     ** already been invalidated. Return SQLITE_ABORT in this case.
     */
@@ -382,4 +403,43 @@ int sqlite3_blob_bytes(sqlite3_blob *pBlob){
   return p ? p->nByte : 0;
 }
 
+/*
+** Move an existing blob handle to point to a different row of the same
+** database table.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a blob or text value, then an error code is returned and the
+** database handle error code and message set. If this happens, then all 
+** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) 
+** immediately return SQLITE_ABORT.
+*/
+int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
+  int rc;
+  Incrblob *p = (Incrblob *)pBlob;
+  sqlite3 *db;
+
+  if( p==0 ) return SQLITE_MISUSE_BKPT;
+  db = p->db;
+  sqlite3_mutex_enter(db->mutex);
+
+  if( p->pStmt==0 ){
+    /* If there is no statement handle, then the blob-handle has
+    ** already been invalidated. Return SQLITE_ABORT in this case.
+    */
+    rc = SQLITE_ABORT;
+  }else{
+    char *zErr;
+    rc = blobSeekToRow(p, iRow, &zErr);
+    if( rc!=SQLITE_OK ){
+      sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
+      sqlite3DbFree(db, zErr);
+    }
+    assert( rc!=SQLITE_SCHEMA );
+  }
+
+  rc = sqlite3ApiExit(db, rc);
+  sqlite3_mutex_leave(db->mutex);
+  return rc;
+}
+
 #endif /* #ifndef SQLITE_OMIT_INCRBLOB */

test/fts3ah.test

@@ -56,15 +56,15 @@ do_test fts3ah-1.2 {
   execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm}
 } {1 3}
 
-do_test fts3ah-1.2 {
+do_test fts3ah-1.3 {
   execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm}
 } {}
 
-do_test fts3ah-1.3 {
+do_test fts3ah-1.4 {
   execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'"
 } {1 3}
 
-do_test fts3ah-1.4 {
+do_test fts3ah-1.5 {
   execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'"
 } {1}
 

test/fts3defer.test

@@ -18,6 +18,8 @@ ifcapable !fts3 {
   return
 }
 
+set sqlite_fts3_enable_parentheses 1
+
 set ::testprefix fts3defer
 
 #--------------------------------------------------------------------------
@@ -187,7 +189,14 @@ lappend data {*}{
   "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki"
 }
 
+#set e [list]
+#foreach d $data {set e [concat $e $d]}
+#puts [lsort -unique $e]
+#exit
 
+set zero_long_doclists {
+  UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000
+}
 
 foreach {tn setup} {
   1 {
@@ -204,11 +213,14 @@ foreach {tn setup} {
     set dmt_modes {0 1 2}
     execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
     foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
-    execsql {
+    execsql $zero_long_doclists
-      UPDATE t1_segments
+  }
-      SET block = zeroblob(length(block)) 
+  4 {
-      WHERE length(block)>10000;
+    set dmt_modes 0
-    }
+    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
+    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
+    execsql "INSERT INTO t1(t1) VALUES('optimize')"
+    execsql $zero_long_doclists
   }
 } {
 
@@ -271,6 +283,17 @@ foreach {tn setup} {
     SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"'
   } {13 17}
 
+  do_select_test 2.8 {
+    SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
+  } {10 13 17 31 35 51 58 88 89 90 93 100}
+  do_select_test 2.9 {
+    SELECT rowid FROM t1 
+    WHERE t1 MATCH '(
+      zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
+      zk OR zkhdvkw OR zm OR zsmhnf
+    ) vgsld'
+  } {10 13 17 31 35 51 58 88 89 90 93 100}
+
   do_select_test 3.1 {
     SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
   } {