Merge in all trunk changes prior to the BTREE_FORDELETE enhancement.

FossilOrigin-Name: 53d5a4add6b60722ad77daf98b6b8983b081e16a

Makefile.msc

@@ -1084,11 +1084,9 @@ TESTEXT = \
   $(TOP)\ext\misc\eval.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
-  fts5.c \
   $(TOP)\ext\fts5\fts5_tcl.c \
   $(TOP)\ext\fts5\fts5_test_mi.c \
   $(TOP)\ext\misc\ieee754.c \
-  $(TOP)\ext\misc\json1.c \
   $(TOP)\ext\misc\nextchar.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \

ext/fts5/fts5Int.h

@@ -224,7 +224,7 @@ struct Fts5Buffer {
   int nSpace;
 };
 
-int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int);
+int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
@@ -232,17 +232,19 @@ void sqlite3Fts5BufferFree(Fts5Buffer*);
 void sqlite3Fts5BufferZero(Fts5Buffer*);
 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
-void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);
 
 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 
 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
-#define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
-#define fts5BufferAppend32(a,b,c)     sqlite3Fts5BufferAppend32(a,b,c)
+
+#define fts5BufferGrow(pRc,pBuf,nn) ( \
+  (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \
+    sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
+)
 
 /* Write and decode big-endian 32-bit integer values */
 void sqlite3Fts5Put32(u8*, int);
@@ -475,7 +477,7 @@ int sqlite3Fts5GetTokenizer(
   char **pzErr
 );
 
-Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, int*);
+Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
 
 /*
 ** End of interface to code in fts5.c.

ext/fts5/fts5_buffer.c

@@ -15,36 +15,30 @@
 
 #include "fts5Int.h"
 
-int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
+int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
-
+  int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
-  if( (pBuf->n + nByte) > pBuf->nSpace ){
+  u8 *pNew;
-    u8 *pNew;
+  while( nNew<nByte ){
-    int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
+    nNew = nNew * 2;
-
+  }
-    /* A no-op if an error has already occurred */
+  pNew = sqlite3_realloc(pBuf->p, nNew);
-    if( *pRc ) return 1;
+  if( pNew==0 ){
-
+    *pRc = SQLITE_NOMEM;
-    while( nNew<(pBuf->n + nByte) ){
+    return 1;
-      nNew = nNew * 2;
+  }else{
-    }
+    pBuf->nSpace = nNew;
-    pNew = sqlite3_realloc(pBuf->p, nNew);
+    pBuf->p = pNew;
-    if( pNew==0 ){
-      *pRc = SQLITE_NOMEM;
-      return 1;
-    }else{
-      pBuf->nSpace = nNew;
-      pBuf->p = pNew;
-    }
   }
   return 0;
 }
 
+
 /*
 ** Encode value iVal as an SQLite varint and append it to the buffer object
 ** pBuf. If an OOM error occurs, set the error code in p.
 */
 void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
+  if( fts5BufferGrow(pRc, pBuf, 9) ) return;
   pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
 }
 
@@ -59,12 +53,6 @@ int sqlite3Fts5Get32(const u8 *aBuf){
   return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
 }
 
-void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, 4) ) return;
-  sqlite3Fts5Put32(&pBuf->p[pBuf->n], iVal);
-  pBuf->n += 4;
-}
-
 /*
 ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
 ** the error code in p. If an error has already occurred when this function
@@ -77,7 +65,7 @@ void sqlite3Fts5BufferAppendBlob(
   const u8 *pData
 ){
   assert( *pRc || nData>=0 );
-  if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return;
+  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
   memcpy(&pBuf->p[pBuf->n], pData, nData);
   pBuf->n += nData;
 }
@@ -227,7 +215,7 @@ int sqlite3Fts5PoslistWriterAppend(
 ){
   static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   int rc = SQLITE_OK;
-  if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
+  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
     if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
       pBuf->p[pBuf->n++] = 1;
       pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));

ext/fts5/fts5_index.c

@@ -374,26 +374,6 @@ struct Fts5SegWriter {
   int iBtPage;                    /* Page number corresponding to btterm */
 };
 
-/*
-** Object for iterating through the merged results of one or more segments,
-** visiting each term/rowid pair in the merged data.
-**
-** nSeg is always a power of two greater than or equal to the number of
-** segments that this object is merging data from. Both the aSeg[] and
-** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded
-** with zeroed objects - these are handled as if they were iterators opened
-** on empty segments.
-**
-** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an
-** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the 
-** comparison in this context is the index of the iterator that currently
-** points to the smaller term/rowid combination. Iterators at EOF are
-** considered to be greater than all other iterators.
-**
-** aFirst[1] contains the index in aSeg[] of the iterator that points to
-** the smallest key overall. aFirst[0] is unused. 
-*/
-
 typedef struct Fts5CResult Fts5CResult;
 struct Fts5CResult {
   u16 iFirst;                     /* aSeg[] index of firstest iterator */
@@ -500,6 +480,24 @@ struct Fts5SegIter {
 #define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p))
 
 /*
+** Object for iterating through the merged results of one or more segments,
+** visiting each term/rowid pair in the merged data.
+**
+** nSeg is always a power of two greater than or equal to the number of
+** segments that this object is merging data from. Both the aSeg[] and
+** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded
+** with zeroed objects - these are handled as if they were iterators opened
+** on empty segments.
+**
+** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an
+** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the 
+** comparison in this context is the index of the iterator that currently
+** points to the smaller term/rowid combination. Iterators at EOF are
+** considered to be greater than all other iterators.
+**
+** aFirst[1] contains the index in aSeg[] of the iterator that points to
+** the smallest key overall. aFirst[0] is unused. 
+**
 ** poslist:
 **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
 **   There is no way to tell if this is populated or not.
@@ -1008,6 +1006,18 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){
 }
 #endif
 
+#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) {     \
+  assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) );             \
+  memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob);             \
+  (pBuf)->n += nBlob;                                      \
+}
+
+#define fts5BufferSafeAppendVarint(pBuf, iVal) {                \
+  (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal));  \
+  assert( (pBuf)->nSpace>=(pBuf)->n );                          \
+}
+
+
 /*
 ** Serialize and store the "structure" record.
 **
@@ -1026,11 +1036,14 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     /* Append the current configuration cookie */
     iCookie = p->pConfig->iCookie;
     if( iCookie<0 ) iCookie = 0;
-    fts5BufferAppend32(&p->rc, &buf, iCookie);
 
-    fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel);
+    if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){
-    fts5BufferAppendVarint(&p->rc, &buf, pStruct->nSegment);
+      sqlite3Fts5Put32(buf.p, iCookie);
-    fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter);
+      buf.n = 4;
+      fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
+      fts5BufferSafeAppendVarint(&buf, pStruct->nSegment);
+      fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter);
+    }
 
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
       int iSeg;                     /* Used to iterate through segments */
@@ -1830,7 +1843,14 @@ static void fts5SegIterNext(
             if( pbNewTerm ) *pbNewTerm = 1;
           }
         }else{
-          fts5SegIterLoadNPos(p, pIter);
+          /* The following could be done by calling fts5SegIterLoadNPos(). But
+          ** this block is particularly performance critical, so equivalent
+          ** code is inlined. */
+          int nSz;
+          assert( p->rc==SQLITE_OK );
+          fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
+          pIter->bDel = (nSz & 0x0001);
+          pIter->nPos = nSz>>1;
         }
       }
     }
@@ -2593,6 +2613,35 @@ static void fts5MultiIterNext(
   }
 }
 
+static void fts5MultiIterNext2(
+  Fts5Index *p, 
+  Fts5IndexIter *pIter,
+  int *pbNewTerm                  /* OUT: True if *might* be new term */
+){
+  assert( pIter->bSkipEmpty );
+  if( p->rc==SQLITE_OK ){
+    do {
+      int iFirst = pIter->aFirst[1].iFirst;
+      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+      int bNewTerm = 0;
+
+      fts5SegIterNext(p, pSeg, &bNewTerm);
+      if( pSeg->pLeaf==0 || bNewTerm 
+       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
+      ){
+        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+        fts5MultiIterSetEof(pIter);
+        *pbNewTerm = 1;
+      }else{
+        *pbNewTerm = 0;
+      }
+      fts5AssertMultiIterSetup(p, pIter);
+
+    }while( fts5MultiIterIsEmpty(p, pIter) );
+  }
+}
+
+
 static Fts5IndexIter *fts5MultiIterAlloc(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   int nSeg
@@ -3346,9 +3395,12 @@ static void fts5WriteInit(
   pWriter->bFirstTermInPage = 1;
   pWriter->iBtPage = 1;
 
+  assert( pWriter->writer.buf.n==0 );
+  assert( pWriter->writer.pgidx.n==0 );
+
   /* Grow the two buffers to pgsz + padding bytes in size. */
-  fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, nBuffer);
+  sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer);
-  fts5BufferGrow(&p->rc, &pWriter->writer.buf, nBuffer);
+  sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer);
 
   if( p->pIdxWriter==0 ){
     Fts5Config *pConfig = p->pConfig;
@@ -3701,17 +3753,6 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
   return ret;
 }
 
-#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) {     \
-  assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) );             \
-  memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob);             \
-  (pBuf)->n += nBlob;                                      \
-}
-
-#define fts5BufferSafeAppendVarint(pBuf, iVal) {                \
-  (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal));  \
-  assert( (pBuf)->nSpace>=(pBuf)->n );                          \
-}
-
 /*
 ** Flush the contents of in-memory hash table iHash to a new level-0 
 ** segment on disk. Also update the corresponding structure record.
@@ -4060,7 +4101,7 @@ static int fts5IndexExtractCol(
   const u8 *pEnd = &p[n];         /* One byte past end of position list */
   u8 prev = 0;
 
-  while( iCol!=iCurrent ){
+  while( iCol>iCurrent ){
     /* Advance pointer p until it points to pEnd or an 0x01 byte that is
     ** not part of a varint */
     while( (prev & 0x80) || *p!=0x01 ){
@@ -4070,6 +4111,7 @@ static int fts5IndexExtractCol(
     *pa = p++;
     p += fts5GetVarint32(p, iCurrent);
   }
+  if( iCol!=iCurrent ) return 0;
 
   /* Advance pointer p until it points to pEnd or an 0x01 byte that is
   ** not part of a varint */
@@ -4107,18 +4149,6 @@ static int fts5AppendPoslist(
     assert( fts5MultiIterEof(p, pMulti)==0 );
     assert( pSeg->nPos>0 );
     if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
-      int iSv1;
-      int iSv2;
-      int iData;
-
-      /* Append iDelta */
-      iSv1 = pBuf->n;
-      fts5BufferSafeAppendVarint(pBuf, iDelta);
-
-      /* WRITEPOSLISTSIZE */
-      iSv2 = pBuf->n;
-      fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
-      iData = pBuf->n;
 
       if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
        && (pColset==0 || pColset->nCol==1)
@@ -4127,27 +4157,45 @@ static int fts5AppendPoslist(
         int nPos;
         if( pColset ){
           nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
+          if( nPos==0 ) return 1;
         }else{
           nPos = pSeg->nPos;
         }
+        assert( nPos>0 );
+        fts5BufferSafeAppendVarint(pBuf, iDelta);
+        fts5BufferSafeAppendVarint(pBuf, nPos*2);
         fts5BufferSafeAppendBlob(pBuf, pPos, nPos);
       }else{
-        fts5SegiterPoslist(p, pSeg, pColset, pBuf);
+        int iSv1;
-      }
+        int iSv2;
+        int iData;
 
-      if( pColset ){
+        /* Append iDelta */
-        int nActual = pBuf->n - iData;
+        iSv1 = pBuf->n;
-        if( nActual!=pSeg->nPos ){
+        fts5BufferSafeAppendVarint(pBuf, iDelta);
-          if( nActual==0 ){
+
-            pBuf->n = iSv1;
+        /* WRITEPOSLISTSIZE */
-            return 1;
+        iSv2 = pBuf->n;
-          }else{
+        fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
-            int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
+        iData = pBuf->n;
-            while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
+
-            sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
+        fts5SegiterPoslist(p, pSeg, pColset, pBuf);
+
+        if( pColset ){
+          int nActual = pBuf->n - iData;
+          if( nActual!=pSeg->nPos ){
+            if( nActual==0 ){
+              pBuf->n = iSv1;
+              return 1;
+            }else{
+              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
+              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
+              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
+            }
           }
         }
       }
+
     }
   }
 
@@ -4236,7 +4284,7 @@ static void fts5MergePrefixLists(
     memset(&out, 0, sizeof(out));
     memset(&tmp, 0, sizeof(tmp));
 
-    sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
+    sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
     fts5DoclistIterInit(p1, &i1);
     fts5DoclistIterInit(p2, &i2);
     while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
@@ -4327,17 +4375,20 @@ static void fts5SetupPrefixIter(
     Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
     Fts5Data *pData;
     Fts5Buffer doclist;
+    int bNewTerm = 0;
 
     memset(&doclist, 0, sizeof(doclist));
     for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
         fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext(p, p1, 0, 0)
+        fts5MultiIterNext2(p, p1, &bNewTerm)
     ){
       i64 iRowid = fts5MultiIterRowid(p1);
       int nTerm;
       const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
       assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
-      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
+      if( bNewTerm ){
+        if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
+      }
 
       if( doclist.n>0 && iRowid<=iLastRowid ){
         for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
@@ -4597,11 +4648,9 @@ int sqlite3Fts5IndexQuery(
   Fts5Buffer buf = {0, 0, 0};
 
   /* If the QUERY_SCAN flag is set, all other flags must be clear. */
-  assert( (flags & FTS5INDEX_QUERY_SCAN)==0
+  assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
-       || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN
-  );
 
-  if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){
+  if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
     memcpy(&buf.p[1], pToken, nToken);
 
 #ifdef SQLITE_DEBUG
@@ -5286,9 +5335,11 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   Fts5IndexIter *pIter;           /* Used to iterate through entire index */
   Fts5Structure *pStruct;         /* Index structure */
 
+#ifdef SQLITE_DEBUG
   /* Used by extra internal tests only run if NDEBUG is not defined */
   u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
   Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
+#endif
   
   /* Load the FTS index structure */
   pStruct = fts5StructureRead(p);
@@ -5344,7 +5395,9 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
 
   fts5StructureRelease(pStruct);
+#ifdef SQLITE_DEBUG
   fts5BufferFree(&term);
+#endif
   fts5BufferFree(&poslist);
   return fts5IndexReturn(p);
 }

ext/fts5/fts5_main.c

@@ -2021,20 +2021,20 @@ static void fts5ApiCallback(
 ** Given cursor id iId, return a pointer to the corresponding Fts5Index 
 ** object. Or NULL If the cursor id does not exist.
 **
-** If successful, set *pnCol to the number of indexed columns in the
+** If successful, set *ppConfig to point to the associated config object 
-** table before returning.
+** before returning.
 */
 Fts5Index *sqlite3Fts5IndexFromCsrid(
-  Fts5Global *pGlobal, 
+  Fts5Global *pGlobal,            /* FTS5 global context for db handle */
-  i64 iCsrId, 
+  i64 iCsrId,                     /* Id of cursor to find */
-  int *pnCol
+  Fts5Config **ppConfig           /* OUT: Configuration object */
 ){
   Fts5Cursor *pCsr;
   Fts5Table *pTab;
 
   pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
   pTab = (Fts5Table*)pCsr->base.pVtab;
-  *pnCol = pTab->pConfig->nCol;
+  *ppConfig = pTab->pConfig;
 
   return pTab->pIndex;
 }

ext/fts5/fts5_storage.c

@@ -914,12 +914,12 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
   /* Check that the %_docsize and %_content tables contain the expected
   ** number of rows.  */
   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
-    i64 nRow;
+    i64 nRow = 0;
     rc = fts5StorageCount(p, "content", &nRow);
     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
   }
   if( rc==SQLITE_OK && pConfig->bColumnsize ){
-    i64 nRow;
+    i64 nRow = 0;
     rc = fts5StorageCount(p, "docsize", &nRow);
     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
   }

ext/fts5/fts5_vocab.c

@@ -55,16 +55,18 @@ struct Fts5VocabCursor {
   int bEof;                       /* True if this cursor is at EOF */
   Fts5IndexIter *pIter;           /* Term/rowid iterator object */
 
+  int nLeTerm;                    /* Size of zLeTerm in bytes */
+  char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
+
   /* These are used by 'col' tables only */
-  int nCol;
+  Fts5Config *pConfig;            /* Fts5 table configuration */
   int iCol;
   i64 *aCnt;
   i64 *aDoc;
 
-  /* Output values */
+  /* Output values used by 'row' and 'col' tables */
   i64 rowid;                      /* This table's current rowid value */
   Fts5Buffer term;                /* Current value of 'term' column */
-  i64 aVal[3];                    /* Up to three columns left of 'term' */
 };
 
 #define FTS5_VOCAB_COL    0
@@ -73,6 +75,14 @@ struct Fts5VocabCursor {
 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
 
+/*
+** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
+*/
+#define FTS5_VOCAB_TERM_EQ 0x01
+#define FTS5_VOCAB_TERM_GE 0x02
+#define FTS5_VOCAB_TERM_LE 0x04
+
+
 /*
 ** Translate a string containing an fts5vocab table type to an 
 ** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
@@ -170,7 +180,7 @@ static int fts5VocabInitVtab(
     const char *zType = bDb ? argv[5] : argv[4];
     int nDb = (int)strlen(zDb)+1; 
     int nTab = (int)strlen(zTab)+1;
-    int eType;
+    int eType = 0;
     
     rc = fts5VocabTableType(zType, pzErr, &eType);
     if( rc==SQLITE_OK ){
@@ -230,6 +240,45 @@ static int fts5VocabBestIndexMethod(
   sqlite3_vtab *pVTab, 
   sqlite3_index_info *pInfo
 ){
+  int i;
+  int iTermEq = -1;
+  int iTermGe = -1;
+  int iTermLe = -1;
+  int idxNum = 0;
+  int nArg = 0;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
+    if( p->usable==0 ) continue;
+    if( p->iColumn==0 ){          /* term column */
+      if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
+      if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
+    }
+  }
+
+  if( iTermEq>=0 ){
+    idxNum |= FTS5_VOCAB_TERM_EQ;
+    pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
+    pInfo->estimatedCost = 100;
+  }else{
+    pInfo->estimatedCost = 1000000;
+    if( iTermGe>=0 ){
+      idxNum |= FTS5_VOCAB_TERM_GE;
+      pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
+      pInfo->estimatedCost = pInfo->estimatedCost / 2;
+    }
+    if( iTermLe>=0 ){
+      idxNum |= FTS5_VOCAB_TERM_LE;
+      pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
+      pInfo->estimatedCost = pInfo->estimatedCost / 2;
+    }
+  }
+
+  pInfo->idxNum = idxNum;
+
   return SQLITE_OK;
 }
 
@@ -242,12 +291,11 @@ static int fts5VocabOpenMethod(
 ){
   Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
   Fts5Index *pIndex = 0;
-  int nCol = 0;
+  Fts5Config *pConfig = 0;
   Fts5VocabCursor *pCsr = 0;
   int rc = SQLITE_OK;
   sqlite3_stmt *pStmt = 0;
   char *zSql = 0;
-  int nByte;
 
   zSql = sqlite3Fts5Mprintf(&rc,
       "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
@@ -262,7 +310,7 @@ static int fts5VocabOpenMethod(
 
   if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
     i64 iId = sqlite3_column_int64(pStmt, 0);
-    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &nCol);
+    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
   }
 
   if( rc==SQLITE_OK && pIndex==0 ){
@@ -276,14 +324,17 @@ static int fts5VocabOpenMethod(
     }
   }
 
-  nByte = nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
+  if( rc==SQLITE_OK ){
-  pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
+    int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
+    pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
+  }
+
   if( pCsr ){
     pCsr->pIndex = pIndex;
     pCsr->pStmt = pStmt;
-    pCsr->nCol = nCol;
+    pCsr->pConfig = pConfig;
     pCsr->aCnt = (i64*)&pCsr[1];
-    pCsr->aDoc = &pCsr->aCnt[nCol];
+    pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
   }else{
     sqlite3_finalize(pStmt);
   }
@@ -296,6 +347,9 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
   pCsr->rowid = 0;
   sqlite3Fts5IterClose(pCsr->pIter);
   pCsr->pIter = 0;
+  sqlite3_free(pCsr->zLeTerm);
+  pCsr->nLeTerm = -1;
+  pCsr->zLeTerm = 0;
 }
 
 /*
@@ -319,16 +373,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
   Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
   int rc = SQLITE_OK;
+  int nCol = pCsr->pConfig->nCol;
 
   pCsr->rowid++;
 
   if( pTab->eType==FTS5_VOCAB_COL ){
-    for(pCsr->iCol++; pCsr->iCol<pCsr->nCol; pCsr->iCol++){
+    for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
       if( pCsr->aCnt[pCsr->iCol] ) break;
     }
   }
 
-  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=pCsr->nCol ){
+  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
     if( sqlite3Fts5IterEof(pCsr->pIter) ){
       pCsr->bEof = 1;
     }else{
@@ -336,10 +391,18 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
       int nTerm;
 
       zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+      if( pCsr->nLeTerm>=0 ){
+        int nCmp = MIN(nTerm, pCsr->nLeTerm);
+        int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
+        if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
+          pCsr->bEof = 1;
+          return SQLITE_OK;
+        }
+      }
+
       sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
-      memset(pCsr->aVal, 0, sizeof(pCsr->aVal));
+      memset(pCsr->aCnt, 0, nCol * sizeof(i64));
-      memset(pCsr->aCnt, 0, pCsr->nCol * sizeof(i64));
+      memset(pCsr->aDoc, 0, nCol * sizeof(i64));
-      memset(pCsr->aDoc, 0, pCsr->nCol * sizeof(i64));
       pCsr->iCol = 0;
 
       assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
@@ -353,9 +416,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
         if( rc==SQLITE_OK ){
           if( pTab->eType==FTS5_VOCAB_ROW ){
             while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-              pCsr->aVal[1]++;
+              pCsr->aCnt[0]++;
             }
-            pCsr->aVal[0]++;
+            pCsr->aDoc[0]++;
           }else{
             int iCol = -1;
             while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
@@ -369,9 +432,12 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
           }
           rc = sqlite3Fts5IterNextScan(pCsr->pIter);
         }
+
         if( rc==SQLITE_OK ){
           zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
-          if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
+          if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
+            break;
+          }
           if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
         }
       }
@@ -380,9 +446,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
   if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
     while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
-    pCsr->aVal[0] = pCsr->iCol;
+    assert( pCsr->iCol<pCsr->pConfig->nCol );
-    pCsr->aVal[1] = pCsr->aDoc[pCsr->iCol];
-    pCsr->aVal[2] = pCsr->aCnt[pCsr->iCol];
   }
   return rc;
 }
@@ -398,11 +462,47 @@ static int fts5VocabFilterMethod(
   sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  int rc;
+  int rc = SQLITE_OK;
-  const int flags = FTS5INDEX_QUERY_SCAN;
+
+  int iVal = 0;
+  int f = FTS5INDEX_QUERY_SCAN;
+  const char *zTerm = 0;
+  int nTerm = 0;
+
+  sqlite3_value *pEq = 0;
+  sqlite3_value *pGe = 0;
+  sqlite3_value *pLe = 0;
 
   fts5VocabResetCursor(pCsr);
-  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter);
+  if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
+  if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
+  if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
+
+  if( pEq ){
+    zTerm = (const char *)sqlite3_value_text(pEq);
+    nTerm = sqlite3_value_bytes(pEq);
+    f = 0;
+  }else{
+    if( pGe ){
+      zTerm = (const char *)sqlite3_value_text(pGe);
+      nTerm = sqlite3_value_bytes(pGe);
+    }
+    if( pLe ){
+      const char *zCopy = (const char *)sqlite3_value_text(pLe);
+      pCsr->nLeTerm = sqlite3_value_bytes(pLe);
+      pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
+      if( pCsr->zLeTerm==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
+      }
+    }
+  }
+
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
+  }
   if( rc==SQLITE_OK ){
     rc = fts5VocabNextMethod(pCursor);
   }
@@ -425,17 +525,29 @@ static int fts5VocabColumnMethod(
   int iCol                        /* Index of column to read value from */
 ){
   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
-  switch( iCol ){
-    case 0: /* term */
-      sqlite3_result_text(
-          pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
-      );
-      break;
 
-    default:
+  if( iCol==0 ){
-      assert( iCol<4 && iCol>0 );
+    sqlite3_result_text(
-      sqlite3_result_int64(pCtx, pCsr->aVal[iCol-1]);
+        pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
-      break;
+    );
+  }
+  else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){
+    assert( iCol==1 || iCol==2 || iCol==3 );
+    if( iCol==1 ){
+      const char *z = pCsr->pConfig->azCol[pCsr->iCol];
+      sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
+    }else if( iCol==2 ){
+      sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]);
+    }else{
+      sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]);
+    }
+  }else{
+    assert( iCol==1 || iCol==2 );
+    if( iCol==1 ){
+      sqlite3_result_int64(pCtx, pCsr->aDoc[0]);
+    }else{
+      sqlite3_result_int64(pCtx, pCsr->aCnt[0]);
+    }
   }
   return SQLITE_OK;
 }

ext/fts5/test/fts5fault5.test

@@ -90,6 +90,14 @@ do_faultsim_test 3.1 -faults oom-t* -body {
   faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}}
 }
 
+do_faultsim_test 3.2 -faults oom-t* -body {
+  db eval {
+    SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
+  }
+} -test {
+  faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
+}
+
 
 
 finish_test

ext/fts5/test/fts5vocab.test

@@ -56,7 +56,7 @@ do_execsql_test 1.4.1 {
 
 do_execsql_test 1.4.2 {
   SELECT * FROM v2;
-} {x 0 2 4  y 0 1 1  z 0 1 1}
+} {x one 2 4  y one 1 1  z one 1 1}
 
 do_execsql_test 1.5.1 {
   BEGIN;
@@ -67,7 +67,7 @@ do_execsql_test 1.5.1 {
 do_execsql_test 1.5.2 {
     SELECT * FROM v2 WHERE term<'d';
   COMMIT;
-} {a 0 1 1  b 0 1 1  c 0 1 1}
+} {a one 1 1  b one 1 1  c one 1 1}
 
 do_execsql_test 1.6 {
   DELETE FROM t1 WHERE one = 'a b c';
@@ -91,14 +91,14 @@ do_execsql_test 2.0 {
 }
 
 set res_col {
-  a 0 6 11    a 1 7 9
+  a a 6 11    a b 7 9
-  b 0 6 7     b 1 7 7 
+  b a 6 7     b b 7 7 
-  c 0 6 12    c 1 5 8 
+  c a 6 12    c b 5 8 
-  d 0 4 6     d 1 9 13 
+  d a 4 6     d b 9 13 
-  e 0 6 7     e 1 6 6 
+  e a 6 7     e b 6 6 
-  f 0 9 10    f 1 7 10 
+  f a 9 10    f b 7 10 
-  g 0 5 7     g 1 5 7
+  g a 5 7     g b 5 7
-  x 0 1 1     y 1 1 1
+  x a 1 1     y b 1 1
 }
 set res_row {
   a 10 20   b 9 14   c 9 20   d 9 19   
@@ -213,5 +213,137 @@ do_catchsql_test 6.2 {
   SELECT * FROM vocab3;
 } {1 {no such fts5 table: main.lll}}
 
+#-------------------------------------------------------------------------
+# Test single term queries on fts5vocab tables (i.e. those with term=?
+# constraints in the WHERE clause).
+#
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE tx USING fts5(one, two);
+  INSERT INTO tx VALUES('g a ggg g a b eee',      'cc d aa ff g ee');
+  INSERT INTO tx VALUES('dd fff i a i jjj',       'f fff hh jj e f');
+  INSERT INTO tx VALUES('ggg a f f fff dd aa',    'd ggg f f j gg ddd');
+  INSERT INTO tx VALUES('e bb h jjj ii gg',       'e aa e f c fff');
+  INSERT INTO tx VALUES('j ff aa a h',            'h a j bbb bb');
+  INSERT INTO tx VALUES('cc i ff c d f',          'dd ii fff f c cc d');
+  INSERT INTO tx VALUES('jjj g i bb cc eee',      'hhh iii aaa b bbb aaa');
+  INSERT INTO tx VALUES('hhh hhh hhh bb fff f',   'fff gg aa ii h a');
+  INSERT INTO tx VALUES('b c cc aaa iii ggg f',   'iii ff ee a ff c cc');
+  INSERT INTO tx VALUES('hhh b hhh aaa j i i',    'dd ee ee aa bbb iii');
+  INSERT INTO tx VALUES('hh dd h b g ff i',       'ccc bb cc ccc f a d');
+  INSERT INTO tx VALUES('g d b ggg jj',           'fff jj ff jj g gg ee');
+  INSERT INTO tx VALUES('g ee ggg ggg cc bb eee', 'aa j jjj bbb dd eee ff');
+  INSERT INTO tx VALUES('c jjj hh ddd dd h',      'e aaa h jjj gg');
+
+  CREATE VIRTUAL TABLE txr USING fts5vocab(tx, row);
+  CREATE VIRTUAL TABLE txc USING fts5vocab(tx, col);
+}
+
+proc cont {L elem} {
+  set n 0
+  foreach e $L { if {$elem==$e} {incr n} }
+  set n
+}
+db func cont cont
+
+foreach {term} {
+  a aa aaa
+  b bb bbb
+  c cc ccc
+  d dd ddd
+  e ee eee
+  f ff fff
+  g gg ggg
+  h hh hhh
+  i ii iii
+  j jj jjj
+} {
+  set resr [db eval {
+    SELECT $term, 
+      sum(cont(one || ' ' || two, $term) > 0),
+      sum(cont(one || ' ' || two, $term))
+    FROM tx
+  }]
+  if {[lindex $resr 1]==0} {set resr [list]}
+
+  set r1 [db eval {
+    SELECT $term, 'one', sum(cont(one, $term)>0), sum(cont(one, $term)) FROM tx
+  }]
+  if {[lindex $r1 2]==0} {set r1 [list]}
+
+  set r2 [db eval {
+    SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx
+  }]
+  if {[lindex $r2 2]==0} {set r2 [list]}
+
+  set resc [concat $r1 $r2]
+  do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc
+  do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr
+}
+
+do_execsql_test 7.1 {
+  CREATE TABLE txr_c AS SELECT * FROM txr;
+  CREATE TABLE txc_c AS SELECT * FROM txc;
+}
+
+# Test range queries on the fts5vocab tables created above.
+#
+foreach {tn a b} {
+  1   a   jjj
+  2   bb  j
+  3   ccc ddd
+  4   dd  xyz
+  5   xzy dd
+  6   h   hh
+} {
+  do_execsql_test 7.2.$tn.1 {
+    SELECT * FROM txr WHERE term>=$a
+  } [db eval {SELECT * FROM txr_c WHERE term>=$a}]
+  do_execsql_test 7.2.$tn.2 {
+    SELECT * FROM txr WHERE term<=$b
+  } [db eval {SELECT * FROM txr_c WHERE term <=$b}]
+  do_execsql_test 7.2.$tn.3 {
+    SELECT * FROM txr WHERE term>=$a AND term<=$b
+  } [db eval {SELECT * FROM txr_c WHERE term>=$a AND term <=$b}]
+
+  do_execsql_test 7.2.$tn.4 {
+    SELECT * FROM txc WHERE term>=$a
+  } [db eval {SELECT * FROM txc_c WHERE term>=$a}]
+  do_execsql_test 7.2.$tn.5 {
+    SELECT * FROM txc WHERE term<=$b
+  } [db eval {SELECT * FROM txc_c WHERE term <=$b}]
+  do_execsql_test 7.2.$tn.6 {
+    SELECT * FROM txc WHERE term>=$a AND term<=$b
+  } [db eval {SELECT * FROM txc_c WHERE term>=$a AND term <=$b}]
+
+  do_execsql_test 7.2.$tn.7 {
+    SELECT * FROM txr WHERE term>$a
+  } [db eval {SELECT * FROM txr_c WHERE term>$a}]
+  do_execsql_test 7.2.$tn.8 {
+    SELECT * FROM txr WHERE term<$b
+  } [db eval {SELECT * FROM txr_c WHERE term<$b}]
+  do_execsql_test 7.2.$tn.9 {
+    SELECT * FROM txr WHERE term>$a AND term<$b
+  } [db eval {SELECT * FROM txr_c WHERE term>$a AND term <$b}]
+
+  do_execsql_test 7.2.$tn.10 {
+    SELECT * FROM txc WHERE term>$a
+  } [db eval {SELECT * FROM txc_c WHERE term>$a}]
+  do_execsql_test 7.2.$tn.11 {
+    SELECT * FROM txc WHERE term<$b
+  } [db eval {SELECT * FROM txc_c WHERE term<$b}]
+  do_execsql_test 7.2.$tn.12 {
+    SELECT * FROM txc WHERE term>$a AND term<$b
+  } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}]
+}
+
+do_execsql_test 7.3.1 {
+  SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term;
+} {30}
+
+do_execsql_test 7.3.2 {
+  SELECT count(*) FROM txc, txc_c 
+  WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
+} {57}
+
 finish_test
 

ext/fts5/tool/fts5txt2db.tcl

@@ -25,7 +25,7 @@ is exhausted.
   exit -1
 }
 
-set O(aColsize)       [list 10 10 10]
+set O(aColSize)       [list 10 10 10]
 set O(tblname)        t1
 set O(fts)            fts5
 
@@ -61,22 +61,7 @@ if {$i > [llength $argv]-2} usage
 set O(db) [lindex $argv $i]
 set O(files) [lrange $argv [expr $i+1] end]
 
-foreach {k v} [lrange $argv 0 end-2] {
-  switch -- $k {
-    -colsize {
-      set O(aColSize) $v
-    }
-
-    -colsize {
-      set O(aColSize) $v
-    }
-  }
-
-}
-
 sqlite3 db $O(db)
-load_static_extension db fts5
-
 
 # Create the FTS table in the db. Return a list of the table columns.
 #
@@ -84,7 +69,7 @@ proc create_table {} {
   global O
   set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
 
-  set nCol [llength $O(aColsize)]
+  set nCol [llength $O(aColSize)]
   set cols [lrange $cols 0 [expr $nCol-1]]
 
   set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
@@ -117,14 +102,14 @@ set i 0
 set cols [create_table]
 set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
 foreach c [lrange $cols 1 end] {
-  append sql ", \$$c"
+  append sql ", \$A($c)"
 }
 append sql ")"
 
 db eval BEGIN
   while {$i < $N} {
-    foreach c $cols s $O(aColsize) {
+    foreach c $cols s $O(aColSize) {
-      set $c [lrange $tokens $i [expr $i+$s-1]]
+      set A($c) [lrange $tokens $i [expr $i+$s-1]]
       incr i $s
     }
     db eval $sql

ext/fts5/tool/mkfts5c.tcl

@@ -78,7 +78,7 @@ proc fts5c_printfile {zIn} {
   global G
   set data [readfile $zIn]
   set zTail [file tail $zIn]
-  puts $G(fd) "#line 2 \"$zTail\""
+  puts $G(fd) "#line 1 \"$zTail\""
 
   set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
   if {$zTail=="fts5parse.c"} {
@@ -86,8 +86,10 @@ proc fts5c_printfile {zIn} {
   }
 
   foreach line [split $data "\n"] {
-    if {[regexp {^#include.*fts5} $line]} continue
+    if {[regexp {^#include.*fts5} $line]} {
-    if { ![regexp { sqlite3Fts5Init\(} $line] 
+      set line "/* $line */"
+    } elseif { 
+         ![regexp { sqlite3Fts5Init\(} $line] 
        && [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]
     } {
       set line "static $line"

ext/rbu/sqlite3rbu.h

@@ -341,6 +341,9 @@ sqlite3rbu *sqlite3rbu_open(
 ** If an error has occurred, either while opening or stepping the RBU object,
 ** this function may return NULL. The error code and message may be collected
 ** when sqlite3rbu_close() is called.
+**
+** Database handles returned by this function remain valid until the next
+** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db().
 */
 sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu);
 

ext/rbu/test_rbu.c

@@ -56,20 +56,34 @@ static int test_sqlite3rbu_cmd(
 ){
   int ret = TCL_OK;
   sqlite3rbu *pRbu = (sqlite3rbu*)clientData;
-  const char *azMethod[] = { 
+  struct RbuCmd {
-    "step", "close", "create_rbu_delta", "savestate", 0 
+    const char *zName;
+    int nArg;
+    const char *zUsage;
+  } aCmd[] = {
+    {"step", 2, ""},              /* 0 */
+    {"close", 2, ""},             /* 1 */
+    {"create_rbu_delta", 2, ""},  /* 2 */
+    {"savestate", 2, ""},         /* 3 */
+    {"dbMain_eval", 3, "SQL"},    /* 4 */
+    {0,0,0}
   };
-  int iMethod;
+  int iCmd;
 
-  if( objc!=2 ){
+  if( objc<2 ){
     Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
     return TCL_ERROR;
   }
-  if( Tcl_GetIndexFromObj(interp, objv[1], azMethod, "method", 0, &iMethod) ){
+  ret = Tcl_GetIndexFromObjStruct(
+      interp, objv[1], aCmd, sizeof(aCmd[0]), "method", 0, &iCmd
+  );
+  if( ret ) return TCL_ERROR;
+  if( objc!=aCmd[iCmd].nArg ){
+    Tcl_WrongNumArgs(interp, 1, objv, aCmd[iCmd].zUsage);
     return TCL_ERROR;
   }
 
-  switch( iMethod ){
+  switch( iCmd ){
     case 0: /* step */ {
       int rc = sqlite3rbu_step(pRbu);
       Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
@@ -112,6 +126,16 @@ static int test_sqlite3rbu_cmd(
       break;
     }
 
+    case 4: /* dbMain_eval */ {
+      sqlite3 *db = sqlite3rbu_db(pRbu, 0);
+      int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
+      if( rc!=SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
+        ret = TCL_ERROR;
+      }
+      break;
+    }
+
     default: /* seems unlikely */
       assert( !"cannot happen" );
       break;

manifest

@@ -1,8 +1,8 @@
-C Merge\srecent\senhancements\sfrom\strunk.\s\sVersion\snow\s3.9.1.
+C Merge\sin\sall\strunk\schanges\sprior\sto\sthe\sBTREE_FORDELETE\senhancement.
-D 2015-10-16T20:53:57.452
+D 2015-10-26T18:51:09.614
 F Makefile.in 2ea961bc09e441874eb3d1bf7398e04feb24f3ee
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 4eb750e0fdf52050a06d881e1b060f4bb116ed7e
+F Makefile.msc 702d3e98f3afc6587a78481257f3c4c900efc3a4
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION a47917b59f38b632b3a8662d14fd20f94956bdd0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -103,21 +103,21 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9
-F ext/fts5/fts5Int.h 38667e39859ff3f3bc91f47efe672023a145a118
+F ext/fts5/fts5Int.h 06594fd3e5a3c74da6df9141e165975dc0ea6ef4
 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35
-F ext/fts5/fts5_buffer.c 9a3aa73a4e7b26b1c805b9c92c1344ba3d19d2a8
+F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2
 F ext/fts5/fts5_config.c 88a77f5d5e4dfbb2355b8f6cc9969b7f02d94685
 F ext/fts5/fts5_expr.c 28b15c9ae296204bc0a2e5cf7a667d840a9d2900
 F ext/fts5/fts5_hash.c a9d4c1efebc2a91d26ad7ebdfcbf2678ceac405f
-F ext/fts5/fts5_index.c c76d636d6cd22e3b82b1dbf3a3993447091c3bed
+F ext/fts5/fts5_index.c 2bd3cb65b36160f7b411e70c4360d0619c965e4e
-F ext/fts5/fts5_main.c d3fde32e1fdd7962dc7062400d1d8c7ec2297acf
+F ext/fts5/fts5_main.c 520a29136ba07448331f73bdc36d0ffa1e9dcfef
-F ext/fts5/fts5_storage.c 9b30115742b758706de70595a8d1d084e940c978
+F ext/fts5/fts5_storage.c 8038a54a88d3beb94dc7f9db6428a3bc08b718bb
 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd
 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf
 F ext/fts5/fts5_tokenize.c 12c5d925286491a71bb3dad7c8924ce9cfd18320
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
-F ext/fts5/fts5_vocab.c 85ebf2e93089c9d3d136cc69163370592fc040f3
+F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2
 F ext/fts5/fts5parse.y e83dca6028e3309178d05b5bd920e372dc295d35
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d
@@ -152,7 +152,7 @@ F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec
 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
-F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
+F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118
 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
 F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
@@ -180,10 +180,10 @@ F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
-F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
+F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0
-F ext/fts5/tool/fts5txt2db.tcl 3d19fb8ffb234031d33d7d2151acfbc55e9cfcc4
+F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
 F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3
-F ext/fts5/tool/mkfts5c.tcl 09ce6a7997440508360f5ba1651ab7e923a8bf31
+F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
 F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
@@ -230,8 +230,8 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/sqlite3rbu.c ea47de615e911b3a69a8e7fb3be3866298403a25
-F ext/rbu/sqlite3rbu.h 1d568cb33738d7900975cc5c72e6f68049f15914
+F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde
-F ext/rbu/test_rbu.c 2a3652241fa45d5eaa141775e4ae68c1d3582c03
+F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c 0f9b595bd0debcbedf1d7a63d0e0678d619e6c9c
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -280,7 +280,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c c3a9c4209439b806c44cf30daf466955727bf46c
 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c 7fc9513b151e3ef30369609151a18177b0b3beda
+F src/btree.c 6e7cafdb9f245e0670c3478136a3c0c39ff5cf7b
 F src/btree.h 9cc758f0f1e0002049d9eb0232f9783f21f25ac5
 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0
 F src/build.c d6162335d690396dfc5c4bd59e8b2b0c14ba6285
@@ -290,7 +290,7 @@ F src/ctime.c 509ef9c64d1321f42448f111da86400b1799218a
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7
 F src/delete.c 35c939eb8bacc9dd8a6715964e5f69feb8c20e44
-F src/expr.c 41c83c44758eba4c2bac868090346ce69d2f2dd7
+F src/expr.c ea3e0c2981ffc7bd358714fa6d5132cb2cd5fc3a
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 31900763094a3736a5fc887469202eb579fef2d0
 F src/func.c ecdd69ec6a1e406f04cc73324be2ebbf6354197f
@@ -303,10 +303,10 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012
 F src/loadext.c 18586e45a215325f15096821e9c082035d4fb810
-F src/main.c fec97668771438033a7559883401067b139729e1
+F src/main.c 1cae029707c323292b5691e4d0a4c5c44561c877
-F src/malloc.c 0a2e42b835c980ba91c17866e9b6a9810ff54d75
+F src/malloc.c 337bbe9c7d436ef9b7d06b5dd10bbfc8f3025972
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
-F src/mem1.c 52485a88f22649c3b3b4f3eb15760505d49ccf71
+F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c c1ab1153bd6443bdf6f71e4213c6fb31221b9eb7
@@ -345,7 +345,7 @@ F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 4b66e3e3435da4b4c8c83696d0349f0c503b3924
 F src/sqliteInt.h af76837cad8fb696e6527ec2745f175de7cb9093
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
-F src/status.c 286f6398a4d2cd1e8ff0771e3d30f8dddb4768ea
+F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
 F src/tclsqlite.c d9439b6a910985b7fff43ba6756bcef00de22649
 F src/test1.c 8fff9c5aa63d6490f516d018b70c12a9cb9a4d8a
@@ -415,7 +415,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 05a178b78c835928d7037c0b2e409b7b656a68a4
+F src/where.c 51cca249502ee71fb77b425687dac611277925b2
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 9aa2043685ec3eb73e87fa6f50aa4ec092f197dd
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -465,7 +465,7 @@ F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea
 F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8
 F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972
 F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf
-F test/autoindex5.test 6f487290ce2a667c24517191651bfb8c7d86b6dc
+F test/autoindex5.test 96f084a5e6024ea07cace5888df3223f3ea86990
 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74
 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85
@@ -1392,7 +1392,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 29444149342fc6b1ea8cd34c2c8e1fcb06eaa7ed 39e8a5d93fa370afb03223bf0c20ea0f3448b9fc
+P 26fa091d68e89a0b6af61ba706d23a9f37e8025a 138783b553602a055b3efdeac5947cf5ccd76b5d
-R 65f1893d9bc2d803f33dd71113ac3287
+R 4b723e99df9abf57da690efface1cef0
 U drh
-Z c857f56fb460d2406ff0431b8f8cb27b
+Z 936e93924f01554045ee6552abb7ef82

manifest.uuid

@@ -1 +1 @@
-26fa091d68e89a0b6af61ba706d23a9f37e8025a
+53d5a4add6b60722ad77daf98b6b8983b081e16a

src/btree.c

@@ -4520,7 +4520,9 @@ static int accessPayload(
 
       /* If required, populate the overflow page-list cache. */
       if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){
-        assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage);
+        assert( pCur->aOverflow[iIdx]==0
+                || pCur->aOverflow[iIdx]==nextPage
+                || CORRUPT_DB );
         pCur->aOverflow[iIdx] = nextPage;
       }
 

src/expr.c

@@ -3372,6 +3372,10 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){
 **
 ** The SQLITE_ECEL_FACTOR argument allows constant arguments to be
 ** factored out into initialization code.
+**
+** The SQLITE_ECEL_REF flag means that expressions in the list with
+** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored
+** in registers at srcReg, and so the value can be copied from there.
 */
 int sqlite3ExprCodeExprList(
   Parse *pParse,     /* Parsing context */

src/main.c

@@ -2954,6 +2954,21 @@ opendb_out:
     void *pArg = sqlite3GlobalConfig.pSqllogArg;
     sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0);
   }
+#endif
+#if defined(SQLITE_HAS_CODEC)
+  if( rc==SQLITE_OK ){
+    const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey");
+    if( zHexKey && zHexKey[0] ){
+      u8 iByte;
+      int i;
+      char zKey[40];
+      for(i=0, iByte=0; i<sizeof(zKey)*2 && sqlite3Isxdigit(zHexKey[i]); i++){
+        iByte = (iByte<<4) + sqlite3HexToInt(zHexKey[i]);
+        if( (i&1)!=0 ) zKey[i/2] = iByte;
+      }
+      sqlite3_key_v2(db, 0, zKey, i/2);
+    }
+  }
 #endif
   return rc & 0xff;
 }

src/malloc.c

@@ -455,7 +455,7 @@ void sqlite3_free(void *p){
 ** *db->pnBytesFreed.
 */
 static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){
-  if( p ) *db->pnBytesFreed += sqlite3DbMallocSize(db,p);
+  *db->pnBytesFreed += sqlite3DbMallocSize(db,p);
 }
 
 /*

src/mem1.c

@@ -171,11 +171,12 @@ static void sqlite3MemFree(void *pPrior){
 ** or xRealloc().
 */
 static int sqlite3MemSize(void *pPrior){
-  assert( pPrior!=0 );
 #ifdef SQLITE_MALLOCSIZE
+  assert( pPrior!=0 );
   return (int)SQLITE_MALLOCSIZE(pPrior);
 #else
   sqlite3_int64 *p;
+  assert( pPrior!=0 );
   p = (sqlite3_int64*)pPrior;
   p--;
   return (int)p[0];

src/status.c

@@ -19,15 +19,15 @@
 /*
 ** Variables in which to record status information.
 */
+#if SQLITE_PTRSIZE>4
+typedef sqlite3_int64 sqlite3StatValueType;
+#else
+typedef u32 sqlite3StatValueType;
+#endif
 typedef struct sqlite3StatType sqlite3StatType;
 static SQLITE_WSD struct sqlite3StatType {
-#if SQLITE_PTRSIZE>4
+  sqlite3StatValueType nowValue[10];  /* Current value */
-  sqlite3_int64 nowValue[10];         /* Current value */
+  sqlite3StatValueType mxValue[10];   /* Maximum value */
-  sqlite3_int64 mxValue[10];          /* Maximum value */
-#else
-  u32 nowValue[10];                   /* Current value */
-  u32 mxValue[10];                    /* Maximum value */
-#endif
 } sqlite3Stat = { {0,}, {0,} };
 
 /*
@@ -112,7 +112,10 @@ void sqlite3StatusDown(int op, int N){
 ** The caller must hold the appropriate mutex.
 */
 void sqlite3StatusHighwater(int op, int X){
+  sqlite3StatValueType newValue;
   wsdStatInit;
+  assert( X>=0 );
+  newValue = (sqlite3StatValueType)X;
   assert( op>=0 && op<ArraySize(wsdStat.nowValue) );
   assert( op>=0 && op<ArraySize(statMutex) );
   assert( sqlite3_mutex_held(statMutex[op] ? sqlite3Pcache1Mutex()
@@ -121,8 +124,8 @@ void sqlite3StatusHighwater(int op, int X){
           || op==SQLITE_STATUS_PAGECACHE_SIZE
           || op==SQLITE_STATUS_SCRATCH_SIZE
           || op==SQLITE_STATUS_PARSER_STACK );
-  if( X>wsdStat.mxValue[op] ){
+  if( newValue>wsdStat.mxValue[op] ){
-    wsdStat.mxValue[op] = X;
+    wsdStat.mxValue[op] = newValue;
   }
 }
 

src/where.c

@@ -485,14 +485,20 @@ static LogEst estLog(LogEst N){
 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
 **
 ** This routine runs over generated VDBE code and translates OP_Column
-** opcodes into OP_Copy, and OP_Rowid into OP_Null, when the table is being
+** opcodes into OP_Copy when the table is being accessed via co-routine 
-** accessed via co-routine instead of via table lookup.
+** instead of via table lookup.
+**
+** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on
+** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero,
+** then each OP_Rowid is transformed into an instruction to increment the
+** value stored in its output register.
 */
 static void translateColumnToCopy(
   Vdbe *v,            /* The VDBE containing code to translate */
   int iStart,         /* Translate from this opcode to the end */
   int iTabCur,        /* OP_Column/OP_Rowid references to this table */
-  int iRegister       /* The first column is in this register */
+  int iRegister,      /* The first column is in this register */
+  int bIncrRowid      /* If non-zero, transform OP_rowid to OP_AddImm(1) */
 ){
   VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
   int iEnd = sqlite3VdbeCurrentAddr(v);
@@ -504,9 +510,16 @@ static void translateColumnToCopy(
       pOp->p2 = pOp->p3;
       pOp->p3 = 0;
     }else if( pOp->opcode==OP_Rowid ){
-      pOp->opcode = OP_Null;
+      if( bIncrRowid ){
-      pOp->p1 = 0;
+        /* Increment the value stored in the P2 operand of the OP_Rowid. */
-      pOp->p3 = 0;
+        pOp->opcode = OP_AddImm;
+        pOp->p1 = pOp->p2;
+        pOp->p2 = 1;
+      }else{
+        pOp->opcode = OP_Null;
+        pOp->p1 = 0;
+        pOp->p3 = 0;
+      }
     }
   }
 }
@@ -614,6 +627,8 @@ static void constructAutomaticIndex(
   Expr *pPartial = 0;         /* Partial Index Expression */
   int iContinue = 0;          /* Jump here to skip excluded rows */
   struct SrcList_item *pTabItem;  /* FROM clause term being indexed */
+  int addrCounter = 0;        /* Address where integer counter is initialized */
+  int regBase;                /* Array of registers where record is assembled */
 
   /* Generate code to skip over the creation and initialization of the
   ** transient index on 2nd and subsequent iterations of the loop. */
@@ -742,6 +757,7 @@ static void constructAutomaticIndex(
   pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
   if( pTabItem->fg.viaCoroutine ){
     int regYield = pTabItem->regReturn;
+    addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
     sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub);
     addrTop =  sqlite3VdbeAddOp1(v, OP_Yield, regYield);
     VdbeCoverage(v);
@@ -755,12 +771,15 @@ static void constructAutomaticIndex(
     pLoop->wsFlags |= WHERE_PARTIALIDX;
   }
   regRecord = sqlite3GetTempReg(pParse);
-  sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0);
+  regBase = sqlite3GenerateIndexKey(
+      pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
+  );
   sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
   if( pTabItem->fg.viaCoroutine ){
-    translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult);
+    sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
+    translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1);
     sqlite3VdbeGoto(v, addrTop);
     pTabItem->fg.viaCoroutine = 0;
   }else{
@@ -4512,7 +4531,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     */
     if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){
       translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur,
-                            pTabItem->regResult);
+                            pTabItem->regResult, 0);
       continue;
     }
 

test/autoindex5.test

@@ -17,6 +17,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix autoindex5
 
 # Schema is from the Debian security database
 #
@@ -103,6 +104,26 @@ do_execsql_test autoindex5-1.1 {
             OR sp.release = 'wheezy' OR sp.release = 'squeeze' )
   ORDER BY sp.name, st.bug_name, sp.release, sp.subrelease;
 } {/SEARCH SUBQUERY 2 USING AUTOMATIC COVERING INDEX .bug_name=/}
+
+#-------------------------------------------------------------------------
+# Test that ticket [8a2adec1] has been fixed.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE one(o);
+  INSERT INTO one DEFAULT VALUES;
+
+  CREATE TABLE t1(x, z);
+  INSERT INTO t1 VALUES('aaa', 4.0);
+  INSERT INTO t1 VALUES('aaa', 4.0);
+  CREATE VIEW vvv AS
+    SELECT * FROM t1
+    UNION ALL
+    SELECT 0, 0 WHERE 0;
+
+  SELECT (
+      SELECT sum(z) FROM vvv WHERE x='aaa'
+  ) FROM one;
+} {8.0}
     
 
 finish_test