Further streamlining of fts5 prefix query code.

FossilOrigin-Name: ca91bd8ac70a5b3fef127364f73ec675e58bb92c

ext/fts5/fts5Int.h

@@ -282,6 +282,7 @@ struct Fts5PoslistWriter {
   i64 iPrev;
 };
 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
+void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 
 int sqlite3Fts5PoslistNext64(
   const u8 *a, int n,             /* Buffer containing poslist */

ext/fts5/fts5_buffer.c

@@ -16,18 +16,20 @@
 #include "fts5Int.h"
 
 int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
-  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
+  if( pBuf->nSpace<nByte ){
-  u8 *pNew;
+    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
-  while( nNew<nByte ){
+    u8 *pNew;
-    nNew = nNew * 2;
+    while( nNew<nByte ){
-  }
+      nNew = nNew * 2;
-  pNew = sqlite3_realloc(pBuf->p, nNew);
+    }
-  if( pNew==0 ){
+    pNew = sqlite3_realloc(pBuf->p, nNew);
-    *pRc = SQLITE_NOMEM;
+    if( pNew==0 ){
-    return 1;
+      *pRc = SQLITE_NOMEM;
-  }else{
+      return 1;
-    pBuf->nSpace = nNew;
+    }else{
-    pBuf->p = pNew;
+      pBuf->nSpace = nNew;
+      pBuf->p = pNew;
+    }
   }
   return 0;
 }
@@ -208,23 +210,37 @@ int sqlite3Fts5PoslistReaderInit(
   return pIter->bEof;
 }
 
+/*
+** Append position iPos to the position list being accumulated in buffer
+** pBuf, which must be already be large enough to hold the new data.
+** The previous position written to this list is *piPrev. *piPrev is set
+** to iPos before returning.
+*/
+void sqlite3Fts5PoslistSafeAppend(
+  Fts5Buffer *pBuf, 
+  i64 *piPrev, 
+  i64 iPos
+){
+  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
+  if( (iPos & colmask) != (*piPrev & colmask) ){
+    pBuf->p[pBuf->n++] = 1;
+    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
+    *piPrev = (iPos & colmask);
+  }
+  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
+  *piPrev = iPos;
+}
+
 int sqlite3Fts5PoslistWriterAppend(
   Fts5Buffer *pBuf, 
   Fts5PoslistWriter *pWriter,
   i64 iPos
 ){
   static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
-  int rc = SQLITE_OK;
+  int rc;
-  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
+  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
-    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
+  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
-      pBuf->p[pBuf->n++] = 1;
+  return SQLITE_OK;
-      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
-      pWriter->iPrev = (iPos & colmask);
-    }
-    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
-    pWriter->iPrev = iPos;
-  }
-  return rc;
 }
 
 void *sqlite3Fts5MallocZero(int *pRc, int nByte){

ext/fts5/fts5_index.c

@@ -4704,6 +4704,7 @@ static void fts5MergePrefixLists(
   Fts5Buffer *p1,                 /* First list to merge */
   Fts5Buffer *p2                  /* Second list to merge */
 ){
+
   if( p2->n ){
     if( p1->n==0 ){
       fts5BufferSwap(p1, p2);
@@ -4719,7 +4720,6 @@ static void fts5MergePrefixLists(
       fts5DoclistIterInit(p2, &i2);
 
       while( 1 ){
-
         if( i1.iRowid<i2.iRowid ){
           /* Copy entry from i1 */
           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
@@ -4742,32 +4742,57 @@ static void fts5MergePrefixLists(
           u8 *a1 = &i1.aPoslist[i1.nSize];
           u8 *a2 = &i2.aPoslist[i2.nSize];
 
+          i64 iPrev = 0;
           Fts5PoslistWriter writer;
           memset(&writer, 0, sizeof(writer));
 
           /* Merge the two position lists. */ 
           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
           fts5BufferZero(&tmp);
+          sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
+          if( p->rc ) break;
 
           sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
           sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+          assert( iPos1>=0 && iPos2>=0 );
 
-          while( iPos1>=0 || iPos2>=0 ){
+          if( iPos1<iPos2 ){
-            i64 iNew;
+            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
-            if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
+            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
-              iNew = iPos1;
+          }else{
-              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
+            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
-            }else{
+            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
-              iNew = iPos2;
+          }
-              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+
-              if( iPos1==iPos2 ){
+          if( iPos1>=0 && iPos2>=0 ){
-                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
+            while( 1 ){
+              if( iPos1<iPos2 ){
+                if( iPos1!=iPrev ){
+                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
+                }
+                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
+                if( iPos1<0 ) break;
+              }else{
+                if( iPos2!=iPrev ){
+                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
+                }
+                sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+                if( iPos2<0 ) break;
               }
             }
-            if( iNew!=writer.iPrev || tmp.n==0 ){
+          }
-              p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
+
-              if( p->rc ) goto error_out;
+          if( iPos1>=0 ){
+            if( iPos1!=iPrev ){
+              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
             }
+            fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
+          }
+          else if( iPos2>=0 ){
+            if( iPos2!=iPrev ){
+              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
+            }
+            fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
           }
 
           /* WRITEPOSLISTSIZE */
@@ -4936,7 +4961,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
 int sqlite3Fts5IndexRollback(Fts5Index *p){
   fts5CloseReader(p);
   fts5IndexDiscardData(p);
-  assert( p->rc==SQLITE_OK );
+  /* assert( p->rc==SQLITE_OK ); */
   return SQLITE_OK;
 }
 

ext/fts5/fts5_main.c

@@ -803,7 +803,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
 */
 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
-  int rc = SQLITE_OK;
+  int rc;
 
   assert( (pCsr->ePlan<3)==
           (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
@@ -820,6 +820,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
     switch( pCsr->ePlan ){
       case FTS5_PLAN_SPECIAL: {
         CsrFlagSet(pCsr, FTS5CSR_EOF);
+        rc = SQLITE_OK;
         break;
       }
   

ext/fts5/test/fts5ad.test

@@ -11,6 +11,8 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing the FTS5 module.
 #
+# More specifically, the focus is on testing prefix queries, both with and
+# without prefix indexes.
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]

ext/fts5/test/fts5detail.test

@@ -85,6 +85,10 @@ do_execsql_test 2.2 {
   SELECT fts5_test_poslist(t2) FROM t2('aa');
 } {0.0.0}
 
+do_execsql_test 2.3 {
+  SELECT fts5_test_collist(t2) FROM t2('aa');
+} {0.0}
+
 set ::pc 0
 #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
 #exit

manifest

@@ -1,5 +1,5 @@
-C Make\ssure\sthe\s"bak.db"\sdatabase\sfile\sdoes\snot\sactually\sexist\sbefore\nstarting\sthe\s"quota.test"\stesting.
+C Further\sstreamlining\sof\sfts5\sprefix\squery\scode.
-D 2016-02-05T17:49:43.402
+D 2016-02-05T19:18:02.811
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 72b7858f02017611c3ac1ddc965251017fed0845
@@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 2095cc38e776f19cc083ca90e00772ea0b204ab3
+F ext/fts5/fts5Int.h efb02807dbe5a2bfb0ea592a472d1171cb553d53
 F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d
-F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
+F ext/fts5/fts5_buffer.c ad4bb545c866eea6add1b0f84c2c5029cd689092
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
 F ext/fts5/fts5_expr.c ff5c451a6d025909639ac0f0d0af0cc595b50feb
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 614c54544f844387ba6728c613a884a1aa151f06
+F ext/fts5/fts5_index.c 9706959f2188d97d72df750519fee7baccef9964
-F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d
+F ext/fts5/fts5_main.c 6e23df904049edb498538bd3e22e53ec1ab6f4f7
 F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
@@ -120,7 +120,7 @@ F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
-F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
+F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
@@ -142,7 +142,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
 F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599
-F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d
+F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
 F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
@@ -1426,7 +1426,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P a6b35273239669189f33402144f58328b133d182
+P 1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
-R a53abbe37c3df437dd159edcdce10b4b
+R 82ed27d217d52be944f2d4bee3dfe639
-U drh
+U dan
-Z b28b558e8ce57b27d1473527e7eaafc5
+Z ad1bc1867d7f59ff72d365283a40efbf

manifest.uuid

@@ -1 +1 @@
-1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
+ca91bd8ac70a5b3fef127364f73ec675e58bb92c