Re-introduce the prefix-search optimization of [feef1b15d6], which was lost in a reorganization of FTS3 code.

FossilOrigin-Name: d692434b4935e8e7858230af1c126b0be8203077

ext/fts3/fts3.c

@@ -1621,10 +1621,58 @@ static int fts3DoclistMerge(
 typedef struct TermSelect TermSelect;
 struct TermSelect {
   int isReqPos;
-  char *aOutput;                  /* Malloc'd output buffer */
+  char *aaOutput[16];             /* Malloc'd output buffer */
-  int nOutput;                    /* Size of output in bytes */
+  int anOutput[16];               /* Size of output in bytes */
 };
 
+/*
+** Merge all doclists in the TermSelect.aaOutput[] array into a single
+** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
+** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
+**
+** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
+** the responsibility of the caller to free any doclists left in the
+** TermSelect.aaOutput[] array.
+*/
+static int fts3TermSelectMerge(TermSelect *pTS){
+  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
+  char *aOut = 0;
+  int nOut = 0;
+  int i;
+
+  /* Loop through the doclists in the aaOutput[] array. Merge them all
+  ** into a single doclist.
+  */
+  for(i=0; i<SizeofArray(pTS->aaOutput); i++){
+    if( pTS->aaOutput[i] ){
+      if( !aOut ){
+        aOut = pTS->aaOutput[i];
+        nOut = pTS->anOutput[i];
+        pTS->aaOutput[0] = 0;
+      }else{
+        int nNew = nOut + pTS->anOutput[i];
+        char *aNew = sqlite3_malloc(nNew);
+        if( !aNew ){
+          sqlite3_free(aOut);
+          return SQLITE_NOMEM;
+        }
+        fts3DoclistMerge(mergetype, 0, 0,
+            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+        );
+        sqlite3_free(pTS->aaOutput[i]);
+        sqlite3_free(aOut);
+        pTS->aaOutput[i] = 0;
+        aOut = aNew;
+        nOut = nNew;
+      }
+    }
+  }
+
+  pTS->aaOutput[0] = aOut;
+  pTS->anOutput[0] = nOut;
+  return SQLITE_OK;
+}
+
 /*
 ** This function is used as the sqlite3Fts3SegReaderIterate() callback when
 ** querying the full-text index for a doclist associated with a term or
@@ -1639,38 +1687,63 @@ static int fts3TermSelectCb(
   int nDoclist
 ){
   TermSelect *pTS = (TermSelect *)pContext;
-  int nNew = pTS->nOutput + nDoclist;
-  char *aNew = sqlite3_malloc(nNew);
 
   UNUSED_PARAMETER(p);
   UNUSED_PARAMETER(zTerm);
   UNUSED_PARAMETER(nTerm);
 
-  if( !aNew ){
+  if( pTS->aaOutput[0]==0 ){
-    return SQLITE_NOMEM;
-  }
-
-  if( pTS->nOutput==0 ){
     /* If this is the first term selected, copy the doclist to the output
     ** buffer using memcpy(). TODO: Add a way to transfer control of the
     ** aDoclist buffer from the caller so as to avoid the memcpy().
     */
-    memcpy(aNew, aDoclist, nDoclist);
+    pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
+    pTS->anOutput[0] = nDoclist;
+    if( pTS->aaOutput[0] ){
+      memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+    }else{
+      return SQLITE_NOMEM;
+    }
   }else{
-    /* The output buffer is not empty. Merge doclist aDoclist with the
-    ** existing output. This can only happen with prefix-searches (as
-    ** searches for exact terms return exactly one doclist).
-    */
     int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
-    fts3DoclistMerge(mergetype, 0, 0,
+    char *aMerge = aDoclist;
-        aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
+    int nMerge = nDoclist;
-    );
+    int iOut;
+
+    for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
+      char *aNew;
+      int nNew;
+      if( pTS->aaOutput[iOut]==0 ){
+        assert( iOut>0 );
+        pTS->aaOutput[iOut] = aMerge;
+        pTS->anOutput[iOut] = nMerge;
+        break;
+      }
+
+      nNew = nMerge + pTS->anOutput[iOut];
+      aNew = sqlite3_malloc(nNew);
+      if( !aNew ){
+        if( aMerge!=aDoclist ){
+          sqlite3_free(aMerge);
+        }
+        return SQLITE_NOMEM;
+      }
+      fts3DoclistMerge(mergetype, 0, 0,
+          aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+      );
+
+      if( iOut>0 ) sqlite3_free(aMerge);
+      sqlite3_free(pTS->aaOutput[iOut]);
+      pTS->aaOutput[iOut] = 0;
+
+      aMerge = aNew;
+      nMerge = nNew;
+      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
+        pTS->aaOutput[iOut] = aMerge;
+        pTS->anOutput[iOut] = nMerge;
+      }
+    }
   }
-
-  sqlite3_free(pTS->aOutput);
-  pTS->aOutput = aNew;
-  pTS->nOutput = nNew;
-
   return SQLITE_OK;
 }
 
@@ -1794,12 +1867,17 @@ static int fts3TermSelect(
   rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
       fts3TermSelectCb, (void *)&tsc
   );
+  if( rc==SQLITE_OK ){
+    rc = fts3TermSelectMerge(&tsc);
+  }
 
   if( rc==SQLITE_OK ){
-    *ppOut = tsc.aOutput;
+    *ppOut = tsc.aaOutput[0];
-    *pnOut = tsc.nOutput;
+    *pnOut = tsc.anOutput[0];
   }else{
-    sqlite3_free(tsc.aOutput);
+    for(i=0; i<SizeofArray(tsc.aaOutput); i++){
+      sqlite3_free(tsc.aaOutput[i]);
+    }
   }
 
 finished:

manifest

@@ -1,5 +1,5 @@
-C Enable\spreviously\sfailing\stests\sin\se_expr.test\sthat\spass\sfollowing\s[3e5975aa3b].
+C Re-introduce\sthe\sprefix-search\soptimization\sof\s[feef1b15d6],\swhich\swas\slost\sin\sa\sreorganization\sof\sFTS3\scode.
-D 2010-07-19T05:27:18
+D 2010-07-19T11:16:37
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -60,7 +60,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 51948505e018316cf0b76d249cdd87e409254e8f
+F ext/fts3/fts3.c 9dec342fa1cf0914da679a3b7c0d4b53a27883ba
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 70528ba8c33991699f96ecc64112122833cdbdb5
 F ext/fts3/fts3_expr.c f4ff02ebe854e97ac03ff00b38b728a9ab57fd4b
@@ -392,7 +392,7 @@ F test/fts2p.test 4b48c35c91e6a7dbf5ac8d1e5691823cc999aafb
 F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682
 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
-F test/fts3.test 471611ee89e815accf5514fe4a00bcec569e373d
+F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654
 F test/fts3_common.tcl 4d8eec9db565fed9098f45c378f28e1657802011
 F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
@@ -407,7 +407,7 @@ F test/fts3aj.test 584facbc9ac4381a7ec624bfde677340ffc2a5a4
 F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d
 F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
-F test/fts3an.test 931fa21bd80641ca594bfa32e105250a8a07918b
+F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9
 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
@@ -546,7 +546,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
 F test/pcache.test 4118a183908ecaed343a06fcef3ba82e87e0129d
 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
-F test/permutations.test 15683aaa9b41fb8d19f8a3c616b409cd77e13bef
+F test/permutations.test 3fe47c21c32b294b2354e702a25bfbff65747bb1
 F test/pragma.test ed78d200f65c6998df51196cb8c39d5300570f24
 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@@ -837,7 +837,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 3e5975aa3bb9df9e1f954bcce99384e0f13cb453
+P 3d59c54a39c4c3149bf01063a91c3db35ec84b31
-R 70e259e7c9124a680075accbca8e9ac0
+R d482059365bac67fc30275f964ac7b3d
 U dan
-Z 4442afa27e00d12db3352013b9fcc5ac
+Z e89aa551deb152bbb2478c332f32e64d

manifest.uuid

@@ -1 +1 @@
-3d59c54a39c4c3149bf01063a91c3db35ec84b31
+d692434b4935e8e7858230af1c126b0be8203077

test/fts3.test

@@ -9,61 +9,11 @@
 #
 # $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $
 
-proc lshift {lvar} {
-  upvar $lvar l
-  set ret [lindex $l 0]
-  set l [lrange $l 1 end]
-  return $ret
-}
-while {[set arg [lshift argv]] != ""} {
-  switch -- $arg {
-    -sharedpagercache {
-      sqlite3_enable_shared_cache 1
-    }
-    -soak {
-       set G(issoak) 1
-    }
-    default {
-      set argv [linsert $argv 0 $arg]
-      break
-    }
-  }
-}
-
 set testdir [file dirname $argv0]
-source $testdir/tester.tcl
+source $testdir/permutations.test
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
-  return
-}
-rename finish_test really_finish_test
-proc finish_test {} {}
-set G(isquick) 1
 
-set EXCLUDE {
+ifcapable fts3 {
-  fts3.test
+  run_test_suite fts3
-  fts3malloc.test
-  fts3rnd.test
 }
 
-# Files to include in the test.  If this list is empty then everything
+finish_test
-# that is not in the EXCLUDE list is run.
-#
-set INCLUDE {
-}
-
-foreach testfile [lsort -dictionary [glob $testdir/fts3*.test]] {
-  set tail [file tail $testfile]
-  if {[lsearch -exact $EXCLUDE $tail]>=0} continue
-  if {[llength $INCLUDE]>0 && [lsearch -exact $INCLUDE $tail]<0} continue
-  source $testfile
-  catch {db close}
-  if {$sqlite_open_file_count>0} {
-    puts "$tail did not close all files: $sqlite_open_file_count"
-    fail_test $tail
-    set sqlite_open_file_count 0
-  }
-}
-
-set sqlite_open_file_count 0
-really_finish_test

test/fts3an.test

@@ -185,12 +185,34 @@ do_test fts3an-3.1 {
   set t
 } $ret
 
-# TODO(shess) It would be useful to test a couple edge cases, but I
+# Test a boundary condition: More than 2^16 terms that match a searched for
-# don't know if we have the precision to manage it from here at this
+# prefix in a single segment.
-# time.  Prefix hits can cross leaves, which the code above _should_
+#
-# hit by virtue of size.  There are two variations on this.  If the
+puts "This next test can take a little while (~ 30 seconds)..."
-# tree is 2 levels high, the code will find the leaf-node extent
+do_test fts3an-4.1 {
-# directly, but if it is higher, the code will have to follow two
+  execsql { CREATE VIRTUAL TABLE ft USING fts3(x) }
-# separate interior branches down the tree.  Both should be tested.
+  execsql BEGIN
+    execsql { INSERT INTO ft VALUES(NULL) }
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#      2
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#      4
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#      8
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#     16
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#     32
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#     64
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#    128
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#    256
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#    512
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#   1024
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#   2048
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#   4096
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#   8192
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#  16384
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#  32768
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;#  65536
+    execsql { INSERT INTO ft SELECT * FROM ft }     ;# 131072
+  execsql COMMIT
+  execsql { UPDATE ft SET x = 'abc' || rowid }
+  execsql { SELECT count(*) FROM ft WHERE x MATCH 'abc*' }
+} {131072}
 
 finish_test

test/permutations.test

@@ -153,6 +153,17 @@ test_suite "threads" -prefix "" -description {
   thread004.test thread005.test walthread.test
 }
 
+test_suite "fts3" -prefix "" -description {
+  All FTS3 tests except fts3malloc.test and fts3rnd.test.
+} -files {
+  fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
+  fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
+  fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
+  fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
+  fts3e.test fts3expr.test fts3expr2.test fts3near.test 
+  fts3query.test fts3snippet.test
+}
+
 
 lappend ::testsuitelist xxx
 #-------------------------------------------------------------------------