Add coverage test cases for fts3.

FossilOrigin-Name: 8fcb0478c82507403165719724b62a308cb83b57

ext/fts3/fts3.c

@@ -621,6 +621,7 @@ static int fts3InitVtab(
   p->azColumn = (char **)&p[1];
   p->pTokenizer = pTokenizer;
   p->nNodeSize = 1000;
+  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
   zCsr = (char *)&p->azColumn[nCol];
 
   fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
@@ -2268,6 +2269,10 @@ int sqlite3Fts3Init(sqlite3 *db){
     }
   }
 
+#ifdef SQLITE_TEST
+  sqlite3Fts3ExprInitTestInterface(db);
+#endif
+
   /* Create the virtual table wrapper around the hash-table and overload 
   ** the two scalar functions. If this is successful, register the
   ** module with sqlite.

ext/fts3/fts3Int.h

@@ -120,10 +120,11 @@ struct Fts3Table {
   /* The following hash table is used to buffer pending index updates during
   ** transactions. Variable nPendingData estimates the memory size of the 
   ** pending data, including hash table overhead, but not malloc overhead. 
-  ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed 
+  ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   ** automatically. Variable iPrevDocid is the docid of the most recently
   ** inserted record.
   */
+  int nMaxPendingData;
   int nPendingData;
   sqlite_int64 iPrevDocid;
   Fts3Hash pendingTerms;

ext/fts3/fts3_write.c

@@ -294,8 +294,8 @@ int sqlite3Fts3ReadBlock(
   
     *pnBlock = sqlite3_column_bytes(pStmt, 0);
     *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
-    if( !*pzBlock ){
+    if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
-      return SQLITE_NOMEM;
+      return SQLITE_CORRUPT;
     }
   }
   return SQLITE_OK;
@@ -510,7 +510,7 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
   ** buffer was half empty, that would let the less frequent terms
   ** generate longer doclists.
   */
-  if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){
+  if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
     int rc = sqlite3Fts3PendingTermsFlush(p);
     if( rc!=SQLITE_OK ) return rc;
   }
@@ -2221,6 +2221,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
   }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
     p->nNodeSize = atoi(&zVal[9]);
     rc = SQLITE_OK;
+  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+    p->nMaxPendingData = atoi(&zVal[11]);
+    rc = SQLITE_OK;
 #endif
   }else{
     rc = SQLITE_ERROR;

manifest

@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
+C Add\scoverage\stest\scases\sfor\sfts3.
-Hash: SHA1
+D 2009-12-12T09:51:25
-
-C Additional\schanges\sto\sC-language\sinterface\sdocumentation.
-D 2009-12-11T23:11:27
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -59,9 +56,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c c059f8e743276b589cd07e311a42d9b699cc7bb7
+F ext/fts3/fts3.c e1828210a55b4b27df2b1f6fa78b1fb12058f56e
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 1419e2973b44ee78f0ae8f7e03abfa2bdaf14f54
+F ext/fts3/fts3Int.h 6fdd41b4f296e5bcc908444dc591397995d4ff5d
 F ext/fts3/fts3_expr.c fcf6812dbfd9cb9a2cabaf50e741411794f83e7e
 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
@@ -71,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
 F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff
 F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
 F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c 3f49f2cde3285c6e3559a756812c4a525abcfb25
+F ext/fts3/fts3_write.c a8f2eb04ac4cc869b7d7ffbfff63a4805e5559a9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@@ -190,7 +187,7 @@ F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
 F src/test_config.c 220a67047af393756f55760fdf442d935d0d88f3
 F src/test_devsym.c de3c9af2bb9a8b1e44525c449e4ec3f88e3d4110
 F src/test_func.c 1c94388a23d4a9e7cd62ec79d612d1bae2451fa2
-F src/test_hexio.c 415adbdb88cd9388831ce10edff76cb9e73d8a0b
+F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2
 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c
 F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99
 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207
@@ -380,7 +377,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682
 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
 F test/fts3.test ae0433b09b12def08105640e57693726c4949338
-F test/fts3_common.tcl dbed2fca2162d695908895680071a2b7e0a2eb55
+F test/fts3_common.tcl 2a2044688ce3addb1dd58d3d846c574cf4b7bbcd
 F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
 F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69
@@ -399,6 +396,7 @@ F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9
 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
+F test/fts3cov.test 1a21eb90d994bc8dee3fb810ee663902d0796fa7
 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da
@@ -781,14 +779,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 87fc0ce151c89beb5a43b65c0b1706f340c0c173
+P 1342916fd350d06e1c1f3d7d380249f0c9282c7b
-R b6b67451b1e0a7b9820d087eb26e450f
+R 513a9c1fbf0e1a704528face3e7e0567
-U drh
+U dan
-Z 763db5cb8f1e7f1ba88fe11712fa307e
+Z d3f9cc9a3767b7f1e5f0d0743941cc08
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFLItGjoxKgR168RlERAhmMAJ9m9BZSx/FxFUNPj9G9NQ3Jc6mFwgCfaZ8y
-dtfT2OxwqJGAKinsW/tudQE=
-=r4RM
------END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-1342916fd350d06e1c1f3d7d380249f0c9282c7b
+8fcb0478c82507403165719724b62a308cb83b57

src/test_hexio.c

@@ -330,12 +330,12 @@ static int getFts3Varint(const char *p, sqlite_int64 *v){
 
 
 /*
-** USAGE:  read_varint BLOB VARNAME
+** USAGE:  read_fts3varint BLOB VARNAME
 **
 ** Read a varint from the start of BLOB. Set variable VARNAME to contain
 ** the interpreted value. Return the number of bytes of BLOB consumed.
 */
-static int read_varint(
+static int read_fts3varint(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -373,7 +373,7 @@ int Sqlitetest_hexio_Init(Tcl_Interp *interp){
      { "hexio_render_int16",           hexio_render_int16    },
      { "hexio_render_int32",           hexio_render_int32    },
      { "utf8_to_utf8",                 utf8_to_utf8          },
-     { "read_varint",                  read_varint           },
+     { "read_fts3varint",              read_fts3varint       },
   };
   int i;
   for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){

test/fts3_common.tcl

@@ -208,7 +208,7 @@ proc fts3_doclist {tbl term where} {
 
 proc gobble_varint {varname} {
   upvar $varname blob
-  set n [read_varint $blob ret]
+  set n [read_fts3varint $blob ret]
   set blob [string range $blob $n end]
   return $ret
 }
@@ -310,38 +310,65 @@ proc fts3_read {tbl where varname} {
 # match the expected results passed via parameter $result.
 #
 proc do_select_test {name sql result} {
-  uplevel [list doPassiveTest $name $sql [list 0 $result]]
+  uplevel [list doPassiveTest 0 $name $sql [list 0 $result]]
+}
+
+proc do_restart_select_test {name sql result} {
+  uplevel [list doPassiveTest 1 $name $sql [list 0 $result]]
 }
 
 proc do_error_test {name sql error} {
-  uplevel [list doPassiveTest $name $sql [list 1 $error]]
+  uplevel [list doPassiveTest 0 $name $sql [list 1 $error]]
 }
 
-proc doPassiveTest {name sql catchres} {
+proc doPassiveTest {isRestart name sql catchres} {
   if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
 
-  if {$::DO_MALLOC_TEST} {
+  switch $::DO_MALLOC_TEST {
-    set answers [list {1 {out of memory}} $catchres]
+    0 { # No malloc failures.
-    set modes [list 100000 transient 1 persistent]
+      do_test $name [list catchsql $sql] $catchres
-  } else {
+      return
-    set answers [list $catchres]
-    set modes [list 0 ""]
     }
+    1 { # Simulate transient failures.
+      set nRepeat 1
+      set zName "transient"
+      set nStartLimit 100000
+      set nBackup 1
+    }
+    2 { # Simulate persistent failures.
+      set nRepeat 1
+      set zName "persistent"
+      set nStartLimit 100000
+      set nBackup 1
+    }
+    3 { # Simulate transient failures with extra brute force.
+      set nRepeat 100000
+      set zName "ridiculous"
+      set nStartLimit 1
+      set nBackup 10
+    }
+  }
+
+  # The set of acceptable results from running [catchsql $sql].
+  #
+  set answers [list {1 {out of memory}} $catchres]
   set str [join $answers " OR "]
 
-  foreach {nRepeat zName} $modes {
+  set nFail 1
-    for {set iFail 1} 1 {incr iFail} {
+  for {set iLimit $nStartLimit} {$nFail} {incr iLimit} {
-      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
+    for {set iFail 1} {$nFail && $iFail<=$iLimit} {incr iFail} {
+      for {set iTest 0} {$iTest<$nBackup && ($iFail-$iTest)>0} {incr iTest} {
 
+        if {$isRestart} { sqlite3 db test.db }
+
+        sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat
         set res [uplevel [list catchsql $sql]]
-      if {[lsearch -exact $answers $res]>=0} {
+        if {[lsearch -exact $answers $res]>=0} { set res $str }
-        set res $str
-      }
         set testname "$name.$zName.$iFail"
-      if {$zName == ""} { set testname $name }
+        do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str
-      do_test $testname [list set {} $res] $str
+
         set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
-      if {$nFail==0} break
+      }
     }
   }
 }

test/fts3cov.test

@@ -0,0 +1,168 @@
+# 2009 December 03
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The tests in this file are structural coverage tests. They are designed
+# to complement the tests in fts3rnd.test and fts3doc.test. Between them,
+# the three files should provide full coverage of the fts3 extension code.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If this build does not include FTS3, skip the tests in this file.
+#
+ifcapable !fts3 { finish_test ; return }
+source $testdir/fts3_common.tcl
+
+set DO_MALLOC_TEST 0
+
+#--------------------------------------------------------------------------
+# When it first needs to read a block from the %_segments table, the FTS3 
+# module compiles an SQL statement for that purpose. The statement is 
+# stored and reused each subsequent time a block is read. This test case 
+# tests the effects of an OOM error occuring while compiling the statement.
+#
+# Similarly, when FTS3 first needs to scan through a set of segment leaves
+# to find a set of documents that matches a term, it allocates a string
+# containing the text of the required SQL, and compiles one or more 
+# statements to traverse the leaves. This test case tests that OOM errors
+# that occur while allocating this string and statement are handled correctly
+# also.
+#
+do_test fts3cov-1.1 {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING fts3(x);
+    INSERT INTO t1(t1) VALUES('nodesize=24');
+    BEGIN;
+      INSERT INTO t1 VALUES('Is the night chilly and dark?');
+      INSERT INTO t1 VALUES('The night is chilly, but not dark.');
+      INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
+      INSERT INTO t1 VALUES('It covers but not hides the sky.');
+    COMMIT;
+    SELECT count(*)>0 FROM t1_segments;
+  }
+} {1}
+
+set DO_MALLOC_TEST 1
+do_restart_select_test fts3cov-1.2 {
+  SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
+} {1 2}
+set DO_MALLOC_TEST 0
+
+#--------------------------------------------------------------------------
+# When querying the full-text index, if an expected internal node block is 
+# missing from the %_segments table, or if a NULL value is stored in the 
+# %_segments table instead of a binary blob, database corruption should be 
+# reported.
+#
+# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
+# segment b-tree that uses the %_segments table to store internal nodes. 
+#
+do_test fts3cov-2.1 {
+  execsql {
+    INSERT INTO t1(t1) VALUES('nodesize=24');
+    BEGIN;
+      INSERT INTO t1 VALUES('The moon is behind, and at the full;');
+      INSERT INTO t1 VALUES('And yet she looks both small and dull.');
+      INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
+      INSERT INTO t1 VALUES('''T is a month before the month of May,');
+      INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
+      INSERT INTO t1 VALUES('The lovely lady, Christabel,');
+      INSERT INTO t1 VALUES('Whom her father loves so well,');
+      INSERT INTO t1 VALUES('What makes her in the wood so late,');
+      INSERT INTO t1 VALUES('A furlong from the castle gate?');
+      INSERT INTO t1 VALUES('She had dreams all yesternight');
+      INSERT INTO t1 VALUES('Of her own betrothed knight;');
+      INSERT INTO t1 VALUES('And she in the midnight wood will pray');
+      INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
+    COMMIT;
+
+    INSERT INTO t1(t1) VALUES('optimize');
+    SELECT substr(hex(root), 1, 2) FROM t1_segdir;
+  }
+} {03}
+
+# Test the "missing entry" case:
+do_test fts3cov-2.1 {
+  set root [db one {SELECT root FROM t1_segdir}]
+  read_fts3varint [string range $root 1 end] left_child
+  execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
+} {}
+do_error_test fts3cov-2.2 {
+  SELECT * FROM t1 WHERE t1 MATCH 'c*'
+} {database disk image is malformed}
+
+# Test the "replaced with NULL" case:
+do_test fts3cov-2.3 {
+  execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
+} {}
+do_error_test fts3cov-2.4 {
+  SELECT * FROM t1 WHERE t1 MATCH 'cloud'
+} {database disk image is malformed}
+
+#--------------------------------------------------------------------------
+# The following tests are to test the effects of OOM errors while storing
+# terms in the pending-hash table. Specifically, while creating doclist
+# blobs to store in the table. More specifically, to test OOM errors while
+# appending column numbers to doclists. For example, if a doclist consists
+# of:
+#
+#   <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
+#
+# The following tests check that malloc errors encountered while appending
+# the "0x01 <column N>" data to the dynamically growable blob used to 
+# accumulate the doclist in memory are handled correctly.
+#
+do_test fts3cov-3.1 {
+  set cols [list]
+  set vals [list]
+  for {set i 0} {$i < 120} {incr i} {
+    lappend cols "col$i"
+    lappend vals "'word'"
+  }
+  execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
+} {}
+set DO_MALLOC_TEST 1 
+do_write_test fts3cov-3.2 t2_content "
+  INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
+"
+do_write_test fts3cov-3.3 t2_content "
+  INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
+"
+do_write_test fts3cov-3.4 t2_content "
+  INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
+"
+
+#-------------------------------------------------------------------------
+# If too much data accumulates in the pending-terms hash table, it is
+# flushed to the database automatically, even if the transaction has not
+# finished. The following tests check the effects of encountering an OOM 
+# while doing this.
+#
+do_test fts3cov-4.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t3 USING fts3(x);
+    INSERT INTO t3(t3) VALUES('nodesize=24');
+    INSERT INTO t3(t3) VALUES('maxpending=100');
+  }
+} {}
+set DO_MALLOC_TEST 1 
+do_write_test fts3cov-4.2 t3_content {
+  INSERT INTO t3(docid, x)
+    SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
+    SELECT 3, 'And comforted fair Geraldine:'             UNION ALL
+    SELECT 4, '''O well, bright dame, may you command'    UNION ALL
+    SELECT 5, 'The service of Sir Leoline;'               UNION ALL
+    SELECT 2, 'And gladly our stout chivalry'             UNION ALL
+    SELECT 7, 'Will he send forth, and friends withal,'   UNION ALL
+    SELECT 8, 'To guide and guard you safe and free'      UNION ALL
+    SELECT 6, 'Home to your noble father''s hall.'''
+}
+
+finish_test
+