Add tests for using tokendata=1 and contentless_delete=1 together.

FossilOrigin-Name: a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10

ext/fts5/test/fts5origintext5.test

@@ -0,0 +1,135 @@
+# 2023 Dec 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for tables that use both tokendata=1 and contentless_delete=1.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} { expr {abs(int(rand()*$n))} }
+
+# Select an element of the list passed as the only argument at random and
+# return it. 
+#
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+# Given a term that consists entirely of alphabet characters, return all
+# permutations of the term using upper and lower case characters. e.g.
+#
+#    "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
+#
+proc casify {term {lRet {{}}}} {
+  if {$term==""} { return $lRet }
+  set t [string range $term 1 end]
+  set f1 [string toupper [string range $term 0 0]]
+  set f2 [string tolower [string range $term 0 0]]
+  set ret [list]
+  foreach x $lRet {
+    lappend ret "$x$f1"
+    lappend ret "$x$f2"
+  }
+  return [casify $t $ret]
+}
+
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+
+# Return a random 3 letter term.
+#
+proc term {} {
+  if {[info exists ::expanded_vocab]==0} {
+    foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
+  }
+
+  select_one $::expanded_vocab
+}
+
+# Return a document - between 3 and 6 terms.
+#
+proc document {} {
+  set nTerm [expr [random 3] + 3]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+#-------------------------------------------------------------------------
+
+set NDOC  200
+set NLOOP 100
+
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", contentless_delete=1, content=,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+}
+do_test 1.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc [document]
+    execsql {
+      INSERT INTO ft(rowid, x) VALUES($ii, $doc);
+      INSERT INTO ctrl(id, x) VALUES($ii, $doc);
+    }
+  }
+} {}
+
+proc do_all_vocab_test {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {SELECT id FROM ctrl WHERE x LIKE '%' || $::v || '%'}]
+    do_execsql_test $tn.$::v {
+      SELECT rowid FROM ft($::v)
+    } $answer
+  }
+}
+
+do_all_vocab_test 1.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft WHERE rowid = $r }
+    execsql { DELETE FROM ctrl WHERE rowid = $r }
+
+    set doc [document]
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
+    execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
+  }
+  do_all_vocab_test 1.3.$ii
+}
+
+
+
+
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Fix\svarious\scompiler\swarnings\sand\sother\sproblems\swith\sthe\snew\scode\son\sthis\sbranch.
-D 2023-12-02T20:35:04.768
+C Add\stests\sfor\susing\stokendata=1\sand\scontentless_delete=1\stogether.
+D 2023-12-04T15:08:21.125
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -194,6 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0
 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
+F ext/fts5/test/fts5origintext5.test 5b9fa1b7d2f8c5f933076000c30aea5b104c00c3f1b767334b87b76d46492e59
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2149,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f
-R a72f98879c56fe0b2489dc56b6289649
+P 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0
+R 977c39d056cdc1226d7f3117e124e785
 U dan
-Z 6a8664529c8f348c40cce12fb229aa10
+Z 8168409535c63203870031d6c1a58cc5
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0
+a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10