Add tests for, and source code comments to, fts5. No changes to code.

FossilOrigin-Name: b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a

ext/fts5/fts5_index.c

@@ -5048,7 +5048,6 @@ static void fts5DoSecureDelete(
   int iPgIdx = pSeg->pLeaf->szLeaf;
 
   u64 iDelta = 0;
-  u64 iNextDelta = 0;
   int iNextOff = 0;
   int iOff = 0;
   int nIdx = 0;
@@ -5056,8 +5055,6 @@ static void fts5DoSecureDelete(
   int bLastInDoclist = 0;
   int iIdx = 0;
   int iStart = 0;
-  int iKeyOff = 0;
-  int iPrevKeyOff = 0;
   int iDelKeyOff = 0;       /* Offset of deleted key, if any */
 
   nIdx = nPg-iPgIdx;
@@ -5082,10 +5079,21 @@ static void fts5DoSecureDelete(
   ** This block sets the following variables:
   **
   **   iStart:
+  **     The offset of the first byte of the rowid or delta-rowid
+  **     value for the doclist entry being removed.
+  **
   **   iDelta:
+  **     The value of the rowid or delta-rowid value for the doclist
+  **     entry being removed.
+  **
+  **   iNextOff:
+  **     The offset of the next entry following the position list
+  **     for the one being removed. If the position list for this
+  **     entry overflows onto the next leaf page, this value will be
+  **     greater than pLeaf->szLeaf.
   */
   {
-    int iSOP;
+    int iSOP;                     /* Start-Of-Position-list */
     if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
       iStart = pSeg->iTermLeafOffset;
     }else{
@@ -5121,14 +5129,20 @@ static void fts5DoSecureDelete(
   }
 
   iOff = iStart;
+
+  /* Set variable bLastInDoclist to true if this entry happens to be
+  ** the last rowid in the doclist for its term.  */
   if( iNextOff>=iPgIdx ){
     int pgno = pSeg->iLeafPgno+1;
     fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
     iNextOff = iPgIdx;
   }else{
-    /* Set bLastInDoclist to true if the entry being removed is the last
+    /* Loop through the page-footer. If iNextOff (offset of the
+    ** entry following the one we are removing) is equal to the 
+    ** offset of a key on this page, then the entry is the last 
     ** in its doclist.  */
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+    int iKeyOff = 0;
+    for(iIdx=0; iIdx<nIdx; /* no-op */){
       u32 iVal = 0;
       iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
       iKeyOff += iVal;
@@ -5138,30 +5152,47 @@ static void fts5DoSecureDelete(
     }
   }
 
+  /* If this is (a) the first rowid on a page and (b) is not followed by
+  ** another position list on the same page, set the "first-rowid" field
+  ** of the header to 0.  */
   if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
     fts5PutU16(&aPg[0], 0);
   }
 
   if( bLastInDoclist==0 ){
     if( iNextOff!=iPgIdx ){
+      u64 iNextDelta = 0;
       iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
       iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
     }
   }else if( 
-      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+      pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+   && iStart==pSeg->iTermLeafOffset 
   ){
     /* The entry being removed was the only position list in its
     ** doclist. Therefore the term needs to be removed as well. */
     int iKey = 0;
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+    int iKeyOff = 0;
+
+    /* Set iKeyOff to the offset of the term that will be removed - the
+    ** last offset in the footer that is not greater than iStart. */
+    for(iIdx=0; iIdx<nIdx; iKey++){
       u32 iVal = 0;
       iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
       if( (iKeyOff+iVal)>(u32)iStart ) break;
       iKeyOff += iVal;
     }
+    assert_nc( iKey>=1 );
 
+    /* Set iDelKeyOff to the value of the footer entry to remove from 
+    ** the page. */
     iDelKeyOff = iOff = iKeyOff;
+
     if( iNextOff!=iPgIdx ){
+      /* This is the only position-list associated with the term, and there
+      ** is another term following it on this page. So the subsequent term
+      ** needs to be moved to replace the term associated with the entry
+      ** being removed. */
       int nPrefix = 0;
       int nSuffix = 0;
       int nPrefix2 = 0;
@@ -5241,16 +5272,17 @@ static void fts5DoSecureDelete(
   }
 
   if( p->rc==SQLITE_OK ){
-      const int nMove = nPg - iNextOff;
+    const int nMove = nPg - iNextOff;     /* Number of bytes to move */
-      int nShift = 0;
+    int nShift = iNextOff - iOff;         /* Distance to move them */
+    int iKeyOff = 0;
+    int iPrevKeyOff = 0;
 
     memmove(&aPg[iOff], &aPg[iNextOff], nMove);
-      iPgIdx -= (iNextOff - iOff);
+    iPgIdx -= nShift;
     nPg = iPgIdx;
     fts5PutU16(&aPg[2], iPgIdx);
 
-      nShift = iNextOff - iOff;
+    for(iIdx=0; iIdx<nIdx; /* no-op */){
-      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
       u32 iVal = 0;
       iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
       iKeyOff += iVal;

ext/fts5/test/fts5secure6.test

@@ -70,5 +70,26 @@ do_execsql_test 2.2 {
   SELECT rowid FROM t1('def')
 } {-100000 -99999 9223372036854775800}
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO t1(rowid, x) 
+      VALUES(51869, 'when whenever where weress what turn'), 
+            (51871, 'to were');
+  COMMIT;
+}
+
+do_execsql_test 3.2 {
+  DELETE FROM t1 WHERE rowid=51871;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
 finish_test
 

ext/fts5/test/fts5secure7.test

@@ -0,0 +1,116 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure7
+
+
+set NVOCAB 500
+set NDOC [expr 1000]
+
+set NREP 100
+set nDeletePerRep [expr 5]
+
+set VOCAB [list]
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [expr {abs(int(rand()*$n))}]
+}
+
+proc init_vocab {} {
+  set L [split "abcdefghijklmnopqrstuvwxyz" {}]
+  set nL [llength $L]
+  for {set i 0} {$i < $::NVOCAB} {incr i} {
+    set n [expr {6 + int(rand()*8)}]
+    set word ""
+    for {set j 0} {$j < $n} {incr j} {
+      append word [select_one $L]
+    }
+    lappend ::VOCAB $word
+  }
+}
+
+proc get_word {} {
+  select_one $::VOCAB
+}
+
+proc get_document {nWord} {
+  set ret [list]
+  for {set i 0} {$i < $nWord} {incr i} {
+    lappend ret [get_word]
+  }
+  return $ret
+}
+
+init_vocab
+
+db func document [list get_document 12]
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 1.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+
+for {set iRep 0} {$iRep < $NREP} {incr iRep} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
+    set idx [select_one $lRowid]
+    db eval {
+      DELETE FROM t1 WHERE rowid=$idx
+    }
+  }
+  db eval {
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
+    )
+    INSERT INTO t1 SELECT document() FROM s;
+  }
+  do_execsql_test 1.2.$iRep {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+reset_db
+db func document [list get_document 12]
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
+}
+do_execsql_test 2.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+for {set ii 0} {$ii < $NDOC} {incr ii} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  set idx [select_one $lRowid]
+  db eval { DELETE FROM t1 WHERE rowid=$idx }
+  do_execsql_test 2.2.$ii {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
+

manifest

@@ -1,5 +1,5 @@
-C Do\snot\smake\sassumptions\sabout\sthe\sbyteorder\sof\sPowerPC\sprocessors.
+C Add\stests\sfor,\sand\ssource\scode\scomments\sto,\sfts5.\sNo\schanges\sto\scode.
-D 2023-09-04T12:50:17.388
+D 2023-09-04T16:48:31.747
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c 77bd70d50cb8397f3d8465cc4894dcdac75aa5e1fb3bbc5a4a5bc15e12fced97
+F ext/fts5/fts5_index.c 47c290589fa38f6a1860b4fc26716ed30d79ddc283b38813d1c8c1d702108ab8
 F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -204,7 +204,8 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd
 F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
-F ext/fts5/test/fts5secure6.test a0a28cfb9bf9721408b65b5d7c7ce369af3d688e273da24d101c25d60cdce05c
+F ext/fts5/test/fts5secure6.test 120feecc8c55b4774f858721e6c62c2094b059ecbcfd7fdc24bde886f55ef6ca
+F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
 F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
@@ -2115,8 +2116,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2aef9af3dd493a82ba090ccb8ab1b8974349392ec3f1c35db7a3dbbec41b5c51
+P 4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61
-R a87cf75daed42b21bba72eb7f67c969b
+R e68bce01327834d099d97be2fe3709dd
-U drh
+U dan
-Z 72c7bea784c6c2657adbd0596be177aa
+Z a7ec9ec1221680dfc975891082f06d30
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61
+b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a