Fix some problems with using fts5 options 'secure-delete' and detail=none together.

FossilOrigin-Name: 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed

ext/fts5/fts5_index.c

@@ -1954,7 +1954,7 @@ static void fts5SegIterNext_None(
   iOff = pIter->iLeafOffset;
 
   /* Next entry is on the next page */
-  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
+  while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( p->rc || pIter->pLeaf==0 ) return;
     pIter->iRowid = 0;
@@ -4601,6 +4601,7 @@ static void fts5SecureDeleteOverflow(
   int iPgno,
   int *pbLastInDoclist
 ){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
   int pgno;
   Fts5Data *pLeaf = 0;
   assert( iPgno!=1 );
@@ -4627,9 +4628,12 @@ static void fts5SecureDeleteOverflow(
       /* The page contains no terms or rowids. Replace it with an empty
       ** page and move on to the right-hand peer.  */
       const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; 
-      fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
+      assert_nc( bDetailNone==0 || pLeaf->nn==4 );
+      if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
       fts5DataRelease(pLeaf);
       pLeaf = 0;
+    }else if( bDetailNone ){
+      break;
     }else{
       int nShift = iNext - 4;
       int nPg;
@@ -4678,34 +4682,16 @@ static void fts5SecureDeleteOverflow(
   fts5DataRelease(pLeaf);
 }
 
-
 /*
-** This is called as part of flushing a delete to disk in 'secure-delete'
+** Completely remove the entry that pSeg currently points to from 
-** mode. It edits the segments within the database described by argument
+** the database.
-** pStruct to remove the entries for term zTerm, rowid iRowid.
 */
-static void fts5FlushSecureDelete(
+static void fts5DoSecureDelete(
   Fts5Index *p,
   Fts5Structure *pStruct,
-  const char *zTerm,
+  Fts5SegIter *pSeg
-  i64 iRowid
 ){
-  const int f = FTS5INDEX_QUERY_SKIPHASH;
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
-  int nTerm = strlen(zTerm);
-  Fts5Iter *pIter = 0;            /* Used to find term instance */
-
-  fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
-  if( fts5MultiIterEof(p, pIter)==0 ){
-    i64 iThis = fts5MultiIterRowid(pIter);
-    if( iThis<iRowid ){
-      fts5MultiIterNextFrom(p, pIter, iRowid);
-    }
-
-    if( p->rc==SQLITE_OK 
-     && fts5MultiIterEof(p, pIter)==0 
-     && iRowid==fts5MultiIterRowid(pIter)
-    ){
-      Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
   int iSegid = pSeg->pSeg->iSegid;
   u8 *aPg = pSeg->pLeaf->p;
   int nPg = pSeg->pLeaf->nn;
@@ -4715,37 +4701,41 @@ static void fts5FlushSecureDelete(
   u64 iNextDelta = 0;
   int iNextOff = 0;
   int iOff = 0;
-      int nMove = 0;
   int nIdx = 0;
-
   u8 *aIdx = 0;
-
-      nIdx = nPg-iPgIdx;
-      aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
-      if( aIdx ){
   int bLastInDoclist = 0;
   int iIdx = 0;
   int iStart = 0;
   int iKeyOff = 0;
   int iPrevKeyOff = 0;
-        int nShift = 0;
   int iDelKeyOff = 0;       /* Offset of deleted key, if any */
+
+  nIdx = nPg-iPgIdx;
+  aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
+  if( p->rc ) return;
   memcpy(aIdx, &aPg[iPgIdx], nIdx);
 
   /* At this point segment iterator pSeg points to the entry
   ** this function should remove from the b-tree segment. 
   **
-        ** More specifically, pSeg->iLeafOffset is the offset of the
+  ** In detail=full or detail=column mode, pSeg->iLeafOffset is the 
-        ** first byte in the position-list for the entry to remove.
+  ** offset of the first byte in the position-list for the entry to 
-        ** Immediately before this comes two varints that will also
+  ** remove. Immediately before this comes two varints that will also
   ** need to be removed:
   **
   **     + the rowid or delta rowid value for the entry, and
   **     + the size of the position list in bytes.
+  **
+  ** Or, in detail=none mode, there is a single varint prior to 
+  ** pSeg->iLeafOffset - the rowid or delta rowid value.
+  **
+  ** This block sets the following variables:
+  **
+  **   iStart:
+  **   iDelta:
   */
   {
     int iSOP;
-          int nPos = 0;
     if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
       iStart = pSeg->iTermLeafOffset;
     }else{
@@ -4753,18 +4743,34 @@ static void fts5FlushSecureDelete(
     }
 
     iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
-          iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
     assert_nc( iSOP<=pSeg->iLeafOffset );
+
+    if( bDetailNone ){
+      while( iSOP<pSeg->iLeafOffset ){
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        iStart = iSOP;
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+      }
+
+      iNextOff = iSOP;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+
+    }else{
+      int nPos = 0;
+      iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
       while( iSOP<pSeg->iLeafOffset ){
         iStart = iSOP + (nPos/2);
         iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
         iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
       }
       assert_nc( iSOP==pSeg->iLeafOffset );
+      iNextOff = pSeg->iLeafOffset + pSeg->nPos;
+    }
   }
 
   iOff = iStart;
-        iNextOff = pSeg->iLeafOffset + pSeg->nPos;
   if( iNextOff>=iPgIdx ){
     int pgno = pSeg->iLeafPgno+1;
     fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
@@ -4831,7 +4837,7 @@ static void fts5FlushSecureDelete(
         }
         iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
         if( nPrefix2>nPrefix ){
-                memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix);
+          memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
           iOff += (nPrefix2-nPrefix);
         }
         memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
@@ -4883,7 +4889,9 @@ static void fts5FlushSecureDelete(
     }
 
     if( p->rc==SQLITE_OK ){
-          nMove = nPg - iNextOff;
+      const int nMove = nPg - iNextOff;
+      int nShift = 0;
+
       memmove(&aPg[iOff], &aPg[iNextOff], nMove);
       iPgIdx -= (iNextOff - iOff);
       nPg = iPgIdx;
@@ -4913,6 +4921,35 @@ static void fts5FlushSecureDelete(
     }
     sqlite3_free(aIdx);
 }
+
+/*
+** This is called as part of flushing a delete to disk in 'secure-delete'
+** mode. It edits the segments within the database described by argument
+** pStruct to remove the entries for term zTerm, rowid iRowid.
+*/
+static void fts5FlushSecureDelete(
+  Fts5Index *p,
+  Fts5Structure *pStruct,
+  const char *zTerm,
+  i64 iRowid
+){
+  const int f = FTS5INDEX_QUERY_SKIPHASH;
+  int nTerm = strlen(zTerm);
+  Fts5Iter *pIter = 0;            /* Used to find term instance */
+
+  fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
+  if( fts5MultiIterEof(p, pIter)==0 ){
+    i64 iThis = fts5MultiIterRowid(pIter);
+    if( iThis<iRowid ){
+      fts5MultiIterNextFrom(p, pIter, iRowid);
+    }
+
+    if( p->rc==SQLITE_OK 
+     && fts5MultiIterEof(p, pIter)==0 
+     && iRowid==fts5MultiIterRowid(pIter)
+    ){
+      Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+      fts5DoSecureDelete(p, pStruct, pSeg);
     }
   }
 
@@ -4998,13 +5035,26 @@ static void fts5FlushOneHash(Fts5Index *p){
           /* If in secure delete mode, and if this entry in the poslist is
           ** in fact a delete, then edit the existing segments directly
           ** using fts5FlushSecureDelete().  */
-          if( bSecureDelete && (pDoclist[iOff] & 0x01) ){
+          if( bSecureDelete ){
+            if( eDetail==FTS5_DETAIL_NONE ){
+              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  iOff++;
+                  nDoclist = 0;
+                }else{
+                  continue;
+                }
+              }
+            }else if( (pDoclist[iOff] & 0x01) ){
               fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
               if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
                 iOff++;
                 continue;
               }
             }
+          }
 
           if( p->rc==SQLITE_OK && bTermWritten==0 ){
             fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);

ext/fts5/test/fts5secure4.test

@@ -90,18 +90,20 @@ do_catchsql_test 1.12 {
 foreach {tn d} {1 full 2 none 3 column} {
   reset_db
   do_execsql_test 2.$tn.1 "
-    CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2');
+    CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20');
     INSERT INTO x1(x1, rank) VALUES('pgsz', 64);
+    INSERT INTO x1(x1, rank) VALUES('secure-delete', 1);
   "
 
   do_execsql_test 2.$tn.2 {
     BEGIN;
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
     COMMIT;
+    INSERT INTO x1(x1) VALUES('integrity-check');
   }
 
   do_execsql_test 2.$tn.3 {

ext/fts5/test/fts5secure5.test

@@ -0,0 +1,129 @@
+# 2023 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set ::testprefix fts5secure5
+return_if_no_fts5
+
+proc dump {} {
+  execsql_pp {
+    SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd');
+  COMMIT;
+}
+do_execsql_test 1.2 {
+  DELETE FROM ft1 WHERE rowid=1;
+}
+do_execsql_test 1.3 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.4 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.5 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.6 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.7 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three');
+  COMMIT;
+}
+
+do_execsql_test 2.2a {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 2.2b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.3a {
+  DELETE FROM ft1 WHERE rowid=2;
+}
+do_execsql_test 2.3b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.4a {
+  DELETE FROM ft1 WHERE rowid=4;
+}
+do_execsql_test 2.4b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+}
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO ft1(a) VALUES('c');
+  COMMIT;
+}
+do_execsql_test 3.2 {
+  DELETE FROM ft1 WHERE rowid IN (1);
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+  )
+  INSERT INTO ft1 SELECT 'abcdefg' FROM s;
+}
+
+do_test 4.1 {
+  for {set i 500} {$i > 0} {incr i -1} {
+    execsql { DELETE FROM ft1 WHERE rowid=$i }
+    execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') }
+  }
+} {}
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Fix\sproblems\shandling\scorrupt\srecords\sin\snew\sfts5\scode.
+C Fix\ssome\sproblems\swith\susing\sfts5\soptions\s'secure-delete'\sand\sdetail=none\stogether.
-D 2023-04-17T11:41:42.438
+D 2023-04-17T18:32:22.329
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd
 F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 13c4ea17f9e6b155d3c6d78b54ae4e5ed22e1692041958f3ec71a5712172d915
+F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce
 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
 F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -190,7 +190,8 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33
 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64
 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc
 F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
-F ext/fts5/test/fts5secure4.test 58638eab7e80571649d08673e7924d85ec293566377a9e7f49160857f001d529
+F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
+F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
 F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
 F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
@@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8
+P 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
-R e73db0b2fec49cf21caca2b40145b9f4
+R 94f7d6e08b83dc7ea2e9a235119af696
 U dan
-Z 8d23b3253d2a52bc5cc6c258ff86d3c2
+Z 7915ab356b23565aa915e2ae0ad516a8
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
+4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed