From 6788c7b7c00372860889216cefd5df2c4587312f Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 10 Jul 2023 20:44:09 +0000
Subject: [PATCH 01/22] Begin adding support for deleting rows from contentless
 fts5 tables.

FossilOrigin-Name: e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
---
 ext/fts5/fts5Int.h                 |   8 +
 ext/fts5/fts5_config.c             |  32 ++
 ext/fts5/fts5_index.c              | 477 ++++++++++++++++++++++++++++-
 ext/fts5/fts5_main.c               |   1 -
 ext/fts5/fts5_storage.c            |  74 ++++-
 ext/fts5/test/fts5aa.test          |   1 +
 ext/fts5/test/fts5contentless.test | 175 +++++++++++
 manifest                           |  28 +-
 manifest.uuid                      |   2 +-
 9 files changed, 769 insertions(+), 29 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless.test

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 5d05da875e..43f78c6d99 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -154,6 +154,10 @@ typedef struct Fts5Config Fts5Config;
 **   attempt to merge together. A value of 1 sets the object to use the 
 **   compile time default. Zero disables auto-merge altogether.
 **
+** bContentlessDelete:
+**   True if the contentless_delete option was present in the CREATE 
+**   VIRTUAL TABLE statement.
+**
 ** zContent:
 **
 ** zContentRowid:
@@ -188,6 +192,7 @@ struct Fts5Config {
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   int eContent;                   /* An FTS5_CONTENT value */
+  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
@@ -531,6 +536,9 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
+int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc);
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 7a4c7b8177..e0c85bbd3f 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -352,6 +352,16 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
+  if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bContentlessDelete = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
   if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
     if( pConfig->zContentRowid ){
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -596,6 +606,28 @@ int sqlite3Fts5ConfigParse(
     sqlite3_free(zTwo);
   }
 
+  /* We only allow contentless_delete=1 if the table is indeed contentless. */
+  if( rc==SQLITE_OK 
+   && pRet->bContentlessDelete 
+   && pRet->eContent!=FTS5_CONTENT_NONE 
+  ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 requires a contentless table"
+    );
+    rc = SQLITE_ERROR;
+  }
+
+  /* We only allow contentless_delete=1 if columnsize=0 is not present. 
+  **
+  ** This restriction may be removed at some point. 
+  */
+  if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 is incompatible with columnsize=0"
+    );
+    rc = SQLITE_ERROR;
+  }
+
   /* If a tokenizer= option was successfully parsed, the tokenizer has
   ** already been allocated. Otherwise, allocate an instance of the default
   ** tokenizer (unicode61) now.  */
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 7b9d21e87f..e86da1c06e 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,8 @@
 
 #define FTS5_MAX_LEVEL 64
 
+#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01"
+
 /*
 ** Details:
 **
@@ -92,6 +94,11 @@
 **         + first leaf page number (often 1, always greater than 0)
 **         + final leaf page number
 **
+**      Then, for V2 structures only:
+**
+**         + lower location counter value,
+**         + upper location counter value
+**
 ** 2. The Averages Record:
 **
 **   A single record within the %_data table. The data is a list of varints.
@@ -241,6 +248,8 @@
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
 
+#define FTS5_TOMBSTONE_ROWID(segid) fts5_dri(segid + (1<<16), 0, 0, 0)
+
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
 #endif
@@ -295,6 +304,7 @@ struct Fts5Index {
 
   /* State used by the fts5DataXXX() functions. */
   sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
+  sqlite3_stmt *pReaderOpt;
   sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
@@ -323,11 +333,20 @@ struct Fts5DoclistIter {
 ** The contents of the "structure" record for each index are represented
 ** using an Fts5Structure record in memory. Which uses instances of the 
 ** other Fts5StructureXXX types as components.
+**
+** nLocCounter:
+**   This value is set to non-zero for structure records created for
+**   contentlessdelete=1 tables only. In that case it represents the
+**   location value to apply to the next top-level segment created.
 */
 struct Fts5StructureSegment {
   int iSegid;                     /* Segment id */
   int pgnoFirst;                  /* First leaf page number in segment */
   int pgnoLast;                   /* Last leaf page number in segment */
+
+  /* contentlessdelete=1 tables only: */
+  u64 iLoc1;
+  u64 iLoc2;
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -337,6 +356,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
+  u64 nLocCounter;
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -433,6 +453,7 @@ struct Fts5SegIter {
   Fts5Data *pLeaf;                /* Current leaf data */
   Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   i64 iLeafOffset;                /* Byte offset within current leaf */
+  Fts5Data *pTombstone;
 
   /* Next method */
   void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
@@ -626,6 +647,7 @@ void sqlite3Fts5IndexCloseReader(Fts5Index *p){
   }
 }
 
+
 /*
 ** Retrieve a record from the %_data table.
 **
@@ -740,6 +762,40 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
+static Fts5Data *fts5DataReadOpt(Fts5Index *p, i64 iRowid){
+  Fts5Data *pRet = 0;
+  int rc2 = SQLITE_OK;
+
+  if( p->pReaderOpt==0 ){
+    Fts5Config *pConfig = p->pConfig;
+    fts5IndexPrepareStmt(p, &p->pReaderOpt, sqlite3_mprintf(
+          "SELECT block FROM '%q'.'%q_data' WHERE id=?",
+          pConfig->zDb, pConfig->zName
+    ));
+  }
+
+  if( p->rc==SQLITE_OK ){
+    sqlite3_bind_int64(p->pReaderOpt, 1, iRowid);
+    if( SQLITE_ROW==sqlite3_step(p->pReaderOpt) ){
+      int nByte = sqlite3_column_bytes(p->pReaderOpt, 0);
+      const u8 *aByte = (const u8*)sqlite3_column_blob(p->pReaderOpt, 0);
+      i64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
+
+      pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
+      if( pRet==0 ){
+        p->rc = SQLITE_NOMEM;
+      }else{
+        pRet->nn = nByte;
+        pRet->p = (u8*)&pRet[1];
+        memcpy(pRet->p, aByte, nByte);
+      }
+    }
+    rc2 = sqlite3_reset(p->pReaderOpt);
+    if( p->rc==SQLITE_OK ) p->rc = rc2;
+  }
+
+  return pRet;
+}
 
 /*
 ** INSERT OR REPLACE a record into the %_data table.
@@ -793,6 +849,10 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
   i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
   i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
   fts5DataDelete(p, iFirst, iLast);
+  if( p->pConfig->bContentlessDelete ){
+    i64 iTomb = FTS5_TOMBSTONE_ROWID(iSegid);
+    fts5DataDelete(p, iTomb, iTomb);
+  }
   if( p->pIdxDeleter==0 ){
     Fts5Config *pConfig = p->pConfig;
     fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
@@ -903,11 +963,18 @@ static int fts5StructureDecode(
   int nSegment = 0;
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
+  int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
+  i64 nLocCounter = 0;
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
   i = 4;
 
+  if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){
+    i += 4;
+    bStructureV2 = 1;
+  }
+
   /* Read the total number of levels and segments from the start of the
   ** structure record.  */
   i += fts5GetVarint32(&pData[i], nLevel);
@@ -958,6 +1025,11 @@ static int fts5StructureDecode(
           i += fts5GetVarint32(&pData[i], pSeg->iSegid);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
+          if( bStructureV2 ){
+            i += fts5GetVarint(&pData[i], &pSeg->iLoc1);
+            i += fts5GetVarint(&pData[i], &pSeg->iLoc2);
+            nLocCounter = MAX(nLocCounter, pSeg->iLoc2);
+          }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
             rc = FTS5_CORRUPT;
             break;
@@ -968,6 +1040,9 @@ static int fts5StructureDecode(
       }
     }
     if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
+    if( bStructureV2 ){
+      pRet->nLocCounter = nLocCounter+1;
+    }
 
     if( rc!=SQLITE_OK ){
       fts5StructureRelease(pRet);
@@ -1180,6 +1255,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     Fts5Buffer buf;               /* Buffer to serialize record into */
     int iLvl;                     /* Used to iterate through levels */
     int iCookie;                  /* Cookie value to store */
+    int nHdr = (pStruct->nLocCounter>0 ? (4+4+9+9+9) : (4+9+9));
 
     assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
     memset(&buf, 0, sizeof(Fts5Buffer));
@@ -1188,9 +1264,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     iCookie = p->pConfig->iCookie;
     if( iCookie<0 ) iCookie = 0;
 
-    if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){
+    if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
       sqlite3Fts5Put32(buf.p, iCookie);
       buf.n = 4;
+      if( pStruct->nLocCounter>0 ){
+        fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4);
+      }
       fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
       fts5BufferSafeAppendVarint(&buf, pStruct->nSegment);
       fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter);
@@ -1207,6 +1286,10 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+        if( pStruct->nLocCounter>0 ){
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc1);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc2);
+        }
       }
     }
 
@@ -1729,6 +1812,11 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
+static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
+  i64 iRowid = FTS5_TOMBSTONE_ROWID(pIter->pSeg->iSegid);
+  pIter->pTombstone = fts5DataReadOpt(p, iRowid);
+}
+
 /*
 ** Initialize the iterator object pIter to iterate through the entries in
 ** segment pSeg. The iterator is left pointing to the first entry when 
@@ -1771,6 +1859,8 @@ static void fts5SegIterInit(
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
   }
+
+  fts5SegIterLoadTombstone(p, pIter);
 }
 
 /*
@@ -2471,6 +2561,7 @@ static void fts5SegIterSeekInit(
   }
 
   fts5SegIterSetNext(p, pIter);
+  fts5SegIterLoadTombstone(p, pIter);
 
   /* Either:
   **
@@ -2558,6 +2649,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
+  fts5DataRelease(pIter->pTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -2895,6 +2987,43 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
+static u64 fts5GetU64(u8 *a){
+  return ((u64)a[0] << 56)
+       + ((u64)a[1] << 48)
+       + ((u64)a[2] << 40)
+       + ((u64)a[3] << 32)
+       + ((u64)a[4] << 24)
+       + ((u64)a[5] << 16)
+       + ((u64)a[6] << 8)
+       + ((u64)a[7] << 0);
+}
+
+static void fts5PutU64(u8 *a, u64 iVal){
+  a[0] = ((iVal >> 56) & 0xFF);
+  a[1] = ((iVal >> 48) & 0xFF);
+  a[2] = ((iVal >> 40) & 0xFF);
+  a[3] = ((iVal >> 32) & 0xFF);
+  a[4] = ((iVal >> 24) & 0xFF);
+  a[5] = ((iVal >> 16) & 0xFF);
+  a[6] = ((iVal >>  8) & 0xFF);
+  a[7] = ((iVal >>  0) & 0xFF);
+}
+
+static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
+  int iFirst = pIter->aFirst[1].iFirst;
+  Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+
+  if( pSeg->pTombstone ){
+    int ii;
+    for(ii=0; ii<pSeg->pTombstone->nn; ii+=8){
+      i64 iVal = (i64)fts5GetU64(&pSeg->pTombstone->p[ii]);
+      if( iVal==pSeg->iRowid ) return 1;
+    }
+  }
+
+  return 0;
+}
+
 /*
 ** Move the iterator to the next entry. 
 **
@@ -2932,7 +3061,9 @@ static void fts5MultiIterNext(
 
     fts5AssertMultiIterSetup(p, pIter);
     assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
-    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
+    if( (pIter->bSkipEmpty==0 || pSeg->nPos) 
+      && 0==fts5MultiIterIsDeleted(pIter)
+    ){
       pIter->xSetOutputs(pIter, pSeg);
       return;
     }
@@ -2964,7 +3095,7 @@ static void fts5MultiIterNext2(
       }
       fts5AssertMultiIterSetup(p, pIter);
 
-    }while( fts5MultiIterIsEmpty(p, pIter) );
+    }while( fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter) );
   }
 }
 
@@ -3519,7 +3650,9 @@ static void fts5MultiIterNew(
     fts5MultiIterSetEof(pNew);
     fts5AssertMultiIterSetup(p, pNew);
 
-    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
+    if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew))
+     || fts5MultiIterIsDeleted(pNew)
+    ){
       fts5MultiIterNext(p, pNew, 0, 0);
     }else if( pNew->base.bEof==0 ){
       Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst];
@@ -4334,6 +4467,12 @@ static void fts5IndexMergeLevel(
 
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
+
+    /* Set the range of locations that will go into the output segment */
+    if( pStruct->nLocCounter>0 ){
+      pSeg->iLoc1 = pLvl->aSeg[0].iLoc1;
+      pSeg->iLoc2 = pLvl->aSeg[pLvl->nSeg-1].iLoc2;
+    }
   }
   bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
 
@@ -5150,6 +5289,11 @@ static void fts5FlushOneHash(Fts5Index *p){
         pSeg->iSegid = iSegid;
         pSeg->pgnoFirst = 1;
         pSeg->pgnoLast = pgnoLast;
+        if( pStruct->nLocCounter>0 ){
+          pSeg->iLoc1 = pStruct->nLocCounter;
+          pSeg->iLoc2 = pStruct->nLocCounter;
+          pStruct->nLocCounter++;
+        }
         pStruct->nSegment++;
       }
       fts5StructurePromote(p, 0, pStruct);
@@ -5212,6 +5356,7 @@ static Fts5Structure *fts5IndexOptimizeStruct(
     pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
     pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
+    pNew->nLocCounter = pStruct->nLocCounter;
     pLvl = &pNew->aLevel[pNew->nLevel-1];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
     if( pLvl->aSeg ){
@@ -5830,6 +5975,9 @@ int sqlite3Fts5IndexReinit(Fts5Index *p){
   fts5StructureInvalidate(p);
   fts5IndexDiscardData(p);
   memset(&s, 0, sizeof(Fts5Structure));
+  if( p->pConfig->bContentlessDelete ){
+    s.nLocCounter = 1;
+  }
   fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
   fts5StructureWrite(p, &s);
   return fts5IndexReturn(p);
@@ -5889,6 +6037,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
     assert( p->pReader==0 );
     fts5StructureInvalidate(p);
     sqlite3_finalize(p->pWriter);
+    sqlite3_finalize(p->pReaderOpt);
     sqlite3_finalize(p->pDeleter);
     sqlite3_finalize(p->pIdxWriter);
     sqlite3_finalize(p->pIdxDeleter);
@@ -6221,6 +6370,70 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
+int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
+  Fts5Structure *pStruct;
+  pStruct = fts5StructureRead(p);
+  if( pStruct ){
+    *piLoc = pStruct->nLocCounter;
+    fts5StructureRelease(pStruct);
+  }
+  return fts5IndexReturn(p);
+}
+
+static void fts5IndexTombstoneAdd(
+  Fts5Index *p, 
+  Fts5StructureSegment *pSeg, 
+  i64 iRowid
+){
+  Fts5Data *pHash = 0;
+  u8 *aNew = 0;
+  int nNew = 0;
+
+  pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
+  if( p->rc ) return;
+
+  if( pHash ){
+    nNew = 8 + pHash->nn;
+  }else{
+    nNew = 8;
+  }
+  aNew = sqlite3_malloc(nNew);
+  if( aNew==0 ){
+    p->rc = SQLITE_NOMEM;
+  }else{
+    if( pHash ){
+      memcpy(aNew, pHash->p, pHash->nn);
+    }
+    fts5PutU64(&aNew[nNew-8], iRowid);
+    fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
+  }
+
+  sqlite3_free(aNew);
+  fts5DataRelease(pHash);
+}
+
+/*
+** Add iRowid to the tombstone list of the segment or segments that contain
+** rows from location iLoc.
+*/
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
+  Fts5Structure *pStruct;
+  pStruct = fts5StructureRead(p);
+  if( pStruct ){
+    int iLvl;
+    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+      int iSeg;
+      for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+        Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
+        if( pSeg->iLoc1<=iLoc && pSeg->iLoc2>=iLoc ){
+          fts5IndexTombstoneAdd(p, pSeg, iRowid);
+        }
+      }
+    }
+    fts5StructureRelease(pStruct);
+  }
+  return fts5IndexReturn(p);
+}
 
 /*************************************************************************
 **************************************************************************
@@ -6832,9 +7045,15 @@ static void fts5DebugStructure(
     );
     for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
       Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", 
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
           pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
       );
+      if( pSeg->iLoc1>0 ){
+        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " loc=%lld..%lld",
+            pSeg->iLoc1, pSeg->iLoc2
+        );
+      }
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
     }
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   }
@@ -7237,6 +7456,224 @@ static void fts5RowidFunction(
 }
 #endif /* SQLITE_TEST */
 
+#ifdef SQLITE_TEST
+
+typedef struct Fts5StructVtab Fts5StructVtab;
+struct Fts5StructVtab {
+  sqlite3_vtab base;
+};
+
+typedef struct Fts5StructVcsr Fts5StructVcsr;
+struct Fts5StructVcsr {
+  sqlite3_vtab_cursor base;
+  Fts5Structure *pStruct;
+  int iLevel;
+  int iSeg;
+  int iRowid;
+};
+
+/*
+** Create a new fts5_structure() table-valued function.
+*/
+static int fts5structConnectMethod(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  Fts5StructVtab *pNew = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_declare_vtab(db, 
+      "CREATE TABLE xyz("
+          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2,"
+      "struct HIDDEN);"
+  );
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+  }
+
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** We must have a single struct=? constraint that will be passed through
+** into the xFilter method.  If there is no valid stmt=? constraint,
+** then return an SQLITE_CONSTRAINT error.
+*/
+static int fts5structBestIndexMethod(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;
+  int rc = SQLITE_CONSTRAINT;
+  struct sqlite3_index_constraint *p;
+  pIdxInfo->estimatedCost = (double)100;
+  pIdxInfo->estimatedRows = 100;
+  pIdxInfo->idxNum = 0;
+  for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
+    if( p->usable==0 ) continue;
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==8 ){
+      rc = SQLITE_OK;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      break;
+    }
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for bytecodevtab objects.
+*/
+static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){
+  Fts5StructVtab *p = (Fts5StructVtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new bytecodevtab_cursor object.
+*/
+static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+  int rc = SQLITE_OK;
+  Fts5StructVcsr *pNew = 0;
+
+  pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+  *ppCsr = (sqlite3_vtab_cursor*)pNew;
+
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a bytecodevtab_cursor.
+*/
+static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  fts5StructureRelease(pCsr->pStruct);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a bytecodevtab_cursor to its next row of output.
+*/
+static int fts5structNextMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+
+  assert( pCsr->pStruct );
+  pCsr->iSeg++;
+  pCsr->iRowid++;
+  while( pCsr->iSeg>=pCsr->pStruct->aLevel[pCsr->iLevel].nSeg ){
+    pCsr->iLevel++;
+    pCsr->iSeg = 0;
+  }
+  if( pCsr->iLevel>=pCsr->pStruct->nLevel ){
+    fts5StructureRelease(pCsr->pStruct);
+    pCsr->pStruct = 0;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fts5structEofMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  return pCsr->pStruct==0;
+}
+
+static int fts5structRowidMethod(
+  sqlite3_vtab_cursor *cur, 
+  sqlite_int64 *piRowid
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  *piRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the bytecodevtab_cursor
+** is currently pointing.
+*/
+static int fts5structColumnMethod(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  Fts5Structure *p = pCsr->pStruct;
+  Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg];
+
+  switch( i ){
+    case 0: /* level */
+      sqlite3_result_int(ctx, pCsr->iLevel);
+      break;
+    case 1: /* segment */
+      sqlite3_result_int(ctx, pCsr->iSeg);
+      break;
+    case 2: /* merge */
+      sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge);
+      break;
+    case 3: /* segid */
+      sqlite3_result_int(ctx, pSeg->iSegid);
+      break;
+    case 4: /* leaf1 */
+      sqlite3_result_int(ctx, pSeg->pgnoFirst);
+      break;
+    case 5: /* leaf2 */
+      sqlite3_result_int(ctx, pSeg->pgnoLast);
+      break;
+    case 6: /* loc1 */
+      sqlite3_result_int(ctx, pSeg->iLoc1);
+      break;
+    case 7: /* loc2 */
+      sqlite3_result_int(ctx, pSeg->iLoc2);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Initialize a cursor.
+**
+**    idxNum==0     means show all subprograms
+**    idxNum==1     means show only the main bytecode and omit subprograms.
+*/
+static int fts5structFilterMethod(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor;
+  int rc = SQLITE_OK;
+
+  const u8 *aBlob = 0;
+  int nBlob = 0;
+
+  assert( argc==1 );
+  fts5StructureRelease(pCsr->pStruct);
+  pCsr->pStruct = 0;
+
+  nBlob = sqlite3_value_bytes(argv[0]);
+  aBlob = (const u8*)sqlite3_value_blob(argv[0]);
+  rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct);
+  if( rc==SQLITE_OK ){
+    pCsr->iLevel = 0;
+    pCsr->iRowid = 0;
+    pCsr->iSeg = -1;
+    rc = fts5structNextMethod(pVtabCursor);
+  }
+
+  return rc;
+}
+
+#endif /* SQLITE_TEST */
+
 /*
 ** This is called as part of registering the FTS5 module with database
 ** connection db. It registers several user-defined scalar functions useful
@@ -7263,6 +7700,36 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
         db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
     );
   }
+
+  if( rc==SQLITE_OK ){
+    static const sqlite3_module fts5structure_module = {
+      0,                           /* iVersion      */
+      0,                           /* xCreate       */
+      fts5structConnectMethod,     /* xConnect      */
+      fts5structBestIndexMethod,   /* xBestIndex    */
+      fts5structDisconnectMethod,  /* xDisconnect   */
+      0,                           /* xDestroy      */
+      fts5structOpenMethod,        /* xOpen         */
+      fts5structCloseMethod,       /* xClose        */
+      fts5structFilterMethod,      /* xFilter       */
+      fts5structNextMethod,        /* xNext         */
+      fts5structEofMethod,         /* xEof          */
+      fts5structColumnMethod,      /* xColumn       */
+      fts5structRowidMethod,       /* xRowid        */
+      0,                           /* xUpdate       */
+      0,                           /* xBegin        */
+      0,                           /* xSync         */
+      0,                           /* xCommit       */
+      0,                           /* xRollback     */
+      0,                           /* xFindFunction */
+      0,                           /* xRename       */
+      0,                           /* xSavepoint    */
+      0,                           /* xRelease      */
+      0,                           /* xRollbackTo   */
+      0                            /* xShadowName   */
+    };
+    rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0);
+  }
   return rc;
 #else
   return SQLITE_OK;
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 13921ce49e..180aba3973 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1624,7 +1624,6 @@ static int fts5UpdateMethod(
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
   int bUpdateOrDelete = 0;
-  
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 02b98d9e44..adc8e750ed 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -77,10 +77,10 @@ static int fts5StorageGetStmt(
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
-      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
+      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)",     /* REPLACE_DOCSIZE  */
       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
 
-      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
+      "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?",    /* LOOKUP_DOCSIZE  */
 
       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
       "SELECT %s FROM %s AS T",                         /* SCAN */
@@ -128,6 +128,19 @@ static int fts5StorageGetStmt(
         break;
       }
 
+      case FTS5_STMT_REPLACE_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
+          (pC->bContentlessDelete ? ",?" : "")
+        );
+        break;
+
+      case FTS5_STMT_LOOKUP_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], 
+            (pC->bContentlessDelete ? ",location" : ""),
+            pC->zDb, pC->zName
+        );
+        break;
+
       default:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
         break;
@@ -317,9 +330,11 @@ int sqlite3Fts5StorageOpen(
     }
 
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
-      rc = sqlite3Fts5CreateTable(
-          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
-      );
+      const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
+      if( pConfig->bContentlessDelete ){
+        zCols = "id INTEGER PRIMARY KEY, sz BLOB, location INTEGER";
+      }
+      rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
     }
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5CreateTable(
@@ -449,6 +464,34 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
+static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
+  i64 iLoc = 0;
+  sqlite3_stmt *pLookup = 0;
+  int rc = SQLITE_OK;
+
+  assert( p->pConfig->bContentlessDelete );
+  assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
+
+  /* Look up the location of the document in the %_docsize table. Store
+  ** this in stack variable iLoc.  */
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pLookup, 1, iDel);
+    if( SQLITE_ROW==sqlite3_step(pLookup) ){
+      iLoc = sqlite3_column_int64(pLookup, 1);
+    }
+    rc = sqlite3_reset(pLookup);
+    if( rc==SQLITE_OK && iLoc==0 ){
+      rc = FTS5_CORRUPT;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
+  }
+
+  return rc;
+}
 
 /*
 ** Insert a record into the %_docsize table. Specifically, do:
@@ -469,10 +512,17 @@ static int fts5StorageInsertDocsize(
     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
-      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-      sqlite3_step(pReplace);
-      rc = sqlite3_reset(pReplace);
-      sqlite3_bind_null(pReplace, 2);
+      if( p->pConfig->bContentlessDelete ){
+        i64 iLoc = 0;
+        rc = sqlite3Fts5IndexGetLocation(p->pIndex, &iLoc);
+        sqlite3_bind_int64(pReplace, 3, iLoc);
+      }
+      if( rc==SQLITE_OK ){
+        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+        sqlite3_step(pReplace);
+        rc = sqlite3_reset(pReplace);
+        sqlite3_bind_null(pReplace, 2);
+      }
     }
   }
   return rc;
@@ -536,7 +586,11 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    if( p->pConfig->bContentlessDelete ){
+      rc = fts5StorageContentlessDelete(p, iDel);
+    }else{
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    }
   }
 
   /* Delete the %_docsize record */
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 59ce4f6a1f..9ae8bab65f 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -50,6 +50,7 @@ do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
 }
 
+breakpoint
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
 } {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
new file mode 100644
index 0000000000..9fc5667589
--- /dev/null
+++ b/ext/fts5/test/fts5contentless.test
@@ -0,0 +1,175 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Check that it is not possible to specify "contentless_delete=1" for 
+# anything other than a contentless table.
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 requires a contentless table}}
+foreach {tn sql bError} {
+  1 "(a, b, contentless_delete=1)"              1
+  2 "(a, b, contentless_delete=1, content=abc)" 1
+  3 "(a, b, contentless_delete=1, content=)"    0
+  4 "(content=, contentless_delete=1, a)"       0
+  5 "(content='', contentless_delete=1, hello)" 0
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that it is not possible to specify "contentless_delete=1" 
+# along with columnsize=1. 
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}}
+foreach {tn sql bError} {
+  2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that if contentless_delete=1 is specified, then the "location" 
+# column is added to the %_docsize table.
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(c, content='');
+  CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1);
+}
+do_execsql_test 3.1 {
+  SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
+} {
+  {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
+  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, location INTEGER)}
+}
+
+do_execsql_test 3.2.1 {
+  SELECT hex(block) FROM x1_data WHERE id=10
+} {00000000000000}
+do_execsql_test 3.2.2 {
+  SELECT hex(block) FROM x2_data WHERE id=10
+} {00000000FF000001000000}
+
+do_execsql_test 3.3 {
+  INSERT INTO x2 VALUES('first text');
+  INSERT INTO x2 VALUES('second text');
+}
+do_execsql_test 3.4 {
+  SELECT id, location FROM x2_docsize
+} {1 1 2 2}
+do_execsql_test 3.5 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  0 0   1 1
+  0 1   2 2
+}
+do_execsql_test 3.6 {
+  INSERT INTO x2(x2) VALUES('optimize');
+}
+do_execsql_test 3.7 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  1 0   1 2
+}
+
+do_execsql_test 3.8 {
+  INSERT INTO x2(x2, rowid) VALUES('delete', 2);
+}
+
+do_execsql_test 3.9 {
+  SELECT rowid FROM x2('text')
+} {1}
+
+#--------------------------------------------------------------------------
+reset_db
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+
+set nRow 1000
+
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x);
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+do_test 4.1 {
+  for {set ii 0} {$ii < $nRow} {incr ii} {
+    set doc [document 6]
+    execsql {
+      INSERT INTO t1 VALUES($doc);
+      INSERT INTO ft VALUES($doc);
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.2.$v { set L1 } $L2
+}
+
+do_test 4.3 {
+  for {set ii 1} {$ii < $nRow} {incr ii 2} { 
+    execsql {
+       INSERT INTO ft(ft, rowid) VALUES('delete', $ii);
+       DELETE FROM t1 WHERE rowid=$ii;
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.4.$v { set L1 } $L2
+}
+
+do_execsql_test 4.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.6.$v { set L1 } $L2
+}
+
+execsql_pp {
+  SELECT fts5_decode(id, block) FROM ft_data
+}
+
+
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 58dc9db6f2..389da73618 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Always\suse\sthe\s"LL"\ssuffix\son\s64-bit\sinteger\sliterals.
-D 2023-07-08T17:42:24.748
+C Begin\sadding\ssupport\sfor\sdeleting\srows\sfrom\scontentless\sfts5\stables.
+D 2023-07-10T20:44:09.251
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,15 +86,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d
+F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f0419c16a
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f
+F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c fe98ebd8835760b9c787d20f6b50d648a761afd8e3b55780e718ee34c694743b
-F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
-F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
+F ext/fts5/fts5_index.c 80fdc17d423f0b881109b397bbfb167830e3c2dc06a8399aded75beba7ef3903
+F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
+F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -105,7 +105,7 @@ F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9
+F ext/fts5/test/fts5aa.test 2106b14aa665cb7e9832cb40fec5b278c404236ed21fdab756484d5f8739b712
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -132,6 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
+F ext/fts5/test/fts5contentless.test e3cee6bac3681707031d2cd5f957178fa43c0d856e90c0ea6fcb3c1bb2fff154
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2043,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P beab3c98639be531744e60440223bb9ee76bc15234aff05e5efb273c8241dfd8
-R 0aeac1606c7bbc75dfae0a73d28601e5
-U drh
-Z 7fe2cdb685f595b3ff5ea313f1ff0511
+P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
+R 04413d3e9bd70379e2afd8eeb162f8d3
+T *branch * fts5-contentless-delete
+T *sym-fts5-contentless-delete *
+T -sym-trunk *
+U dan
+Z 17a9272f129a5cd1a6eee3f0f514cf63
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1161b8bd7f..0166f4e233 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
\ No newline at end of file
+e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
\ No newline at end of file

From d23f2103771159853b09aec39a4ace445e4ee136 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 18:55:19 +0000
Subject: [PATCH 02/22] Use a hash-table instead of a flat list to store
 tombstone rowids.

FossilOrigin-Name: 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
---
 ext/fts5/fts5_index.c              | 173 +++++++++++++++++++++++------
 ext/fts5/test/fts5contentless.test |  16 ++-
 manifest                           |  17 ++-
 manifest.uuid                      |   2 +-
 4 files changed, 158 insertions(+), 50 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e86da1c06e..3ae709f622 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -583,6 +583,41 @@ static u16 fts5GetU16(const u8 *aIn){
   return ((u16)aIn[0] << 8) + aIn[1];
 } 
 
+static u64 fts5GetU64(u8 *a){
+  return ((u64)a[0] << 56)
+       + ((u64)a[1] << 48)
+       + ((u64)a[2] << 40)
+       + ((u64)a[3] << 32)
+       + ((u64)a[4] << 24)
+       + ((u64)a[5] << 16)
+       + ((u64)a[6] << 8)
+       + ((u64)a[7] << 0);
+}
+
+static void fts5PutU64(u8 *a, u64 iVal){
+  a[0] = ((iVal >> 56) & 0xFF);
+  a[1] = ((iVal >> 48) & 0xFF);
+  a[2] = ((iVal >> 40) & 0xFF);
+  a[3] = ((iVal >> 32) & 0xFF);
+  a[4] = ((iVal >> 24) & 0xFF);
+  a[5] = ((iVal >> 16) & 0xFF);
+  a[6] = ((iVal >>  8) & 0xFF);
+  a[7] = ((iVal >>  0) & 0xFF);
+}
+
+static u32 fts5GetU32(const u8 *a){
+  return ((u32)a[0] << 24)
+       + ((u32)a[1] << 16)
+       + ((u32)a[2] << 8)
+       + ((u32)a[3] << 0);
+} 
+static void fts5PutU32(u8 *a, u32 iVal){
+  a[0] = ((iVal >> 24) & 0xFF);
+  a[1] = ((iVal >> 16) & 0xFF);
+  a[2] = ((iVal >>  8) & 0xFF);
+  a[3] = ((iVal >>  0) & 0xFF);
+}
+
 /*
 ** Allocate and return a buffer at least nByte bytes in size.
 **
@@ -2987,26 +3022,26 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
-static u64 fts5GetU64(u8 *a){
-  return ((u64)a[0] << 56)
-       + ((u64)a[1] << 48)
-       + ((u64)a[2] << 40)
-       + ((u64)a[3] << 32)
-       + ((u64)a[4] << 24)
-       + ((u64)a[5] << 16)
-       + ((u64)a[6] << 8)
-       + ((u64)a[7] << 0);
-}
+static int fts5IndexTombstoneQuery(u8 *aHash, int nHash, u64 iRowid){
+  int szKey = aHash[3] ? 8 : 4;
+  int nSlot = (nHash - 8) / szKey;
+  int iSlot = iRowid % nSlot;
 
-static void fts5PutU64(u8 *a, u64 iVal){
-  a[0] = ((iVal >> 56) & 0xFF);
-  a[1] = ((iVal >> 48) & 0xFF);
-  a[2] = ((iVal >> 40) & 0xFF);
-  a[3] = ((iVal >> 32) & 0xFF);
-  a[4] = ((iVal >> 24) & 0xFF);
-  a[5] = ((iVal >> 16) & 0xFF);
-  a[6] = ((iVal >>  8) & 0xFF);
-  a[7] = ((iVal >>  0) & 0xFF);
+  if( szKey==4 ){
+    u32 *aSlot = (u32*)&aHash[8];
+    while( aSlot[iSlot] ){
+      if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      iSlot = (iSlot+1)%nSlot;
+    }
+  }else{
+    u64 *aSlot = (u64*)&aHash[8];
+    while( aSlot[iSlot] ){
+      if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      iSlot = (iSlot+1)%nSlot;
+    }
+  }
+
+  return 0;
 }
 
 static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
@@ -3014,11 +3049,9 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
   if( pSeg->pTombstone ){
-    int ii;
-    for(ii=0; ii<pSeg->pTombstone->nn; ii+=8){
-      i64 iVal = (i64)fts5GetU64(&pSeg->pTombstone->p[ii]);
-      if( iVal==pSeg->iRowid ) return 1;
-    }
+    return fts5IndexTombstoneQuery(
+        pSeg->pTombstone->p, pSeg->pTombstone->nn, pSeg->iRowid
+    );
   }
 
   return 0;
@@ -6380,35 +6413,103 @@ int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
   return fts5IndexReturn(p);
 }
 
+/*
+** Add a tombstone for rowid iRowid to segment pSeg.
+**
+** All tombstones for a single segment are stored in a blob formatted to 
+** contain a hash table. The format is:
+**
+**   * 32-bit integer. 1 for 64-bit unsigned keys, 0 for 32-bit unsigned keys.
+**   * 32-bit integer. The number of entries currently in the hash table.
+**
+** Then an array of entries. The number of entries can be calculated based
+** on the size of the blob in the database and the size of the keys as 
+** specified by the first 32-bit field of the hash table header.
+**
+** All values in the hash table are stored as big-endian integers.
+*/
 static void fts5IndexTombstoneAdd(
   Fts5Index *p, 
   Fts5StructureSegment *pSeg, 
-  i64 iRowid
+  u64 iRowid
 ){
   Fts5Data *pHash = 0;
+  u8 *aFree = 0;
   u8 *aNew = 0;
   int nNew = 0;
+  int szKey = 0;
+  int nSlot = 0;
+  int bKey64 = (iRowid>0xFFFFFFFF);
+  u32 nHash = 0;
 
+  /* Load the current hash table, if any */
   pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
   if( p->rc ) return;
 
   if( pHash ){
-    nNew = 8 + pHash->nn;
-  }else{
-    nNew = 8;
+    szKey = pHash->p[3] ? 8 : 4;
+    nSlot = (pHash->nn - 8) / szKey;
+    nHash = fts5GetU32(&pHash->p[4]);
   }
-  aNew = sqlite3_malloc(nNew);
-  if( aNew==0 ){
-    p->rc = SQLITE_NOMEM;
-  }else{
-    if( pHash ){
-      memcpy(aNew, pHash->p, pHash->nn);
+
+  /* Check if the current hash table needs to be rebuilt. Either because
+  ** (a) it does not yet exist, (b) it is full, or (c) it is using the
+  ** wrong sized keys. */
+  if( pHash==0 || nSlot<=(nHash*2) || (bKey64 && szKey==4) ){
+    int szNewKey = (bKey64 || szKey==8) ? 8 : 4;
+    int nNewSlot = (nSlot ? nSlot*2 : 16);
+
+    nNew = 8 + (nNewSlot * szNewKey);
+    aFree = aNew = (u8*)sqlite3Fts5MallocZero(&p->rc, nNew);
+    if( aNew ){
+      int iSlot = 0;
+      int ii;
+      fts5PutU32(aNew, (szNewKey==8 ? 1 : 0));
+      for(ii=0; ii<nSlot; ii++){
+        u64 iVal = 0;
+        if( szKey==4 ){
+          iVal = (u64)fts5GetU32(&pHash->p[8 + ii*szKey]);
+        }else{
+          iVal = fts5GetU64(&pHash->p[8 + ii*szKey]);
+        }
+
+        iSlot = iVal % nNewSlot;
+        if( szNewKey==4 ){
+          u32 *aSlot = (u32*)&aNew[8];
+          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
+          fts5PutU32((u8*)&aSlot[iSlot], (u32)iVal);
+        }else{
+          u64 *aSlot = (u64*)&aNew[8];
+          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
+          fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+        }
+      }
     }
-    fts5PutU64(&aNew[nNew-8], iRowid);
+    szKey = szNewKey;
+    nSlot = nNewSlot;
+  }else{
+    aNew = pHash->p;
+    nNew = pHash->nn;
+  }
+
+  if( aNew ){
+    int iSlot = (iRowid % nSlot);
+    if( szKey==4 ){
+      u32 *aSlot = (u32*)&aNew[8];
+      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
+      fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
+    }else{
+      u64 *aSlot = (u64*)&aNew[8];
+      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
+      fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+    }
+
+    fts5PutU32((u8*)&aNew[4], nHash+1);
+    assert( nNew>8 );
     fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
   }
 
-  sqlite3_free(aNew);
+  sqlite3_free(aFree);
   fts5DataRelease(pHash);
 }
 
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 9fc5667589..b9b7a2a4a1 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -164,11 +164,21 @@ foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
   do_test 4.6.$v { set L1 } $L2
 }
 
-execsql_pp {
-  SELECT fts5_decode(id, block) FROM ft_data
+#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data }
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+  INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
 }
 
-
+breakpoint
+do_execsql_test 5.1 {
+  INSERT INTO ft(ft, rowid) VALUES('delete', 2);
+}
 
 
 finish_test
diff --git a/manifest b/manifest
index 389da73618..8f2c159d4e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Begin\sadding\ssupport\sfor\sdeleting\srows\sfrom\scontentless\sfts5\stables.
-D 2023-07-10T20:44:09.251
+C Use\sa\shash-table\sinstead\sof\sa\sflat\slist\sto\sstore\stombstone\srowids.
+D 2023-07-11T18:55:19.003
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 80fdc17d423f0b881109b397bbfb167830e3c2dc06a8399aded75beba7ef3903
+F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
 F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
 F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test e3cee6bac3681707031d2cd5f957178fa43c0d856e90c0ea6fcb3c1bb2fff154
+F ext/fts5/test/fts5contentless.test b807a15020dfae84f215370f08d7270aa01bbcc5abdb54a42ec2dee8998e4842
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,11 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
-R 04413d3e9bd70379e2afd8eeb162f8d3
-T *branch * fts5-contentless-delete
-T *sym-fts5-contentless-delete *
-T -sym-trunk *
+P e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
+R e240ab2da08f49c28fa1cdca40ac2a12
 U dan
-Z 17a9272f129a5cd1a6eee3f0f514cf63
+Z d2566e3569908687c69b9bc52ae7c980
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 0166f4e233..1aeb8a479d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
\ No newline at end of file
+948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
\ No newline at end of file

From b92669db7361bd81599d249e7f6ed2e337b65e50 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 20:19:46 +0000
Subject: [PATCH 03/22] Have contentless_delete=1 tables support regular DELETE
 statements, instead of just the special INSERT syntax.

FossilOrigin-Name: fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
---
 ext/fts5/fts5_main.c               |  8 ++++++--
 ext/fts5/test/fts5contentless.test | 15 +++++++++++----
 manifest                           | 14 +++++++-------
 manifest.uuid                      |  2 +-
 4 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 180aba3973..7b1f4a6d08 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1678,8 +1678,12 @@ static int fts5UpdateMethod(
     assert( nArg!=1 || eType0==SQLITE_INTEGER );
 
     /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported.  */
-    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
+    ** This is not suported. Except - DELETE is supported if the CREATE
+    ** VIRTUAL TABLE statement contained "contentless_delete=1". */
+    if( eType0==SQLITE_INTEGER 
+     && pConfig->eContent==FTS5_CONTENT_NONE 
+     && (nArg>1 || pConfig->bContentlessDelete==0)
+    ){
       pTab->p.base.zErrMsg = sqlite3_mprintf(
           "cannot %s contentless fts5 table: %s", 
           (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index b9b7a2a4a1..a55aa84021 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -99,7 +99,7 @@ do_execsql_test 3.7 {
 }
 
 do_execsql_test 3.8 {
-  INSERT INTO x2(x2, rowid) VALUES('delete', 2);
+  DELETE FROM x2 WHERE rowid=2;
 }
 
 do_execsql_test 3.9 {
@@ -142,7 +142,7 @@ foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
 do_test 4.3 {
   for {set ii 1} {$ii < $nRow} {incr ii 2} { 
     execsql {
-       INSERT INTO ft(ft, rowid) VALUES('delete', $ii);
+       DELETE FROM ft WHERE rowid=$ii;
        DELETE FROM t1 WHERE rowid=$ii;
     }
   }
@@ -175,11 +175,18 @@ do_execsql_test 5.0 {
   INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
 }
 
-breakpoint
 do_execsql_test 5.1 {
-  INSERT INTO ft(ft, rowid) VALUES('delete', 2);
+  DELETE FROM ft WHERE rowid=2
 }
 
+do_execsql_test 5.2 {
+  SELECT rowid FROM ft
+} {1 3}
+
+do_catchsql_test 5.3 {
+  UPDATE ft SET x='four six' WHERE rowid=3
+} {1 {cannot UPDATE contentless fts5 table: ft}}
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 8f2c159d4e..963270545a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sa\shash-table\sinstead\sof\sa\sflat\slist\sto\sstore\stombstone\srowids.
-D 2023-07-11T18:55:19.003
+C Have\scontentless_delete=1\stables\ssupport\sregular\sDELETE\sstatements,\sinstead\sof\sjust\sthe\sspecial\sINSERT\ssyntax.
+D 2023-07-11T20:19:46.857
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,7 +93,7 @@ F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
-F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
+F ext/fts5/fts5_main.c 843a6223397afb07c515aaf074c4bd9079cd7390b3b6d3de585f600afe294d5c
 F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test b807a15020dfae84f215370f08d7270aa01bbcc5abdb54a42ec2dee8998e4842
+F ext/fts5/test/fts5contentless.test 5ffce3185e40c8025e1fc0a2a484f331f76f9a8e8b8441d169bbe1276b76ae2c
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
-R e240ab2da08f49c28fa1cdca40ac2a12
+P 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
+R 3eb31b5966f6d22024a48d705ddecf8e
 U dan
-Z d2566e3569908687c69b9bc52ae7c980
+Z ecb4a545638f553899d716cc379fd8e4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1aeb8a479d..06acf1e66a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
\ No newline at end of file
+fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
\ No newline at end of file

From b184c91076dd1131678482398eb6725172c4ddad Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 20:57:19 +0000
Subject: [PATCH 04/22] Have contentless_delete=1 tables support REPLACE
 statements.

FossilOrigin-Name: 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
---
 ext/fts5/fts5_main.c               |  2 +-
 ext/fts5/fts5_storage.c            |  5 +----
 ext/fts5/test/fts5contentless.test | 17 ++++++++++++++++-
 manifest                           | 16 ++++++++--------
 manifest.uuid                      |  2 +-
 5 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 7b1f4a6d08..844b5c57ef 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1670,7 +1670,7 @@ static int fts5UpdateMethod(
     ** Cases 3 and 4 may violate the rowid constraint.
     */
     int eConflict = SQLITE_ABORT;
-    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){
       eConflict = sqlite3_vtab_on_conflict(pConfig->db);
     }
 
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index adc8e750ed..35c30d06e6 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -481,12 +481,9 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
       iLoc = sqlite3_column_int64(pLookup, 1);
     }
     rc = sqlite3_reset(pLookup);
-    if( rc==SQLITE_OK && iLoc==0 ){
-      rc = FTS5_CORRUPT;
-    }
   }
 
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && iLoc!=0 ){
     rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
   }
 
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index a55aa84021..3179b52e92 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -173,6 +173,8 @@ do_execsql_test 5.0 {
   INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
   INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
   INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
+  INSERT INTO ft(rowid, x) VALUES(4, 'one two seven');
+  INSERT INTO ft(rowid, x) VALUES(5, 'one two eight');
 }
 
 do_execsql_test 5.1 {
@@ -181,12 +183,25 @@ do_execsql_test 5.1 {
 
 do_execsql_test 5.2 {
   SELECT rowid FROM ft
-} {1 3}
+} {1 3 4 5}
 
 do_catchsql_test 5.3 {
   UPDATE ft SET x='four six' WHERE rowid=3
 } {1 {cannot UPDATE contentless fts5 table: ft}}
 
+do_execsql_test 5.4 {
+  SELECT rowid FROM ft('one');
+} {1 3 4 5}
+
+do_execsql_test 5.5 {
+  REPLACE INTO ft(rowid, x) VALUES(3, 'four six');
+  SELECT rowid FROM ft('one');
+} {1 4 5}
+
+do_execsql_test 5.6 {
+  REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven');
+  SELECT rowid FROM ft('one');
+} {1 4 5 6}
 
 finish_test
 
diff --git a/manifest b/manifest
index 963270545a..9f5b8a7cfa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\scontentless_delete=1\stables\ssupport\sregular\sDELETE\sstatements,\sinstead\sof\sjust\sthe\sspecial\sINSERT\ssyntax.
-D 2023-07-11T20:19:46.857
+C Have\scontentless_delete=1\stables\ssupport\sREPLACE\sstatements.
+D 2023-07-11T20:57:19.305
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,8 +93,8 @@ F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
-F ext/fts5/fts5_main.c 843a6223397afb07c515aaf074c4bd9079cd7390b3b6d3de585f600afe294d5c
-F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
+F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
+F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 5ffce3185e40c8025e1fc0a2a484f331f76f9a8e8b8441d169bbe1276b76ae2c
+F ext/fts5/test/fts5contentless.test feb27b077be771141c298b68798b93efd0e2d6fd3cb759201628b69576d69228
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
-R 3eb31b5966f6d22024a48d705ddecf8e
+P fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
+R cbe733514e57856ac29410eaba1125b8
 U dan
-Z ecb4a545638f553899d716cc379fd8e4
+Z 9ded30fbafe538528320232b2760b444
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 06acf1e66a..40b428d76e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
\ No newline at end of file
+2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
\ No newline at end of file

From 9faa3bbdbbae6a74aa242f344a326b3530edfb16 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 12 Jul 2023 18:38:47 +0000
Subject: [PATCH 05/22] Get access to SQL functions and virtual tables used for
 debugging and analysis of FTS5 using SQLITE_FTS5_DEBUG and without the need
 for SQLITE_TEST.

FossilOrigin-Name: 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
---
 ext/fts5/fts5_expr.c  |  6 +++---
 ext/fts5/fts5_index.c | 46 +++++++++++++++++++++----------------------
 manifest              | 16 +++++++--------
 manifest.uuid         |  2 +-
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 0e018420d0..0e07b9246c 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -2477,7 +2477,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
   return pRet;
 }
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
   sqlite3_int64 nByte = 0;
   Fts5ExprTerm *p;
@@ -2844,14 +2844,14 @@ static void fts5ExprFold(
     sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
   }
 }
-#endif /* ifdef SQLITE_TEST */
+#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 /*
 ** This is called during initialization to register the fts5_expr() scalar
 ** UDF with the SQLite handle passed as the only argument.
 */
 int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
   struct Fts5ExprFunc {
     const char *z;
     void (*x)(sqlite3_context*,int,sqlite3_value**);
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3ae709f622..94f97891ac 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -7086,7 +7086,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
 ** function only.
 */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** Decode a segment-data rowid from the %_data table. This function is
 ** the opposite of macro FTS5_SEGMENT_ROWID().
@@ -7109,9 +7109,9 @@ static void fts5DecodeRowid(
 
   *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
   int iSegid, iHeight, iPgno, bDlidx;       /* Rowid compenents */
   fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
@@ -7129,9 +7129,9 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
     );
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugStructure(
   int *pRc,                       /* IN/OUT: error code */
   Fts5Buffer *pBuf,
@@ -7159,9 +7159,9 @@ static void fts5DebugStructure(
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This is part of the fts5_decode() debugging aid.
 **
@@ -7186,9 +7186,9 @@ static void fts5DecodeStructure(
   fts5DebugStructure(pRc, pBuf, p);
   fts5StructureRelease(p);
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This is part of the fts5_decode() debugging aid.
 **
@@ -7211,9 +7211,9 @@ static void fts5DecodeAverages(
     zSpace = " ";
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** Buffer (a/n) is assumed to contain a list of serialized varints. Read
 ** each varint and append its string representation to buffer pBuf. Return
@@ -7230,9 +7230,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   }
   return iOff;
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** The start of buffer (a/n) contains the start of a doclist. The doclist
 ** may or may not finish within the buffer. This function appends a text
@@ -7265,9 +7265,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
 
   return iOff;
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This function is part of the fts5_decode() debugging function. It is 
 ** only ever used with detail=none tables.
@@ -7308,9 +7308,9 @@ static void fts5DecodeRowidList(
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** The implementation of user-defined scalar function fts5_decode().
 */
@@ -7519,9 +7519,9 @@ static void fts5DecodeFunction(
   }
   fts5BufferFree(&s);
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST 
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) 
 /*
 ** The implementation of user-defined scalar function fts5_rowid().
 */
@@ -7555,9 +7555,9 @@ static void fts5RowidFunction(
     }
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 
 typedef struct Fts5StructVtab Fts5StructVtab;
 struct Fts5StructVtab {
@@ -7773,7 +7773,7 @@ static int fts5structFilterMethod(
   return rc;
 }
 
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 /*
 ** This is called as part of registering the FTS5 module with database
@@ -7784,7 +7784,7 @@ static int fts5structFilterMethod(
 ** SQLite error code is returned instead.
 */
 int sqlite3Fts5IndexInit(sqlite3 *db){
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
   int rc = sqlite3_create_function(
       db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
   );
diff --git a/manifest b/manifest
index 9f5b8a7cfa..dac59f9187 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\scontentless_delete=1\stables\ssupport\sREPLACE\sstatements.
-D 2023-07-11T20:57:19.305
+C Get\saccess\sto\sSQL\sfunctions\sand\svirtual\stables\sused\sfor\sdebugging\sand\sanalysis\nof\sFTS5\susing\sSQLITE_FTS5_DEBUG\sand\swithout\sthe\sneed\sfor\sSQLITE_TEST.
+D 2023-07-12T18:38:47.529
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -90,9 +90,9 @@ F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f041
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
-F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
+F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
+F ext/fts5/fts5_index.c 05415638e4cb77cda7ead90e3cdc5f8c0dd91770e6596da9ba8e4068395a3490
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
-R cbe733514e57856ac29410eaba1125b8
-U dan
-Z 9ded30fbafe538528320232b2760b444
+P 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
+R 25b067c7b898a5633244bd9d8061a59f
+U drh
+Z 48c04c5430ca6d0e76f1576b44c1f771
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 40b428d76e..d59b7466a9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
\ No newline at end of file
+383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
\ No newline at end of file

From 9f6c0d0027d4b37dab21852f1c11d3509ae0e6d5 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 12 Jul 2023 19:34:32 +0000
Subject: [PATCH 06/22] Fix a crash that may occur when handling corrupt
 records in contentless_delete=1 mode.

FossilOrigin-Name: 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
---
 ext/fts5/fts5_index.c |  3 +--
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 94f97891ac..984a4852ba 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1893,9 +1893,8 @@ static void fts5SegIterInit(
     pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
+    fts5SegIterLoadTombstone(p, pIter);
   }
-
-  fts5SegIterLoadTombstone(p, pIter);
 }
 
 /*
diff --git a/manifest b/manifest
index dac59f9187..6e9efcd348 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\saccess\sto\sSQL\sfunctions\sand\svirtual\stables\sused\sfor\sdebugging\sand\sanalysis\nof\sFTS5\susing\sSQLITE_FTS5_DEBUG\sand\swithout\sthe\sneed\sfor\sSQLITE_TEST.
-D 2023-07-12T18:38:47.529
+C Fix\sa\scrash\sthat\smay\soccur\swhen\shandling\scorrupt\srecords\sin\scontentless_delete=1\smode.
+D 2023-07-12T19:34:32.103
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 05415638e4cb77cda7ead90e3cdc5f8c0dd91770e6596da9ba8e4068395a3490
+F ext/fts5/fts5_index.c 508abd2340e138c7de07b180748f011927553f14f3329f1499e71302183c38d3
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
-R 25b067c7b898a5633244bd9d8061a59f
-U drh
-Z 48c04c5430ca6d0e76f1576b44c1f771
+P 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
+R 1327e4474941bd216c5977cab00c437b
+U dan
+Z ed477a224293640a589dbee38d43f3b4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d59b7466a9..3ff0cdbe56 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
\ No newline at end of file
+74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
\ No newline at end of file

From fc158d3870d46222994df77c1a8c302fb56c4b2d Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 12 Jul 2023 20:24:23 +0000
Subject: [PATCH 07/22] Avoid a case of an infinite loop in fts5 when dealing
 with corrupt records.

FossilOrigin-Name: 0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
---
 ext/fts5/fts5_index.c |  4 +++-
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 984a4852ba..e3b4139d9e 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3127,7 +3127,9 @@ static void fts5MultiIterNext2(
       }
       fts5AssertMultiIterSetup(p, pIter);
 
-    }while( fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter) );
+    }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) 
+         && (p->rc==SQLITE_OK)
+    );
   }
 }
 
diff --git a/manifest b/manifest
index 6e9efcd348..4c89bf87f9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scrash\sthat\smay\soccur\swhen\shandling\scorrupt\srecords\sin\scontentless_delete=1\smode.
-D 2023-07-12T19:34:32.103
+C Avoid\sa\scase\sof\san\sinfinite\sloop\sin\sfts5\swhen\sdealing\swith\scorrupt\srecords.
+D 2023-07-12T20:24:23.049
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 508abd2340e138c7de07b180748f011927553f14f3329f1499e71302183c38d3
+F ext/fts5/fts5_index.c 7b9dd923301f8649db2f13a0d383c9270511555fc779e727e69f92c8d2e97b99
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
-R 1327e4474941bd216c5977cab00c437b
+P 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
+R b3f674c48ef86903b0134e340f9975df
 U dan
-Z ed477a224293640a589dbee38d43f3b4
+Z 4a7fc420576ceeaedd11216cbfd17eb6
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3ff0cdbe56..b56caf54da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
\ No newline at end of file
+0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
\ No newline at end of file

From 0dac350f3c412f97dad1a6e9f565eec3f34505a7 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Sat, 15 Jul 2023 18:57:57 +0000
Subject: [PATCH 08/22] Store large tombstone hash tables in multiple database
 records. Ensure the same hash tables handle rowid 0.

FossilOrigin-Name: 4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3
---
 ext/fts5/fts5_index.c              | 502 +++++++++++++++++++----------
 ext/fts5/test/fts5contentless.test |  44 +++
 manifest                           |  14 +-
 manifest.uuid                      |   2 +-
 4 files changed, 392 insertions(+), 170 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e3b4139d9e..2ed9eee1fd 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -248,7 +248,7 @@
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
 
-#define FTS5_TOMBSTONE_ROWID(segid) fts5_dri(segid + (1<<16), 0, 0, 0)
+#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid + (1<<16), 0, 0, ipg)
 
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
@@ -334,7 +334,7 @@ struct Fts5DoclistIter {
 ** using an Fts5Structure record in memory. Which uses instances of the 
 ** other Fts5StructureXXX types as components.
 **
-** nLocCounter:
+** nOriginCntr:
 **   This value is set to non-zero for structure records created for
 **   contentlessdelete=1 tables only. In that case it represents the
 **   location value to apply to the next top-level segment created.
@@ -345,8 +345,9 @@ struct Fts5StructureSegment {
   int pgnoLast;                   /* Last leaf page number in segment */
 
   /* contentlessdelete=1 tables only: */
-  u64 iLoc1;
-  u64 iLoc2;
+  u64 iOrigin1;
+  u64 iOrigin2;
+  int nPgTombstone;               /* Number of tombstone hash table pages */
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -356,7 +357,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
-  u64 nLocCounter;
+  u64 nOriginCntr;
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -453,7 +454,8 @@ struct Fts5SegIter {
   Fts5Data *pLeaf;                /* Current leaf data */
   Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   i64 iLeafOffset;                /* Byte offset within current leaf */
-  Fts5Data *pTombstone;
+  Fts5Data **apTombstone;         /* Array of tombstone pages */
+  int nTombstone;
 
   /* Next method */
   void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
@@ -797,41 +799,6 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
-static Fts5Data *fts5DataReadOpt(Fts5Index *p, i64 iRowid){
-  Fts5Data *pRet = 0;
-  int rc2 = SQLITE_OK;
-
-  if( p->pReaderOpt==0 ){
-    Fts5Config *pConfig = p->pConfig;
-    fts5IndexPrepareStmt(p, &p->pReaderOpt, sqlite3_mprintf(
-          "SELECT block FROM '%q'.'%q_data' WHERE id=?",
-          pConfig->zDb, pConfig->zName
-    ));
-  }
-
-  if( p->rc==SQLITE_OK ){
-    sqlite3_bind_int64(p->pReaderOpt, 1, iRowid);
-    if( SQLITE_ROW==sqlite3_step(p->pReaderOpt) ){
-      int nByte = sqlite3_column_bytes(p->pReaderOpt, 0);
-      const u8 *aByte = (const u8*)sqlite3_column_blob(p->pReaderOpt, 0);
-      i64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
-
-      pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
-      if( pRet==0 ){
-        p->rc = SQLITE_NOMEM;
-      }else{
-        pRet->nn = nByte;
-        pRet->p = (u8*)&pRet[1];
-        memcpy(pRet->p, aByte, nByte);
-      }
-    }
-    rc2 = sqlite3_reset(p->pReaderOpt);
-    if( p->rc==SQLITE_OK ) p->rc = rc2;
-  }
-
-  return pRet;
-}
-
 /*
 ** INSERT OR REPLACE a record into the %_data table.
 */
@@ -880,13 +847,16 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
 /*
 ** Remove all records associated with segment iSegid.
 */
-static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
+static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){
+  int iSegid = pSeg->iSegid;
   i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
   i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
   fts5DataDelete(p, iFirst, iLast);
-  if( p->pConfig->bContentlessDelete ){
-    i64 iTomb = FTS5_TOMBSTONE_ROWID(iSegid);
-    fts5DataDelete(p, iTomb, iTomb);
+
+  if( pSeg->nPgTombstone ){
+    i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0);
+    i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1);
+    fts5DataDelete(p, iTomb1, iTomb2);
   }
   if( p->pIdxDeleter==0 ){
     Fts5Config *pConfig = p->pConfig;
@@ -999,7 +969,7 @@ static int fts5StructureDecode(
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
   int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
-  i64 nLocCounter = 0;
+  u64 nOriginCntr = 0;
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
@@ -1061,9 +1031,10 @@ static int fts5StructureDecode(
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
           if( bStructureV2 ){
-            i += fts5GetVarint(&pData[i], &pSeg->iLoc1);
-            i += fts5GetVarint(&pData[i], &pSeg->iLoc2);
-            nLocCounter = MAX(nLocCounter, pSeg->iLoc2);
+            i += fts5GetVarint(&pData[i], &pSeg->iOrigin1);
+            i += fts5GetVarint(&pData[i], &pSeg->iOrigin2);
+            i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone);
+            nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2);
           }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
             rc = FTS5_CORRUPT;
@@ -1076,7 +1047,7 @@ static int fts5StructureDecode(
     }
     if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
     if( bStructureV2 ){
-      pRet->nLocCounter = nLocCounter+1;
+      pRet->nOriginCntr = nOriginCntr+1;
     }
 
     if( rc!=SQLITE_OK ){
@@ -1290,7 +1261,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     Fts5Buffer buf;               /* Buffer to serialize record into */
     int iLvl;                     /* Used to iterate through levels */
     int iCookie;                  /* Cookie value to store */
-    int nHdr = (pStruct->nLocCounter>0 ? (4+4+9+9+9) : (4+9+9));
+    int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9));
 
     assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
     memset(&buf, 0, sizeof(Fts5Buffer));
@@ -1302,7 +1273,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
       sqlite3Fts5Put32(buf.p, iCookie);
       buf.n = 4;
-      if( pStruct->nLocCounter>0 ){
+      if( pStruct->nOriginCntr>0 ){
         fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4);
       }
       fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
@@ -1321,9 +1292,10 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
-        if( pStruct->nLocCounter>0 ){
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc1);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc2);
+        if( pStruct->nOriginCntr>0 ){
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin1);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin2);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nPgTombstone);
         }
       }
     }
@@ -1848,8 +1820,15 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
 }
 
 static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
-  i64 iRowid = FTS5_TOMBSTONE_ROWID(pIter->pSeg->iSegid);
-  pIter->pTombstone = fts5DataReadOpt(p, iRowid);
+  const int nTomb = pIter->pSeg->nPgTombstone;
+  if( nTomb>0 ){
+    Fts5Data **apTomb = 0;
+    apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb);
+    if( apTomb ){
+      pIter->apTombstone = apTomb;
+      pIter->nTombstone = nTomb;
+    }
+  }
 }
 
 /*
@@ -2680,10 +2659,14 @@ static void fts5SegIterHashInit(
 ** Zero the iterator passed as the only argument.
 */
 static void fts5SegIterClear(Fts5SegIter *pIter){
+  int ii;
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
-  fts5DataRelease(pIter->pTombstone);
+  for(ii=0; ii<pIter->nTombstone; ii++){
+    fts5DataRelease(pIter->apTombstone[ii]);
+  }
+  sqlite3_free(pIter->apTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -3021,19 +3004,31 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
-static int fts5IndexTombstoneQuery(u8 *aHash, int nHash, u64 iRowid){
-  int szKey = aHash[3] ? 8 : 4;
-  int nSlot = (nHash - 8) / szKey;
-  int iSlot = iRowid % nSlot;
+#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
-  if( szKey==4 ){
-    u32 *aSlot = (u32*)&aHash[8];
+/*
+** Query a single tombstone hash table for rowid iRowid. The tombstone hash
+** table is one of nHashTable tables.
+*/
+static int fts5IndexTombstoneQuery(
+  Fts5Data *pHash,
+  int nHashTable,
+  u64 iRowid
+){
+  int szKey = TOMBSTONE_KEYSIZE(pHash);
+  int nSlot = (pHash->nn - 8) / szKey;
+  int iSlot = (iRowid / nHashTable) % nSlot;
+
+  if( iRowid==0 ){
+    return pHash->p[1];
+  }else if( szKey==4 ){
+    u32 *aSlot = (u32*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
       iSlot = (iSlot+1)%nSlot;
     }
   }else{
-    u64 *aSlot = (u64*)&aHash[8];
+    u64 *aSlot = (u64*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
       iSlot = (iSlot+1)%nSlot;
@@ -3047,9 +3042,21 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   int iFirst = pIter->aFirst[1].iFirst;
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
-  if( pSeg->pTombstone ){
+  if( pSeg->nTombstone ){
+    /* Figure out which page the rowid might be present on. */
+    int iPg = pSeg->iRowid % pSeg->nTombstone;
+
+    if( pSeg->apTombstone[iPg]==0 ){
+      pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
+          FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)
+      );
+      if( pSeg->apTombstone[iPg]==0 ) return 0;
+    }
+
     return fts5IndexTombstoneQuery(
-        pSeg->pTombstone->p, pSeg->pTombstone->nn, pSeg->iRowid
+        pSeg->apTombstone[iPg],
+        pSeg->nTombstone,
+        pSeg->iRowid
     );
   }
 
@@ -4503,9 +4510,9 @@ static void fts5IndexMergeLevel(
     nInput = pLvl->nSeg;
 
     /* Set the range of locations that will go into the output segment */
-    if( pStruct->nLocCounter>0 ){
-      pSeg->iLoc1 = pLvl->aSeg[0].iLoc1;
-      pSeg->iLoc2 = pLvl->aSeg[pLvl->nSeg-1].iLoc2;
+    if( pStruct->nOriginCntr>0 ){
+      pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
+      pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
     }
   }
   bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
@@ -4567,7 +4574,7 @@ static void fts5IndexMergeLevel(
 
     /* Remove the redundant segments from the %_data table */
     for(i=0; i<nInput; i++){
-      fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid);
+      fts5DataRemoveSegment(p, &pLvl->aSeg[i]);
     }
 
     /* Remove the redundant segments from the input level */
@@ -5323,10 +5330,10 @@ static void fts5FlushOneHash(Fts5Index *p){
         pSeg->iSegid = iSegid;
         pSeg->pgnoFirst = 1;
         pSeg->pgnoLast = pgnoLast;
-        if( pStruct->nLocCounter>0 ){
-          pSeg->iLoc1 = pStruct->nLocCounter;
-          pSeg->iLoc2 = pStruct->nLocCounter;
-          pStruct->nLocCounter++;
+        if( pStruct->nOriginCntr>0 ){
+          pSeg->iOrigin1 = pStruct->nOriginCntr;
+          pSeg->iOrigin2 = pStruct->nOriginCntr;
+          pStruct->nOriginCntr++;
         }
         pStruct->nSegment++;
       }
@@ -5390,7 +5397,7 @@ static Fts5Structure *fts5IndexOptimizeStruct(
     pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
     pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
-    pNew->nLocCounter = pStruct->nLocCounter;
+    pNew->nOriginCntr = pStruct->nOriginCntr;
     pLvl = &pNew->aLevel[pNew->nLevel-1];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
     if( pLvl->aSeg ){
@@ -6010,7 +6017,7 @@ int sqlite3Fts5IndexReinit(Fts5Index *p){
   fts5IndexDiscardData(p);
   memset(&s, 0, sizeof(Fts5Structure));
   if( p->pConfig->bContentlessDelete ){
-    s.nLocCounter = 1;
+    s.nOriginCntr = 1;
   }
   fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
   fts5StructureWrite(p, &s);
@@ -6408,20 +6415,214 @@ int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
-    *piLoc = pStruct->nLocCounter;
+    *piLoc = pStruct->nOriginCntr;
     fts5StructureRelease(pStruct);
   }
   return fts5IndexReturn(p);
 }
 
+/*
+** Buffer pPg contains a page of a tombstone hash table - one of nPg.
+*/
+static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
+  int szKey = TOMBSTONE_KEYSIZE(pPg);
+  int nSlot = (pPg->nn - 8) / szKey;
+  int iSlot = (iRowid / nPg) % nSlot;
+  int nElem = fts5GetU32(&pPg->p[4]);
+
+  if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
+  if( iRowid==0 ){
+    pPg->p[1] = 0x01;
+    return 0;
+  }
+
+  fts5PutU32(&pPg->p[4], nElem+1);
+  if( szKey==4 ){
+    u32 *aSlot = (u32*)&pPg->p[8];
+    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
+  }else{
+    u64 *aSlot = (u64*)&pPg->p[8];
+    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+  }
+
+  return nElem >= (nSlot/2);
+}
+
+/*
+** Return 0 if the hash is successfully rebuilt using nOut pages. Or 
+** non-zero if it is not. In this case the caller should retry with a 
+** larger nOut parameter.
+*/
+static int fts5IndexTombstoneRehash(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,     /* Segment to rebuild hash of */
+  Fts5Data *pData1,               /* One page of current hash - or NULL */
+  int iPg1,                       /* Which page of the current hash is pData1 */
+  int szKey,                      /* 4 or 8, the keysize */
+  int nOut,                       /* Number of output pages */
+  Fts5Data **apOut                /* Array of output hash pages */
+){
+  int ii;
+  int res = 0;
+
+  /* Initialize the headers of all the output pages */
+  for(ii=0; ii<nOut; ii++){
+    apOut[ii]->p[0] = szKey;
+    fts5PutU32(&apOut[ii]->p[4], 0);
+  }
+
+  /* Loop through the current pages of the hash table. */ 
+  for(ii=0; res==0 && ii<pSeg->nPgTombstone; ii++){
+    Fts5Data *pData = 0;          /* Page ii of the current hash table */
+    Fts5Data *pFree = 0;          /* Free this at the end of the loop */
+
+    if( iPg1==ii ){
+      pData = pData1;
+    }else{
+      pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii));
+    }
+
+    if( pData ){
+      int szKeyIn = pData->p[3] ? 8 : 4;
+      int nSlotIn = (pData->nn - 8) / szKeyIn;
+      int iIn;
+      for(iIn=0; iIn<nSlotIn; iIn++){
+        u64 iVal = 0;
+
+        /* Read the value from slot iIn of the input page into iVal. */
+        if( szKeyIn==4 ){
+          u32 *aSlot = (u32*)&pData->p[8];
+          if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]);
+        }else{
+          u64 *aSlot = (u64*)&pData->p[8];
+          if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]);
+        }
+
+        /* If iVal is not 0 at this point, insert it into the new hash table */
+        if( iVal ){
+          Fts5Data *pPg = apOut[(iVal % nOut)];
+          res = fts5IndexTombstoneAddToPage(pPg, nOut, iVal);
+          if( res ) break;
+        }
+      }
+
+      /* If this is page 0 of the old hash, copy the rowid-0-flag from the
+      ** old hash to the new.  */
+      if( ii==0 ){
+        apOut[0]->p[1] = pData->p[1];
+      }
+    }
+    fts5DataRelease(pFree);
+  }
+
+  return res;
+}
+
+static void fts5IndexTombstoneFreeArray(Fts5Data **ap, int n){
+  int ii;
+  for(ii=0; ii<n; ii++){
+    fts5DataRelease(ap[ii]);
+  }
+  sqlite3_free(ap);
+}
+
+/*
+** This is called to rebuild the hash table belonging to segment pSeg.
+*/
+static void fts5IndexTombstoneRebuild(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,     /* Segment to rebuild hash of */
+  Fts5Data *pData1,               /* One page of current hash - or NULL */
+  int iPg1,                       /* Which page of the current hash is pData1 */
+  int szKey,                      /* 4 or 8, the keysize */
+  int *pnOut,                     /* OUT: Number of output pages */
+  Fts5Data ***papOut              /* OUT: Output hash pages */
+){
+  const int MINSLOT = 32;
+  int nSlotPerPage = (p->pConfig->pgsz - 8) / szKey;
+  int nSlot = MINSLOT;            /* Number of slots in each output page */
+  int nOut = 0;
+
+  /* Figure out how many output pages (nOut) and how many slots per 
+  ** page (nSlot).  */
+  if( pSeg->nPgTombstone==0 ){
+    nOut = 1;
+    nSlot = MINSLOT;
+  }else if( pSeg->nPgTombstone==1 ){
+    int nElem = (int)fts5GetU32(&pData1->p[4]);
+    assert( pData1 && iPg1==0 );
+
+    nOut = 1;
+    while( nSlot<nElem*2 ){
+      nSlot = nSlot * 2;
+      if( nSlot>nSlotPerPage ){ 
+        nOut = 0; 
+        break;
+      }
+    }
+    if( nOut && nSlot>nSlotPerPage/2 ){
+      nSlot = nSlotPerPage;
+    }
+  }
+  if( nOut==0 ){
+    nOut = (pSeg->nPgTombstone * 2 + 1);
+    nSlot = nSlotPerPage;
+  }
+
+  /* Allocate the required array and output pages */
+  while( 1 ){
+    int res = 0;
+    int ii = 0;
+    int szPage = 0;
+    Fts5Data **apOut = 0;
+
+    apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
+    szPage = 8 + nSlot*szKey;
+    for(ii=0; ii<nOut; ii++){
+      Fts5Data *pNew = (Fts5Data*)sqlite3Fts5MallocZero(&p->rc, 
+          sizeof(Fts5Data)+szPage
+      );
+      if( pNew ){
+        pNew->nn = szPage;
+        pNew->p = (u8*)&pNew[1];
+      }
+      apOut[ii] = pNew;
+    }
+
+    res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    if( res==0 ){
+      if( p->rc ){
+        fts5IndexTombstoneFreeArray(apOut, nOut);
+        apOut = 0;
+        nOut = 0;
+      }
+      *pnOut = nOut;
+      *papOut = apOut;
+      break;
+    }
+    assert( p->rc==SQLITE_OK );
+
+    fts5IndexTombstoneFreeArray(apOut, nOut);
+    nSlot = nSlotPerPage;
+    nOut = nOut*2 + 1;
+  }
+}
+
+
 /*
 ** Add a tombstone for rowid iRowid to segment pSeg.
 **
 ** All tombstones for a single segment are stored in a blob formatted to 
 ** contain a hash table. The format is:
 **
-**   * 32-bit integer. 1 for 64-bit unsigned keys, 0 for 32-bit unsigned keys.
-**   * 32-bit integer. The number of entries currently in the hash table.
+**   * Key-size: 1 byte. Either 4 or 8.
+**   * rowid-0-flag: 1 byte. Either 0 or 1.
+**   * UNUSED: 2 bytes.
+**   * 32-bit big-endian integer. The number of entries currently in the hash
+**     table. This does not change when the rowid-0-flag is set - it only
+**     includes entries in the hash table.
 **
 ** Then an array of entries. The number of entries can be calculated based
 ** on the size of the blob in the database and the size of the keys as 
@@ -6434,84 +6635,51 @@ static void fts5IndexTombstoneAdd(
   Fts5StructureSegment *pSeg, 
   u64 iRowid
 ){
-  Fts5Data *pHash = 0;
-  u8 *aFree = 0;
-  u8 *aNew = 0;
-  int nNew = 0;
+  Fts5Data *pPg = 0;
+  int iPg = -1;
   int szKey = 0;
-  int nSlot = 0;
-  int bKey64 = (iRowid>0xFFFFFFFF);
-  u32 nHash = 0;
 
-  /* Load the current hash table, if any */
-  pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
-  if( p->rc ) return;
+  int nHash = 0;
+  Fts5Data **apHash = 0;
 
-  if( pHash ){
-    szKey = pHash->p[3] ? 8 : 4;
-    nSlot = (pHash->nn - 8) / szKey;
-    nHash = fts5GetU32(&pHash->p[4]);
-  }
-
-  /* Check if the current hash table needs to be rebuilt. Either because
-  ** (a) it does not yet exist, (b) it is full, or (c) it is using the
-  ** wrong sized keys. */
-  if( pHash==0 || nSlot<=(nHash*2) || (bKey64 && szKey==4) ){
-    int szNewKey = (bKey64 || szKey==8) ? 8 : 4;
-    int nNewSlot = (nSlot ? nSlot*2 : 16);
-
-    nNew = 8 + (nNewSlot * szNewKey);
-    aFree = aNew = (u8*)sqlite3Fts5MallocZero(&p->rc, nNew);
-    if( aNew ){
-      int iSlot = 0;
-      int ii;
-      fts5PutU32(aNew, (szNewKey==8 ? 1 : 0));
-      for(ii=0; ii<nSlot; ii++){
-        u64 iVal = 0;
-        if( szKey==4 ){
-          iVal = (u64)fts5GetU32(&pHash->p[8 + ii*szKey]);
-        }else{
-          iVal = fts5GetU64(&pHash->p[8 + ii*szKey]);
-        }
-
-        iSlot = iVal % nNewSlot;
-        if( szNewKey==4 ){
-          u32 *aSlot = (u32*)&aNew[8];
-          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
-          fts5PutU32((u8*)&aSlot[iSlot], (u32)iVal);
-        }else{
-          u64 *aSlot = (u64*)&aNew[8];
-          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
-          fts5PutU64((u8*)&aSlot[iSlot], iRowid);
-        }
-      }
-    }
-    szKey = szNewKey;
-    nSlot = nNewSlot;
-  }else{
-    aNew = pHash->p;
-    nNew = pHash->nn;
-  }
-
-  if( aNew ){
-    int iSlot = (iRowid % nSlot);
-    if( szKey==4 ){
-      u32 *aSlot = (u32*)&aNew[8];
-      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
-      fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
-    }else{
-      u64 *aSlot = (u64*)&aNew[8];
-      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
-      fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+  if( pSeg->nPgTombstone>0 ){
+    iPg = iRowid % pSeg->nPgTombstone;
+    pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg));
+    if( pPg==0 ){
+      assert( p->rc!=SQLITE_OK );
+      return;
     }
 
-    fts5PutU32((u8*)&aNew[4], nHash+1);
-    assert( nNew>8 );
-    fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
+    if( 0==fts5IndexTombstoneAddToPage(pPg, pSeg->nPgTombstone, iRowid) ){
+      fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn);
+      fts5DataRelease(pPg);
+      return;
+    }
   }
 
-  sqlite3_free(aFree);
-  fts5DataRelease(pHash);
+  /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */
+  szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4;
+  if( iRowid>0xFFFFFFFF ) szKey = 8;
+
+  /* Rebuild the hash table */
+  fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash);
+  assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) );
+
+  /* If all has succeeded, write the new rowid into one of the new hash
+  ** table pages, then write them all out to disk. */
+  if( nHash ){
+    int ii = 0;
+    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid);
+    for(ii=0; ii<nHash; ii++){
+      i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii);
+      fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
+    }
+    pSeg->nPgTombstone = nHash;
+    fts5StructureWrite(p, p->pStruct);
+  }
+
+  fts5DataRelease(pPg);
+  fts5IndexTombstoneFreeArray(apHash, nHash);
 }
 
 /*
@@ -6527,7 +6695,7 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
       int iSeg;
       for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
-        if( pSeg->iLoc1<=iLoc && pSeg->iLoc2>=iLoc ){
+        if( pSeg->iOrigin1<=(u64)iLoc && pSeg->iOrigin2>=(u64)iLoc ){
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
@@ -7094,6 +7262,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
 */
 static void fts5DecodeRowid(
   i64 iRowid,                     /* Rowid from %_data table */
+  int *pbTombstone,               /* OUT: Tombstone hash flag */
   int *piSegid,                   /* OUT: Segment id */
   int *pbDlidx,                   /* OUT: Dlidx flag */
   int *piHeight,                  /* OUT: Height */
@@ -7109,13 +7278,16 @@ static void fts5DecodeRowid(
   iRowid >>= FTS5_DATA_DLI_B;
 
   *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
+  iRowid >>= FTS5_DATA_ID_B;
+
+  *pbTombstone = (int)(iRowid & 0x0001);
 }
 #endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 #if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
-  int iSegid, iHeight, iPgno, bDlidx;       /* Rowid compenents */
-  fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
+  int iSegid, iHeight, iPgno, bDlidx, bTomb;     /* Rowid compenents */
+  fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
 
   if( iSegid==0 ){
     if( iKey==FTS5_AVERAGES_ROWID ){
@@ -7125,8 +7297,10 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
     }
   }
   else{
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}",
-        bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}",
+        bDlidx ? "dlidx " : "", 
+        bTomb ? "tombstone " : "", 
+        iSegid, iHeight, iPgno
     );
   }
 }
@@ -7150,9 +7324,9 @@ static void fts5DebugStructure(
       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
           pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
       );
-      if( pSeg->iLoc1>0 ){
-        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " loc=%lld..%lld",
-            pSeg->iLoc1, pSeg->iLoc2
+      if( pSeg->iOrigin1>0 ){
+        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld",
+            pSeg->iOrigin1, pSeg->iOrigin2
         );
       }
       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
@@ -7322,6 +7496,7 @@ static void fts5DecodeFunction(
 ){
   i64 iRowid;                     /* Rowid for record being decoded */
   int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
+  int bTomb;
   const u8 *aBlob; int n;         /* Record to decode */
   u8 *a = 0;
   Fts5Buffer s;                   /* Build up text to return here */
@@ -7344,7 +7519,7 @@ static void fts5DecodeFunction(
   if( a==0 ) goto decode_out;
   if( n>0 ) memcpy(a, aBlob, n);
 
-  fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);
+  fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
 
   fts5DebugRowid(&rc, &s, iRowid);
   if( bDlidx ){
@@ -7363,6 +7538,9 @@ static void fts5DecodeFunction(
           " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
       );
     }
+  }else if( bTomb ){
+    u32 nElem  = fts5GetU32(&a[4]);
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
   }else if( iSegid==0 ){
     if( iRowid==FTS5_AVERAGES_ROWID ){
       fts5DecodeAverages(&rc, &s, a, n);
@@ -7731,10 +7909,10 @@ static int fts5structColumnMethod(
       sqlite3_result_int(ctx, pSeg->pgnoLast);
       break;
     case 6: /* loc1 */
-      sqlite3_result_int(ctx, pSeg->iLoc1);
+      sqlite3_result_int(ctx, pSeg->iOrigin1);
       break;
     case 7: /* loc2 */
-      sqlite3_result_int(ctx, pSeg->iLoc2);
+      sqlite3_result_int(ctx, pSeg->iOrigin2);
       break;
   }
   return SQLITE_OK;
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 3179b52e92..18f05da887 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -122,6 +122,7 @@ set nRow 1000
 do_execsql_test 4.0 {
   CREATE TABLE t1(x);
   CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 100);
 }
 do_test 4.1 {
   for {set ii 0} {$ii < $nRow} {incr ii} {
@@ -203,5 +204,48 @@ do_execsql_test 5.6 {
   SELECT rowid FROM ft('one');
 } {1 4 5 6}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+}
+
+do_test 6.1 {
+  db eval { SELECT rowid FROM ft('one two') } {
+    if {$rowid==1} {
+      db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') }
+    }
+  }
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+
+set lRowid [list -450 0 1 2 42]
+
+do_test 7.1 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); }
+  }
+  execsql COMMIT
+} {}
+
+do_test 7.2 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
+do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
+
 finish_test
 
diff --git a/manifest b/manifest
index 4c89bf87f9..0316fa86f1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sa\scase\sof\san\sinfinite\sloop\sin\sfts5\swhen\sdealing\swith\scorrupt\srecords.
-D 2023-07-12T20:24:23.049
+C Store\slarge\stombstone\shash\stables\sin\smultiple\sdatabase\srecords.\sEnsure\sthe\ssame\shash\stables\shandle\srowid\s0.
+D 2023-07-15T18:57:57.087
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 7b9dd923301f8649db2f13a0d383c9270511555fc779e727e69f92c8d2e97b99
+F ext/fts5/fts5_index.c c38e8892905e9e57e22ca4441cb6fdb519f188d786efe541e9d4df24c34e9197
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test feb27b077be771141c298b68798b93efd0e2d6fd3cb759201628b69576d69228
+F ext/fts5/test/fts5contentless.test 08af7954537cf89acbdef4fd79f39b3a36f382683a1eb98d5c827c187f51e113
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
-R b3f674c48ef86903b0134e340f9975df
+P 0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
+R 681d2015e734a53885da51bd1dfb0cfa
 U dan
-Z 4a7fc420576ceeaedd11216cbfd17eb6
+Z bb27d80790c5d5605515450ca29c8287
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index b56caf54da..8729b7fc92 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
\ No newline at end of file
+4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3
\ No newline at end of file

From 55e0fd4a9d81bde6452f276eb05532c60ad1d49e Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 17 Jul 2023 17:59:58 +0000
Subject: [PATCH 09/22] Do not allow the 'delete' command to be used on
 contentless_delete=1 fts5 tables.

FossilOrigin-Name: cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
---
 ext/fts5/fts5Int.h                 |  4 ++--
 ext/fts5/fts5_index.c              | 18 +++++++++---------
 ext/fts5/fts5_main.c               |  9 ++++++++-
 ext/fts5/fts5_storage.c            | 12 ++++++------
 ext/fts5/test/fts5contentless.test | 19 ++++++++++++++++---
 manifest                           | 24 ++++++++++++------------
 manifest.uuid                      |  2 +-
 7 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 43f78c6d99..da2f90f230 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -536,8 +536,8 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
-int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc);
-int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid);
+int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
 
 /*
 ** End of interface to code in fts5_index.c.
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 2ed9eee1fd..0bb204ea92 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -96,8 +96,8 @@
 **
 **      Then, for V2 structures only:
 **
-**         + lower location counter value,
-**         + upper location counter value
+**         + lower origin counter value,
+**         + upper origin counter value
 **
 ** 2. The Averages Record:
 **
@@ -337,7 +337,7 @@ struct Fts5DoclistIter {
 ** nOriginCntr:
 **   This value is set to non-zero for structure records created for
 **   contentlessdelete=1 tables only. In that case it represents the
-**   location value to apply to the next top-level segment created.
+**   origin value to apply to the next top-level segment created.
 */
 struct Fts5StructureSegment {
   int iSegid;                     /* Segment id */
@@ -4509,7 +4509,7 @@ static void fts5IndexMergeLevel(
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
 
-    /* Set the range of locations that will go into the output segment */
+    /* Set the range of origins that will go into the output segment */
     if( pStruct->nOriginCntr>0 ){
       pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
       pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
@@ -6411,11 +6411,11 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
-int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
+int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
-    *piLoc = pStruct->nOriginCntr;
+    *piOrigin = pStruct->nOriginCntr;
     fts5StructureRelease(pStruct);
   }
   return fts5IndexReturn(p);
@@ -6684,9 +6684,9 @@ static void fts5IndexTombstoneAdd(
 
 /*
 ** Add iRowid to the tombstone list of the segment or segments that contain
-** rows from location iLoc.
+** rows from origin iOrigin.
 */
-int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
@@ -6695,7 +6695,7 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
       int iSeg;
       for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
-        if( pSeg->iOrigin1<=(u64)iLoc && pSeg->iOrigin2>=(u64)iLoc ){
+        if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 844b5c57ef..b10da49c75 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1653,7 +1653,14 @@ static int fts5UpdateMethod(
     if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
     ){
-      rc = fts5SpecialDelete(pTab, apVal);
+      if( pConfig->bContentlessDelete ){
+        fts5SetVtabError(pTab, 
+            "'delete' may not be used with a contentless_delete=1 table"
+        );
+        rc = SQLITE_ERROR;
+      }else{
+        rc = fts5SpecialDelete(pTab, apVal);
+      }
     }else{
       rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
     }
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 35c30d06e6..f28e4d7dd2 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -136,7 +136,7 @@ static int fts5StorageGetStmt(
 
       case FTS5_STMT_LOOKUP_DOCSIZE: 
         zSql = sqlite3_mprintf(azStmt[eStmt], 
-            (pC->bContentlessDelete ? ",location" : ""),
+            (pC->bContentlessDelete ? ",origin" : ""),
             pC->zDb, pC->zName
         );
         break;
@@ -332,7 +332,7 @@ int sqlite3Fts5StorageOpen(
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
       const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
       if( pConfig->bContentlessDelete ){
-        zCols = "id INTEGER PRIMARY KEY, sz BLOB, location INTEGER";
+        zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
       }
       rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
     }
@@ -472,7 +472,7 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
   assert( p->pConfig->bContentlessDelete );
   assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
 
-  /* Look up the location of the document in the %_docsize table. Store
+  /* Look up the origin of the document in the %_docsize table. Store
   ** this in stack variable iLoc.  */
   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
   if( rc==SQLITE_OK ){
@@ -510,9 +510,9 @@ static int fts5StorageInsertDocsize(
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
       if( p->pConfig->bContentlessDelete ){
-        i64 iLoc = 0;
-        rc = sqlite3Fts5IndexGetLocation(p->pIndex, &iLoc);
-        sqlite3_bind_int64(pReplace, 3, iLoc);
+        i64 iOrigin = 0;
+        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
+        sqlite3_bind_int64(pReplace, 3, iOrigin);
       }
       if( rc==SQLITE_OK ){
         sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 18f05da887..181ccb177e 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -51,7 +51,7 @@ foreach {tn sql bError} {
   execsql { ROLLBACK }
 }
 
-# Check that if contentless_delete=1 is specified, then the "location" 
+# Check that if contentless_delete=1 is specified, then the "origin" 
 # column is added to the %_docsize table.
 reset_db
 do_execsql_test 3.0 {
@@ -62,7 +62,7 @@ do_execsql_test 3.1 {
   SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
 } {
   {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
-  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, location INTEGER)}
+  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)}
 }
 
 do_execsql_test 3.2.1 {
@@ -77,7 +77,7 @@ do_execsql_test 3.3 {
   INSERT INTO x2 VALUES('second text');
 }
 do_execsql_test 3.4 {
-  SELECT id, location FROM x2_docsize
+  SELECT id, origin FROM x2_docsize
 } {1 1 2 2}
 do_execsql_test 3.5 {
   SELECT level, segment, loc1, loc2 FROM fts5_structure(
@@ -247,5 +247,18 @@ do_test 7.2 {
 do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
 do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
 
+#-------------------------------------------------------------------------
+reset_db
+
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft VALUES('hello world');
+}
+
+do_catchsql_test 8.1 {
+  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
+} {1 {'delete' may not be used with a contentless_delete=1 table}}
+
 finish_test
 
diff --git a/manifest b/manifest
index 07be41e1ec..2bd8a2d3d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\schanges\sinto\sthis\sbranch.
-D 2023-07-17T11:47:42.362
+C Do\snot\sallow\sthe\s'delete'\scommand\sto\sbe\sused\son\scontentless_delete=1\sfts5\stables.
+D 2023-07-17T17:59:58.252
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,15 +86,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f0419c16a
+F ext/fts5/fts5Int.h fa9dd8ecbda6340f406c6f21b9b524b4666817aa89850e60a42937eea87cef6a
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c c38e8892905e9e57e22ca4441cb6fdb519f188d786efe541e9d4df24c34e9197
-F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
-F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
+F ext/fts5/fts5_index.c 8e5fd1f1eb9489f1ba2eff2d3667ffd78a8d936c1180d284ccc6c37f25953a8f
+F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
+F ext/fts5/fts5_storage.c ee69d6c8195d2bf584ac8c5433e5c685812f39b48c7464f8aa76d9ed8b489119
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 08af7954537cf89acbdef4fd79f39b3a36f382683a1eb98d5c827c187f51e113
+F ext/fts5/test/fts5contentless.test 063249b7d0e92dc7ea54b4a179c466828927d720ee054031cbf6e06009c972bf
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -535,8 +535,8 @@ F ext/wasm/index.html b768e8659b4fe311912e54d42906449d51c0f84b7f036cca47ec1f93bf
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10
-F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 w ext/wasm/scratchpad-wasmfs-main.html
-F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 w ext/wasm/scratchpad-wasmfs-main.js
+F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96
+F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
 F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
-R 3a33d523b0a231ecfa8d84bbc59989ba
+P c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
+R 461e0bbee60dbc3aadc58386c3f130fe
 U dan
-Z cf32871cae7a0d476c03b4d509e36da6
+Z 81dd61cf0a290b6d176bc4273c289b56
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 8d642d98f8..7ecfbd954a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
\ No newline at end of file
+cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
\ No newline at end of file

From 2cccced14054fe03ebd5cb22af83416a70b14006 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 17 Jul 2023 18:40:39 +0000
Subject: [PATCH 10/22] Fix cases where a row is inserted into a
 contentless_delete=1 fts5 table and then deleted within the same transaction.

FossilOrigin-Name: d928856a226fb7f001e55ff7e8eb58a656b982f1efa811de46c382b8b7cd778c
---
 ext/fts5/fts5_storage.c            |  7 +++++--
 ext/fts5/test/fts5contentless.test | 11 +++++++++--
 manifest                           | 14 +++++++-------
 manifest.uuid                      |  2 +-
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index f28e4d7dd2..c2f0ca9e44 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -411,7 +411,7 @@ static int fts5StorageDeleteFromIndex(
 ){
   Fts5Config *pConfig = p->pConfig;
   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
-  int rc;                         /* Return code */
+  int rc = SQLITE_OK;             /* Return code */
   int rc2;                        /* sqlite3_reset() return code */
   int iCol;
   Fts5InsertCtx ctx;
@@ -427,7 +427,6 @@ static int fts5StorageDeleteFromIndex(
 
   ctx.pStorage = p;
   ctx.iCol = -1;
-  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
       const char *zText;
@@ -582,6 +581,10 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   rc = fts5StorageLoadTotals(p, 1);
 
   /* Delete the index records */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  }
+
   if( rc==SQLITE_OK ){
     if( p->pConfig->bContentlessDelete ){
       rc = fts5StorageContentlessDelete(p, iDel);
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 181ccb177e..41a78e9c8f 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -248,17 +248,24 @@ do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
 do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
 
 #-------------------------------------------------------------------------
-reset_db
-
 reset_db
 do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
   INSERT INTO ft VALUES('hello world');
+  INSERT INTO ft VALUES('one two three');
 }
 
 do_catchsql_test 8.1 {
   INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
 } {1 {'delete' may not be used with a contentless_delete=1 table}}
 
+do_execsql_test 8.2 {
+  BEGIN;
+    INSERT INTO ft(rowid, x) VALUES(3, 'four four four');
+    DELETE FROM ft WHERE rowid=3;
+  COMMIT;
+  SELECT rowid FROM ft('four');
+} {}
+
 finish_test
 
diff --git a/manifest b/manifest
index 2bd8a2d3d4..a45f1f64f3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sallow\sthe\s'delete'\scommand\sto\sbe\sused\son\scontentless_delete=1\sfts5\stables.
-D 2023-07-17T17:59:58.252
+C Fix\scases\swhere\sa\srow\sis\sinserted\sinto\sa\scontentless_delete=1\sfts5\stable\sand\sthen\sdeleted\swithin\sthe\ssame\stransaction.
+D 2023-07-17T18:40:39.967
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 8e5fd1f1eb9489f1ba2eff2d3667ffd78a8d936c1180d284ccc6c37f25953a8f
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
-F ext/fts5/fts5_storage.c ee69d6c8195d2bf584ac8c5433e5c685812f39b48c7464f8aa76d9ed8b489119
+F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 063249b7d0e92dc7ea54b4a179c466828927d720ee054031cbf6e06009c972bf
+F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
-R 461e0bbee60dbc3aadc58386c3f130fe
+P cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
+R e5c49e6a91d60244d934bac112d65223
 U dan
-Z 81dd61cf0a290b6d176bc4273c289b56
+Z 153b54340cf10dbebadd96c0477be869
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 7ecfbd954a..6b9ee4a956 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
\ No newline at end of file
+d928856a226fb7f001e55ff7e8eb58a656b982f1efa811de46c382b8b7cd778c
\ No newline at end of file

From d05bf0fe617ef8e1245eb260b1a547cec0771212 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 18 Jul 2023 19:52:32 +0000
Subject: [PATCH 11/22] Fix various problems with fts5 contentless_delete=1
 tables.

FossilOrigin-Name: 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
---
 ext/fts5/fts5_index.c               |  51 ++++++++--
 ext/fts5/test/fts5contentless2.test | 152 ++++++++++++++++++++++++++++
 manifest                            |  15 +--
 manifest.uuid                       |   2 +-
 4 files changed, 203 insertions(+), 17 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless2.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ed919298e7..8804e94bb0 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3038,13 +3038,19 @@ static int fts5IndexTombstoneQuery(
   return 0;
 }
 
+/*
+** Return true if the iterator passed as the only argument points
+** to an segment entry for which there is a tombstone. Return false
+** if there is no tombstone or if the iterator is already at EOF.
+*/
 static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   int iFirst = pIter->aFirst[1].iFirst;
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
-  if( pSeg->nTombstone ){
+  if( pSeg->pLeaf && pSeg->nTombstone ){
     /* Figure out which page the rowid might be present on. */
-    int iPg = pSeg->iRowid % pSeg->nTombstone;
+    int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone;
+    assert( iPg>=0 );
 
     if( pSeg->apTombstone[iPg]==0 ){
       pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
@@ -6428,7 +6434,12 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
 /*
 ** Buffer pPg contains a page of a tombstone hash table - one of nPg.
 */
-static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
+static int fts5IndexTombstoneAddToPage(
+  Fts5Data *pPg, 
+  int bForce,
+  int nPg, 
+  u64 iRowid
+){
   int szKey = TOMBSTONE_KEYSIZE(pPg);
   int nSlot = (pPg->nn - 8) / szKey;
   int iSlot = (iRowid / nPg) % nSlot;
@@ -6440,6 +6451,10 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
     return 0;
   }
 
+  if( bForce==0 && nElem>=(nSlot/2) ){
+    return 1;
+  }
+
   fts5PutU32(&pPg->p[4], nElem+1);
   if( szKey==4 ){
     u32 *aSlot = (u32*)&pPg->p[8];
@@ -6451,7 +6466,7 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
     fts5PutU64((u8*)&aSlot[iSlot], iRowid);
   }
 
-  return nElem >= (nSlot/2);
+  return 0;
 }
 
 /*
@@ -6489,7 +6504,7 @@ static int fts5IndexTombstoneRehash(
     }
 
     if( pData ){
-      int szKeyIn = pData->p[3] ? 8 : 4;
+      int szKeyIn = TOMBSTONE_KEYSIZE(pData);
       int nSlotIn = (pData->nn - 8) / szKeyIn;
       int iIn;
       for(iIn=0; iIn<nSlotIn; iIn++){
@@ -6507,7 +6522,7 @@ static int fts5IndexTombstoneRehash(
         /* If iVal is not 0 at this point, insert it into the new hash table */
         if( iVal ){
           Fts5Data *pPg = apOut[(iVal % nOut)];
-          res = fts5IndexTombstoneAddToPage(pPg, nOut, iVal);
+          res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal);
           if( res ) break;
         }
       }
@@ -6642,7 +6657,6 @@ static void fts5IndexTombstoneAdd(
   Fts5Data *pPg = 0;
   int iPg = -1;
   int szKey = 0;
-
   int nHash = 0;
   Fts5Data **apHash = 0;
 
@@ -6654,7 +6668,7 @@ static void fts5IndexTombstoneAdd(
       return;
     }
 
-    if( 0==fts5IndexTombstoneAddToPage(pPg, pSeg->nPgTombstone, iRowid) ){
+    if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){
       fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn);
       fts5DataRelease(pPg);
       return;
@@ -6673,7 +6687,7 @@ static void fts5IndexTombstoneAdd(
   ** table pages, then write them all out to disk. */
   if( nHash ){
     int ii = 0;
-    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid);
+    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid);
     for(ii=0; ii<nHash; ii++){
       i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii);
       fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
@@ -7544,7 +7558,26 @@ static void fts5DecodeFunction(
     }
   }else if( bTomb ){
     u32 nElem  = fts5GetU32(&a[4]);
+    int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8;
+    int nSlot = (n - 8) / szKey;
+    int ii;
     sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
+    if( aBlob[1] ){
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0");
+    }
+    for(ii=0; ii<nSlot; ii++){
+      u64 iVal = 0;
+      if( szKey==4 ){
+        u32 *aSlot = (u32*)&aBlob[8];
+        if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]);
+      }else{
+        u64 *aSlot = (u64*)&aBlob[8];
+        if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]);
+      }
+      if( iVal!=0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal);
+      }
+    }
   }else if( iSegid==0 ){
     if( iRowid==FTS5_AVERAGES_ROWID ){
       fts5DecodeAverages(&rc, &s, a, n);
diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test
new file mode 100644
index 0000000000..c9e93c6600
--- /dev/null
+++ b/ext/fts5/test/fts5contentless2.test
@@ -0,0 +1,152 @@
+# 2023 July 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc vocab {} {
+  list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp
+}
+
+proc document {nToken} {
+  set doc [list]
+  set vocab [vocab]
+  for {set ii 0} {$ii < $nToken} {incr ii} {
+    lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set doc
+}
+db func document document
+
+proc contains {doc token} {
+  expr {[lsearch $doc $token]>=0}
+}
+db func contains contains
+
+proc do_compare_tables_test {tn} {
+  uplevel [list do_test $tn {
+    foreach v [vocab] {
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }]
+      set l2 [execsql { SELECT rowid FROM t2($v) }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 1]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 0]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set l1 [execsql { 
+        SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC 
+      }]
+      set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+    }
+    set {} {}
+  } {}]
+}
+
+proc lshuffle {in} {
+  set L [list]
+  set ret [list]
+  foreach elem $in { lappend L [list [expr rand()] $elem] }
+  foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] }
+  set ret
+}
+
+expr srand(0)
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      doc, prefix=2, content=, contentless_delete=1
+  );
+
+  CREATE TABLE t1(doc);
+  CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
+    DELETE FROM t2 WHERE rowid = old.rowid;
+  END;
+}
+
+set SMALLEST64 -9223372036854775808
+set LARGEST64   9223372036854775807
+
+foreach {tn r1 r2} {
+  1   0               50
+  2   $SMALLEST64     $SMALLEST64+50
+  3   $LARGEST64-50   $LARGEST64
+  4   -50             -1
+} {
+  set r1 [expr $r1]
+  set r2 [expr $r2]
+
+  do_test 1.1.$tn {
+    execsql BEGIN
+    for {set ii $r1} {$ii <= $r2} {incr ii} {
+      execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); }
+    }
+    execsql COMMIT
+  } {}
+}
+do_test 1.2 {
+  db eval { SELECT rowid, doc FROM t1 } {
+    execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) }
+  }
+} {}
+
+foreach {tn rowid} {
+  1  $SMALLEST64
+  2  0
+  3  -5
+  4  -30
+  5  $LARGEST64
+  6  $LARGEST64-1
+} {
+  set rowid [expr $rowid]
+  do_execsql_test 1.3.$tn.1 {
+    DELETE FROM t1 WHERE rowid=$rowid
+  }
+  do_compare_tables_test 1.3.$tn.2
+}
+
+set iTest 1
+foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] {
+  if {($iTest % 50)==0} {
+    execsql { INSERT INTO t2(t2) VALUES('optimize') }
+  }
+  if {($iTest % 5)==0} {
+    execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) }
+  }
+  do_execsql_test 1.4.$iTest.1($r) {
+    DELETE FROM t1 WHERE rowid=$r
+  }
+  do_compare_tables_test 1.4.$iTest.2
+  incr iTest
+}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t1
+} {}
+
+finish_test
+
diff --git a/manifest b/manifest
index b2e058d24d..53072385c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\sFTS5\smemory\sleak\sfix\sfrom\strunk.
-D 2023-07-18T17:43:47.072
+C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables.
+D 2023-07-18T19:52:32.733
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 78c234c47a4870dd8dad7289f457188b8b8c039ce6ecf0e1d8aea0d369857522
+F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -133,6 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
+F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5f66eb4e2603278dcc9dbfe4bf506cba1aa03180cfb492a0dfc3a8be32cc994b 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a
-R 6df4842f998a408f262f6f4830fd595f
-U drh
-Z 63584765e02f480fe2927dfa6f8c1662
+P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
+R 0b35403fc3d2abc67db6df3c073d3f31
+U dan
+Z c153ae75657e3c69ee3dd162b0a421d3
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 28852ffb55..aa3336730e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
\ No newline at end of file
+0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
\ No newline at end of file

From d1fbaa071bac376206cc009ecdce95b13e131b62 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 19 Jul 2023 18:47:02 +0000
Subject: [PATCH 12/22] Fix various issues with code added to this branch.

FossilOrigin-Name: 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
---
 ext/fts5/fts5_index.c     | 261 ++++++++++++++++++++++++++++----------
 ext/fts5/fts5_storage.c   |  16 ++-
 ext/fts5/test/fts5aa.test |   1 -
 manifest                  |  16 +--
 manifest.uuid             |   2 +-
 5 files changed, 213 insertions(+), 83 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 8804e94bb0..614d6d31c3 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,22 @@
 
 #define FTS5_MAX_LEVEL 64
 
+/*
+** There are two versions of the format used for the structure record:
+**
+**   1. the legacy format, that may be read by all fts5 versions, and
+**
+**   2. the V2 format, which is used by contentless_delete=1 databases.
+**
+** Both begin with a 4-byte "configuration cookie" value. Then, a legacy
+** format structure record contains a varint - the number of levels in
+** the structure. Whereas a V2 structure record contains the constant
+** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a
+** varint has to be at least 16256 to begin with "0xFF". And the default
+** maximum number of levels is 64. 
+**
+** See below for more on structure record formats.
+*/
 #define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01"
 
 /*
@@ -65,7 +81,7 @@
 **
 **     CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
 **
-** , contains the following 5 types of records. See the comments surrounding
+** , contains the following 6 types of records. See the comments surrounding
 ** the FTS5_*_ROWID macros below for a description of how %_data rowids are 
 ** assigned to each fo them.
 **
@@ -73,13 +89,13 @@
 **
 **   The set of segments that make up an index - the index structure - are
 **   recorded in a single record within the %_data table. The record consists
-**   of a single 32-bit configuration cookie value followed by a list of 
-**   SQLite varints. If the FTS table features more than one index (because
-**   there are one or more prefix indexes), it is guaranteed that all share
-**   the same cookie value.
+**   of a single 32-bit configuration cookie value followed by a list of
+**   SQLite varints. 
 **
-**   Immediately following the configuration cookie, the record begins with
-**   three varints:
+**   If the structure record is a V2 record, the configuration cookie is 
+**   followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. 
+**
+**   Next, the record continues with three varints:
 **
 **     + number of levels,
 **     + total number of segments on all levels,
@@ -97,7 +113,8 @@
 **      Then, for V2 structures only:
 **
 **         + lower origin counter value,
-**         + upper origin counter value
+**         + upper origin counter value,
+**         + the number of tombstone hash pages.
 **
 ** 2. The Averages Record:
 **
@@ -214,6 +231,38 @@
 **     * A list of delta-encoded varints - the first rowid on each subsequent
 **       child page. 
 **
+** 6. Tombstone Hash Page
+**
+**   These records are only ever present in contentless_delete=1 tables. 
+**   There are zero or more of these associated with each segment. They
+**   are used to store the tombstone rowids for rows contained in the
+**   associated segments.
+**
+**   The set of nHashPg tombstone hash pages associated with a single 
+**   segment together form a single hash table containing tombstone rowids.
+**   To find the page of the hash on which a key might be stored:
+**
+**       iPg = (rowid % nHashPg)
+**
+**   Then, within page iPg, which has nSlot slots:
+**
+**       iSlot = (rowid / nHashPg) % nSlot
+**
+**   Each tombstone hash page begins with an 8 byte header: 
+**
+**     1-byte:  Key-size (the size in bytes of each slot). Either 4 or 8.
+**     1-byte:  rowid-0-tombstone flag. This flag is only valid on the 
+**              first tombstone hash page for each segment (iPg=0). If set,
+**              the hash table contains rowid 0. If clear, it does not.
+**              Rowid 0 is handled specially.
+**     2-bytes: unused.
+**     4-bytes: Big-endian integer containing number of entries on page.
+**
+**   Following this are nSlot 4 or 8 byte slots (depending on the key-size
+**   in the first byte of the page header). The number of slots may be
+**   determined based on the size of the page record and the key-size:
+**
+**     nSlot = (nByte - 8) / key-size
 */
 
 /*
@@ -247,8 +296,7 @@
 
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
-
-#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid + (1<<16), 0, 0, ipg)
+#define FTS5_TOMBSTONE_ROWID(segid,ipg)       fts5_dri(segid+(1<<16), 0, 0, ipg)
 
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
@@ -304,7 +352,6 @@ struct Fts5Index {
 
   /* State used by the fts5DataXXX() functions. */
   sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
-  sqlite3_stmt *pReaderOpt;
   sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
@@ -357,7 +404,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
-  u64 nOriginCntr;
+  u64 nOriginCntr;                /* Origin value for next top-level segment */
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -446,6 +493,13 @@ struct Fts5CResult {
 **
 ** iTermIdx:
 **     Index of current term on iTermLeafPgno.
+**
+** apTombstone/nTombstone:
+**     These are used for contentless_delete=1 tables only. When the cursor
+**     is first allocated, the apTombstone[] array is allocated so that it
+**     is large enough for all tombstones hash pages associated with the
+**     segment. The pages themselves are loaded lazily from the database as
+**     they are required.
 */
 struct Fts5SegIter {
   Fts5StructureSegment *pSeg;     /* Segment to iterate through */
@@ -585,6 +639,11 @@ static u16 fts5GetU16(const u8 *aIn){
   return ((u16)aIn[0] << 8) + aIn[1];
 } 
 
+/*
+** The only argument points to a buffer at least 8 bytes in size. This
+** function interprets the first 8 bytes of the buffer as a 64-bit big-endian
+** unsigned integer and returns the result.
+*/
 static u64 fts5GetU64(u8 *a){
   return ((u64)a[0] << 56)
        + ((u64)a[1] << 48)
@@ -596,6 +655,22 @@ static u64 fts5GetU64(u8 *a){
        + ((u64)a[7] << 0);
 }
 
+/*
+** The only argument points to a buffer at least 4 bytes in size. This
+** function interprets the first 4 bytes of the buffer as a 32-bit big-endian
+** unsigned integer and returns the result.
+*/
+static u32 fts5GetU32(const u8 *a){
+  return ((u32)a[0] << 24)
+       + ((u32)a[1] << 16)
+       + ((u32)a[2] << 8)
+       + ((u32)a[3] << 0);
+} 
+
+/*
+** Write iVal, formated as a 64-bit big-endian unsigned integer, to the
+** buffer indicated by the first argument.
+*/
 static void fts5PutU64(u8 *a, u64 iVal){
   a[0] = ((iVal >> 56) & 0xFF);
   a[1] = ((iVal >> 48) & 0xFF);
@@ -607,12 +682,10 @@ static void fts5PutU64(u8 *a, u64 iVal){
   a[7] = ((iVal >>  0) & 0xFF);
 }
 
-static u32 fts5GetU32(const u8 *a){
-  return ((u32)a[0] << 24)
-       + ((u32)a[1] << 16)
-       + ((u32)a[2] << 8)
-       + ((u32)a[3] << 0);
-} 
+/*
+** Write iVal, formated as a 32-bit big-endian unsigned integer, to the
+** buffer indicated by the first argument.
+*/
 static void fts5PutU32(u8 *a, u32 iVal){
   a[0] = ((iVal >> 24) & 0xFF);
   a[1] = ((iVal >> 16) & 0xFF);
@@ -684,7 +757,6 @@ void sqlite3Fts5IndexCloseReader(Fts5Index *p){
   }
 }
 
-
 /*
 ** Retrieve a record from the %_data table.
 **
@@ -799,6 +871,7 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
+
 /*
 ** INSERT OR REPLACE a record into the %_data table.
 */
@@ -969,12 +1042,13 @@ static int fts5StructureDecode(
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
   int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
-  u64 nOriginCntr = 0;
+  u64 nOriginCntr = 0;            /* Largest origin value seen so far */
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
   i = 4;
 
+  /* Check if this is a V2 structure record. Set bStructureV2 if it is. */
   if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){
     i += 4;
     bStructureV2 = 1;
@@ -1819,7 +1893,12 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
-static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
+/*
+** Allocate a tombstone hash page array (pIter->apTombstone) for the 
+** iterator passed as the second argument. If an OOM error occurs, leave
+** an error in the Fts5Index object.
+*/
+static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){
   const int nTomb = pIter->pSeg->nPgTombstone;
   if( nTomb>0 ){
     Fts5Data **apTomb = 0;
@@ -1872,7 +1951,7 @@ static void fts5SegIterInit(
     pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
-    fts5SegIterLoadTombstone(p, pIter);
+    fts5SegIterAllocTombstone(p, pIter);
   }
 }
 
@@ -2574,7 +2653,7 @@ static void fts5SegIterSeekInit(
   }
 
   fts5SegIterSetNext(p, pIter);
-  fts5SegIterLoadTombstone(p, pIter);
+  fts5SegIterAllocTombstone(p, pIter);
 
   /* Either:
   **
@@ -2655,18 +2734,26 @@ static void fts5SegIterHashInit(
   fts5SegIterSetNext(p, pIter);
 }
 
+/*
+** Array ap[] contains n elements. Release each of these elements using
+** fts5DataRelease(). Then free the array itself using sqlite3_free().
+*/
+static void fts5IndexFreeArray(Fts5Data **ap, int n){
+  int ii;
+  for(ii=0; ii<n; ii++){
+    fts5DataRelease(ap[ii]);
+  }
+  sqlite3_free(ap);
+}
+
 /*
 ** Zero the iterator passed as the only argument.
 */
 static void fts5SegIterClear(Fts5SegIter *pIter){
-  int ii;
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
-  for(ii=0; ii<pIter->nTombstone; ii++){
-    fts5DataRelease(pIter->apTombstone[ii]);
-  }
-  sqlite3_free(pIter->apTombstone);
+  fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -3004,16 +3091,21 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
+/*
+** The argument to this macro must be an Fts5Data structure containing a
+** tombstone hash page. This macro returns the key-size of the hash-page.
+*/
 #define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
 /*
-** Query a single tombstone hash table for rowid iRowid. The tombstone hash
-** table is one of nHashTable tables.
+** Query a single tombstone hash table for rowid iRowid. Return true if
+** it is found or false otherwise. The tombstone hash table is one of
+** nHashTable tables.
 */
 static int fts5IndexTombstoneQuery(
-  Fts5Data *pHash,
-  int nHashTable,
-  u64 iRowid
+  Fts5Data *pHash,                /* Hash table page to query */
+  int nHashTable,                 /* Number of pages attached to segment */
+  u64 iRowid                      /* Rowid to query hash for */
 ){
   int szKey = TOMBSTONE_KEYSIZE(pHash);
   int nSlot = (pHash->nn - 8) / szKey;
@@ -3052,6 +3144,8 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
     int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone;
     assert( iPg>=0 );
 
+    /* If tombstone hash page iPg has not yet been loaded from the 
+    ** database, load it now. */
     if( pSeg->apTombstone[iPg]==0 ){
       pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
           FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)
@@ -6088,7 +6182,6 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
     assert( p->pReader==0 );
     fts5StructureInvalidate(p);
     sqlite3_finalize(p->pWriter);
-    sqlite3_finalize(p->pReaderOpt);
     sqlite3_finalize(p->pDeleter);
     sqlite3_finalize(p->pIdxWriter);
     sqlite3_finalize(p->pIdxDeleter);
@@ -6421,6 +6514,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
+/*
+** Retrieve the origin value that will be used for the segment currently
+** being accumulated in the in-memory hash table when it is flushed to
+** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to
+** the queried value. Or, if an error occurs, an error code is returned
+** and the final value of (*piOrigin) is undefined.
+*/
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
@@ -6432,7 +6532,15 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
 }
 
 /*
-** Buffer pPg contains a page of a tombstone hash table - one of nPg.
+** Buffer pPg contains a page of a tombstone hash table - one of nPg pages
+** associated with the same segment. This function adds rowid iRowid to
+** the hash table. The caller is required to guarantee that there is at
+** least one free slot on the page.
+**
+** If parameter bForce is false and the hash table is deemed to be full
+** (more than half of the slots are occupied), then non-zero is returned
+** and iRowid not inserted. Or, if bForce is true or if the hash table page
+** is not full, iRowid is inserted and zero returned.
 */
 static int fts5IndexTombstoneAddToPage(
   Fts5Data *pPg, 
@@ -6470,9 +6578,16 @@ static int fts5IndexTombstoneAddToPage(
 }
 
 /*
-** Return 0 if the hash is successfully rebuilt using nOut pages. Or 
-** non-zero if it is not. In this case the caller should retry with a 
-** larger nOut parameter.
+** This function attempts to build a new hash containing all the keys 
+** currently in the tombstone hash table for segment pSeg. The new
+** hash will be stored in the nOut buffers passed in array apOut[].
+** All pages of the new hash use key-size szKey (4 or 8).
+**
+** Return 0 if the hash is successfully rebuilt into the nOut pages. 
+** Or non-zero if it is not (because one page became overfull). In this 
+** case the caller should retry with a larger nOut parameter.
+**
+** Parameter pData1 is page iPg1 of the hash table being rebuilt.
 */
 static int fts5IndexTombstoneRehash(
   Fts5Index *p,
@@ -6539,16 +6654,19 @@ static int fts5IndexTombstoneRehash(
   return res;
 }
 
-static void fts5IndexTombstoneFreeArray(Fts5Data **ap, int n){
-  int ii;
-  for(ii=0; ii<n; ii++){
-    fts5DataRelease(ap[ii]);
-  }
-  sqlite3_free(ap);
-}
-
 /*
 ** This is called to rebuild the hash table belonging to segment pSeg.
+** If parameter pData1 is not NULL, then one page of the existing hash table
+** has already been loaded - pData1, which is page iPg1. The key-size for
+** the new hash table is szKey (4 or 8).
+**
+** If successful, the new hash table is not written to disk. Instead, 
+** output parameter (*pnOut) is set to the number of pages in the new
+** hash table, and (*papOut) to point to an array of buffers containing
+** the new page data.
+**
+** If an error occurs, an error code is left in the Fts5Index object and
+** both output parameters set to 0 before returning.
 */
 static void fts5IndexTombstoneRebuild(
   Fts5Index *p,
@@ -6565,11 +6683,28 @@ static void fts5IndexTombstoneRebuild(
   int nOut = 0;
 
   /* Figure out how many output pages (nOut) and how many slots per 
-  ** page (nSlot).  */
+  ** page (nSlot).  There are three possibilities: 
+  **
+  **   1. The hash table does not yet exist. In this case the new hash
+  **      table will consist of a single page with MINSLOT slots.
+  **
+  **   2. The hash table exists but is currently a single page. In this
+  **      case an attempt is made to grow the page to accommodate the new
+  **      entry. The page is allowed to grow up to nSlotPerPage (see above)
+  **      slots.
+  **
+  **   3. The hash table already consists of more than one page, or of
+  **      a single page already so large that it cannot be grown. In this
+  **      case the new hash consists of (nPg*2+1) pages of nSlotPerPage
+  **      slots each, where nPg is the current number of pages in the 
+  **      hash table.
+  */
   if( pSeg->nPgTombstone==0 ){
+    /* Case 1. */
     nOut = 1;
     nSlot = MINSLOT;
   }else if( pSeg->nPgTombstone==1 ){
+    /* Case 2. */
     int nElem = (int)fts5GetU32(&pData1->p[4]);
     assert( pData1 && iPg1==0 );
 
@@ -6586,6 +6721,7 @@ static void fts5IndexTombstoneRebuild(
     }
   }
   if( nOut==0 ){
+    /* Case 3. */
     nOut = (pSeg->nPgTombstone * 2 + 1);
     nSlot = nSlotPerPage;
   }
@@ -6597,6 +6733,7 @@ static void fts5IndexTombstoneRebuild(
     int szPage = 0;
     Fts5Data **apOut = 0;
 
+    /* Allocate space for the new hash table */
     apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
     szPage = 8 + nSlot*szKey;
     for(ii=0; ii<nOut; ii++){
@@ -6610,10 +6747,11 @@ static void fts5IndexTombstoneRebuild(
       apOut[ii] = pNew;
     }
 
+    /* Rebuild the hash table. */
     res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
     if( res==0 ){
       if( p->rc ){
-        fts5IndexTombstoneFreeArray(apOut, nOut);
+        fts5IndexFreeArray(apOut, nOut);
         apOut = 0;
         nOut = 0;
       }
@@ -6621,9 +6759,11 @@ static void fts5IndexTombstoneRebuild(
       *papOut = apOut;
       break;
     }
+    
+    /* If control flows to here, it was not possible to rebuild the hash
+    ** table. Free all buffers and then try again with more pages. */
     assert( p->rc==SQLITE_OK );
-
-    fts5IndexTombstoneFreeArray(apOut, nOut);
+    fts5IndexFreeArray(apOut, nOut);
     nSlot = nSlotPerPage;
     nOut = nOut*2 + 1;
   }
@@ -6632,22 +6772,6 @@ static void fts5IndexTombstoneRebuild(
 
 /*
 ** Add a tombstone for rowid iRowid to segment pSeg.
-**
-** All tombstones for a single segment are stored in a blob formatted to 
-** contain a hash table. The format is:
-**
-**   * Key-size: 1 byte. Either 4 or 8.
-**   * rowid-0-flag: 1 byte. Either 0 or 1.
-**   * UNUSED: 2 bytes.
-**   * 32-bit big-endian integer. The number of entries currently in the hash
-**     table. This does not change when the rowid-0-flag is set - it only
-**     includes entries in the hash table.
-**
-** Then an array of entries. The number of entries can be calculated based
-** on the size of the blob in the database and the size of the keys as 
-** specified by the first 32-bit field of the hash table header.
-**
-** All values in the hash table are stored as big-endian integers.
 */
 static void fts5IndexTombstoneAdd(
   Fts5Index *p, 
@@ -6697,12 +6821,13 @@ static void fts5IndexTombstoneAdd(
   }
 
   fts5DataRelease(pPg);
-  fts5IndexTombstoneFreeArray(apHash, nHash);
+  fts5IndexFreeArray(apHash, nHash);
 }
 
 /*
 ** Add iRowid to the tombstone list of the segment or segments that contain
-** rows from origin iOrigin.
+** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite
+** error code otherwise.
 */
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index c2f0ca9e44..0a0af9d4b5 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -463,8 +463,14 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
+/*
+** This function is called to process a DELETE on a contentless_delete=1
+** table. It adds the tombstone required to delete the entry with rowid 
+** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
+** an SQLite error code.
+*/
 static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
-  i64 iLoc = 0;
+  i64 iOrigin = 0;
   sqlite3_stmt *pLookup = 0;
   int rc = SQLITE_OK;
 
@@ -472,18 +478,18 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
   assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
 
   /* Look up the origin of the document in the %_docsize table. Store
-  ** this in stack variable iLoc.  */
+  ** this in stack variable iOrigin.  */
   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
   if( rc==SQLITE_OK ){
     sqlite3_bind_int64(pLookup, 1, iDel);
     if( SQLITE_ROW==sqlite3_step(pLookup) ){
-      iLoc = sqlite3_column_int64(pLookup, 1);
+      iOrigin = sqlite3_column_int64(pLookup, 1);
     }
     rc = sqlite3_reset(pLookup);
   }
 
-  if( rc==SQLITE_OK && iLoc!=0 ){
-    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
+  if( rc==SQLITE_OK && iOrigin!=0 ){
+    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
   }
 
   return rc;
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 9ae8bab65f..59ce4f6a1f 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -50,7 +50,6 @@ do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
 }
 
-breakpoint
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
 } {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
diff --git a/manifest b/manifest
index 53072385c5..1b02848d2b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables.
-D 2023-07-18T19:52:32.733
+C Fix\svarious\sissues\swith\scode\sadded\sto\sthis\sbranch.
+D 2023-07-19T18:47:02.359
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,9 +92,9 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48
+F ext/fts5/fts5_index.c 67f3a9fc321cdbf43545980d3ae848f6e8f8b066359f0bc08a6b1e9753c9490f
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
-F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
+F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -105,7 +105,7 @@ F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test 2106b14aa665cb7e9832cb40fec5b278c404236ed21fdab756484d5f8739b712
+F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
-R 0b35403fc3d2abc67db6df3c073d3f31
+P 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
+R 221d8a4262bccaa94e920ab3984953a0
 U dan
-Z c153ae75657e3c69ee3dd162b0a421d3
+Z c2c5b391f8fc6df06c3245812d127c81
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index aa3336730e..c6a3d16d89 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
\ No newline at end of file
+8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
\ No newline at end of file

From 0b3791b537dca9bee9cb73546b8657f22637526e Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 16:07:04 +0000
Subject: [PATCH 13/22] Add tests and fixes for the new code on this branch.

FossilOrigin-Name: 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
---
 ext/fts5/fts5_index.c               |  29 +++-----
 ext/fts5/test/fts5contentless2.test |  56 ++++++++++++++
 ext/fts5/test/fts5faultF.test       | 111 ++++++++++++++++++++++++++++
 manifest                            |  15 ++--
 manifest.uuid                       |   2 +-
 5 files changed, 188 insertions(+), 25 deletions(-)
 create mode 100644 ext/fts5/test/fts5faultF.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 614d6d31c3..c1acf4acd9 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2739,11 +2739,13 @@ static void fts5SegIterHashInit(
 ** fts5DataRelease(). Then free the array itself using sqlite3_free().
 */
 static void fts5IndexFreeArray(Fts5Data **ap, int n){
-  int ii;
-  for(ii=0; ii<n; ii++){
-    fts5DataRelease(ap[ii]);
+  if( ap ){
+    int ii;
+    for(ii=0; ii<n; ii++){
+      fts5DataRelease(ap[ii]);
+    }
+    sqlite3_free(ap);
   }
-  sqlite3_free(ap);
 }
 
 /*
@@ -6707,18 +6709,9 @@ static void fts5IndexTombstoneRebuild(
     /* Case 2. */
     int nElem = (int)fts5GetU32(&pData1->p[4]);
     assert( pData1 && iPg1==0 );
-
     nOut = 1;
-    while( nSlot<nElem*2 ){
-      nSlot = nSlot * 2;
-      if( nSlot>nSlotPerPage ){ 
-        nOut = 0; 
-        break;
-      }
-    }
-    if( nOut && nSlot>nSlotPerPage/2 ){
-      nSlot = nSlotPerPage;
-    }
+    nSlot = MAX(nElem*4, MINSLOT);
+    if( nSlot>nSlotPerPage ) nOut = 0; 
   }
   if( nOut==0 ){
     /* Case 3. */
@@ -6743,12 +6736,14 @@ static void fts5IndexTombstoneRebuild(
       if( pNew ){
         pNew->nn = szPage;
         pNew->p = (u8*)&pNew[1];
+        apOut[ii] = pNew;
       }
-      apOut[ii] = pNew;
     }
 
     /* Rebuild the hash table. */
-    res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    if( p->rc==SQLITE_OK ){
+      res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    }
     if( res==0 ){
       if( p->rc ){
         fts5IndexFreeArray(apOut, nOut);
diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test
index c9e93c6600..fbb857ab38 100644
--- a/ext/fts5/test/fts5contentless2.test
+++ b/ext/fts5/test/fts5contentless2.test
@@ -148,5 +148,61 @@ do_execsql_test 1.5 {
   SELECT * FROM t1
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    DELETE FROM t2 WHERE rowid=32;
+    DELETE FROM t2 WHERE rowid=64;
+    DELETE FROM t2 WHERE rowid=96;
+    DELETE FROM t2 WHERE rowid=128;
+    DELETE FROM t2 WHERE rowid=160;
+    DELETE FROM t2 WHERE rowid=192;
+  COMMIT;
+}
+
+do_execsql_test 2.2 {
+  SELECT * FROM t2('128');
+} {}
+
+#-------------------------------------------------------------------------
+
+foreach {tn step} {
+  1     3 
+  2     7
+  3     15
+} {
+  set step [expr $step]
+
+  reset_db
+  db func document document
+  do_execsql_test 3.$tn.0 {
+    CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+    INSERT INTO t2(t2, rank) VALUES('pgsz', 100);
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+  }
+  do_execsql_test 3.$tn.1 {
+    DELETE FROM t2 WHERE (rowid % $step)==0
+  }
+  do_execsql_test 3.$tn.2 {
+    SELECT * FROM t2( $step * 5 )
+  } {}
+}
+
+
+
 finish_test
 
diff --git a/ext/fts5/test/fts5faultF.test b/ext/fts5/test/fts5faultF.test
new file mode 100644
index 0000000000..96cc2b083f
--- /dev/null
+++ b/ext/fts5/test/fts5faultF.test
@@ -0,0 +1,111 @@
+# 2023 July 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors. Particularly those that may occur
+# when using contentless_delete=1 databases.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultF
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) 
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 
+}
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+  DELETE FROM t1 WHERE rowid IN (2, 4);
+}
+
+do_faultsim_test 2 -prep {
+  sqlite3 db test.db
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    SELECT rowid FROM t1('b c');
+  }
+} -test {
+  faultsim_test_result {0 {1 3}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+}
+
+faultsim_save_and_close
+do_faultsim_test 3 -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s;
+}
+
+do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 }
+
+faultsim_save_and_close
+do_faultsim_test 4 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    DELETE FROM t1 WHERE rowid < 100
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 1b02848d2b..658e5239df 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sissues\swith\scode\sadded\sto\sthis\sbranch.
-D 2023-07-19T18:47:02.359
+C Add\stests\sand\sfixes\sfor\sthe\snew\scode\son\sthis\sbranch.
+D 2023-07-20T16:07:04.821
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 67f3a9fc321cdbf43545980d3ae848f6e8f8b066359f0bc08a6b1e9753c9490f
+F ext/fts5/fts5_index.c 69d44e0358de3db791165c1accbbabcb0e41b7942b12d644f3d515c9a5434386
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -133,7 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
-F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a
+F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -161,6 +161,7 @@ F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c
 F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5
 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
+F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -2045,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
-R 221d8a4262bccaa94e920ab3984953a0
+P 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
+R 8705dc9e0fc623331a29c540c6694db8
 U dan
-Z c2c5b391f8fc6df06c3245812d127c81
+Z 5d09d15bf916dffd1a9ed46964991145
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index c6a3d16d89..02e8325d20 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
\ No newline at end of file
+5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
\ No newline at end of file

From d6f5aa824ec77fe59ce9874dc5b26e595f2bfb4f Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 20:09:26 +0000
Subject: [PATCH 14/22] Avoid an infinite loop that could be entered when
 dealing with corrupt fts5 tombstone hash pages.

FossilOrigin-Name: 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
---
 ext/fts5/fts5_index.c               | 24 ++++++---
 ext/fts5/test/fts5contentless3.test | 76 +++++++++++++++++++++++++++++
 manifest                            | 13 ++---
 manifest.uuid                       |  2 +-
 4 files changed, 101 insertions(+), 14 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless3.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index c1acf4acd9..99ff4c3b5f 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3109,9 +3109,10 @@ static int fts5IndexTombstoneQuery(
   int nHashTable,                 /* Number of pages attached to segment */
   u64 iRowid                      /* Rowid to query hash for */
 ){
-  int szKey = TOMBSTONE_KEYSIZE(pHash);
-  int nSlot = (pHash->nn - 8) / szKey;
+  const int szKey = TOMBSTONE_KEYSIZE(pHash);
+  const int nSlot = (pHash->nn - 8) / szKey;
   int iSlot = (iRowid / nHashTable) % nSlot;
+  int nCollide = nSlot;
 
   if( iRowid==0 ){
     return pHash->p[1];
@@ -3119,12 +3120,14 @@ static int fts5IndexTombstoneQuery(
     u32 *aSlot = (u32*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      if( nCollide--==0 ) break;
       iSlot = (iSlot+1)%nSlot;
     }
   }else{
     u64 *aSlot = (u64*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      if( nCollide--==0 ) break;
       iSlot = (iSlot+1)%nSlot;
     }
   }
@@ -6550,10 +6553,11 @@ static int fts5IndexTombstoneAddToPage(
   int nPg, 
   u64 iRowid
 ){
-  int szKey = TOMBSTONE_KEYSIZE(pPg);
-  int nSlot = (pPg->nn - 8) / szKey;
+  const int szKey = TOMBSTONE_KEYSIZE(pPg);
+  const int nSlot = (pPg->nn - 8) / szKey;
+  const int nElem = fts5GetU32(&pPg->p[4]);
   int iSlot = (iRowid / nPg) % nSlot;
-  int nElem = fts5GetU32(&pPg->p[4]);
+  int nCollide = nSlot;
 
   if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
   if( iRowid==0 ){
@@ -6568,11 +6572,17 @@ static int fts5IndexTombstoneAddToPage(
   fts5PutU32(&pPg->p[4], nElem+1);
   if( szKey==4 ){
     u32 *aSlot = (u32*)&pPg->p[8];
-    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    while( aSlot[iSlot] ){
+      iSlot = (iSlot + 1) % nSlot;
+      if( nCollide--==0 ) return 0;
+    }
     fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
   }else{
     u64 *aSlot = (u64*)&pPg->p[8];
-    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    while( aSlot[iSlot] ){
+      iSlot = (iSlot + 1) % nSlot;
+      if( nCollide--==0 ) return 0;
+    }
     fts5PutU64((u8*)&aSlot[iSlot], iRowid);
   }
 
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
new file mode 100644
index 0000000000..34b7b23da4
--- /dev/null
+++ b/ext/fts5/test/fts5contentless3.test
@@ -0,0 +1,76 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO ft VALUES('one one one');
+    INSERT INTO ft VALUES('two two two');
+    INSERT INTO ft VALUES('three three three');
+    INSERT INTO ft VALUES('four four four');
+    INSERT INTO ft VALUES('five five five');
+  COMMIT;
+
+  DELETE FROM ft WHERE rowid=3;
+}
+
+proc myhex {hex} { binary decode hex $hex }
+db func myhex myhex
+
+do_execsql_test 1.1 {
+  UPDATE ft_data SET block = 
+    myhex('04000000 00000001'                   ||
+          '01020304 01020304 01020304 01020304' ||
+          '01020304 01020304 01020304 01020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM ft WHERE rowid=1
+}
+
+do_execsql_test 1.3 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.3 {
+  UPDATE ft_data SET block = 
+    myhex('08000000 00000001'                                               ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304' ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.5 {
+  DELETE FROM ft WHERE rowid=4
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 658e5239df..08dc29ea64 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sand\sfixes\sfor\sthe\snew\scode\son\sthis\sbranch.
-D 2023-07-20T16:07:04.821
+C Avoid\san\sinfinite\sloop\sthat\scould\sbe\sentered\swhen\sdealing\swith\scorrupt\sfts5\stombstone\shash\spages.
+D 2023-07-20T20:09:26.394
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 69d44e0358de3db791165c1accbbabcb0e41b7942b12d644f3d515c9a5434386
+F ext/fts5/fts5_index.c 5b5b9944ef97bf9aa0726fdbd82552d488b6fcee32e086e6d32ea9934e1809e2
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,6 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
+F ext/fts5/test/fts5contentless3.test db92625a2e3f9e5ba047ce8ded58c825affce45cc711ed6d7c96fe54b95894a1
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2046,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
-R 8705dc9e0fc623331a29c540c6694db8
+P 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
+R a2bc7a639e489f4200810e70aeab413a
 U dan
-Z 5d09d15bf916dffd1a9ed46964991145
+Z 36cb6c12a15fce5364a14d77012ed267
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 02e8325d20..87b279a730 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
\ No newline at end of file
+69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
\ No newline at end of file

From 3a51f8c3078bacc8c8059f985d511042f8561da3 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 20:29:56 +0000
Subject: [PATCH 15/22] Fix some divide-by-zero errors that could occur when
 handling corrupt tombstone hash records.

FossilOrigin-Name: 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
---
 ext/fts5/fts5_index.c               |  7 +++++--
 ext/fts5/test/fts5contentless3.test | 20 ++++++++++++++++++++
 manifest                            | 14 +++++++-------
 manifest.uuid                       |  2 +-
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 99ff4c3b5f..d5235b2496 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3099,6 +3099,9 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
 */
 #define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
+#define TOMBSTONE_NSLOT(pPg)   \
+  ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1)
+
 /*
 ** Query a single tombstone hash table for rowid iRowid. Return true if
 ** it is found or false otherwise. The tombstone hash table is one of
@@ -3110,7 +3113,7 @@ static int fts5IndexTombstoneQuery(
   u64 iRowid                      /* Rowid to query hash for */
 ){
   const int szKey = TOMBSTONE_KEYSIZE(pHash);
-  const int nSlot = (pHash->nn - 8) / szKey;
+  const int nSlot = TOMBSTONE_NSLOT(pHash);
   int iSlot = (iRowid / nHashTable) % nSlot;
   int nCollide = nSlot;
 
@@ -6554,7 +6557,7 @@ static int fts5IndexTombstoneAddToPage(
   u64 iRowid
 ){
   const int szKey = TOMBSTONE_KEYSIZE(pPg);
-  const int nSlot = (pPg->nn - 8) / szKey;
+  const int nSlot = TOMBSTONE_NSLOT(pPg);
   const int nElem = fts5GetU32(&pPg->p[4]);
   int iSlot = (iRowid / nPg) % nSlot;
   int nCollide = nSlot;
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index 34b7b23da4..ec5477e676 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -29,6 +29,10 @@ do_execsql_test 1.0 {
     INSERT INTO ft VALUES('three three three');
     INSERT INTO ft VALUES('four four four');
     INSERT INTO ft VALUES('five five five');
+    INSERT INTO ft VALUES('six six six');
+    INSERT INTO ft VALUES('seven seven seven');
+    INSERT INTO ft VALUES('eight eight eight');
+    INSERT INTO ft VALUES('nine nine nine');
   COMMIT;
 
   DELETE FROM ft WHERE rowid=3;
@@ -71,6 +75,22 @@ do_execsql_test 1.5 {
   DELETE FROM ft WHERE rowid=4
 }
 
+do_execsql_test 1.6 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.7 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.8 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.9 {
+  DELETE FROM ft WHERE rowid=8
+} {}
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 08dc29ea64..9e392f9f22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\san\sinfinite\sloop\sthat\scould\sbe\sentered\swhen\sdealing\swith\scorrupt\sfts5\stombstone\shash\spages.
-D 2023-07-20T20:09:26.394
+C Fix\ssome\sdivide-by-zero\serrors\sthat\scould\soccur\swhen\shandling\scorrupt\stombstone\shash\srecords.
+D 2023-07-20T20:29:56.954
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 5b5b9944ef97bf9aa0726fdbd82552d488b6fcee32e086e6d32ea9934e1809e2
+F ext/fts5/fts5_index.c db653198e9bb76d285df246df1f14ebf3f1cfcbca0605f2bf272026f52021011
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test db92625a2e3f9e5ba047ce8ded58c825affce45cc711ed6d7c96fe54b95894a1
+F ext/fts5/test/fts5contentless3.test b773267d7d5434d0a503b9d73de240f5769efb0e3576e133dd76eccb318116bc
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
-R a2bc7a639e489f4200810e70aeab413a
+P 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
+R 716993a5a4c359e4b4afccc9c55270f5
 U dan
-Z 36cb6c12a15fce5364a14d77012ed267
+Z f01327fd76f39e2a6bc2b85c23603eff
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 87b279a730..a6af9f94f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
\ No newline at end of file
+7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
\ No newline at end of file

From 4e0c157d02c1116006701982a6fde8ab2a502eaf Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 21 Jul 2023 19:33:35 +0000
Subject: [PATCH 16/22] Ensure the fts5 'optimize' command correctly rewrites
 any index that consists of a single segment and one or more tombstone hash
 pages.

FossilOrigin-Name: f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
---
 ext/fts5/fts5_index.c               | 19 +++++++-----
 ext/fts5/test/fts5contentless3.test | 47 +++++++++++++++++++++++++++++
 manifest                            | 14 ++++-----
 manifest.uuid                       |  2 +-
 4 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index d5235b2496..9488c687e5 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5483,17 +5483,22 @@ static Fts5Structure *fts5IndexOptimizeStruct(
   /* Figure out if this structure requires optimization. A structure does
   ** not require optimization if either:
   **
-  **  + it consists of fewer than two segments, or 
-  **  + all segments are on the same level, or
-  **  + all segments except one are currently inputs to a merge operation.
+  **  1. it consists of fewer than two segments, or 
+  **  2. all segments are on the same level, or
+  **  3. all segments except one are currently inputs to a merge operation.
   **
-  ** In the first case, return NULL. In the second, increment the ref-count
-  ** on *pStruct and return a copy of the pointer to it.
+  ** In the first case, if there are no tombstone hash pages, return NULL. In
+  ** the second, increment the ref-count on *pStruct and return a copy of the
+  ** pointer to it.
   */
-  if( nSeg<2 ) return 0;
+  if( nSeg==0 ) return 0;
   for(i=0; i<pStruct->nLevel; i++){
     int nThis = pStruct->aLevel[i].nSeg;
-    if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
+    int nMerge = pStruct->aLevel[i].nMerge;
+    if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){
+      if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){
+        return 0;
+      }
       fts5StructureRef(pStruct);
       return pStruct;
     }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index ec5477e676..a478f795f0 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -91,6 +91,53 @@ do_execsql_test 1.9 {
   DELETE FROM ft WHERE rowid=8
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+    INSERT INTO ft VALUES('one one one');
+    INSERT INTO ft VALUES('two two two');
+    INSERT INTO ft VALUES('three three three');
+    INSERT INTO ft VALUES('four four four');
+    INSERT INTO ft VALUES('five five five');
+    INSERT INTO ft VALUES('six six six');
+    INSERT INTO ft VALUES('seven seven seven');
+    INSERT INTO ft VALUES('eight eight eight');
+    INSERT INTO ft VALUES('nine nine nine');
+}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.2 {
+  SELECT count(*) FROM ft_data
+} {3}
+do_execsql_test 2.3 {
+  DELETE FROM ft WHERE rowid=5
+}
+do_execsql_test 2.4 {
+  SELECT count(*) FROM ft_data
+} {4}
+
+# Check that an 'optimize' works (rewrites the index) if there is a single
+# segment with one or more tombstone hash pages.
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.6 {
+  SELECT count(*) FROM ft_data
+} {3}
+
+# Check that an 'optimize' is a no-op if there is a single segment
+# and no tombstone hash pages.
+do_execsql_test 2.7 {
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+
+
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 9e392f9f22..7e0682938e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sdivide-by-zero\serrors\sthat\scould\soccur\swhen\shandling\scorrupt\stombstone\shash\srecords.
-D 2023-07-20T20:29:56.954
+C Ensure\sthe\sfts5\s'optimize'\scommand\scorrectly\srewrites\sany\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\sand\sone\sor\smore\stombstone\shash\spages.
+D 2023-07-21T19:33:35.722
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c db653198e9bb76d285df246df1f14ebf3f1cfcbca0605f2bf272026f52021011
+F ext/fts5/fts5_index.c 3817749456b9c4e59a172cfef8cbf8d7b02e3d11d03c85da749dce0edc1af0ca
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test b773267d7d5434d0a503b9d73de240f5769efb0e3576e133dd76eccb318116bc
+F ext/fts5/test/fts5contentless3.test bd1521137ecda248ffc27f126eb07442c03a9c29ed9aed01fc405ae86b5e62db
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
-R 716993a5a4c359e4b4afccc9c55270f5
+P 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
+R 6e42e7eceee582bdc020aa8182af6a3a
 U dan
-Z f01327fd76f39e2a6bc2b85c23603eff
+Z f1b6971f3b1dbfa08900d3843d873627
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a6af9f94f9..2830d8dc7f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
\ No newline at end of file
+f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
\ No newline at end of file

From 330e36c2c60ab97f297e5d6b2cca4eb998f4dcea Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 21 Jul 2023 21:10:33 +0000
Subject: [PATCH 17/22] Fix fts5 incremental optimization so that it too can
 handle an index that consists of a single segment with one or more tombstone
 hash pages.

FossilOrigin-Name: e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
---
 ext/fts5/fts5_index.c               | 18 +++++---
 ext/fts5/test/fts5contentless3.test | 72 +++++++++++++++++++++++++----
 manifest                            | 14 +++---
 manifest.uuid                       |  2 +-
 4 files changed, 83 insertions(+), 23 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9488c687e5..9bd23a41f4 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4759,6 +4759,8 @@ static int fts5IndexMerge(
     if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
       fts5StructurePromote(p, iBestLvl+1, pStruct);
     }
+
+    if( nMin==1 ) nMin = 2;
   }
   *ppStruct = pStruct;
   return bRet;
@@ -5582,7 +5584,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
       Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
       fts5StructureRelease(pStruct);
       pStruct = pNew;
-      nMin = 2;
+      nMin = 1;
       nMerge = nMerge*-1;
     }
     if( pStruct && pStruct->nLevel ){
@@ -6698,8 +6700,8 @@ static void fts5IndexTombstoneRebuild(
   Fts5Data ***papOut              /* OUT: Output hash pages */
 ){
   const int MINSLOT = 32;
-  int nSlotPerPage = (p->pConfig->pgsz - 8) / szKey;
-  int nSlot = MINSLOT;            /* Number of slots in each output page */
+  int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey);
+  int nSlot = 0;                  /* Number of slots in each output page */
   int nOut = 0;
 
   /* Figure out how many output pages (nOut) and how many slots per 
@@ -6745,6 +6747,7 @@ static void fts5IndexTombstoneRebuild(
     Fts5Data **apOut = 0;
 
     /* Allocate space for the new hash table */
+    assert( nSlot>=MINSLOT );
     apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
     szPage = 8 + nSlot*szKey;
     for(ii=0; ii<nOut; ii++){
@@ -7942,8 +7945,8 @@ static int fts5structConnectMethod(
 
   rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE xyz("
-          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2,"
-      "struct HIDDEN);"
+          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
+          "npgtombstone, struct HIDDEN);"
   );
   if( rc==SQLITE_OK ){
     pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
@@ -7970,7 +7973,7 @@ static int fts5structBestIndexMethod(
   pIdxInfo->idxNum = 0;
   for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
     if( p->usable==0 ) continue;
-    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==8 ){
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==9 ){
       rc = SQLITE_OK;
       pIdxInfo->aConstraintUsage[i].omit = 1;
       pIdxInfo->aConstraintUsage[i].argvIndex = 1;
@@ -8089,6 +8092,9 @@ static int fts5structColumnMethod(
     case 7: /* loc2 */
       sqlite3_result_int(ctx, pSeg->iOrigin2);
       break;
+    case 8: /* npgtombstone */
+      sqlite3_result_int(ctx, pSeg->nPgTombstone);
+      break;
   }
   return SQLITE_OK;
 }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index a478f795f0..6316628c55 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -95,15 +95,15 @@ do_execsql_test 1.9 {
 reset_db
 do_execsql_test 2.0 {
   CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
-    INSERT INTO ft VALUES('one one one');
-    INSERT INTO ft VALUES('two two two');
-    INSERT INTO ft VALUES('three three three');
-    INSERT INTO ft VALUES('four four four');
-    INSERT INTO ft VALUES('five five five');
-    INSERT INTO ft VALUES('six six six');
-    INSERT INTO ft VALUES('seven seven seven');
-    INSERT INTO ft VALUES('eight eight eight');
-    INSERT INTO ft VALUES('nine nine nine');
+  INSERT INTO ft VALUES('one one one');
+  INSERT INTO ft VALUES('two two two');
+  INSERT INTO ft VALUES('three three three');
+  INSERT INTO ft VALUES('four four four');
+  INSERT INTO ft VALUES('five five five');
+  INSERT INTO ft VALUES('six six six');
+  INSERT INTO ft VALUES('seven seven seven');
+  INSERT INTO ft VALUES('eight eight eight');
+  INSERT INTO ft VALUES('nine nine nine');
 }
 
 do_execsql_test 2.1 {
@@ -135,7 +135,61 @@ do_execsql_test 2.7 {
   SELECT rowid FROM ft_data;
 } [db eval {SELECT rowid FROM ft_data}]
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
 
+do_execsql_test 3.1 {
+  SELECT count(*) FROM ft_data
+} {200}
+
+do_execsql_test 3.2 {
+  DELETE FROM ft WHERE (rowid % 50)==0;
+  SELECT count(*) FROM ft_data;
+} {203}
+
+do_execsql_test 3.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 500);
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+do_execsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('merge', -1000);
+  SELECT count(*) FROM ft_data;
+} {197}
+
+do_execsql_test 3.5 {
+  DELETE FROM ft WHERE (rowid % 50)==1;
+  SELECT count(*) FROM ft_data;
+} {200}
+
+do_execsql_test 3.6 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {1 0 3}
+
+do_test 3.6 {
+  while 1 {
+    set nChange [db total_changes]
+    execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) }
+    if {([db total_changes] - $nChange)<2} break
+  }
+} {}
+
+do_execsql_test 3.7 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {2 0 0}
 
 
 
diff --git a/manifest b/manifest
index 7e0682938e..3e4343c8ea 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthe\sfts5\s'optimize'\scommand\scorrectly\srewrites\sany\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\sand\sone\sor\smore\stombstone\shash\spages.
-D 2023-07-21T19:33:35.722
+C Fix\sfts5\sincremental\soptimization\sso\sthat\sit\stoo\scan\shandle\san\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\swith\sone\sor\smore\stombstone\shash\spages.
+D 2023-07-21T21:10:33.579
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 3817749456b9c4e59a172cfef8cbf8d7b02e3d11d03c85da749dce0edc1af0ca
+F ext/fts5/fts5_index.c e500a5d33ae312c2ebab91123f0ef5f9bbc3eb555252c7d40ba4d8688780c7ca
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test bd1521137ecda248ffc27f126eb07442c03a9c29ed9aed01fc405ae86b5e62db
+F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
-R 6e42e7eceee582bdc020aa8182af6a3a
+P f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
+R 8117856750f55ecca6e7594df2bcd37e
 U dan
-Z f1b6971f3b1dbfa08900d3843d873627
+Z 3675210f196caeb196ef6ee2658c36e9
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 2830d8dc7f..92b4b794ac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
\ No newline at end of file
+e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
\ No newline at end of file

From 2159292ce07565d6791395ac2ff9205e14a7b16a Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Sat, 22 Jul 2023 19:47:46 +0000
Subject: [PATCH 18/22] Integrate contentless delete with auto-merge.

FossilOrigin-Name: 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
---
 ext/fts5/fts5Int.h                  |   6 +
 ext/fts5/fts5_hash.c                |   7 +
 ext/fts5/fts5_index.c               | 358 +++++++++++++++-------------
 ext/fts5/fts5_main.c                |   3 +-
 ext/fts5/test/fts5contentless4.test |  61 +++++
 manifest                            |  19 +-
 manifest.uuid                       |   2 +-
 7 files changed, 282 insertions(+), 174 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless4.test

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index da2f90f230..24417483c1 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -623,6 +623,11 @@ int sqlite3Fts5HashWrite(
 */
 void sqlite3Fts5HashClear(Fts5Hash*);
 
+/*
+** Return true if the hash is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash*);
+
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
   int nPre,
@@ -644,6 +649,7 @@ void sqlite3Fts5HashScanEntry(Fts5Hash *,
 );
 
 
+
 /*
 ** End of interface to code in fts5_hash.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index bc9244fc01..2010e4ff9c 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -529,6 +529,13 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
+/*
+** Return true if the hash table is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
+  return pHash->nEntry==0;
+}
+
 void sqlite3Fts5HashScanNext(Fts5Hash *p){
   assert( !sqlite3Fts5HashScanEof(p) );
   p->pScan = p->pScan->pScanNext;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9bd23a41f4..392218864c 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,8 @@
 
 #define FTS5_MAX_LEVEL 64
 
+#define FTS5_MERGE_TOMBSTONE_WEIGHT 5
+
 /*
 ** There are two versions of the format used for the structure record:
 **
@@ -332,6 +334,12 @@ struct Fts5Data {
 
 /*
 ** One object per %_data table.
+**
+** nContentlessDelete:
+**   The number of contentless delete operations since the most recent
+**   call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked
+**   so that extra auto-merge work can be done by fts5IndexFlush() to
+**   account for the delete operations.
 */
 struct Fts5Index {
   Fts5Config *pConfig;            /* Virtual table configuration */
@@ -346,6 +354,7 @@ struct Fts5Index {
   int nPendingData;               /* Current bytes of pending data */
   i64 iWriteRowid;                /* Rowid for current doc being written */
   int bDelete;                    /* Current write is a delete */
+  int nContentlessDelete;         /* Number of contentless delete ops */
 
   /* Error state. */
   int rc;                         /* Current error code */
@@ -3980,6 +3989,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
     sqlite3Fts5HashClear(p->pHash);
     p->nPendingData = 0;
   }
+  p->nContentlessDelete = 0;
 }
 
 /*
@@ -4722,6 +4732,7 @@ static int fts5IndexMerge(
   int nRem = nPg;
   int bRet = 0;
   Fts5Structure *pStruct = *ppStruct;
+  int bTombstone = 0;
   while( nRem>0 && p->rc==SQLITE_OK ){
     int iLvl;                   /* To iterate through levels */
     int iBestLvl = 0;           /* Level offering the most input segments */
@@ -4731,6 +4742,7 @@ static int fts5IndexMerge(
     assert( pStruct->nLevel>0 );
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
       Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
+      int nThisSeg = 0;
       if( pLvl->nMerge ){
         if( pLvl->nMerge>nBest ){
           iBestLvl = iLvl;
@@ -4738,8 +4750,20 @@ static int fts5IndexMerge(
         }
         break;
       }
-      if( pLvl->nSeg>nBest ){
-        nBest = pLvl->nSeg;
+      nThisSeg = pLvl->nSeg;
+      if( bTombstone && nThisSeg ){
+        int iSeg;
+        int nPg = 0;
+        int nTomb = 0;
+        for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+          nPg += pSeg->pgnoLast;
+          nTomb += pSeg->nPgTombstone;
+        }
+        nThisSeg += ((nTomb*FTS5_MERGE_TOMBSTONE_WEIGHT) / nPg);
+      }
+      if( nThisSeg>nBest ){
+        nBest = nThisSeg;
         iBestLvl = iLvl;
       }
     }
@@ -4752,7 +4776,12 @@ static int fts5IndexMerge(
 #endif
 
     if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
-      break;
+      if( bTombstone || p->pConfig->bContentlessDelete==0 ){
+        break;
+      }else{
+        bTombstone = 1;
+        continue;
+      }
     }
     bRet = 1;
     fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
@@ -5273,192 +5302,195 @@ static void fts5FlushOneHash(Fts5Index *p){
   /* Obtain a reference to the index structure and allocate a new segment-id
   ** for the new level-0 segment.  */
   pStruct = fts5StructureRead(p);
-  iSegid = fts5AllocateSegid(p, pStruct);
   fts5StructureInvalidate(p);
 
-  if( iSegid ){
-    const int pgsz = p->pConfig->pgsz;
-    int eDetail = p->pConfig->eDetail;
-    int bSecureDelete = p->pConfig->bSecureDelete;
-    Fts5StructureSegment *pSeg;   /* New segment within pStruct */
-    Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
-    Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
-
-    Fts5SegWriter writer;
-    fts5WriteInit(p, &writer, iSegid);
-
-    pBuf = &writer.writer.buf;
-    pPgidx = &writer.writer.pgidx;
-
-    /* fts5WriteInit() should have initialized the buffers to (most likely)
-    ** the maximum space required. */
-    assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) );
-    assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) );
-
-    /* Begin scanning through hash table entries. This loop runs once for each
-    ** term/doclist currently stored within the hash table. */
-    if( p->rc==SQLITE_OK ){
-      p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
-    }
-    while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
-      const char *zTerm;          /* Buffer containing term */
-      int nTerm;                  /* Size of zTerm in bytes */
-      const u8 *pDoclist;         /* Pointer to doclist for this term */
-      int nDoclist;               /* Size of doclist in bytes */
-
-      /* Get the term and doclist for this entry. */
-      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
-      nTerm = (int)strlen(zTerm);
-      if( bSecureDelete==0 ){
-        fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
-        if( p->rc!=SQLITE_OK ) break;
-        assert( writer.bFirstRowidInPage==0 );
+  if( sqlite3Fts5HashIsEmpty(pHash)==0 ){
+    iSegid = fts5AllocateSegid(p, pStruct);
+    if( iSegid ){
+      const int pgsz = p->pConfig->pgsz;
+      int eDetail = p->pConfig->eDetail;
+      int bSecureDelete = p->pConfig->bSecureDelete;
+      Fts5StructureSegment *pSeg; /* New segment within pStruct */
+      Fts5Buffer *pBuf;           /* Buffer in which to assemble leaf page */
+      Fts5Buffer *pPgidx;         /* Buffer in which to assemble pgidx */
+  
+      Fts5SegWriter writer;
+      fts5WriteInit(p, &writer, iSegid);
+  
+      pBuf = &writer.writer.buf;
+      pPgidx = &writer.writer.pgidx;
+  
+      /* fts5WriteInit() should have initialized the buffers to (most likely)
+      ** the maximum space required. */
+      assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) );
+      assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) );
+  
+      /* Begin scanning through hash table entries. This loop runs once for each
+      ** term/doclist currently stored within the hash table. */
+      if( p->rc==SQLITE_OK ){
+        p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
       }
-
-      if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
-        /* The entire doclist will fit on the current leaf. */
-        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
-      }else{
-        int bTermWritten = !bSecureDelete;
-        i64 iRowid = 0;
-        i64 iPrev = 0;
-        int iOff = 0;
-
-        /* The entire doclist will not fit on this leaf. The following 
-        ** loop iterates through the poslists that make up the current 
-        ** doclist.  */
-        while( p->rc==SQLITE_OK && iOff<nDoclist ){
-          u64 iDelta = 0;
-          iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
-          iRowid += iDelta;
-
-          /* If in secure delete mode, and if this entry in the poslist is
-          ** in fact a delete, then edit the existing segments directly
-          ** using fts5FlushSecureDelete().  */
-          if( bSecureDelete ){
-            if( eDetail==FTS5_DETAIL_NONE ){
-              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
-                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
-                iOff++;
+      while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
+        const char *zTerm;        /* Buffer containing term */
+        int nTerm;                /* Size of zTerm in bytes */
+        const u8 *pDoclist;       /* Pointer to doclist for this term */
+        int nDoclist;             /* Size of doclist in bytes */
+  
+        /* Get the term and doclist for this entry. */
+        sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
+        nTerm = (int)strlen(zTerm);
+        if( bSecureDelete==0 ){
+          fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+          if( p->rc!=SQLITE_OK ) break;
+          assert( writer.bFirstRowidInPage==0 );
+        }
+  
+        if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
+          /* The entire doclist will fit on the current leaf. */
+          fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
+        }else{
+          int bTermWritten = !bSecureDelete;
+          i64 iRowid = 0;
+          i64 iPrev = 0;
+          int iOff = 0;
+  
+          /* The entire doclist will not fit on this leaf. The following 
+          ** loop iterates through the poslists that make up the current 
+          ** doclist.  */
+          while( p->rc==SQLITE_OK && iOff<nDoclist ){
+            u64 iDelta = 0;
+            iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
+            iRowid += iDelta;
+  
+            /* If in secure delete mode, and if this entry in the poslist is
+            ** in fact a delete, then edit the existing segments directly
+            ** using fts5FlushSecureDelete().  */
+            if( bSecureDelete ){
+              if( eDetail==FTS5_DETAIL_NONE ){
                 if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                  iOff++;
+                  if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                    iOff++;
+                    nDoclist = 0;
+                  }else{
+                    continue;
+                  }
+                }
+              }else if( (pDoclist[iOff] & 0x01) ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
                   iOff++;
-                  nDoclist = 0;
-                }else{
                   continue;
                 }
               }
-            }else if( (pDoclist[iOff] & 0x01) ){
-              fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
-              if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
-                iOff++;
-                continue;
-              }
             }
-          }
-
-          if( p->rc==SQLITE_OK && bTermWritten==0 ){
-            fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
-            bTermWritten = 1;
-            assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
-          }
-          
-          if( writer.bFirstRowidInPage ){
-            fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
-            writer.bFirstRowidInPage = 0;
-            fts5WriteDlidxAppend(p, &writer, iRowid);
-          }else{
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
-          }
-          if( p->rc!=SQLITE_OK ) break;
-          assert( pBuf->n<=pBuf->nSpace );
-          iPrev = iRowid;
-
-          if( eDetail==FTS5_DETAIL_NONE ){
-            if( iOff<nDoclist && pDoclist[iOff]==0 ){
-              pBuf->p[pBuf->n++] = 0;
-              iOff++;
+  
+            if( p->rc==SQLITE_OK && bTermWritten==0 ){
+              fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+              bTermWritten = 1;
+              assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
+            }
+            
+            if( writer.bFirstRowidInPage ){
+              fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
+              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
+              writer.bFirstRowidInPage = 0;
+              fts5WriteDlidxAppend(p, &writer, iRowid);
+            }else{
+              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
+            }
+            if( p->rc!=SQLITE_OK ) break;
+            assert( pBuf->n<=pBuf->nSpace );
+            iPrev = iRowid;
+  
+            if( eDetail==FTS5_DETAIL_NONE ){
               if( iOff<nDoclist && pDoclist[iOff]==0 ){
                 pBuf->p[pBuf->n++] = 0;
                 iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0 ){
+                  pBuf->p[pBuf->n++] = 0;
+                  iOff++;
+                }
+              }
+              if( (pBuf->n + pPgidx->n)>=pgsz ){
+                fts5WriteFlushLeaf(p, &writer);
               }
-            }
-            if( (pBuf->n + pPgidx->n)>=pgsz ){
-              fts5WriteFlushLeaf(p, &writer);
-            }
-          }else{
-            int bDummy;
-            int nPos;
-            int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
-            nCopy += nPos;
-            if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
-              /* The entire poslist will fit on the current leaf. So copy
-              ** it in one go. */
-              fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
             }else{
-              /* The entire poslist will not fit on this leaf. So it needs
-              ** to be broken into sections. The only qualification being
-              ** that each varint must be stored contiguously.  */
-              const u8 *pPoslist = &pDoclist[iOff];
-              int iPos = 0;
-              while( p->rc==SQLITE_OK ){
-                int nSpace = pgsz - pBuf->n - pPgidx->n;
-                int n = 0;
-                if( (nCopy - iPos)<=nSpace ){
-                  n = nCopy - iPos;
-                }else{
-                  n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
+              int bDummy;
+              int nPos;
+              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
+              nCopy += nPos;
+              if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
+                /* The entire poslist will fit on the current leaf. So copy
+                ** it in one go. */
+                fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
+              }else{
+                /* The entire poslist will not fit on this leaf. So it needs
+                ** to be broken into sections. The only qualification being
+                ** that each varint must be stored contiguously.  */
+                const u8 *pPoslist = &pDoclist[iOff];
+                int iPos = 0;
+                while( p->rc==SQLITE_OK ){
+                  int nSpace = pgsz - pBuf->n - pPgidx->n;
+                  int n = 0;
+                  if( (nCopy - iPos)<=nSpace ){
+                    n = nCopy - iPos;
+                  }else{
+                    n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
+                  }
+                  assert( n>0 );
+                  fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
+                  iPos += n;
+                  if( (pBuf->n + pPgidx->n)>=pgsz ){
+                    fts5WriteFlushLeaf(p, &writer);
+                  }
+                  if( iPos>=nCopy ) break;
                 }
-                assert( n>0 );
-                fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
-                iPos += n;
-                if( (pBuf->n + pPgidx->n)>=pgsz ){
-                  fts5WriteFlushLeaf(p, &writer);
-                }
-                if( iPos>=nCopy ) break;
               }
+              iOff += nCopy;
             }
-            iOff += nCopy;
           }
         }
+  
+        /* TODO2: Doclist terminator written here. */
+        /* pBuf->p[pBuf->n++] = '\0'; */
+        assert( pBuf->n<=pBuf->nSpace );
+        if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
       }
-
-      /* TODO2: Doclist terminator written here. */
-      /* pBuf->p[pBuf->n++] = '\0'; */
-      assert( pBuf->n<=pBuf->nSpace );
-      if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
-    }
-    sqlite3Fts5HashClear(pHash);
-    fts5WriteFinish(p, &writer, &pgnoLast);
-
-    assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
-    if( pgnoLast>0 ){
-      /* Update the Fts5Structure. It is written back to the database by the
-      ** fts5StructureRelease() call below.  */
-      if( pStruct->nLevel==0 ){
-        fts5StructureAddLevel(&p->rc, &pStruct);
-      }
-      fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
-      if( p->rc==SQLITE_OK ){
-        pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
-        pSeg->iSegid = iSegid;
-        pSeg->pgnoFirst = 1;
-        pSeg->pgnoLast = pgnoLast;
-        if( pStruct->nOriginCntr>0 ){
-          pSeg->iOrigin1 = pStruct->nOriginCntr;
-          pSeg->iOrigin2 = pStruct->nOriginCntr;
-          pStruct->nOriginCntr++;
+      sqlite3Fts5HashClear(pHash);
+      fts5WriteFinish(p, &writer, &pgnoLast);
+  
+      assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
+      if( pgnoLast>0 ){
+        /* Update the Fts5Structure. It is written back to the database by the
+        ** fts5StructureRelease() call below.  */
+        if( pStruct->nLevel==0 ){
+          fts5StructureAddLevel(&p->rc, &pStruct);
         }
-        pStruct->nSegment++;
+        fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
+        if( p->rc==SQLITE_OK ){
+          pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
+          pSeg->iSegid = iSegid;
+          pSeg->pgnoFirst = 1;
+          pSeg->pgnoLast = pgnoLast;
+          if( pStruct->nOriginCntr>0 ){
+            pSeg->iOrigin1 = pStruct->nOriginCntr;
+            pSeg->iOrigin2 = pStruct->nOriginCntr;
+            pStruct->nOriginCntr++;
+          }
+          pStruct->nSegment++;
+        }
+        fts5StructurePromote(p, 0, pStruct);
       }
-      fts5StructurePromote(p, 0, pStruct);
     }
   }
 
-  fts5IndexAutomerge(p, &pStruct, pgnoLast);
+  fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete);
   fts5IndexCrisismerge(p, &pStruct);
   fts5StructureWrite(p, pStruct);
   fts5StructureRelease(pStruct);
+  p->nContentlessDelete = 0;
 }
 
 /*
@@ -5466,7 +5498,7 @@ static void fts5FlushOneHash(Fts5Index *p){
 */
 static void fts5IndexFlush(Fts5Index *p){
   /* Unless it is empty, flush the hash table to disk */
-  if( p->nPendingData ){
+  if( p->nPendingData || (p->nContentlessDelete && p->pConfig->nAutomerge>0) ){
     assert( p->pHash );
     p->nPendingData = 0;
     fts5FlushOneHash(p);
@@ -6800,6 +6832,8 @@ static void fts5IndexTombstoneAdd(
   int nHash = 0;
   Fts5Data **apHash = 0;
 
+  p->nContentlessDelete++;
+
   if( pSeg->nPgTombstone>0 ){
     iPg = iRowid % pSeg->nPgTombstone;
     pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg));
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index b10da49c75..d1c79d3604 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1777,8 +1777,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
-  fts5TripCursors(pTab);
-  rc = sqlite3Fts5StorageSync(pTab->pStorage);
+  rc = sqlite3Fts5FlushToDisk(&pTab->p);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
new file mode 100644
index 0000000000..21eba71b1a
--- /dev/null
+++ b/ext/fts5/test/fts5contentless4.test
@@ -0,0 +1,61 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+db func document document
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 240);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+do_execsql_test 1.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM ft WHERE rowid < 1000
+}
+
+execsql_pp {
+  SELECT * FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+}
+
+finish_test
+
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 3e4343c8ea..f9de77a7fc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sfts5\sincremental\soptimization\sso\sthat\sit\stoo\scan\shandle\san\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\swith\sone\sor\smore\stombstone\shash\spages.
-D 2023-07-21T21:10:33.579
+C Integrate\scontentless\sdelete\swith\sauto-merge.
+D 2023-07-22T19:47:46.359
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,14 +86,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h fa9dd8ecbda6340f406c6f21b9b524b4666817aa89850e60a42937eea87cef6a
+F ext/fts5/fts5Int.h f59c14f725ad0fcb8a81b9bf012e5021c6501bf43e73aa00b00d728e2ac7efaf
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
-F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c e500a5d33ae312c2ebab91123f0ef5f9bbc3eb555252c7d40ba4d8688780c7ca
-F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
+F ext/fts5/fts5_hash.c 60224220ccfb2846b741b6dbb1b8872094ec6d87b3118c04244dafc83e6f9c40
+F ext/fts5/fts5_index.c 31b8c8dd6913d76d6d7755342e36816495e5ad177d253f6bf39e0efdb9dc31e0
+F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -135,6 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
+F ext/fts5/test/fts5contentless4.test 3b11ccbbe928d45eb8f985c0137a8fe2c69b70b940b10de31540040de5674311
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
-R 8117856750f55ecca6e7594df2bcd37e
+P e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
+R 223fbfa40ff414b8abee5011e5d1533a
 U dan
-Z 3675210f196caeb196ef6ee2658c36e9
+Z 8cb202caf797f67b895f2a9a207e2618
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 92b4b794ac..13263786f5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
\ No newline at end of file
+85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
\ No newline at end of file

From 24730de8d14efb28c0be0b109401e131dc60fc61 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 24 Jul 2023 19:13:06 +0000
Subject: [PATCH 19/22] Add the fts5 'delete-automerge' integer option. A level
 is eligible for auto-merging if it has a greater than or equal percentage of
 its entries deleted by tombstones than the 'delete-automerge' option. Default
 value is 10.

FossilOrigin-Name: b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
---
 ext/fts5/fts5Int.h                  |   1 +
 ext/fts5/fts5_config.c              |  15 +++
 ext/fts5/fts5_hash.c                |  16 ++-
 ext/fts5/fts5_index.c               | 148 ++++++++++++++++++----------
 ext/fts5/test/fts5contentless3.test |   1 -
 ext/fts5/test/fts5contentless4.test |  23 ++++-
 manifest                            |  22 ++---
 manifest.uuid                       |   2 +-
 8 files changed, 159 insertions(+), 69 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 24417483c1..89b553161b 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -214,6 +214,7 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
+  int nDeleteAutomerge;           /* 'delete-automerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index e0c85bbd3f..af2ef40bac 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -22,6 +22,8 @@
 #define FTS5_DEFAULT_CRISISMERGE   16
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
 
+#define FTS5_DEFAULT_DELETE_AUTOMERGE 10      /* default 10% */
+
 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (64*1024)
 
@@ -922,6 +924,18 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
+  else if( 0==sqlite3_stricmp(zKey, "delete-automerge") ){
+    int nVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nVal = sqlite3_value_int(pVal);
+    }else{
+      *pbBadkey = 1;
+    }
+    if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
+    if( nVal>100 ) nVal = 0;
+    pConfig->nDeleteAutomerge = nVal;
+  }
+
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
     const char *zIn = (const char*)sqlite3_value_text(pVal);
     char *zRank;
@@ -970,6 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
+  pConfig->nDeleteAutomerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index 2010e4ff9c..7e50c36608 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -475,7 +475,6 @@ static int fts5HashEntrySort(
     pList = fts5HashEntryMerge(pList, ap[i]);
   }
 
-  pHash->nEntry = 0;
   sqlite3_free(ap);
   *ppSorted = pList;
   return SQLITE_OK;
@@ -529,10 +528,25 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
+#ifdef SQLITE_DEBUG
+static int fts5HashCount(Fts5Hash *pHash){
+  int nEntry = 0;
+  int ii;
+  for(ii=0; ii<pHash->nSlot; ii++){
+    Fts5HashEntry *p = 0;
+    for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
+      nEntry++;
+    }
+  }
+  return nEntry;
+}
+#endif
+
 /*
 ** Return true if the hash table is empty, false otherwise.
 */
 int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
+  assert( pHash->nEntry==fts5HashCount(pHash) );
   return pHash->nEntry==0;
 }
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 392218864c..ce52825533 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,8 +56,6 @@
 
 #define FTS5_MAX_LEVEL 64
 
-#define FTS5_MERGE_TOMBSTONE_WEIGHT 5
-
 /*
 ** There are two versions of the format used for the structure record:
 **
@@ -355,6 +353,7 @@ struct Fts5Index {
   i64 iWriteRowid;                /* Rowid for current doc being written */
   int bDelete;                    /* Current write is a delete */
   int nContentlessDelete;         /* Number of contentless delete ops */
+  int nPendingRow;                /* Number of INSERT in hash table */
 
   /* Error state. */
   int rc;                         /* Current error code */
@@ -404,6 +403,8 @@ struct Fts5StructureSegment {
   u64 iOrigin1;
   u64 iOrigin2;
   int nPgTombstone;               /* Number of tombstone hash table pages */
+  i64 nEntryTombstone;            /* Number of tombstone entries that "count" */
+  i64 nEntry;                     /* Number of rows in this segment */
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -1117,6 +1118,8 @@ static int fts5StructureDecode(
             i += fts5GetVarint(&pData[i], &pSeg->iOrigin1);
             i += fts5GetVarint(&pData[i], &pSeg->iOrigin2);
             i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone);
+            i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone);
+            i += fts5GetVarint(&pData[i], &pSeg->nEntry);
             nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2);
           }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
@@ -1372,13 +1375,16 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
       assert( pLvl->nMerge<=pLvl->nSeg );
 
       for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid);
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst);
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast);
         if( pStruct->nOriginCntr>0 ){
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin1);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin2);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nPgTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry);
         }
       }
     }
@@ -3988,6 +3994,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
   if( p->pHash ){
     sqlite3Fts5HashClear(p->pHash);
     p->nPendingData = 0;
+    p->nPendingRow = 0;
   }
   p->nContentlessDelete = 0;
 }
@@ -4627,7 +4634,7 @@ static void fts5IndexMergeLevel(
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
 
-    /* Set the range of origins that will go into the output segment */
+    /* Set the range of origins that will go into the output segment. */
     if( pStruct->nOriginCntr>0 ){
       pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
       pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
@@ -4691,8 +4698,11 @@ static void fts5IndexMergeLevel(
     int i;
 
     /* Remove the redundant segments from the %_data table */
+    assert( pSeg->nEntry==0 );
     for(i=0; i<nInput; i++){
-      fts5DataRemoveSegment(p, &pLvl->aSeg[i]);
+      Fts5StructureSegment *pOld = &pLvl->aSeg[i];
+      pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone);
+      fts5DataRemoveSegment(p, pOld);
     }
 
     /* Remove the redundant segments from the input level */
@@ -4718,6 +4728,43 @@ static void fts5IndexMergeLevel(
   if( pnRem ) *pnRem -= writer.nLeafWritten;
 }
 
+/*
+** If this is not a contentless_delete=1 table, or if the 'delete-automerge'
+** configuration option is set to 0, then this function always returns -1.
+** Otherwise, it searches the structure object passed as the second argument
+** for a level suitable for merging due to having a large number of 
+** tombstones in the tombstone hash. If one is found, its index is returned.
+** Otherwise, if there is no suitable level, -1.
+*/
+static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
+  Fts5Config *pConfig = p->pConfig;
+  int iRet = -1;
+  if( pConfig->bContentlessDelete && pConfig->nDeleteAutomerge>0 ){
+    int ii;
+    int nBest = 0;
+
+    for(ii=0; ii<pStruct->nLevel; ii++){
+      Fts5StructureLevel *pLvl = &pStruct->aLevel[ii];
+      i64 nEntry = 0;
+      i64 nTomb = 0;
+      int iSeg;
+      for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+        nEntry += pLvl->aSeg[iSeg].nEntry;
+        nTomb += pLvl->aSeg[iSeg].nEntryTombstone;
+      }
+      assert( nEntry>0 || pLvl->nSeg==0 );
+      if( nEntry>0 ){
+        int nPercent = (nTomb * 100) / nEntry;
+        if( nPercent>=pConfig->nDeleteAutomerge && nPercent>nBest ){
+          iRet = ii;
+          nBest = nPercent;
+        }
+      }
+    }
+  }
+  return iRet;
+}
+
 /*
 ** Do up to nPg pages of automerge work on the index.
 **
@@ -4738,51 +4785,28 @@ static int fts5IndexMerge(
     int iBestLvl = 0;           /* Level offering the most input segments */
     int nBest = 0;              /* Number of input segments on best level */
 
-    /* Set iBestLvl to the level to read input segments from. */
+    /* Set iBestLvl to the level to read input segments from. Or to -1 if
+    ** there is no level suitable to merge segments from.  */
     assert( pStruct->nLevel>0 );
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
       Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
-      int nThisSeg = 0;
       if( pLvl->nMerge ){
         if( pLvl->nMerge>nBest ){
           iBestLvl = iLvl;
-          nBest = pLvl->nMerge;
+          nBest = nMin;
         }
         break;
       }
-      nThisSeg = pLvl->nSeg;
-      if( bTombstone && nThisSeg ){
-        int iSeg;
-        int nPg = 0;
-        int nTomb = 0;
-        for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-          nPg += pSeg->pgnoLast;
-          nTomb += pSeg->nPgTombstone;
-        }
-        nThisSeg += ((nTomb*FTS5_MERGE_TOMBSTONE_WEIGHT) / nPg);
-      }
-      if( nThisSeg>nBest ){
-        nBest = nThisSeg;
+      if( pLvl->nSeg>nBest ){
+        nBest = pLvl->nSeg;
         iBestLvl = iLvl;
       }
     }
-
-    /* If nBest is still 0, then the index must be empty. */
-#ifdef SQLITE_DEBUG
-    for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
-      assert( pStruct->aLevel[iLvl].nSeg==0 );
+    if( nBest<nMin ){
+      iBestLvl = fts5IndexFindDeleteMerge(p, pStruct);
     }
-#endif
 
-    if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
-      if( bTombstone || p->pConfig->bContentlessDelete==0 ){
-        break;
-      }else{
-        bTombstone = 1;
-        continue;
-      }
-    }
+    if( iBestLvl<0 ) break;
     bRet = 1;
     fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
     if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
@@ -5477,6 +5501,7 @@ static void fts5FlushOneHash(Fts5Index *p){
           if( pStruct->nOriginCntr>0 ){
             pSeg->iOrigin1 = pStruct->nOriginCntr;
             pSeg->iOrigin2 = pStruct->nOriginCntr;
+            pSeg->nEntry = p->nPendingRow;
             pStruct->nOriginCntr++;
           }
           pStruct->nSegment++;
@@ -5498,10 +5523,11 @@ static void fts5FlushOneHash(Fts5Index *p){
 */
 static void fts5IndexFlush(Fts5Index *p){
   /* Unless it is empty, flush the hash table to disk */
-  if( p->nPendingData || (p->nContentlessDelete && p->pConfig->nAutomerge>0) ){
+  if( p->nPendingData || p->nContentlessDelete ){
     assert( p->pHash );
-    p->nPendingData = 0;
     fts5FlushOneHash(p);
+    p->nPendingData = 0;
+    p->nPendingRow = 0;
   }
 }
 
@@ -5579,6 +5605,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
 
   assert( p->rc==SQLITE_OK );
   fts5IndexFlush(p);
+  assert( p->nContentlessDelete==0 );
   pStruct = fts5StructureRead(p);
   fts5StructureInvalidate(p);
 
@@ -5608,7 +5635,10 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
 ** INSERT command.
 */
 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
-  Fts5Structure *pStruct = fts5StructureRead(p);
+  Fts5Structure *pStruct = 0;
+
+  fts5IndexFlush(p);
+  pStruct = fts5StructureRead(p);
   if( pStruct ){
     int nMin = p->pConfig->nUsermerge;
     fts5StructureInvalidate(p);
@@ -6130,6 +6160,9 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
 
   p->iWriteRowid = iRowid;
   p->bDelete = bDelete;
+  if( bDelete==0 ){
+    p->nPendingRow++;
+  }
   return fts5IndexReturn(p);
 }
 
@@ -6883,12 +6916,17 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
+    int bFound = 0;               /* True after pSeg->nEntryTombstone incr. */
     int iLvl;
-    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+    for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
       int iSeg;
-      for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+      for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
         if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
+          if( bFound==0 ){
+            pSeg->nEntryTombstone++;
+            bFound = 1;
+          }
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
@@ -7980,7 +8018,7 @@ static int fts5structConnectMethod(
   rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE xyz("
           "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
-          "npgtombstone, struct HIDDEN);"
+          "npgtombstone, nentrytombstone, nentry, struct HIDDEN);"
   );
   if( rc==SQLITE_OK ){
     pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
@@ -8007,7 +8045,7 @@ static int fts5structBestIndexMethod(
   pIdxInfo->idxNum = 0;
   for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
     if( p->usable==0 ) continue;
-    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==9 ){
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){
       rc = SQLITE_OK;
       pIdxInfo->aConstraintUsage[i].omit = 1;
       pIdxInfo->aConstraintUsage[i].argvIndex = 1;
@@ -8120,15 +8158,21 @@ static int fts5structColumnMethod(
     case 5: /* leaf2 */
       sqlite3_result_int(ctx, pSeg->pgnoLast);
       break;
-    case 6: /* loc1 */
-      sqlite3_result_int(ctx, pSeg->iOrigin1);
+    case 6: /* origin1 */
+      sqlite3_result_int64(ctx, pSeg->iOrigin1);
       break;
-    case 7: /* loc2 */
-      sqlite3_result_int(ctx, pSeg->iOrigin2);
+    case 7: /* origin2 */
+      sqlite3_result_int64(ctx, pSeg->iOrigin2);
       break;
     case 8: /* npgtombstone */
       sqlite3_result_int(ctx, pSeg->nPgTombstone);
       break;
+    case 9: /* nentrytombstone */
+      sqlite3_result_int64(ctx, pSeg->nEntryTombstone);
+      break;
+    case 10: /* nentry */
+      sqlite3_result_int64(ctx, pSeg->nEntry);
+      break;
   }
   return SQLITE_OK;
 }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index 6316628c55..a44311e45a 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -192,6 +192,5 @@ do_execsql_test 3.7 {
 } {2 0 0}
 
 
-
 finish_test
 
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 21eba71b1a..9eb1900ee3 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -39,19 +39,36 @@ do_execsql_test 1.0 {
   )
   INSERT INTO ft SELECT document(12) FROM s;
 }
+
 do_execsql_test 1.1 {
   INSERT INTO ft(ft) VALUES('optimize');
 }
 
 do_execsql_test 1.2 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 0}
+
+do_execsql_test 1.3 {
+  DELETE FROM ft WHERE rowid < 50
+}
+
+do_execsql_test 1.4 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 49}
+
+do_execsql_test 1.5 {
   DELETE FROM ft WHERE rowid < 1000
 }
 
-execsql_pp {
-  SELECT * FROM fts5_structure((
+do_execsql_test 1.6 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
     SELECT block FROM ft_data WHERE id=10
   ))
-}
+} {1 0 1 0}
 
 finish_test
 
diff --git a/manifest b/manifest
index f9de77a7fc..8a6f302dd1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Integrate\scontentless\sdelete\swith\sauto-merge.
-D 2023-07-22T19:47:46.359
+C Add\sthe\sfts5\s'delete-automerge'\sinteger\soption.\sA\slevel\sis\seligible\sfor\sauto-merging\sif\sit\shas\sa\sgreater\sthan\sor\sequal\spercentage\sof\sits\sentries\sdeleted\sby\stombstones\sthan\sthe\s'delete-automerge'\soption.\sDefault\svalue\sis\s10.
+D 2023-07-24T19:13:06.235
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,13 +86,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h f59c14f725ad0fcb8a81b9bf012e5021c6501bf43e73aa00b00d728e2ac7efaf
+F ext/fts5/fts5Int.h 7decc306406187d1826c5eba9b8e8e6661b85580e5da1203760c0c2de9bc4a5e
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
+F ext/fts5/fts5_config.c c35f3433586f9152af2f444356020bf5c0f2e1d0fae29e67ab45de81277d07c9
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
-F ext/fts5/fts5_hash.c 60224220ccfb2846b741b6dbb1b8872094ec6d87b3118c04244dafc83e6f9c40
-F ext/fts5/fts5_index.c 31b8c8dd6913d76d6d7755342e36816495e5ad177d253f6bf39e0efdb9dc31e0
+F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
+F ext/fts5/fts5_index.c f5d50e218db3d32dd12c83b3f700bf79ed7f24e3f60f43f5b56e62146e2c31b5
 F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,8 +134,8 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
-F ext/fts5/test/fts5contentless4.test 3b11ccbbe928d45eb8f985c0137a8fe2c69b70b940b10de31540040de5674311
+F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
+F ext/fts5/test/fts5contentless4.test 4403cbbbb5021b36b24d914addb22a782699d1a03a98fede705793f7184dbcd8
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
-R 223fbfa40ff414b8abee5011e5d1533a
+P 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
+R 067f23316c8bda4fd7b0c9766119547d
 U dan
-Z 8cb202caf797f67b895f2a9a207e2618
+Z b4aabcdd1189dc9cd9c674268a107cec
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 13263786f5..d3ab0c925e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
\ No newline at end of file
+b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
\ No newline at end of file

From 039d494d5fe9ab68d458eb64241ec356da89f1f8 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 13:53:42 +0000
Subject: [PATCH 20/22] Further tests for 'delete-automerge'.

FossilOrigin-Name: ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
---
 ext/fts5/test/fts5contentless4.test | 57 ++++++++++++++++++++++++++++-
 manifest                            | 12 +++---
 manifest.uuid                       |  2 +-
 3 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 9eb1900ee3..d707003c6f 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -70,9 +70,64 @@ do_execsql_test 1.6 {
   ))
 } {1 0 1 0}
 
-finish_test
+#--------------------------------------------------------------------------
+reset_db
+db func document document
 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
 
+do_test 2.1 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {5000}
+
+for {set ii 5000} {$ii >= 0} {incr ii -100} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid > $ii
+  }
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } $ii
+}
+
+execsql_pp {
+  SELECT * FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+}
+
+do_test 2.4 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+for {set ii 1} {$ii <= 5000} {incr ii 10} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid = $ii;
+    INSERT INTO ft VALUES( document(12) );
+    INSERT INTO ft(ft, rank) VALUES('merge', -10);
+  }
+
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } 5000
+}
 
 finish_test
 
diff --git a/manifest b/manifest
index 8a6f302dd1..c125c285b2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sfts5\s'delete-automerge'\sinteger\soption.\sA\slevel\sis\seligible\sfor\sauto-merging\sif\sit\shas\sa\sgreater\sthan\sor\sequal\spercentage\sof\sits\sentries\sdeleted\sby\stombstones\sthan\sthe\s'delete-automerge'\soption.\sDefault\svalue\sis\s10.
-D 2023-07-24T19:13:06.235
+C Further\stests\sfor\s'delete-automerge'.
+D 2023-07-25T13:53:42.704
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test 4403cbbbb5021b36b24d914addb22a782699d1a03a98fede705793f7184dbcd8
+F ext/fts5/test/fts5contentless4.test b42b76b4b4c7d6c8dcc4d681fe98f804241bed22e31d1c125f1dbab3104cadab
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
-R 067f23316c8bda4fd7b0c9766119547d
+P b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
+R 57589268405704f94631ba59ff450fcb
 U dan
-Z b4aabcdd1189dc9cd9c674268a107cec
+Z 4728fe3315208b53f9df48d307acc67e
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d3ab0c925e..f1f62f3312 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
\ No newline at end of file
+ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
\ No newline at end of file

From 3f874b58fbfd05a1cf5881a365812fe4869be0ac Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 15:48:58 +0000
Subject: [PATCH 21/22] Change the name of the fts5 'delete-automerge' option
 to 'deletemerge'. And add tests for it.

FossilOrigin-Name: 1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
---
 ext/fts5/fts5Int.h                  |  2 +-
 ext/fts5/fts5_config.c              |  6 +--
 ext/fts5/fts5_index.c               |  6 +--
 ext/fts5/test/fts5contentless4.test | 66 +++++++++++++++++++++++++++++
 manifest                            | 18 ++++----
 manifest.uuid                       |  2 +-
 6 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 89b553161b..8bbafbaaf4 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -214,7 +214,7 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
-  int nDeleteAutomerge;           /* 'delete-automerge' */
+  int nDeleteMerge;           /* 'deletemerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index af2ef40bac..5d0770502e 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -924,7 +924,7 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
-  else if( 0==sqlite3_stricmp(zKey, "delete-automerge") ){
+  else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
     int nVal = -1;
     if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
       nVal = sqlite3_value_int(pVal);
@@ -933,7 +933,7 @@ int sqlite3Fts5ConfigSetValue(
     }
     if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
     if( nVal>100 ) nVal = 0;
-    pConfig->nDeleteAutomerge = nVal;
+    pConfig->nDeleteMerge = nVal;
   }
 
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
@@ -984,7 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
-  pConfig->nDeleteAutomerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
+  pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ce52825533..5e20da2b04 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4729,7 +4729,7 @@ static void fts5IndexMergeLevel(
 }
 
 /*
-** If this is not a contentless_delete=1 table, or if the 'delete-automerge'
+** If this is not a contentless_delete=1 table, or if the 'deletemerge'
 ** configuration option is set to 0, then this function always returns -1.
 ** Otherwise, it searches the structure object passed as the second argument
 ** for a level suitable for merging due to having a large number of 
@@ -4739,7 +4739,7 @@ static void fts5IndexMergeLevel(
 static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
   Fts5Config *pConfig = p->pConfig;
   int iRet = -1;
-  if( pConfig->bContentlessDelete && pConfig->nDeleteAutomerge>0 ){
+  if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){
     int ii;
     int nBest = 0;
 
@@ -4755,7 +4755,7 @@ static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
       assert( nEntry>0 || pLvl->nSeg==0 );
       if( nEntry>0 ){
         int nPercent = (nTomb * 100) / nEntry;
-        if( nPercent>=pConfig->nDeleteAutomerge && nPercent>nBest ){
+        if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){
           iRet = ii;
           nBest = nPercent;
         }
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index d707003c6f..6463b18d00 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -129,5 +129,71 @@ for {set ii 1} {$ii <= 5000} {incr ii 10} {
   } 5000
 }
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text');
+} {1 {SQL logic error}}
+do_catchsql_test 3.2 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+} {0 {}}
+do_execsql_test 3.3 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 50}
+do_catchsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 101);
+} {0 {}}
+do_execsql_test 3.5 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 101}
+
+do_execsql_test 3.6 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.7 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {94 100}
+
+do_execsql_test 3.8 {
+  DELETE FROM ft WHERE rowid=95
+}
+
+do_execsql_test 3.9 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {95 100}
+
+do_execsql_test 3.10 {
+  DELETE FROM ft;
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+}
+
+do_execsql_test 3.11 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.12 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 6}
+
 finish_test
 
diff --git a/manifest b/manifest
index c125c285b2..3cacac4565 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\stests\sfor\s'delete-automerge'.
-D 2023-07-25T13:53:42.704
+C Change\sthe\sname\sof\sthe\sfts5\s'delete-automerge'\soption\sto\s'deletemerge'.\sAnd\sadd\stests\sfor\sit.
+D 2023-07-25T15:48:58.767
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,13 +86,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h 7decc306406187d1826c5eba9b8e8e6661b85580e5da1203760c0c2de9bc4a5e
+F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c c35f3433586f9152af2f444356020bf5c0f2e1d0fae29e67ab45de81277d07c9
+F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c f5d50e218db3d32dd12c83b3f700bf79ed7f24e3f60f43f5b56e62146e2c31b5
+F ext/fts5/fts5_index.c 182cf576bae17682adf2fe8cefb8bc3b46bea27e6a227e444859b3c065322687
 F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test b42b76b4b4c7d6c8dcc4d681fe98f804241bed22e31d1c125f1dbab3104cadab
+F ext/fts5/test/fts5contentless4.test 52aad02fe9eb1bfa9272ee35ac95f6dc31eab57ba470c2df0ed63f0bfbe95bc2
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
-R 57589268405704f94631ba59ff450fcb
+P ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
+R 4c7400d78b496d9cb420c06c5a4d6bad
 U dan
-Z 4728fe3315208b53f9df48d307acc67e
+Z a03778de7e1285f066cc1d73e2bb1632
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index f1f62f3312..736b30b29f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
\ No newline at end of file
+1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
\ No newline at end of file

From 5326953e5799542071373922c1da43e96f65335b Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 16:48:54 +0000
Subject: [PATCH 22/22] Add extra test for 'deletemerge'.

FossilOrigin-Name: bc33cff4203cef12518e0f43d380a06d53d67c725fb96cfe6e934b7dc97a7efd
---
 ext/fts5/test/fts5contentless4.test | 49 +++++++++++++++++++++++++++++
 manifest                            | 12 +++----
 manifest.uuid                       |  2 +-
 3 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 6463b18d00..1c2666dcf8 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -195,5 +195,54 @@ do_execsql_test 3.12 {
   ))
 } {0 6}
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO x1(x1, rank) VALUES('usermerge', 16);
+  INSERT INTO x1(x1, rank) VALUES('deletemerge', 40);
+  INSERT INTO x1 VALUES('one');
+  INSERT INTO x1 VALUES('two');
+  INSERT INTO x1 VALUES('three');
+  INSERT INTO x1 VALUES('four');
+  INSERT INTO x1 VALUES('five');
+  INSERT INTO x1 VALUES('six');
+  INSERT INTO x1 VALUES('seven');
+  INSERT INTO x1 VALUES('eight');
+  INSERT INTO x1 VALUES('nine');
+  INSERT INTO x1 VALUES('ten');
+}
+
+do_execsql_test 4.1 {
+  SELECT level, segment FROM fts5_structure((
+    SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+}
+
+for {set ii 1} {$ii < 4} {incr ii} {
+  do_execsql_test 4.2.$ii {
+    DELETE FROM x1 WHERE rowid = $ii;
+    INSERT INTO x1(x1, rank) VALUES('merge', 5);
+    SELECT level, segment FROM fts5_structure((
+          SELECT block FROM x1_data WHERE id=10
+    ))
+  } {
+    0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+  }
+}
+
+do_execsql_test 4.3 {
+  DELETE FROM x1 WHERE rowid = $ii;
+  INSERT INTO x1(x1, rank) VALUES('merge', 5);
+  SELECT level, segment, nentry FROM fts5_structure((
+        SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  1 0   6
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index 3cacac4565..34b86f367f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sthe\sfts5\s'delete-automerge'\soption\sto\s'deletemerge'.\sAnd\sadd\stests\sfor\sit.
-D 2023-07-25T15:48:58.767
+C Add\sextra\stest\sfor\s'deletemerge'.
+D 2023-07-25T16:48:54.280
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test 52aad02fe9eb1bfa9272ee35ac95f6dc31eab57ba470c2df0ed63f0bfbe95bc2
+F ext/fts5/test/fts5contentless4.test 0f43ededc2874f65d7da99b641a82239854d98d3fa43db729f284b723f23b69f
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
-R 4c7400d78b496d9cb420c06c5a4d6bad
+P 1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
+R 5e50972831b0957645434d1b66948061
 U dan
-Z a03778de7e1285f066cc1d73e2bb1632
+Z 17e8bfe851f3a2d36e6b484a74a5eecf
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 736b30b29f..20b6bcd047 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
\ No newline at end of file
+bc33cff4203cef12518e0f43d380a06d53d67c725fb96cfe6e934b7dc97a7efd
\ No newline at end of file