From aec0aa83a9a4ad1c380707594fea1da191bdcb9b Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 30 Jun 2023 18:59:10 +0000
Subject: [PATCH 01/66] Add the "nexec" and "ncycle" columns to the bytecode
 virtual table. For accessing counters collected when
 SQLITE_DBCONFIG_STMT_SCANSTATUS is enabled.

FossilOrigin-Name: f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006
---
 manifest       | 16 +++++++++-------
 manifest.uuid  |  2 +-
 src/vdbevtab.c | 29 +++++++++++++++++++++++------
 3 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 0b84625ebf..b4de521228 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\ssqlite3_stmt_scanstatus_v2()\sprofiling\sof\sGROUP\sBY\sclauses\sthat\suse\sa\stemp\sb-tree,\sand\sfor\ssub-queries\simplemented\sas\sco-routines.
-D 2023-06-30T18:31:37.545
+C Add\sthe\s"nexec"\sand\s"ncycle"\scolumns\sto\sthe\sbytecode\svirtual\stable.\sFor\saccessing\scounters\scollected\swhen\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sis\senabled.
+D 2023-06-30T18:59:10.790
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -716,7 +716,7 @@ F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2c
 F src/vdbemem.c aed58a560caab12540f7c14c43ee188636017814e21247a97902f78de2d43117
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
-F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
+F src/vdbevtab.c c3085edc48b123ca1c30d478f2a7cb326b5e4c2960b65ca8f0f0d6ebf497140a
 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c cbfeeb7415baa545efa244dd34bb5af4ae953a206fed720c6fa7f1ef763ec122
@@ -2041,9 +2041,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f936f101406069b29218c89a36581b4497226fb61906782ea368f12d943c901c 4e8718dc35dbbaf75f17265a88d14acd9750dc75efbadf41377f9c97e732009c
-R 4da01a7efae23c104b98c2eba70ec2ef
-T +closed 4e8718dc35dbbaf75f17265a88d14acd9750dc75efbadf41377f9c97e732009c
+P 7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1
+R 4dba3e78e9c48b15e9dce867cfd3b67a
+T *branch * scanstatus-exp
+T *sym-scanstatus-exp *
+T -sym-trunk *
 U dan
-Z 8ca5997f8884b68f398b8d0f077d59ae
+Z 262a3b7e30cd989cb3c51edc832f13b0
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index f482cd32b9..848125017e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1
\ No newline at end of file
+f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006
\ No newline at end of file
diff --git a/src/vdbevtab.c b/src/vdbevtab.c
index 6557d8cb01..b95852fb5d 100644
--- a/src/vdbevtab.c
+++ b/src/vdbevtab.c
@@ -69,6 +69,8 @@ static int bytecodevtabConnect(
       "p5 INT,"
       "comment TEXT,"
       "subprog TEXT," 
+      "nexec INT,"
+      "ncycle INT,"
       "stmt HIDDEN"
     ");",
 
@@ -231,7 +233,7 @@ static int bytecodevtabColumn(
           }
         }
       }
-      i += 10;
+      i += 20;
     }
   }
   switch( i ){
@@ -281,16 +283,31 @@ static int bytecodevtabColumn(
       }
       break;
     }
-    case 10:  /* tables_used.type */
+
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+    case 9:     /* nexec */
+      sqlite3_result_int(ctx, pCur->aOp[0].nExec);
+      break;
+    case 10:    /* ncycle */
+      sqlite3_result_int(ctx, pCur->aOp[0].nCycle);
+      break;
+#else
+    case 9:     /* nexec */
+    case 10:    /* ncycle */
+      sqlite3_result_int(ctx, 0);
+      break;
+#endif
+
+    case 20:  /* tables_used.type */
       sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC);
       break;
-    case 11:  /* tables_used.schema */
+    case 21:  /* tables_used.schema */
       sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC);
       break;
-    case 12:  /* tables_used.name */
+    case 22:  /* tables_used.name */
       sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC);
       break;
-    case 13:  /* tables_used.wr */
+    case 23:  /* tables_used.wr */
       sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite);
       break;
   }
@@ -364,7 +381,7 @@ static int bytecodevtabBestIndex(
   int rc = SQLITE_CONSTRAINT;
   struct sqlite3_index_constraint *p;
   bytecodevtab *pVTab = (bytecodevtab*)tab;
-  int iBaseCol = pVTab->bTablesUsed ? 4 : 8;
+  int iBaseCol = pVTab->bTablesUsed ? 4 : 10;
   pIdxInfo->estimatedCost = (double)100;
   pIdxInfo->estimatedRows = 100;
   pIdxInfo->idxNum = 0;

From e6d7ae24e3682a8f98d1a34bdfdab3caf0f85371 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 30 Jun 2023 19:13:27 +0000
Subject: [PATCH 02/66] Fix an error in the previous commit on this branch.

FossilOrigin-Name: 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2
---
 manifest       | 15 ++++++---------
 manifest.uuid  |  2 +-
 src/vdbevtab.c |  4 ++--
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index b4de521228..af6823742c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"nexec"\sand\s"ncycle"\scolumns\sto\sthe\sbytecode\svirtual\stable.\sFor\saccessing\scounters\scollected\swhen\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sis\senabled.
-D 2023-06-30T18:59:10.790
+C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch.
+D 2023-06-30T19:13:27.228
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -716,7 +716,7 @@ F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2c
 F src/vdbemem.c aed58a560caab12540f7c14c43ee188636017814e21247a97902f78de2d43117
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
-F src/vdbevtab.c c3085edc48b123ca1c30d478f2a7cb326b5e4c2960b65ca8f0f0d6ebf497140a
+F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254
 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c cbfeeb7415baa545efa244dd34bb5af4ae953a206fed720c6fa7f1ef763ec122
@@ -2041,11 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1
-R 4dba3e78e9c48b15e9dce867cfd3b67a
-T *branch * scanstatus-exp
-T *sym-scanstatus-exp *
-T -sym-trunk *
+P f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006
+R 24e32cd4f6a370917cd11359ca963cf5
 U dan
-Z 262a3b7e30cd989cb3c51edc832f13b0
+Z dd437f2b1d92b46f41dc47026405c30b
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 848125017e..48abfa6e16 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006
\ No newline at end of file
+47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2
\ No newline at end of file
diff --git a/src/vdbevtab.c b/src/vdbevtab.c
index b95852fb5d..59030e0e12 100644
--- a/src/vdbevtab.c
+++ b/src/vdbevtab.c
@@ -286,10 +286,10 @@ static int bytecodevtabColumn(
 
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     case 9:     /* nexec */
-      sqlite3_result_int(ctx, pCur->aOp[0].nExec);
+      sqlite3_result_int(ctx, pOp->nExec);
       break;
     case 10:    /* ncycle */
-      sqlite3_result_int(ctx, pCur->aOp[0].nCycle);
+      sqlite3_result_int(ctx, pOp->nCycle);
       break;
 #else
     case 9:     /* nexec */

From f8be243fefc5459355f4febd82f9b75004c4746f Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 30 Jun 2023 19:14:35 +0000
Subject: [PATCH 03/66] Add experimental ".scanstats vm" command to the shell
 tool.

FossilOrigin-Name: e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 44 ++++++++++++++++++++++++++++++++++++--------
 3 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index af6823742c..f50f10ae04 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch.
-D 2023-06-30T19:13:27.228
+C Add\sexperimental\s".scanstats\svm"\scommand\sto\sthe\sshell\stool.
+D 2023-06-30T19:14:35.552
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -638,7 +638,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
-F src/shell.c.in 2c02c819349de410d63fcc0217763dfe5a42dbe58f2d68046d4ea8a376d12c26
+F src/shell.c.in 6266eac6a51f1b931d72b3ea95b9f688f390cad7b43bbdb465f216b39eaaa0d3
 F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
@@ -2041,8 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006
-R 24e32cd4f6a370917cd11359ca963cf5
+P 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2
+R 1fc4f436f0d8b18e387b07b97d0465bf
 U dan
-Z dd437f2b1d92b46f41dc47026405c30b
+Z 39b6ffde042c1822fbacb7543ed0038f
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 48abfa6e16..df83ac764a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2
\ No newline at end of file
+e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 2b0884e1c0..04407e54ae 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3332,17 +3332,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){
 }
 #endif
 
-/*
-** Display scan stats.
-*/
-static void display_scanstats(
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+static void display_explain_scanstats(
   sqlite3 *db,                    /* Database to query */
   ShellState *pArg                /* Pointer to ShellState */
 ){
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-  UNUSED_PARAMETER(db);
-  UNUSED_PARAMETER(pArg);
-#else
   static const int f = SQLITE_SCANSTAT_COMPLEX;
   sqlite3_stmt *p = pArg->pStmt;
   int ii = 0;
@@ -3414,6 +3408,37 @@ static void display_scanstats(
   }
 
   eqp_render(pArg, nTotal);
+}
+#endif
+
+static void exec_prepared_stmt(ShellState*, sqlite3_stmt*);
+
+/*
+** Display scan stats.
+*/
+static void display_scanstats(
+  sqlite3 *db,                    /* Database to query */
+  ShellState *pArg                /* Pointer to ShellState */
+){
+#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
+  UNUSED_PARAMETER(db);
+  UNUSED_PARAMETER(pArg);
+#else
+  if( pArg->scanstatsOn==3 ){
+    int rc = SQLITE_OK;
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(db, "SELECT * FROM bytecode(?)", -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pSave = pArg->pStmt;
+      pArg->pStmt = pStmt;
+      sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0);
+      exec_prepared_stmt(pArg, pStmt);
+      sqlite3_finalize(pStmt);
+      pArg->pStmt = pSave;
+    }
+  }else{
+    display_explain_scanstats(db, pArg);
+  }
 #endif
 }
 
@@ -9933,6 +9958,9 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
+      if( cli_strcmp(azArg[1], "vm")==0 ){
+        p->scanstatsOn = 3;
+      }else
       if( cli_strcmp(azArg[1], "est")==0 ){
         p->scanstatsOn = 2;
       }else{

From 6788c7b7c00372860889216cefd5df2c4587312f Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 10 Jul 2023 20:44:09 +0000
Subject: [PATCH 04/66] Begin adding support for deleting rows from contentless
 fts5 tables.

FossilOrigin-Name: e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
---
 ext/fts5/fts5Int.h                 |   8 +
 ext/fts5/fts5_config.c             |  32 ++
 ext/fts5/fts5_index.c              | 477 ++++++++++++++++++++++++++++-
 ext/fts5/fts5_main.c               |   1 -
 ext/fts5/fts5_storage.c            |  74 ++++-
 ext/fts5/test/fts5aa.test          |   1 +
 ext/fts5/test/fts5contentless.test | 175 +++++++++++
 manifest                           |  28 +-
 manifest.uuid                      |   2 +-
 9 files changed, 769 insertions(+), 29 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless.test

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 5d05da875e..43f78c6d99 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -154,6 +154,10 @@ typedef struct Fts5Config Fts5Config;
 **   attempt to merge together. A value of 1 sets the object to use the 
 **   compile time default. Zero disables auto-merge altogether.
 **
+** bContentlessDelete:
+**   True if the contentless_delete option was present in the CREATE 
+**   VIRTUAL TABLE statement.
+**
 ** zContent:
 **
 ** zContentRowid:
@@ -188,6 +192,7 @@ struct Fts5Config {
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   int eContent;                   /* An FTS5_CONTENT value */
+  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
@@ -531,6 +536,9 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
+int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc);
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 7a4c7b8177..e0c85bbd3f 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -352,6 +352,16 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
+  if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bContentlessDelete = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
   if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
     if( pConfig->zContentRowid ){
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -596,6 +606,28 @@ int sqlite3Fts5ConfigParse(
     sqlite3_free(zTwo);
   }
 
+  /* We only allow contentless_delete=1 if the table is indeed contentless. */
+  if( rc==SQLITE_OK 
+   && pRet->bContentlessDelete 
+   && pRet->eContent!=FTS5_CONTENT_NONE 
+  ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 requires a contentless table"
+    );
+    rc = SQLITE_ERROR;
+  }
+
+  /* We only allow contentless_delete=1 if columnsize=0 is not present. 
+  **
+  ** This restriction may be removed at some point. 
+  */
+  if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_delete=1 is incompatible with columnsize=0"
+    );
+    rc = SQLITE_ERROR;
+  }
+
   /* If a tokenizer= option was successfully parsed, the tokenizer has
   ** already been allocated. Otherwise, allocate an instance of the default
   ** tokenizer (unicode61) now.  */
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 7b9d21e87f..e86da1c06e 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,8 @@
 
 #define FTS5_MAX_LEVEL 64
 
+#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01"
+
 /*
 ** Details:
 **
@@ -92,6 +94,11 @@
 **         + first leaf page number (often 1, always greater than 0)
 **         + final leaf page number
 **
+**      Then, for V2 structures only:
+**
+**         + lower location counter value,
+**         + upper location counter value
+**
 ** 2. The Averages Record:
 **
 **   A single record within the %_data table. The data is a list of varints.
@@ -241,6 +248,8 @@
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
 
+#define FTS5_TOMBSTONE_ROWID(segid) fts5_dri(segid + (1<<16), 0, 0, 0)
+
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
 #endif
@@ -295,6 +304,7 @@ struct Fts5Index {
 
   /* State used by the fts5DataXXX() functions. */
   sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
+  sqlite3_stmt *pReaderOpt;
   sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
@@ -323,11 +333,20 @@ struct Fts5DoclistIter {
 ** The contents of the "structure" record for each index are represented
 ** using an Fts5Structure record in memory. Which uses instances of the 
 ** other Fts5StructureXXX types as components.
+**
+** nLocCounter:
+**   This value is set to non-zero for structure records created for
+**   contentlessdelete=1 tables only. In that case it represents the
+**   location value to apply to the next top-level segment created.
 */
 struct Fts5StructureSegment {
   int iSegid;                     /* Segment id */
   int pgnoFirst;                  /* First leaf page number in segment */
   int pgnoLast;                   /* Last leaf page number in segment */
+
+  /* contentlessdelete=1 tables only: */
+  u64 iLoc1;
+  u64 iLoc2;
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -337,6 +356,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
+  u64 nLocCounter;
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -433,6 +453,7 @@ struct Fts5SegIter {
   Fts5Data *pLeaf;                /* Current leaf data */
   Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   i64 iLeafOffset;                /* Byte offset within current leaf */
+  Fts5Data *pTombstone;
 
   /* Next method */
   void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
@@ -626,6 +647,7 @@ void sqlite3Fts5IndexCloseReader(Fts5Index *p){
   }
 }
 
+
 /*
 ** Retrieve a record from the %_data table.
 **
@@ -740,6 +762,40 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
+static Fts5Data *fts5DataReadOpt(Fts5Index *p, i64 iRowid){
+  Fts5Data *pRet = 0;
+  int rc2 = SQLITE_OK;
+
+  if( p->pReaderOpt==0 ){
+    Fts5Config *pConfig = p->pConfig;
+    fts5IndexPrepareStmt(p, &p->pReaderOpt, sqlite3_mprintf(
+          "SELECT block FROM '%q'.'%q_data' WHERE id=?",
+          pConfig->zDb, pConfig->zName
+    ));
+  }
+
+  if( p->rc==SQLITE_OK ){
+    sqlite3_bind_int64(p->pReaderOpt, 1, iRowid);
+    if( SQLITE_ROW==sqlite3_step(p->pReaderOpt) ){
+      int nByte = sqlite3_column_bytes(p->pReaderOpt, 0);
+      const u8 *aByte = (const u8*)sqlite3_column_blob(p->pReaderOpt, 0);
+      i64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
+
+      pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
+      if( pRet==0 ){
+        p->rc = SQLITE_NOMEM;
+      }else{
+        pRet->nn = nByte;
+        pRet->p = (u8*)&pRet[1];
+        memcpy(pRet->p, aByte, nByte);
+      }
+    }
+    rc2 = sqlite3_reset(p->pReaderOpt);
+    if( p->rc==SQLITE_OK ) p->rc = rc2;
+  }
+
+  return pRet;
+}
 
 /*
 ** INSERT OR REPLACE a record into the %_data table.
@@ -793,6 +849,10 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
   i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
   i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
   fts5DataDelete(p, iFirst, iLast);
+  if( p->pConfig->bContentlessDelete ){
+    i64 iTomb = FTS5_TOMBSTONE_ROWID(iSegid);
+    fts5DataDelete(p, iTomb, iTomb);
+  }
   if( p->pIdxDeleter==0 ){
     Fts5Config *pConfig = p->pConfig;
     fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
@@ -903,11 +963,18 @@ static int fts5StructureDecode(
   int nSegment = 0;
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
+  int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
+  i64 nLocCounter = 0;
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
   i = 4;
 
+  if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){
+    i += 4;
+    bStructureV2 = 1;
+  }
+
   /* Read the total number of levels and segments from the start of the
   ** structure record.  */
   i += fts5GetVarint32(&pData[i], nLevel);
@@ -958,6 +1025,11 @@ static int fts5StructureDecode(
           i += fts5GetVarint32(&pData[i], pSeg->iSegid);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
+          if( bStructureV2 ){
+            i += fts5GetVarint(&pData[i], &pSeg->iLoc1);
+            i += fts5GetVarint(&pData[i], &pSeg->iLoc2);
+            nLocCounter = MAX(nLocCounter, pSeg->iLoc2);
+          }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
             rc = FTS5_CORRUPT;
             break;
@@ -968,6 +1040,9 @@ static int fts5StructureDecode(
       }
     }
     if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
+    if( bStructureV2 ){
+      pRet->nLocCounter = nLocCounter+1;
+    }
 
     if( rc!=SQLITE_OK ){
       fts5StructureRelease(pRet);
@@ -1180,6 +1255,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     Fts5Buffer buf;               /* Buffer to serialize record into */
     int iLvl;                     /* Used to iterate through levels */
     int iCookie;                  /* Cookie value to store */
+    int nHdr = (pStruct->nLocCounter>0 ? (4+4+9+9+9) : (4+9+9));
 
     assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
     memset(&buf, 0, sizeof(Fts5Buffer));
@@ -1188,9 +1264,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     iCookie = p->pConfig->iCookie;
     if( iCookie<0 ) iCookie = 0;
 
-    if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){
+    if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
       sqlite3Fts5Put32(buf.p, iCookie);
       buf.n = 4;
+      if( pStruct->nLocCounter>0 ){
+        fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4);
+      }
       fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
       fts5BufferSafeAppendVarint(&buf, pStruct->nSegment);
       fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter);
@@ -1207,6 +1286,10 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+        if( pStruct->nLocCounter>0 ){
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc1);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc2);
+        }
       }
     }
 
@@ -1729,6 +1812,11 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
+static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
+  i64 iRowid = FTS5_TOMBSTONE_ROWID(pIter->pSeg->iSegid);
+  pIter->pTombstone = fts5DataReadOpt(p, iRowid);
+}
+
 /*
 ** Initialize the iterator object pIter to iterate through the entries in
 ** segment pSeg. The iterator is left pointing to the first entry when 
@@ -1771,6 +1859,8 @@ static void fts5SegIterInit(
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
   }
+
+  fts5SegIterLoadTombstone(p, pIter);
 }
 
 /*
@@ -2471,6 +2561,7 @@ static void fts5SegIterSeekInit(
   }
 
   fts5SegIterSetNext(p, pIter);
+  fts5SegIterLoadTombstone(p, pIter);
 
   /* Either:
   **
@@ -2558,6 +2649,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
+  fts5DataRelease(pIter->pTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -2895,6 +2987,43 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
+static u64 fts5GetU64(u8 *a){
+  return ((u64)a[0] << 56)
+       + ((u64)a[1] << 48)
+       + ((u64)a[2] << 40)
+       + ((u64)a[3] << 32)
+       + ((u64)a[4] << 24)
+       + ((u64)a[5] << 16)
+       + ((u64)a[6] << 8)
+       + ((u64)a[7] << 0);
+}
+
+static void fts5PutU64(u8 *a, u64 iVal){
+  a[0] = ((iVal >> 56) & 0xFF);
+  a[1] = ((iVal >> 48) & 0xFF);
+  a[2] = ((iVal >> 40) & 0xFF);
+  a[3] = ((iVal >> 32) & 0xFF);
+  a[4] = ((iVal >> 24) & 0xFF);
+  a[5] = ((iVal >> 16) & 0xFF);
+  a[6] = ((iVal >>  8) & 0xFF);
+  a[7] = ((iVal >>  0) & 0xFF);
+}
+
+static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
+  int iFirst = pIter->aFirst[1].iFirst;
+  Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+
+  if( pSeg->pTombstone ){
+    int ii;
+    for(ii=0; ii<pSeg->pTombstone->nn; ii+=8){
+      i64 iVal = (i64)fts5GetU64(&pSeg->pTombstone->p[ii]);
+      if( iVal==pSeg->iRowid ) return 1;
+    }
+  }
+
+  return 0;
+}
+
 /*
 ** Move the iterator to the next entry. 
 **
@@ -2932,7 +3061,9 @@ static void fts5MultiIterNext(
 
     fts5AssertMultiIterSetup(p, pIter);
     assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
-    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
+    if( (pIter->bSkipEmpty==0 || pSeg->nPos) 
+      && 0==fts5MultiIterIsDeleted(pIter)
+    ){
       pIter->xSetOutputs(pIter, pSeg);
       return;
     }
@@ -2964,7 +3095,7 @@ static void fts5MultiIterNext2(
       }
       fts5AssertMultiIterSetup(p, pIter);
 
-    }while( fts5MultiIterIsEmpty(p, pIter) );
+    }while( fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter) );
   }
 }
 
@@ -3519,7 +3650,9 @@ static void fts5MultiIterNew(
     fts5MultiIterSetEof(pNew);
     fts5AssertMultiIterSetup(p, pNew);
 
-    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
+    if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew))
+     || fts5MultiIterIsDeleted(pNew)
+    ){
       fts5MultiIterNext(p, pNew, 0, 0);
     }else if( pNew->base.bEof==0 ){
       Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst];
@@ -4334,6 +4467,12 @@ static void fts5IndexMergeLevel(
 
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
+
+    /* Set the range of locations that will go into the output segment */
+    if( pStruct->nLocCounter>0 ){
+      pSeg->iLoc1 = pLvl->aSeg[0].iLoc1;
+      pSeg->iLoc2 = pLvl->aSeg[pLvl->nSeg-1].iLoc2;
+    }
   }
   bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
 
@@ -5150,6 +5289,11 @@ static void fts5FlushOneHash(Fts5Index *p){
         pSeg->iSegid = iSegid;
         pSeg->pgnoFirst = 1;
         pSeg->pgnoLast = pgnoLast;
+        if( pStruct->nLocCounter>0 ){
+          pSeg->iLoc1 = pStruct->nLocCounter;
+          pSeg->iLoc2 = pStruct->nLocCounter;
+          pStruct->nLocCounter++;
+        }
         pStruct->nSegment++;
       }
       fts5StructurePromote(p, 0, pStruct);
@@ -5212,6 +5356,7 @@ static Fts5Structure *fts5IndexOptimizeStruct(
     pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
     pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
+    pNew->nLocCounter = pStruct->nLocCounter;
     pLvl = &pNew->aLevel[pNew->nLevel-1];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
     if( pLvl->aSeg ){
@@ -5830,6 +5975,9 @@ int sqlite3Fts5IndexReinit(Fts5Index *p){
   fts5StructureInvalidate(p);
   fts5IndexDiscardData(p);
   memset(&s, 0, sizeof(Fts5Structure));
+  if( p->pConfig->bContentlessDelete ){
+    s.nLocCounter = 1;
+  }
   fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
   fts5StructureWrite(p, &s);
   return fts5IndexReturn(p);
@@ -5889,6 +6037,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
     assert( p->pReader==0 );
     fts5StructureInvalidate(p);
     sqlite3_finalize(p->pWriter);
+    sqlite3_finalize(p->pReaderOpt);
     sqlite3_finalize(p->pDeleter);
     sqlite3_finalize(p->pIdxWriter);
     sqlite3_finalize(p->pIdxDeleter);
@@ -6221,6 +6370,70 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
+int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
+  Fts5Structure *pStruct;
+  pStruct = fts5StructureRead(p);
+  if( pStruct ){
+    *piLoc = pStruct->nLocCounter;
+    fts5StructureRelease(pStruct);
+  }
+  return fts5IndexReturn(p);
+}
+
+static void fts5IndexTombstoneAdd(
+  Fts5Index *p, 
+  Fts5StructureSegment *pSeg, 
+  i64 iRowid
+){
+  Fts5Data *pHash = 0;
+  u8 *aNew = 0;
+  int nNew = 0;
+
+  pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
+  if( p->rc ) return;
+
+  if( pHash ){
+    nNew = 8 + pHash->nn;
+  }else{
+    nNew = 8;
+  }
+  aNew = sqlite3_malloc(nNew);
+  if( aNew==0 ){
+    p->rc = SQLITE_NOMEM;
+  }else{
+    if( pHash ){
+      memcpy(aNew, pHash->p, pHash->nn);
+    }
+    fts5PutU64(&aNew[nNew-8], iRowid);
+    fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
+  }
+
+  sqlite3_free(aNew);
+  fts5DataRelease(pHash);
+}
+
+/*
+** Add iRowid to the tombstone list of the segment or segments that contain
+** rows from location iLoc.
+*/
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
+  Fts5Structure *pStruct;
+  pStruct = fts5StructureRead(p);
+  if( pStruct ){
+    int iLvl;
+    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+      int iSeg;
+      for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+        Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
+        if( pSeg->iLoc1<=iLoc && pSeg->iLoc2>=iLoc ){
+          fts5IndexTombstoneAdd(p, pSeg, iRowid);
+        }
+      }
+    }
+    fts5StructureRelease(pStruct);
+  }
+  return fts5IndexReturn(p);
+}
 
 /*************************************************************************
 **************************************************************************
@@ -6832,9 +7045,15 @@ static void fts5DebugStructure(
     );
     for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
       Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", 
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
           pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
       );
+      if( pSeg->iLoc1>0 ){
+        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " loc=%lld..%lld",
+            pSeg->iLoc1, pSeg->iLoc2
+        );
+      }
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
     }
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   }
@@ -7237,6 +7456,224 @@ static void fts5RowidFunction(
 }
 #endif /* SQLITE_TEST */
 
+#ifdef SQLITE_TEST
+
+typedef struct Fts5StructVtab Fts5StructVtab;
+struct Fts5StructVtab {
+  sqlite3_vtab base;
+};
+
+typedef struct Fts5StructVcsr Fts5StructVcsr;
+struct Fts5StructVcsr {
+  sqlite3_vtab_cursor base;
+  Fts5Structure *pStruct;
+  int iLevel;
+  int iSeg;
+  int iRowid;
+};
+
+/*
+** Create a new fts5_structure() table-valued function.
+*/
+static int fts5structConnectMethod(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  Fts5StructVtab *pNew = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_declare_vtab(db, 
+      "CREATE TABLE xyz("
+          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2,"
+      "struct HIDDEN);"
+  );
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+  }
+
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** We must have a single struct=? constraint that will be passed through
+** into the xFilter method.  If there is no valid stmt=? constraint,
+** then return an SQLITE_CONSTRAINT error.
+*/
+static int fts5structBestIndexMethod(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;
+  int rc = SQLITE_CONSTRAINT;
+  struct sqlite3_index_constraint *p;
+  pIdxInfo->estimatedCost = (double)100;
+  pIdxInfo->estimatedRows = 100;
+  pIdxInfo->idxNum = 0;
+  for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
+    if( p->usable==0 ) continue;
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==8 ){
+      rc = SQLITE_OK;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      break;
+    }
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for bytecodevtab objects.
+*/
+static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){
+  Fts5StructVtab *p = (Fts5StructVtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new bytecodevtab_cursor object.
+*/
+static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+  int rc = SQLITE_OK;
+  Fts5StructVcsr *pNew = 0;
+
+  pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+  *ppCsr = (sqlite3_vtab_cursor*)pNew;
+
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a bytecodevtab_cursor.
+*/
+static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  fts5StructureRelease(pCsr->pStruct);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a bytecodevtab_cursor to its next row of output.
+*/
+static int fts5structNextMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+
+  assert( pCsr->pStruct );
+  pCsr->iSeg++;
+  pCsr->iRowid++;
+  while( pCsr->iSeg>=pCsr->pStruct->aLevel[pCsr->iLevel].nSeg ){
+    pCsr->iLevel++;
+    pCsr->iSeg = 0;
+  }
+  if( pCsr->iLevel>=pCsr->pStruct->nLevel ){
+    fts5StructureRelease(pCsr->pStruct);
+    pCsr->pStruct = 0;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fts5structEofMethod(sqlite3_vtab_cursor *cur){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  return pCsr->pStruct==0;
+}
+
+static int fts5structRowidMethod(
+  sqlite3_vtab_cursor *cur, 
+  sqlite_int64 *piRowid
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  *piRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the bytecodevtab_cursor
+** is currently pointing.
+*/
+static int fts5structColumnMethod(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+  Fts5Structure *p = pCsr->pStruct;
+  Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg];
+
+  switch( i ){
+    case 0: /* level */
+      sqlite3_result_int(ctx, pCsr->iLevel);
+      break;
+    case 1: /* segment */
+      sqlite3_result_int(ctx, pCsr->iSeg);
+      break;
+    case 2: /* merge */
+      sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge);
+      break;
+    case 3: /* segid */
+      sqlite3_result_int(ctx, pSeg->iSegid);
+      break;
+    case 4: /* leaf1 */
+      sqlite3_result_int(ctx, pSeg->pgnoFirst);
+      break;
+    case 5: /* leaf2 */
+      sqlite3_result_int(ctx, pSeg->pgnoLast);
+      break;
+    case 6: /* loc1 */
+      sqlite3_result_int(ctx, pSeg->iLoc1);
+      break;
+    case 7: /* loc2 */
+      sqlite3_result_int(ctx, pSeg->iLoc2);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Initialize a cursor.
+**
+**    idxNum==0     means show all subprograms
+**    idxNum==1     means show only the main bytecode and omit subprograms.
+*/
+static int fts5structFilterMethod(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor;
+  int rc = SQLITE_OK;
+
+  const u8 *aBlob = 0;
+  int nBlob = 0;
+
+  assert( argc==1 );
+  fts5StructureRelease(pCsr->pStruct);
+  pCsr->pStruct = 0;
+
+  nBlob = sqlite3_value_bytes(argv[0]);
+  aBlob = (const u8*)sqlite3_value_blob(argv[0]);
+  rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct);
+  if( rc==SQLITE_OK ){
+    pCsr->iLevel = 0;
+    pCsr->iRowid = 0;
+    pCsr->iSeg = -1;
+    rc = fts5structNextMethod(pVtabCursor);
+  }
+
+  return rc;
+}
+
+#endif /* SQLITE_TEST */
+
 /*
 ** This is called as part of registering the FTS5 module with database
 ** connection db. It registers several user-defined scalar functions useful
@@ -7263,6 +7700,36 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
         db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
     );
   }
+
+  if( rc==SQLITE_OK ){
+    static const sqlite3_module fts5structure_module = {
+      0,                           /* iVersion      */
+      0,                           /* xCreate       */
+      fts5structConnectMethod,     /* xConnect      */
+      fts5structBestIndexMethod,   /* xBestIndex    */
+      fts5structDisconnectMethod,  /* xDisconnect   */
+      0,                           /* xDestroy      */
+      fts5structOpenMethod,        /* xOpen         */
+      fts5structCloseMethod,       /* xClose        */
+      fts5structFilterMethod,      /* xFilter       */
+      fts5structNextMethod,        /* xNext         */
+      fts5structEofMethod,         /* xEof          */
+      fts5structColumnMethod,      /* xColumn       */
+      fts5structRowidMethod,       /* xRowid        */
+      0,                           /* xUpdate       */
+      0,                           /* xBegin        */
+      0,                           /* xSync         */
+      0,                           /* xCommit       */
+      0,                           /* xRollback     */
+      0,                           /* xFindFunction */
+      0,                           /* xRename       */
+      0,                           /* xSavepoint    */
+      0,                           /* xRelease      */
+      0,                           /* xRollbackTo   */
+      0                            /* xShadowName   */
+    };
+    rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0);
+  }
   return rc;
 #else
   return SQLITE_OK;
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 13921ce49e..180aba3973 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1624,7 +1624,6 @@ static int fts5UpdateMethod(
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
   int bUpdateOrDelete = 0;
-  
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 02b98d9e44..adc8e750ed 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -77,10 +77,10 @@ static int fts5StorageGetStmt(
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
-      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
+      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)",     /* REPLACE_DOCSIZE  */
       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
 
-      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
+      "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?",    /* LOOKUP_DOCSIZE  */
 
       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
       "SELECT %s FROM %s AS T",                         /* SCAN */
@@ -128,6 +128,19 @@ static int fts5StorageGetStmt(
         break;
       }
 
+      case FTS5_STMT_REPLACE_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
+          (pC->bContentlessDelete ? ",?" : "")
+        );
+        break;
+
+      case FTS5_STMT_LOOKUP_DOCSIZE: 
+        zSql = sqlite3_mprintf(azStmt[eStmt], 
+            (pC->bContentlessDelete ? ",location" : ""),
+            pC->zDb, pC->zName
+        );
+        break;
+
       default:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
         break;
@@ -317,9 +330,11 @@ int sqlite3Fts5StorageOpen(
     }
 
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
-      rc = sqlite3Fts5CreateTable(
-          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
-      );
+      const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
+      if( pConfig->bContentlessDelete ){
+        zCols = "id INTEGER PRIMARY KEY, sz BLOB, location INTEGER";
+      }
+      rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
     }
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5CreateTable(
@@ -449,6 +464,34 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
+static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
+  i64 iLoc = 0;
+  sqlite3_stmt *pLookup = 0;
+  int rc = SQLITE_OK;
+
+  assert( p->pConfig->bContentlessDelete );
+  assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
+
+  /* Look up the location of the document in the %_docsize table. Store
+  ** this in stack variable iLoc.  */
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pLookup, 1, iDel);
+    if( SQLITE_ROW==sqlite3_step(pLookup) ){
+      iLoc = sqlite3_column_int64(pLookup, 1);
+    }
+    rc = sqlite3_reset(pLookup);
+    if( rc==SQLITE_OK && iLoc==0 ){
+      rc = FTS5_CORRUPT;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
+  }
+
+  return rc;
+}
 
 /*
 ** Insert a record into the %_docsize table. Specifically, do:
@@ -469,10 +512,17 @@ static int fts5StorageInsertDocsize(
     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
-      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-      sqlite3_step(pReplace);
-      rc = sqlite3_reset(pReplace);
-      sqlite3_bind_null(pReplace, 2);
+      if( p->pConfig->bContentlessDelete ){
+        i64 iLoc = 0;
+        rc = sqlite3Fts5IndexGetLocation(p->pIndex, &iLoc);
+        sqlite3_bind_int64(pReplace, 3, iLoc);
+      }
+      if( rc==SQLITE_OK ){
+        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+        sqlite3_step(pReplace);
+        rc = sqlite3_reset(pReplace);
+        sqlite3_bind_null(pReplace, 2);
+      }
     }
   }
   return rc;
@@ -536,7 +586,11 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    if( p->pConfig->bContentlessDelete ){
+      rc = fts5StorageContentlessDelete(p, iDel);
+    }else{
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+    }
   }
 
   /* Delete the %_docsize record */
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 59ce4f6a1f..9ae8bab65f 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -50,6 +50,7 @@ do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
 }
 
+breakpoint
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
 } {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
new file mode 100644
index 0000000000..9fc5667589
--- /dev/null
+++ b/ext/fts5/test/fts5contentless.test
@@ -0,0 +1,175 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Check that it is not possible to specify "contentless_delete=1" for 
+# anything other than a contentless table.
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 requires a contentless table}}
+foreach {tn sql bError} {
+  1 "(a, b, contentless_delete=1)"              1
+  2 "(a, b, contentless_delete=1, content=abc)" 1
+  3 "(a, b, contentless_delete=1, content=)"    0
+  4 "(content=, contentless_delete=1, a)"       0
+  5 "(content='', contentless_delete=1, hello)" 0
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that it is not possible to specify "contentless_delete=1" 
+# along with columnsize=1. 
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}}
+foreach {tn sql bError} {
+  2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 
+} {
+  execsql { BEGIN }
+  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+  execsql { ROLLBACK }
+}
+
+# Check that if contentless_delete=1 is specified, then the "location" 
+# column is added to the %_docsize table.
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(c, content='');
+  CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1);
+}
+do_execsql_test 3.1 {
+  SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
+} {
+  {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
+  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, location INTEGER)}
+}
+
+do_execsql_test 3.2.1 {
+  SELECT hex(block) FROM x1_data WHERE id=10
+} {00000000000000}
+do_execsql_test 3.2.2 {
+  SELECT hex(block) FROM x2_data WHERE id=10
+} {00000000FF000001000000}
+
+do_execsql_test 3.3 {
+  INSERT INTO x2 VALUES('first text');
+  INSERT INTO x2 VALUES('second text');
+}
+do_execsql_test 3.4 {
+  SELECT id, location FROM x2_docsize
+} {1 1 2 2}
+do_execsql_test 3.5 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  0 0   1 1
+  0 1   2 2
+}
+do_execsql_test 3.6 {
+  INSERT INTO x2(x2) VALUES('optimize');
+}
+do_execsql_test 3.7 {
+  SELECT level, segment, loc1, loc2 FROM fts5_structure(
+    (SELECT block FROM x2_data WHERE id=10)
+  )
+} {
+  1 0   1 2
+}
+
+do_execsql_test 3.8 {
+  INSERT INTO x2(x2, rowid) VALUES('delete', 2);
+}
+
+do_execsql_test 3.9 {
+  SELECT rowid FROM x2('text')
+} {1}
+
+#--------------------------------------------------------------------------
+reset_db
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+
+set nRow 1000
+
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x);
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+do_test 4.1 {
+  for {set ii 0} {$ii < $nRow} {incr ii} {
+    set doc [document 6]
+    execsql {
+      INSERT INTO t1 VALUES($doc);
+      INSERT INTO ft VALUES($doc);
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.2.$v { set L1 } $L2
+}
+
+do_test 4.3 {
+  for {set ii 1} {$ii < $nRow} {incr ii 2} { 
+    execsql {
+       INSERT INTO ft(ft, rowid) VALUES('delete', $ii);
+       DELETE FROM t1 WHERE rowid=$ii;
+    }
+  }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.4.$v { set L1 } $L2
+}
+
+do_execsql_test 4.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+  set L2 [execsql {SELECT rowid FROM ft($v)}]
+  do_test 4.6.$v { set L1 } $L2
+}
+
+execsql_pp {
+  SELECT fts5_decode(id, block) FROM ft_data
+}
+
+
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 58dc9db6f2..389da73618 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Always\suse\sthe\s"LL"\ssuffix\son\s64-bit\sinteger\sliterals.
-D 2023-07-08T17:42:24.748
+C Begin\sadding\ssupport\sfor\sdeleting\srows\sfrom\scontentless\sfts5\stables.
+D 2023-07-10T20:44:09.251
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,15 +86,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d
+F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f0419c16a
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f
+F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c fe98ebd8835760b9c787d20f6b50d648a761afd8e3b55780e718ee34c694743b
-F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
-F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
+F ext/fts5/fts5_index.c 80fdc17d423f0b881109b397bbfb167830e3c2dc06a8399aded75beba7ef3903
+F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
+F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -105,7 +105,7 @@ F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9
+F ext/fts5/test/fts5aa.test 2106b14aa665cb7e9832cb40fec5b278c404236ed21fdab756484d5f8739b712
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -132,6 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
+F ext/fts5/test/fts5contentless.test e3cee6bac3681707031d2cd5f957178fa43c0d856e90c0ea6fcb3c1bb2fff154
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2043,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P beab3c98639be531744e60440223bb9ee76bc15234aff05e5efb273c8241dfd8
-R 0aeac1606c7bbc75dfae0a73d28601e5
-U drh
-Z 7fe2cdb685f595b3ff5ea313f1ff0511
+P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
+R 04413d3e9bd70379e2afd8eeb162f8d3
+T *branch * fts5-contentless-delete
+T *sym-fts5-contentless-delete *
+T -sym-trunk *
+U dan
+Z 17a9272f129a5cd1a6eee3f0f514cf63
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1161b8bd7f..0166f4e233 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
\ No newline at end of file
+e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
\ No newline at end of file

From d23f2103771159853b09aec39a4ace445e4ee136 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 18:55:19 +0000
Subject: [PATCH 05/66] Use a hash-table instead of a flat list to store
 tombstone rowids.

FossilOrigin-Name: 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
---
 ext/fts5/fts5_index.c              | 173 +++++++++++++++++++++++------
 ext/fts5/test/fts5contentless.test |  16 ++-
 manifest                           |  17 ++-
 manifest.uuid                      |   2 +-
 4 files changed, 158 insertions(+), 50 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e86da1c06e..3ae709f622 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -583,6 +583,41 @@ static u16 fts5GetU16(const u8 *aIn){
   return ((u16)aIn[0] << 8) + aIn[1];
 } 
 
+static u64 fts5GetU64(u8 *a){
+  return ((u64)a[0] << 56)
+       + ((u64)a[1] << 48)
+       + ((u64)a[2] << 40)
+       + ((u64)a[3] << 32)
+       + ((u64)a[4] << 24)
+       + ((u64)a[5] << 16)
+       + ((u64)a[6] << 8)
+       + ((u64)a[7] << 0);
+}
+
+static void fts5PutU64(u8 *a, u64 iVal){
+  a[0] = ((iVal >> 56) & 0xFF);
+  a[1] = ((iVal >> 48) & 0xFF);
+  a[2] = ((iVal >> 40) & 0xFF);
+  a[3] = ((iVal >> 32) & 0xFF);
+  a[4] = ((iVal >> 24) & 0xFF);
+  a[5] = ((iVal >> 16) & 0xFF);
+  a[6] = ((iVal >>  8) & 0xFF);
+  a[7] = ((iVal >>  0) & 0xFF);
+}
+
+static u32 fts5GetU32(const u8 *a){
+  return ((u32)a[0] << 24)
+       + ((u32)a[1] << 16)
+       + ((u32)a[2] << 8)
+       + ((u32)a[3] << 0);
+} 
+static void fts5PutU32(u8 *a, u32 iVal){
+  a[0] = ((iVal >> 24) & 0xFF);
+  a[1] = ((iVal >> 16) & 0xFF);
+  a[2] = ((iVal >>  8) & 0xFF);
+  a[3] = ((iVal >>  0) & 0xFF);
+}
+
 /*
 ** Allocate and return a buffer at least nByte bytes in size.
 **
@@ -2987,26 +3022,26 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
-static u64 fts5GetU64(u8 *a){
-  return ((u64)a[0] << 56)
-       + ((u64)a[1] << 48)
-       + ((u64)a[2] << 40)
-       + ((u64)a[3] << 32)
-       + ((u64)a[4] << 24)
-       + ((u64)a[5] << 16)
-       + ((u64)a[6] << 8)
-       + ((u64)a[7] << 0);
-}
+static int fts5IndexTombstoneQuery(u8 *aHash, int nHash, u64 iRowid){
+  int szKey = aHash[3] ? 8 : 4;
+  int nSlot = (nHash - 8) / szKey;
+  int iSlot = iRowid % nSlot;
 
-static void fts5PutU64(u8 *a, u64 iVal){
-  a[0] = ((iVal >> 56) & 0xFF);
-  a[1] = ((iVal >> 48) & 0xFF);
-  a[2] = ((iVal >> 40) & 0xFF);
-  a[3] = ((iVal >> 32) & 0xFF);
-  a[4] = ((iVal >> 24) & 0xFF);
-  a[5] = ((iVal >> 16) & 0xFF);
-  a[6] = ((iVal >>  8) & 0xFF);
-  a[7] = ((iVal >>  0) & 0xFF);
+  if( szKey==4 ){
+    u32 *aSlot = (u32*)&aHash[8];
+    while( aSlot[iSlot] ){
+      if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      iSlot = (iSlot+1)%nSlot;
+    }
+  }else{
+    u64 *aSlot = (u64*)&aHash[8];
+    while( aSlot[iSlot] ){
+      if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      iSlot = (iSlot+1)%nSlot;
+    }
+  }
+
+  return 0;
 }
 
 static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
@@ -3014,11 +3049,9 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
   if( pSeg->pTombstone ){
-    int ii;
-    for(ii=0; ii<pSeg->pTombstone->nn; ii+=8){
-      i64 iVal = (i64)fts5GetU64(&pSeg->pTombstone->p[ii]);
-      if( iVal==pSeg->iRowid ) return 1;
-    }
+    return fts5IndexTombstoneQuery(
+        pSeg->pTombstone->p, pSeg->pTombstone->nn, pSeg->iRowid
+    );
   }
 
   return 0;
@@ -6380,35 +6413,103 @@ int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
   return fts5IndexReturn(p);
 }
 
+/*
+** Add a tombstone for rowid iRowid to segment pSeg.
+**
+** All tombstones for a single segment are stored in a blob formatted to 
+** contain a hash table. The format is:
+**
+**   * 32-bit integer. 1 for 64-bit unsigned keys, 0 for 32-bit unsigned keys.
+**   * 32-bit integer. The number of entries currently in the hash table.
+**
+** Then an array of entries. The number of entries can be calculated based
+** on the size of the blob in the database and the size of the keys as 
+** specified by the first 32-bit field of the hash table header.
+**
+** All values in the hash table are stored as big-endian integers.
+*/
 static void fts5IndexTombstoneAdd(
   Fts5Index *p, 
   Fts5StructureSegment *pSeg, 
-  i64 iRowid
+  u64 iRowid
 ){
   Fts5Data *pHash = 0;
+  u8 *aFree = 0;
   u8 *aNew = 0;
   int nNew = 0;
+  int szKey = 0;
+  int nSlot = 0;
+  int bKey64 = (iRowid>0xFFFFFFFF);
+  u32 nHash = 0;
 
+  /* Load the current hash table, if any */
   pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
   if( p->rc ) return;
 
   if( pHash ){
-    nNew = 8 + pHash->nn;
-  }else{
-    nNew = 8;
+    szKey = pHash->p[3] ? 8 : 4;
+    nSlot = (pHash->nn - 8) / szKey;
+    nHash = fts5GetU32(&pHash->p[4]);
   }
-  aNew = sqlite3_malloc(nNew);
-  if( aNew==0 ){
-    p->rc = SQLITE_NOMEM;
-  }else{
-    if( pHash ){
-      memcpy(aNew, pHash->p, pHash->nn);
+
+  /* Check if the current hash table needs to be rebuilt. Either because
+  ** (a) it does not yet exist, (b) it is full, or (c) it is using the
+  ** wrong sized keys. */
+  if( pHash==0 || nSlot<=(nHash*2) || (bKey64 && szKey==4) ){
+    int szNewKey = (bKey64 || szKey==8) ? 8 : 4;
+    int nNewSlot = (nSlot ? nSlot*2 : 16);
+
+    nNew = 8 + (nNewSlot * szNewKey);
+    aFree = aNew = (u8*)sqlite3Fts5MallocZero(&p->rc, nNew);
+    if( aNew ){
+      int iSlot = 0;
+      int ii;
+      fts5PutU32(aNew, (szNewKey==8 ? 1 : 0));
+      for(ii=0; ii<nSlot; ii++){
+        u64 iVal = 0;
+        if( szKey==4 ){
+          iVal = (u64)fts5GetU32(&pHash->p[8 + ii*szKey]);
+        }else{
+          iVal = fts5GetU64(&pHash->p[8 + ii*szKey]);
+        }
+
+        iSlot = iVal % nNewSlot;
+        if( szNewKey==4 ){
+          u32 *aSlot = (u32*)&aNew[8];
+          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
+          fts5PutU32((u8*)&aSlot[iSlot], (u32)iVal);
+        }else{
+          u64 *aSlot = (u64*)&aNew[8];
+          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
+          fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+        }
+      }
     }
-    fts5PutU64(&aNew[nNew-8], iRowid);
+    szKey = szNewKey;
+    nSlot = nNewSlot;
+  }else{
+    aNew = pHash->p;
+    nNew = pHash->nn;
+  }
+
+  if( aNew ){
+    int iSlot = (iRowid % nSlot);
+    if( szKey==4 ){
+      u32 *aSlot = (u32*)&aNew[8];
+      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
+      fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
+    }else{
+      u64 *aSlot = (u64*)&aNew[8];
+      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
+      fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+    }
+
+    fts5PutU32((u8*)&aNew[4], nHash+1);
+    assert( nNew>8 );
     fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
   }
 
-  sqlite3_free(aNew);
+  sqlite3_free(aFree);
   fts5DataRelease(pHash);
 }
 
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 9fc5667589..b9b7a2a4a1 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -164,11 +164,21 @@ foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
   do_test 4.6.$v { set L1 } $L2
 }
 
-execsql_pp {
-  SELECT fts5_decode(id, block) FROM ft_data
+#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data }
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+  INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
 }
 
-
+breakpoint
+do_execsql_test 5.1 {
+  INSERT INTO ft(ft, rowid) VALUES('delete', 2);
+}
 
 
 finish_test
diff --git a/manifest b/manifest
index 389da73618..8f2c159d4e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Begin\sadding\ssupport\sfor\sdeleting\srows\sfrom\scontentless\sfts5\stables.
-D 2023-07-10T20:44:09.251
+C Use\sa\shash-table\sinstead\sof\sa\sflat\slist\sto\sstore\stombstone\srowids.
+D 2023-07-11T18:55:19.003
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 80fdc17d423f0b881109b397bbfb167830e3c2dc06a8399aded75beba7ef3903
+F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
 F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
 F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test e3cee6bac3681707031d2cd5f957178fa43c0d856e90c0ea6fcb3c1bb2fff154
+F ext/fts5/test/fts5contentless.test b807a15020dfae84f215370f08d7270aa01bbcc5abdb54a42ec2dee8998e4842
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,11 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7
-R 04413d3e9bd70379e2afd8eeb162f8d3
-T *branch * fts5-contentless-delete
-T *sym-fts5-contentless-delete *
-T -sym-trunk *
+P e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
+R e240ab2da08f49c28fa1cdca40ac2a12
 U dan
-Z 17a9272f129a5cd1a6eee3f0f514cf63
+Z d2566e3569908687c69b9bc52ae7c980
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 0166f4e233..1aeb8a479d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
\ No newline at end of file
+948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
\ No newline at end of file

From b92669db7361bd81599d249e7f6ed2e337b65e50 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 20:19:46 +0000
Subject: [PATCH 06/66] Have contentless_delete=1 tables support regular DELETE
 statements, instead of just the special INSERT syntax.

FossilOrigin-Name: fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
---
 ext/fts5/fts5_main.c               |  8 ++++++--
 ext/fts5/test/fts5contentless.test | 15 +++++++++++----
 manifest                           | 14 +++++++-------
 manifest.uuid                      |  2 +-
 4 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 180aba3973..7b1f4a6d08 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1678,8 +1678,12 @@ static int fts5UpdateMethod(
     assert( nArg!=1 || eType0==SQLITE_INTEGER );
 
     /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported.  */
-    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
+    ** This is not suported. Except - DELETE is supported if the CREATE
+    ** VIRTUAL TABLE statement contained "contentless_delete=1". */
+    if( eType0==SQLITE_INTEGER 
+     && pConfig->eContent==FTS5_CONTENT_NONE 
+     && (nArg>1 || pConfig->bContentlessDelete==0)
+    ){
       pTab->p.base.zErrMsg = sqlite3_mprintf(
           "cannot %s contentless fts5 table: %s", 
           (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index b9b7a2a4a1..a55aa84021 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -99,7 +99,7 @@ do_execsql_test 3.7 {
 }
 
 do_execsql_test 3.8 {
-  INSERT INTO x2(x2, rowid) VALUES('delete', 2);
+  DELETE FROM x2 WHERE rowid=2;
 }
 
 do_execsql_test 3.9 {
@@ -142,7 +142,7 @@ foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
 do_test 4.3 {
   for {set ii 1} {$ii < $nRow} {incr ii 2} { 
     execsql {
-       INSERT INTO ft(ft, rowid) VALUES('delete', $ii);
+       DELETE FROM ft WHERE rowid=$ii;
        DELETE FROM t1 WHERE rowid=$ii;
     }
   }
@@ -175,11 +175,18 @@ do_execsql_test 5.0 {
   INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
 }
 
-breakpoint
 do_execsql_test 5.1 {
-  INSERT INTO ft(ft, rowid) VALUES('delete', 2);
+  DELETE FROM ft WHERE rowid=2
 }
 
+do_execsql_test 5.2 {
+  SELECT rowid FROM ft
+} {1 3}
+
+do_catchsql_test 5.3 {
+  UPDATE ft SET x='four six' WHERE rowid=3
+} {1 {cannot UPDATE contentless fts5 table: ft}}
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 8f2c159d4e..963270545a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sa\shash-table\sinstead\sof\sa\sflat\slist\sto\sstore\stombstone\srowids.
-D 2023-07-11T18:55:19.003
+C Have\scontentless_delete=1\stables\ssupport\sregular\sDELETE\sstatements,\sinstead\sof\sjust\sthe\sspecial\sINSERT\ssyntax.
+D 2023-07-11T20:19:46.857
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,7 +93,7 @@ F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
-F ext/fts5/fts5_main.c 0f4d21152f23fb5182310d1cb2565bbdf2a8085888185a0f1f9117d2c265cc10
+F ext/fts5/fts5_main.c 843a6223397afb07c515aaf074c4bd9079cd7390b3b6d3de585f600afe294d5c
 F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test b807a15020dfae84f215370f08d7270aa01bbcc5abdb54a42ec2dee8998e4842
+F ext/fts5/test/fts5contentless.test 5ffce3185e40c8025e1fc0a2a484f331f76f9a8e8b8441d169bbe1276b76ae2c
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e513bea84dfaf2280f7429c9a528b3a1354a46c36e58ab178ca45478975634e0
-R e240ab2da08f49c28fa1cdca40ac2a12
+P 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
+R 3eb31b5966f6d22024a48d705ddecf8e
 U dan
-Z d2566e3569908687c69b9bc52ae7c980
+Z ecb4a545638f553899d716cc379fd8e4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1aeb8a479d..06acf1e66a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
\ No newline at end of file
+fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
\ No newline at end of file

From b184c91076dd1131678482398eb6725172c4ddad Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 11 Jul 2023 20:57:19 +0000
Subject: [PATCH 07/66] Have contentless_delete=1 tables support REPLACE
 statements.

FossilOrigin-Name: 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
---
 ext/fts5/fts5_main.c               |  2 +-
 ext/fts5/fts5_storage.c            |  5 +----
 ext/fts5/test/fts5contentless.test | 17 ++++++++++++++++-
 manifest                           | 16 ++++++++--------
 manifest.uuid                      |  2 +-
 5 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 7b1f4a6d08..844b5c57ef 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1670,7 +1670,7 @@ static int fts5UpdateMethod(
     ** Cases 3 and 4 may violate the rowid constraint.
     */
     int eConflict = SQLITE_ABORT;
-    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){
       eConflict = sqlite3_vtab_on_conflict(pConfig->db);
     }
 
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index adc8e750ed..35c30d06e6 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -481,12 +481,9 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
       iLoc = sqlite3_column_int64(pLookup, 1);
     }
     rc = sqlite3_reset(pLookup);
-    if( rc==SQLITE_OK && iLoc==0 ){
-      rc = FTS5_CORRUPT;
-    }
   }
 
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && iLoc!=0 ){
     rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
   }
 
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index a55aa84021..3179b52e92 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -173,6 +173,8 @@ do_execsql_test 5.0 {
   INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
   INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
   INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
+  INSERT INTO ft(rowid, x) VALUES(4, 'one two seven');
+  INSERT INTO ft(rowid, x) VALUES(5, 'one two eight');
 }
 
 do_execsql_test 5.1 {
@@ -181,12 +183,25 @@ do_execsql_test 5.1 {
 
 do_execsql_test 5.2 {
   SELECT rowid FROM ft
-} {1 3}
+} {1 3 4 5}
 
 do_catchsql_test 5.3 {
   UPDATE ft SET x='four six' WHERE rowid=3
 } {1 {cannot UPDATE contentless fts5 table: ft}}
 
+do_execsql_test 5.4 {
+  SELECT rowid FROM ft('one');
+} {1 3 4 5}
+
+do_execsql_test 5.5 {
+  REPLACE INTO ft(rowid, x) VALUES(3, 'four six');
+  SELECT rowid FROM ft('one');
+} {1 4 5}
+
+do_execsql_test 5.6 {
+  REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven');
+  SELECT rowid FROM ft('one');
+} {1 4 5 6}
 
 finish_test
 
diff --git a/manifest b/manifest
index 963270545a..9f5b8a7cfa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\scontentless_delete=1\stables\ssupport\sregular\sDELETE\sstatements,\sinstead\sof\sjust\sthe\sspecial\sINSERT\ssyntax.
-D 2023-07-11T20:19:46.857
+C Have\scontentless_delete=1\stables\ssupport\sREPLACE\sstatements.
+D 2023-07-11T20:57:19.305
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,8 +93,8 @@ F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607
 F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
-F ext/fts5/fts5_main.c 843a6223397afb07c515aaf074c4bd9079cd7390b3b6d3de585f600afe294d5c
-F ext/fts5/fts5_storage.c beff4be2a53c530676d59355b408733ab28202ae351a0840fa211df17b103c4a
+F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
+F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 5ffce3185e40c8025e1fc0a2a484f331f76f9a8e8b8441d169bbe1276b76ae2c
+F ext/fts5/test/fts5contentless.test feb27b077be771141c298b68798b93efd0e2d6fd3cb759201628b69576d69228
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 948267b066d0dbe667881b3d26a007fa24576da6e57c112676fadeb846c13f0b
-R 3eb31b5966f6d22024a48d705ddecf8e
+P fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
+R cbe733514e57856ac29410eaba1125b8
 U dan
-Z ecb4a545638f553899d716cc379fd8e4
+Z 9ded30fbafe538528320232b2760b444
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 06acf1e66a..40b428d76e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
\ No newline at end of file
+2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
\ No newline at end of file

From 9faa3bbdbbae6a74aa242f344a326b3530edfb16 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 12 Jul 2023 18:38:47 +0000
Subject: [PATCH 08/66] Get access to SQL functions and virtual tables used for
 debugging and analysis of FTS5 using SQLITE_FTS5_DEBUG and without the need
 for SQLITE_TEST.

FossilOrigin-Name: 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
---
 ext/fts5/fts5_expr.c  |  6 +++---
 ext/fts5/fts5_index.c | 46 +++++++++++++++++++++----------------------
 manifest              | 16 +++++++--------
 manifest.uuid         |  2 +-
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 0e018420d0..0e07b9246c 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -2477,7 +2477,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
   return pRet;
 }
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
   sqlite3_int64 nByte = 0;
   Fts5ExprTerm *p;
@@ -2844,14 +2844,14 @@ static void fts5ExprFold(
     sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
   }
 }
-#endif /* ifdef SQLITE_TEST */
+#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 /*
 ** This is called during initialization to register the fts5_expr() scalar
 ** UDF with the SQLite handle passed as the only argument.
 */
 int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
   struct Fts5ExprFunc {
     const char *z;
     void (*x)(sqlite3_context*,int,sqlite3_value**);
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3ae709f622..94f97891ac 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -7086,7 +7086,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
 ** function only.
 */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** Decode a segment-data rowid from the %_data table. This function is
 ** the opposite of macro FTS5_SEGMENT_ROWID().
@@ -7109,9 +7109,9 @@ static void fts5DecodeRowid(
 
   *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
   int iSegid, iHeight, iPgno, bDlidx;       /* Rowid compenents */
   fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
@@ -7129,9 +7129,9 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
     );
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugStructure(
   int *pRc,                       /* IN/OUT: error code */
   Fts5Buffer *pBuf,
@@ -7159,9 +7159,9 @@ static void fts5DebugStructure(
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This is part of the fts5_decode() debugging aid.
 **
@@ -7186,9 +7186,9 @@ static void fts5DecodeStructure(
   fts5DebugStructure(pRc, pBuf, p);
   fts5StructureRelease(p);
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This is part of the fts5_decode() debugging aid.
 **
@@ -7211,9 +7211,9 @@ static void fts5DecodeAverages(
     zSpace = " ";
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** Buffer (a/n) is assumed to contain a list of serialized varints. Read
 ** each varint and append its string representation to buffer pBuf. Return
@@ -7230,9 +7230,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   }
   return iOff;
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** The start of buffer (a/n) contains the start of a doclist. The doclist
 ** may or may not finish within the buffer. This function appends a text
@@ -7265,9 +7265,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
 
   return iOff;
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** This function is part of the fts5_decode() debugging function. It is 
 ** only ever used with detail=none tables.
@@ -7308,9 +7308,9 @@ static void fts5DecodeRowidList(
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 /*
 ** The implementation of user-defined scalar function fts5_decode().
 */
@@ -7519,9 +7519,9 @@ static void fts5DecodeFunction(
   }
   fts5BufferFree(&s);
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST 
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) 
 /*
 ** The implementation of user-defined scalar function fts5_rowid().
 */
@@ -7555,9 +7555,9 @@ static void fts5RowidFunction(
     }
   }
 }
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 
 typedef struct Fts5StructVtab Fts5StructVtab;
 struct Fts5StructVtab {
@@ -7773,7 +7773,7 @@ static int fts5structFilterMethod(
   return rc;
 }
 
-#endif /* SQLITE_TEST */
+#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 /*
 ** This is called as part of registering the FTS5 module with database
@@ -7784,7 +7784,7 @@ static int fts5structFilterMethod(
 ** SQLite error code is returned instead.
 */
 int sqlite3Fts5IndexInit(sqlite3 *db){
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
   int rc = sqlite3_create_function(
       db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
   );
diff --git a/manifest b/manifest
index 9f5b8a7cfa..dac59f9187 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\scontentless_delete=1\stables\ssupport\sREPLACE\sstatements.
-D 2023-07-11T20:57:19.305
+C Get\saccess\sto\sSQL\sfunctions\sand\svirtual\stables\sused\sfor\sdebugging\sand\sanalysis\nof\sFTS5\susing\sSQLITE_FTS5_DEBUG\sand\swithout\sthe\sneed\sfor\sSQLITE_TEST.
+D 2023-07-12T18:38:47.529
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -90,9 +90,9 @@ F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f041
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
-F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1
+F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 60c815859589d279ea237a4fdb88386cd5e154288c1c7963e2834ff1edf24915
+F ext/fts5/fts5_index.c 05415638e4cb77cda7ead90e3cdc5f8c0dd91770e6596da9ba8e4068395a3490
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fffb8616905501669a94231d5d9f53446bf09553353f2cdab7c43ca54bbb7fa6
-R cbe733514e57856ac29410eaba1125b8
-U dan
-Z 9ded30fbafe538528320232b2760b444
+P 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
+R 25b067c7b898a5633244bd9d8061a59f
+U drh
+Z 48c04c5430ca6d0e76f1576b44c1f771
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 40b428d76e..d59b7466a9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
\ No newline at end of file
+383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
\ No newline at end of file

From 9f6c0d0027d4b37dab21852f1c11d3509ae0e6d5 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 12 Jul 2023 19:34:32 +0000
Subject: [PATCH 09/66] Fix a crash that may occur when handling corrupt
 records in contentless_delete=1 mode.

FossilOrigin-Name: 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
---
 ext/fts5/fts5_index.c |  3 +--
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 94f97891ac..984a4852ba 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1893,9 +1893,8 @@ static void fts5SegIterInit(
     pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
+    fts5SegIterLoadTombstone(p, pIter);
   }
-
-  fts5SegIterLoadTombstone(p, pIter);
 }
 
 /*
diff --git a/manifest b/manifest
index dac59f9187..6e9efcd348 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\saccess\sto\sSQL\sfunctions\sand\svirtual\stables\sused\sfor\sdebugging\sand\sanalysis\nof\sFTS5\susing\sSQLITE_FTS5_DEBUG\sand\swithout\sthe\sneed\sfor\sSQLITE_TEST.
-D 2023-07-12T18:38:47.529
+C Fix\sa\scrash\sthat\smay\soccur\swhen\shandling\scorrupt\srecords\sin\scontentless_delete=1\smode.
+D 2023-07-12T19:34:32.103
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 05415638e4cb77cda7ead90e3cdc5f8c0dd91770e6596da9ba8e4068395a3490
+F ext/fts5/fts5_index.c 508abd2340e138c7de07b180748f011927553f14f3329f1499e71302183c38d3
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2f553a660e00564e51bc4209c92bd3628fb1266f4a52832792fbf91e4234a0ba
-R 25b067c7b898a5633244bd9d8061a59f
-U drh
-Z 48c04c5430ca6d0e76f1576b44c1f771
+P 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
+R 1327e4474941bd216c5977cab00c437b
+U dan
+Z ed477a224293640a589dbee38d43f3b4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d59b7466a9..3ff0cdbe56 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
\ No newline at end of file
+74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
\ No newline at end of file

From fc158d3870d46222994df77c1a8c302fb56c4b2d Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 12 Jul 2023 20:24:23 +0000
Subject: [PATCH 10/66] Avoid a case of an infinite loop in fts5 when dealing
 with corrupt records.

FossilOrigin-Name: 0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
---
 ext/fts5/fts5_index.c |  4 +++-
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 984a4852ba..e3b4139d9e 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3127,7 +3127,9 @@ static void fts5MultiIterNext2(
       }
       fts5AssertMultiIterSetup(p, pIter);
 
-    }while( fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter) );
+    }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) 
+         && (p->rc==SQLITE_OK)
+    );
   }
 }
 
diff --git a/manifest b/manifest
index 6e9efcd348..4c89bf87f9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scrash\sthat\smay\soccur\swhen\shandling\scorrupt\srecords\sin\scontentless_delete=1\smode.
-D 2023-07-12T19:34:32.103
+C Avoid\sa\scase\sof\san\sinfinite\sloop\sin\sfts5\swhen\sdealing\swith\scorrupt\srecords.
+D 2023-07-12T20:24:23.049
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 508abd2340e138c7de07b180748f011927553f14f3329f1499e71302183c38d3
+F ext/fts5/fts5_index.c 7b9dd923301f8649db2f13a0d383c9270511555fc779e727e69f92c8d2e97b99
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 383de8e2259adb3f8881a618b8b5e06c1f3b2bd7ac274bd047410ad7d2a18f0f
-R 1327e4474941bd216c5977cab00c437b
+P 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
+R b3f674c48ef86903b0134e340f9975df
 U dan
-Z ed477a224293640a589dbee38d43f3b4
+Z 4a7fc420576ceeaedd11216cbfd17eb6
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3ff0cdbe56..b56caf54da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
\ No newline at end of file
+0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
\ No newline at end of file

From e393f6eff00bd78bbda6ecfaf1feb957a845226b Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 15 Jul 2023 16:48:14 +0000
Subject: [PATCH 11/66] Add the experimental sqlite3_stmt_explain(S,E)
 interface.

FossilOrigin-Name: 5683743ddf0bb051f2fe5d137cc18407e000e77e9faa9010a22e3134b575638b
---
 manifest        | 27 +++++++++++++++------------
 manifest.uuid   |  2 +-
 src/parse.y     |  4 ++--
 src/prepare.c   |  7 ++++++-
 src/shell.c.in  | 29 ++++++++---------------------
 src/sqlite.h.in | 31 +++++++++++++++++++++++++++++++
 src/test1.c     | 29 +++++++++++++++++++++++++++++
 src/vdbeapi.c   | 16 ++++++++++++++++
 8 files changed, 108 insertions(+), 37 deletions(-)

diff --git a/manifest b/manifest
index 918e28c08b..560436430f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\sinternal\scleanups\sin\sthe\sOPFS\sVFS.
-D 2023-07-14T21:06:00.870
+C Add\sthe\sexperimental\ssqlite3_stmt_explain(S,E)\sinterface.
+D 2023-07-15T16:48:14.281
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -626,20 +626,20 @@ F src/os_win.c 7038223a1cda0a47e2ab4db47f63bf1833fe53ba0542f0f283a062ea13894103
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 5ddf3a74c633a008ea6b2f5b3186167e88e2c8ca8a252ecab06ab3f1eb48e60f
 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3
-F src/parse.y 8828f9e15f04d469eab9c0f2aed504e534b1c97c68836bed6f07afab29c2ac0b
+F src/parse.y aeb7760d41cfa86465e3adba506500c021597049fd55f82a30e5b7045862c28c
 F src/pcache.c 4cd4a0043167da9ba7e19b4d179a0e6354e7fe32c16f781ecf9bf0a5ff63b40b
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
-F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
+F src/prepare.c 80548297dc0e1fb3139cdebffb5a1bcac3dfac66d791012dd74838e70445072d
 F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
-F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db818
-F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
+F src/shell.c.in 3f125426a25c717fbf8b84b3b75a8b60fa989bf7a039ed38926d455329f9dd0f
+F src/sqlite.h.in 13d5458cd23e7b4759cff4d978ad09591a457b4a2821993579953a9a4257ce0f
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
 F src/sqliteInt.h dcb1a885e8b6cb78df618944b89d44361a99d0fe33e1bba2c150a855f7dc5599
@@ -647,7 +647,7 @@ F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd
-F src/test1.c 9111b12427b3b94429aff68bbcf5b125ae70438f6ce2f3f033e2a69626ec26c9
+F src/test1.c dfd07574689ee7c57af219c5f627ff32dae65dba478c124ba0359a1bde9983be
 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef
 F src/test3.c e5178558c41ff53236ae0271e9acb3d6885a94981d2eb939536ee6474598840e
 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664
@@ -710,7 +710,7 @@ F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
 F src/vdbe.c 74282a947234513872a83b0bab1b8c644ece64b3e27b053ef17677c8ff9c81e0
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae
-F src/vdbeapi.c de9703f8705afc393cc2864669ce28cf9516983c8331d59aa2b978de01634365
+F src/vdbeapi.c c528ef4fafc8be172cbe4f48d8f15c9526bebde9f90185917fcc43fad264bcb6
 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
 F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033
@@ -2043,8 +2043,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 816b503f093c4e6d92d0eb2f9fbd841acd01cc9bc89ee58d961b56c64f71406a
-R 6597235c929ad6ed0b90d6716177ff1b
-U stephan
-Z b7b5463364c5290e11704a96855173c7
+P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
+R d7676f37f2e59b0f85cc021dadb3d7bc
+T *branch * sqlite3_stmt_explain
+T *sym-sqlite3_stmt_explain *
+T -sym-trunk *
+U drh
+Z bc12630434faa3e2cbe8d73f4285c221
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d9fb0905d5..8363f6b593 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
\ No newline at end of file
+5683743ddf0bb051f2fe5d137cc18407e000e77e9faa9010a22e3134b575638b
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index 6085c4bbe2..867b62aa7a 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -148,8 +148,8 @@ ecmd ::= SEMI.
 ecmd ::= cmdx SEMI.
 %ifndef SQLITE_OMIT_EXPLAIN
 ecmd ::= explain cmdx SEMI.       {NEVER-REDUCE}
-explain ::= EXPLAIN.              { pParse->explain = 1; }
-explain ::= EXPLAIN QUERY PLAN.   { pParse->explain = 2; }
+explain ::= EXPLAIN.              { if( pParse->pReprepare==0 ) pParse->explain = 1; }
+explain ::= EXPLAIN QUERY PLAN.   { if( pParse->pReprepare==0 ) pParse->explain = 2; }
 %endif  SQLITE_OMIT_EXPLAIN
 cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
 
diff --git a/src/prepare.c b/src/prepare.c
index 39e8dcf655..9f843faa86 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -700,7 +700,12 @@ static int sqlite3Prepare(
   sParse.pOuterParse = db->pParse;
   db->pParse = &sParse;
   sParse.db = db;
-  sParse.pReprepare = pReprepare;
+  if( pReprepare ){
+    sParse.pReprepare = pReprepare;
+    sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare);
+  }else{
+    assert( sParse.pReprepare==0 );
+  }
   assert( ppStmt && *ppStmt==0 );
   if( db->mallocFailed ){
     sqlite3ErrorMsg(&sParse, "out of memory");
diff --git a/src/shell.c.in b/src/shell.c.in
index 9165f21f71..ed1d8e6c4b 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3453,8 +3453,6 @@ static int str_in_array(const char *zStr, const char **azArray){
 **       and "Goto" by 2 spaces.
 */
 static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
-  const char *zSql;               /* The text of the SQL statement */
-  const char *z;                  /* Used to check if this is an EXPLAIN */
   int *abYield = 0;               /* True if op is an OP_Yield */
   int nAlloc = 0;                 /* Allocated size of p->aiIndent[], abYield */
   int iOp;                        /* Index of operation in p->aiIndent[] */
@@ -3471,10 +3469,7 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
     p->cMode = p->mode;
     return;
   }
-  zSql = sqlite3_sql(pSql);
-  if( zSql==0 ) return;
-  for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++);
-  if( sqlite3_strnicmp(z, "explain", 7) ){
+  if( sqlite3_stmt_isexplain(pSql)!=1 ){
     p->cMode = p->mode;
     return;
   }
@@ -4332,16 +4327,15 @@ static int shell_exec(
       /* Show the EXPLAIN QUERY PLAN if .eqp is on */
       if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){
         sqlite3_stmt *pExplain;
-        char *zEQP;
         int triggerEQP = 0;
         disable_debug_trace_modes();
         sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
         if( pArg->autoEQP>=AUTOEQP_trigger ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
         }
-        zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
-        shell_check_oom(zEQP);
-        rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+        pExplain = pStmt;
+        sqlite3_reset(pExplain);
+        rc = sqlite3_stmt_explain(pExplain, 2);
         if( rc==SQLITE_OK ){
           while( sqlite3_step(pExplain)==SQLITE_ROW ){
             const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
@@ -4353,29 +4347,22 @@ static int shell_exec(
           }
           eqp_render(pArg, 0);
         }
-        sqlite3_finalize(pExplain);
-        sqlite3_free(zEQP);
         if( pArg->autoEQP>=AUTOEQP_full ){
           /* Also do an EXPLAIN for ".eqp full" mode */
-          zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
-          shell_check_oom(zEQP);
-          rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+          sqlite3_reset(pExplain);
+          rc = sqlite3_stmt_explain(pExplain, 1);
           if( rc==SQLITE_OK ){
             pArg->cMode = MODE_Explain;
             explain_data_prepare(pArg, pExplain);
             exec_prepared_stmt(pArg, pExplain);
             explain_data_delete(pArg);
           }
-          sqlite3_finalize(pExplain);
-          sqlite3_free(zEQP);
         }
         if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0);
-          /* Reprepare pStmt before reactivating trace modes */
-          sqlite3_finalize(pStmt);
-          sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-          if( pArg ) pArg->pStmt = pStmt;
         }
+        sqlite3_reset(pStmt);
+        sqlite3_stmt_explain(pStmt, 0);
         restore_debug_trace_modes();
       }
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 59d9986c52..9e165c1e55 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -4421,6 +4421,37 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 */
 int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
 
+/*
+** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** ^The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** setting for prepared statement S.  If E is zero, then S becomes
+** a normal prepared statement.  If E is 1, then S behaves as if
+** its SQL text began with "EXPLAIN".  If E is 2, then S behaves as if
+** its SQL text began with "EXPLAIN QUERY PLAN".
+**
+** Calling sqlite3_stmt_explain(S,E) causes prepared statement S
+** to be reprepared.  A call to sqlite3_stmt_explain(S,E) will fail
+** with SQLITE_ERROR if S cannot be re-prepared because it was created
+** using sqlite3_prepare() instead of the newer sqlite_prepare_v2() or
+** sqlite3_prepare_v3() interfaces and hence has no saved SQL text with
+** which to reprepare. Changing the explain setting for a prepared
+** statement does not change the original SQL text for the statement.
+** Hence, if the SQL text originally began with EXPLAIN or EXPLAIN
+** QUERY PLAN, but sqlite3_stmt_explain(S,0) is called to convert the
+** statement into an ordinary statement, the EXPLAIN or EXPLAIN QUERY
+** PLAN keywords still appear in the sqlite3_sql(S) output, even though
+** the statement now acts like a normal SQL statement.
+**
+** This routine returns SQLITE_OK if the explain mode is successfully
+** changed, or an error code if the explain mode could not be changed.
+** The explain mode cannot be changed while a statement is active.
+** Hence, it is good practice to call [sqlite3_reset(S)]
+** immediately prior to calling sqlite3_stmt_explain(S,E).
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode);
+
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
 ** METHOD: sqlite3_stmt
diff --git a/src/test1.c b/src/test1.c
index adc862156f..520508d1ca 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -2922,6 +2922,34 @@ static int SQLITE_TCLAPI test_stmt_isexplain(
   return TCL_OK;
 }
 
+/*
+** Usage:  sqlite3_stmt_explain  STMT  INT
+**
+** Set the explain to normal (0), EXPLAIN (1) or EXPLAIN QUERY PLAN (2).
+*/
+static int SQLITE_TCLAPI test_stmt_explain(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  int eMode = 0;
+  int rc;
+
+  if( objc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"",
+        Tcl_GetStringFromObj(objv[0], 0), " STMT INT", 0);
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  if( Tcl_GetIntFromObj(interp, objv[2], &eMode) ) return TCL_ERROR;
+  rc = sqlite3_stmt_explain(pStmt, eMode);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_stmt_busy  STMT
 **
@@ -8991,6 +9019,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_isexplain",        test_stmt_isexplain,0 },
+     { "sqlite3_stmt_explain",          test_stmt_explain  ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
      { "uses_stmt_journal",             uses_stmt_journal ,0 },
 
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 920780a896..885b17f762 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -1814,6 +1814,22 @@ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){
   return pStmt ? ((Vdbe*)pStmt)->explain : 0;
 }
 
+/*
+** Set the explain mode for a statement.
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){
+  Vdbe *v = (Vdbe*)pStmt;
+  int rc;
+  if( v->eVdbeState!=VDBE_READY_STATE ) return SQLITE_BUSY; 
+  if( v->explain==eMode ) return SQLITE_OK;
+  if( v->zSql==0 || eMode<0 || eMode>2 ) return SQLITE_ERROR;
+  sqlite3_mutex_enter(v->db->mutex);
+  v->explain = eMode;
+  rc = sqlite3Reprepare(v);
+  sqlite3_mutex_leave(v->db->mutex);
+  return rc;
+}
+
 /*
 ** Return true if the prepared statement is in need of being reset.
 */

From 0dac350f3c412f97dad1a6e9f565eec3f34505a7 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Sat, 15 Jul 2023 18:57:57 +0000
Subject: [PATCH 12/66] Store large tombstone hash tables in multiple database
 records. Ensure the same hash tables handle rowid 0.

FossilOrigin-Name: 4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3
---
 ext/fts5/fts5_index.c              | 502 +++++++++++++++++++----------
 ext/fts5/test/fts5contentless.test |  44 +++
 manifest                           |  14 +-
 manifest.uuid                      |   2 +-
 4 files changed, 392 insertions(+), 170 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e3b4139d9e..2ed9eee1fd 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -248,7 +248,7 @@
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
 
-#define FTS5_TOMBSTONE_ROWID(segid) fts5_dri(segid + (1<<16), 0, 0, 0)
+#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid + (1<<16), 0, 0, ipg)
 
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
@@ -334,7 +334,7 @@ struct Fts5DoclistIter {
 ** using an Fts5Structure record in memory. Which uses instances of the 
 ** other Fts5StructureXXX types as components.
 **
-** nLocCounter:
+** nOriginCntr:
 **   This value is set to non-zero for structure records created for
 **   contentlessdelete=1 tables only. In that case it represents the
 **   location value to apply to the next top-level segment created.
@@ -345,8 +345,9 @@ struct Fts5StructureSegment {
   int pgnoLast;                   /* Last leaf page number in segment */
 
   /* contentlessdelete=1 tables only: */
-  u64 iLoc1;
-  u64 iLoc2;
+  u64 iOrigin1;
+  u64 iOrigin2;
+  int nPgTombstone;               /* Number of tombstone hash table pages */
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -356,7 +357,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
-  u64 nLocCounter;
+  u64 nOriginCntr;
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -453,7 +454,8 @@ struct Fts5SegIter {
   Fts5Data *pLeaf;                /* Current leaf data */
   Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   i64 iLeafOffset;                /* Byte offset within current leaf */
-  Fts5Data *pTombstone;
+  Fts5Data **apTombstone;         /* Array of tombstone pages */
+  int nTombstone;
 
   /* Next method */
   void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
@@ -797,41 +799,6 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
-static Fts5Data *fts5DataReadOpt(Fts5Index *p, i64 iRowid){
-  Fts5Data *pRet = 0;
-  int rc2 = SQLITE_OK;
-
-  if( p->pReaderOpt==0 ){
-    Fts5Config *pConfig = p->pConfig;
-    fts5IndexPrepareStmt(p, &p->pReaderOpt, sqlite3_mprintf(
-          "SELECT block FROM '%q'.'%q_data' WHERE id=?",
-          pConfig->zDb, pConfig->zName
-    ));
-  }
-
-  if( p->rc==SQLITE_OK ){
-    sqlite3_bind_int64(p->pReaderOpt, 1, iRowid);
-    if( SQLITE_ROW==sqlite3_step(p->pReaderOpt) ){
-      int nByte = sqlite3_column_bytes(p->pReaderOpt, 0);
-      const u8 *aByte = (const u8*)sqlite3_column_blob(p->pReaderOpt, 0);
-      i64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
-
-      pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
-      if( pRet==0 ){
-        p->rc = SQLITE_NOMEM;
-      }else{
-        pRet->nn = nByte;
-        pRet->p = (u8*)&pRet[1];
-        memcpy(pRet->p, aByte, nByte);
-      }
-    }
-    rc2 = sqlite3_reset(p->pReaderOpt);
-    if( p->rc==SQLITE_OK ) p->rc = rc2;
-  }
-
-  return pRet;
-}
-
 /*
 ** INSERT OR REPLACE a record into the %_data table.
 */
@@ -880,13 +847,16 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
 /*
 ** Remove all records associated with segment iSegid.
 */
-static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
+static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){
+  int iSegid = pSeg->iSegid;
   i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
   i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
   fts5DataDelete(p, iFirst, iLast);
-  if( p->pConfig->bContentlessDelete ){
-    i64 iTomb = FTS5_TOMBSTONE_ROWID(iSegid);
-    fts5DataDelete(p, iTomb, iTomb);
+
+  if( pSeg->nPgTombstone ){
+    i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0);
+    i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1);
+    fts5DataDelete(p, iTomb1, iTomb2);
   }
   if( p->pIdxDeleter==0 ){
     Fts5Config *pConfig = p->pConfig;
@@ -999,7 +969,7 @@ static int fts5StructureDecode(
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
   int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
-  i64 nLocCounter = 0;
+  u64 nOriginCntr = 0;
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
@@ -1061,9 +1031,10 @@ static int fts5StructureDecode(
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
           if( bStructureV2 ){
-            i += fts5GetVarint(&pData[i], &pSeg->iLoc1);
-            i += fts5GetVarint(&pData[i], &pSeg->iLoc2);
-            nLocCounter = MAX(nLocCounter, pSeg->iLoc2);
+            i += fts5GetVarint(&pData[i], &pSeg->iOrigin1);
+            i += fts5GetVarint(&pData[i], &pSeg->iOrigin2);
+            i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone);
+            nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2);
           }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
             rc = FTS5_CORRUPT;
@@ -1076,7 +1047,7 @@ static int fts5StructureDecode(
     }
     if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
     if( bStructureV2 ){
-      pRet->nLocCounter = nLocCounter+1;
+      pRet->nOriginCntr = nOriginCntr+1;
     }
 
     if( rc!=SQLITE_OK ){
@@ -1290,7 +1261,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     Fts5Buffer buf;               /* Buffer to serialize record into */
     int iLvl;                     /* Used to iterate through levels */
     int iCookie;                  /* Cookie value to store */
-    int nHdr = (pStruct->nLocCounter>0 ? (4+4+9+9+9) : (4+9+9));
+    int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9));
 
     assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
     memset(&buf, 0, sizeof(Fts5Buffer));
@@ -1302,7 +1273,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
     if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
       sqlite3Fts5Put32(buf.p, iCookie);
       buf.n = 4;
-      if( pStruct->nLocCounter>0 ){
+      if( pStruct->nOriginCntr>0 ){
         fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4);
       }
       fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
@@ -1321,9 +1292,10 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
         fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
-        if( pStruct->nLocCounter>0 ){
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc1);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc2);
+        if( pStruct->nOriginCntr>0 ){
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin1);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin2);
+          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nPgTombstone);
         }
       }
     }
@@ -1848,8 +1820,15 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
 }
 
 static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
-  i64 iRowid = FTS5_TOMBSTONE_ROWID(pIter->pSeg->iSegid);
-  pIter->pTombstone = fts5DataReadOpt(p, iRowid);
+  const int nTomb = pIter->pSeg->nPgTombstone;
+  if( nTomb>0 ){
+    Fts5Data **apTomb = 0;
+    apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb);
+    if( apTomb ){
+      pIter->apTombstone = apTomb;
+      pIter->nTombstone = nTomb;
+    }
+  }
 }
 
 /*
@@ -2680,10 +2659,14 @@ static void fts5SegIterHashInit(
 ** Zero the iterator passed as the only argument.
 */
 static void fts5SegIterClear(Fts5SegIter *pIter){
+  int ii;
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
-  fts5DataRelease(pIter->pTombstone);
+  for(ii=0; ii<pIter->nTombstone; ii++){
+    fts5DataRelease(pIter->apTombstone[ii]);
+  }
+  sqlite3_free(pIter->apTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -3021,19 +3004,31 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
-static int fts5IndexTombstoneQuery(u8 *aHash, int nHash, u64 iRowid){
-  int szKey = aHash[3] ? 8 : 4;
-  int nSlot = (nHash - 8) / szKey;
-  int iSlot = iRowid % nSlot;
+#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
-  if( szKey==4 ){
-    u32 *aSlot = (u32*)&aHash[8];
+/*
+** Query a single tombstone hash table for rowid iRowid. The tombstone hash
+** table is one of nHashTable tables.
+*/
+static int fts5IndexTombstoneQuery(
+  Fts5Data *pHash,
+  int nHashTable,
+  u64 iRowid
+){
+  int szKey = TOMBSTONE_KEYSIZE(pHash);
+  int nSlot = (pHash->nn - 8) / szKey;
+  int iSlot = (iRowid / nHashTable) % nSlot;
+
+  if( iRowid==0 ){
+    return pHash->p[1];
+  }else if( szKey==4 ){
+    u32 *aSlot = (u32*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
       iSlot = (iSlot+1)%nSlot;
     }
   }else{
-    u64 *aSlot = (u64*)&aHash[8];
+    u64 *aSlot = (u64*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
       iSlot = (iSlot+1)%nSlot;
@@ -3047,9 +3042,21 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   int iFirst = pIter->aFirst[1].iFirst;
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
-  if( pSeg->pTombstone ){
+  if( pSeg->nTombstone ){
+    /* Figure out which page the rowid might be present on. */
+    int iPg = pSeg->iRowid % pSeg->nTombstone;
+
+    if( pSeg->apTombstone[iPg]==0 ){
+      pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
+          FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)
+      );
+      if( pSeg->apTombstone[iPg]==0 ) return 0;
+    }
+
     return fts5IndexTombstoneQuery(
-        pSeg->pTombstone->p, pSeg->pTombstone->nn, pSeg->iRowid
+        pSeg->apTombstone[iPg],
+        pSeg->nTombstone,
+        pSeg->iRowid
     );
   }
 
@@ -4503,9 +4510,9 @@ static void fts5IndexMergeLevel(
     nInput = pLvl->nSeg;
 
     /* Set the range of locations that will go into the output segment */
-    if( pStruct->nLocCounter>0 ){
-      pSeg->iLoc1 = pLvl->aSeg[0].iLoc1;
-      pSeg->iLoc2 = pLvl->aSeg[pLvl->nSeg-1].iLoc2;
+    if( pStruct->nOriginCntr>0 ){
+      pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
+      pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
     }
   }
   bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
@@ -4567,7 +4574,7 @@ static void fts5IndexMergeLevel(
 
     /* Remove the redundant segments from the %_data table */
     for(i=0; i<nInput; i++){
-      fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid);
+      fts5DataRemoveSegment(p, &pLvl->aSeg[i]);
     }
 
     /* Remove the redundant segments from the input level */
@@ -5323,10 +5330,10 @@ static void fts5FlushOneHash(Fts5Index *p){
         pSeg->iSegid = iSegid;
         pSeg->pgnoFirst = 1;
         pSeg->pgnoLast = pgnoLast;
-        if( pStruct->nLocCounter>0 ){
-          pSeg->iLoc1 = pStruct->nLocCounter;
-          pSeg->iLoc2 = pStruct->nLocCounter;
-          pStruct->nLocCounter++;
+        if( pStruct->nOriginCntr>0 ){
+          pSeg->iOrigin1 = pStruct->nOriginCntr;
+          pSeg->iOrigin2 = pStruct->nOriginCntr;
+          pStruct->nOriginCntr++;
         }
         pStruct->nSegment++;
       }
@@ -5390,7 +5397,7 @@ static Fts5Structure *fts5IndexOptimizeStruct(
     pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
     pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
-    pNew->nLocCounter = pStruct->nLocCounter;
+    pNew->nOriginCntr = pStruct->nOriginCntr;
     pLvl = &pNew->aLevel[pNew->nLevel-1];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
     if( pLvl->aSeg ){
@@ -6010,7 +6017,7 @@ int sqlite3Fts5IndexReinit(Fts5Index *p){
   fts5IndexDiscardData(p);
   memset(&s, 0, sizeof(Fts5Structure));
   if( p->pConfig->bContentlessDelete ){
-    s.nLocCounter = 1;
+    s.nOriginCntr = 1;
   }
   fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
   fts5StructureWrite(p, &s);
@@ -6408,20 +6415,214 @@ int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
-    *piLoc = pStruct->nLocCounter;
+    *piLoc = pStruct->nOriginCntr;
     fts5StructureRelease(pStruct);
   }
   return fts5IndexReturn(p);
 }
 
+/*
+** Buffer pPg contains a page of a tombstone hash table - one of nPg.
+*/
+static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
+  int szKey = TOMBSTONE_KEYSIZE(pPg);
+  int nSlot = (pPg->nn - 8) / szKey;
+  int iSlot = (iRowid / nPg) % nSlot;
+  int nElem = fts5GetU32(&pPg->p[4]);
+
+  if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
+  if( iRowid==0 ){
+    pPg->p[1] = 0x01;
+    return 0;
+  }
+
+  fts5PutU32(&pPg->p[4], nElem+1);
+  if( szKey==4 ){
+    u32 *aSlot = (u32*)&pPg->p[8];
+    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
+  }else{
+    u64 *aSlot = (u64*)&pPg->p[8];
+    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+  }
+
+  return nElem >= (nSlot/2);
+}
+
+/*
+** Return 0 if the hash is successfully rebuilt using nOut pages. Or 
+** non-zero if it is not. In this case the caller should retry with a 
+** larger nOut parameter.
+*/
+static int fts5IndexTombstoneRehash(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,     /* Segment to rebuild hash of */
+  Fts5Data *pData1,               /* One page of current hash - or NULL */
+  int iPg1,                       /* Which page of the current hash is pData1 */
+  int szKey,                      /* 4 or 8, the keysize */
+  int nOut,                       /* Number of output pages */
+  Fts5Data **apOut                /* Array of output hash pages */
+){
+  int ii;
+  int res = 0;
+
+  /* Initialize the headers of all the output pages */
+  for(ii=0; ii<nOut; ii++){
+    apOut[ii]->p[0] = szKey;
+    fts5PutU32(&apOut[ii]->p[4], 0);
+  }
+
+  /* Loop through the current pages of the hash table. */ 
+  for(ii=0; res==0 && ii<pSeg->nPgTombstone; ii++){
+    Fts5Data *pData = 0;          /* Page ii of the current hash table */
+    Fts5Data *pFree = 0;          /* Free this at the end of the loop */
+
+    if( iPg1==ii ){
+      pData = pData1;
+    }else{
+      pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii));
+    }
+
+    if( pData ){
+      int szKeyIn = pData->p[3] ? 8 : 4;
+      int nSlotIn = (pData->nn - 8) / szKeyIn;
+      int iIn;
+      for(iIn=0; iIn<nSlotIn; iIn++){
+        u64 iVal = 0;
+
+        /* Read the value from slot iIn of the input page into iVal. */
+        if( szKeyIn==4 ){
+          u32 *aSlot = (u32*)&pData->p[8];
+          if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]);
+        }else{
+          u64 *aSlot = (u64*)&pData->p[8];
+          if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]);
+        }
+
+        /* If iVal is not 0 at this point, insert it into the new hash table */
+        if( iVal ){
+          Fts5Data *pPg = apOut[(iVal % nOut)];
+          res = fts5IndexTombstoneAddToPage(pPg, nOut, iVal);
+          if( res ) break;
+        }
+      }
+
+      /* If this is page 0 of the old hash, copy the rowid-0-flag from the
+      ** old hash to the new.  */
+      if( ii==0 ){
+        apOut[0]->p[1] = pData->p[1];
+      }
+    }
+    fts5DataRelease(pFree);
+  }
+
+  return res;
+}
+
+static void fts5IndexTombstoneFreeArray(Fts5Data **ap, int n){
+  int ii;
+  for(ii=0; ii<n; ii++){
+    fts5DataRelease(ap[ii]);
+  }
+  sqlite3_free(ap);
+}
+
+/*
+** This is called to rebuild the hash table belonging to segment pSeg.
+*/
+static void fts5IndexTombstoneRebuild(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,     /* Segment to rebuild hash of */
+  Fts5Data *pData1,               /* One page of current hash - or NULL */
+  int iPg1,                       /* Which page of the current hash is pData1 */
+  int szKey,                      /* 4 or 8, the keysize */
+  int *pnOut,                     /* OUT: Number of output pages */
+  Fts5Data ***papOut              /* OUT: Output hash pages */
+){
+  const int MINSLOT = 32;
+  int nSlotPerPage = (p->pConfig->pgsz - 8) / szKey;
+  int nSlot = MINSLOT;            /* Number of slots in each output page */
+  int nOut = 0;
+
+  /* Figure out how many output pages (nOut) and how many slots per 
+  ** page (nSlot).  */
+  if( pSeg->nPgTombstone==0 ){
+    nOut = 1;
+    nSlot = MINSLOT;
+  }else if( pSeg->nPgTombstone==1 ){
+    int nElem = (int)fts5GetU32(&pData1->p[4]);
+    assert( pData1 && iPg1==0 );
+
+    nOut = 1;
+    while( nSlot<nElem*2 ){
+      nSlot = nSlot * 2;
+      if( nSlot>nSlotPerPage ){ 
+        nOut = 0; 
+        break;
+      }
+    }
+    if( nOut && nSlot>nSlotPerPage/2 ){
+      nSlot = nSlotPerPage;
+    }
+  }
+  if( nOut==0 ){
+    nOut = (pSeg->nPgTombstone * 2 + 1);
+    nSlot = nSlotPerPage;
+  }
+
+  /* Allocate the required array and output pages */
+  while( 1 ){
+    int res = 0;
+    int ii = 0;
+    int szPage = 0;
+    Fts5Data **apOut = 0;
+
+    apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
+    szPage = 8 + nSlot*szKey;
+    for(ii=0; ii<nOut; ii++){
+      Fts5Data *pNew = (Fts5Data*)sqlite3Fts5MallocZero(&p->rc, 
+          sizeof(Fts5Data)+szPage
+      );
+      if( pNew ){
+        pNew->nn = szPage;
+        pNew->p = (u8*)&pNew[1];
+      }
+      apOut[ii] = pNew;
+    }
+
+    res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    if( res==0 ){
+      if( p->rc ){
+        fts5IndexTombstoneFreeArray(apOut, nOut);
+        apOut = 0;
+        nOut = 0;
+      }
+      *pnOut = nOut;
+      *papOut = apOut;
+      break;
+    }
+    assert( p->rc==SQLITE_OK );
+
+    fts5IndexTombstoneFreeArray(apOut, nOut);
+    nSlot = nSlotPerPage;
+    nOut = nOut*2 + 1;
+  }
+}
+
+
 /*
 ** Add a tombstone for rowid iRowid to segment pSeg.
 **
 ** All tombstones for a single segment are stored in a blob formatted to 
 ** contain a hash table. The format is:
 **
-**   * 32-bit integer. 1 for 64-bit unsigned keys, 0 for 32-bit unsigned keys.
-**   * 32-bit integer. The number of entries currently in the hash table.
+**   * Key-size: 1 byte. Either 4 or 8.
+**   * rowid-0-flag: 1 byte. Either 0 or 1.
+**   * UNUSED: 2 bytes.
+**   * 32-bit big-endian integer. The number of entries currently in the hash
+**     table. This does not change when the rowid-0-flag is set - it only
+**     includes entries in the hash table.
 **
 ** Then an array of entries. The number of entries can be calculated based
 ** on the size of the blob in the database and the size of the keys as 
@@ -6434,84 +6635,51 @@ static void fts5IndexTombstoneAdd(
   Fts5StructureSegment *pSeg, 
   u64 iRowid
 ){
-  Fts5Data *pHash = 0;
-  u8 *aFree = 0;
-  u8 *aNew = 0;
-  int nNew = 0;
+  Fts5Data *pPg = 0;
+  int iPg = -1;
   int szKey = 0;
-  int nSlot = 0;
-  int bKey64 = (iRowid>0xFFFFFFFF);
-  u32 nHash = 0;
 
-  /* Load the current hash table, if any */
-  pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
-  if( p->rc ) return;
+  int nHash = 0;
+  Fts5Data **apHash = 0;
 
-  if( pHash ){
-    szKey = pHash->p[3] ? 8 : 4;
-    nSlot = (pHash->nn - 8) / szKey;
-    nHash = fts5GetU32(&pHash->p[4]);
-  }
-
-  /* Check if the current hash table needs to be rebuilt. Either because
-  ** (a) it does not yet exist, (b) it is full, or (c) it is using the
-  ** wrong sized keys. */
-  if( pHash==0 || nSlot<=(nHash*2) || (bKey64 && szKey==4) ){
-    int szNewKey = (bKey64 || szKey==8) ? 8 : 4;
-    int nNewSlot = (nSlot ? nSlot*2 : 16);
-
-    nNew = 8 + (nNewSlot * szNewKey);
-    aFree = aNew = (u8*)sqlite3Fts5MallocZero(&p->rc, nNew);
-    if( aNew ){
-      int iSlot = 0;
-      int ii;
-      fts5PutU32(aNew, (szNewKey==8 ? 1 : 0));
-      for(ii=0; ii<nSlot; ii++){
-        u64 iVal = 0;
-        if( szKey==4 ){
-          iVal = (u64)fts5GetU32(&pHash->p[8 + ii*szKey]);
-        }else{
-          iVal = fts5GetU64(&pHash->p[8 + ii*szKey]);
-        }
-
-        iSlot = iVal % nNewSlot;
-        if( szNewKey==4 ){
-          u32 *aSlot = (u32*)&aNew[8];
-          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
-          fts5PutU32((u8*)&aSlot[iSlot], (u32)iVal);
-        }else{
-          u64 *aSlot = (u64*)&aNew[8];
-          while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nNewSlot;
-          fts5PutU64((u8*)&aSlot[iSlot], iRowid);
-        }
-      }
-    }
-    szKey = szNewKey;
-    nSlot = nNewSlot;
-  }else{
-    aNew = pHash->p;
-    nNew = pHash->nn;
-  }
-
-  if( aNew ){
-    int iSlot = (iRowid % nSlot);
-    if( szKey==4 ){
-      u32 *aSlot = (u32*)&aNew[8];
-      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
-      fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
-    }else{
-      u64 *aSlot = (u64*)&aNew[8];
-      while( aSlot[iSlot]!=0 ) iSlot = (iSlot+1) % nSlot;
-      fts5PutU64((u8*)&aSlot[iSlot], iRowid);
+  if( pSeg->nPgTombstone>0 ){
+    iPg = iRowid % pSeg->nPgTombstone;
+    pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg));
+    if( pPg==0 ){
+      assert( p->rc!=SQLITE_OK );
+      return;
     }
 
-    fts5PutU32((u8*)&aNew[4], nHash+1);
-    assert( nNew>8 );
-    fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
+    if( 0==fts5IndexTombstoneAddToPage(pPg, pSeg->nPgTombstone, iRowid) ){
+      fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn);
+      fts5DataRelease(pPg);
+      return;
+    }
   }
 
-  sqlite3_free(aFree);
-  fts5DataRelease(pHash);
+  /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */
+  szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4;
+  if( iRowid>0xFFFFFFFF ) szKey = 8;
+
+  /* Rebuild the hash table */
+  fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash);
+  assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) );
+
+  /* If all has succeeded, write the new rowid into one of the new hash
+  ** table pages, then write them all out to disk. */
+  if( nHash ){
+    int ii = 0;
+    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid);
+    for(ii=0; ii<nHash; ii++){
+      i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii);
+      fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
+    }
+    pSeg->nPgTombstone = nHash;
+    fts5StructureWrite(p, p->pStruct);
+  }
+
+  fts5DataRelease(pPg);
+  fts5IndexTombstoneFreeArray(apHash, nHash);
 }
 
 /*
@@ -6527,7 +6695,7 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
       int iSeg;
       for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
-        if( pSeg->iLoc1<=iLoc && pSeg->iLoc2>=iLoc ){
+        if( pSeg->iOrigin1<=(u64)iLoc && pSeg->iOrigin2>=(u64)iLoc ){
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
@@ -7094,6 +7262,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
 */
 static void fts5DecodeRowid(
   i64 iRowid,                     /* Rowid from %_data table */
+  int *pbTombstone,               /* OUT: Tombstone hash flag */
   int *piSegid,                   /* OUT: Segment id */
   int *pbDlidx,                   /* OUT: Dlidx flag */
   int *piHeight,                  /* OUT: Height */
@@ -7109,13 +7278,16 @@ static void fts5DecodeRowid(
   iRowid >>= FTS5_DATA_DLI_B;
 
   *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
+  iRowid >>= FTS5_DATA_ID_B;
+
+  *pbTombstone = (int)(iRowid & 0x0001);
 }
 #endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
 
 #if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
 static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
-  int iSegid, iHeight, iPgno, bDlidx;       /* Rowid compenents */
-  fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
+  int iSegid, iHeight, iPgno, bDlidx, bTomb;     /* Rowid compenents */
+  fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
 
   if( iSegid==0 ){
     if( iKey==FTS5_AVERAGES_ROWID ){
@@ -7125,8 +7297,10 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
     }
   }
   else{
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}",
-        bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}",
+        bDlidx ? "dlidx " : "", 
+        bTomb ? "tombstone " : "", 
+        iSegid, iHeight, iPgno
     );
   }
 }
@@ -7150,9 +7324,9 @@ static void fts5DebugStructure(
       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
           pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
       );
-      if( pSeg->iLoc1>0 ){
-        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " loc=%lld..%lld",
-            pSeg->iLoc1, pSeg->iLoc2
+      if( pSeg->iOrigin1>0 ){
+        sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld",
+            pSeg->iOrigin1, pSeg->iOrigin2
         );
       }
       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
@@ -7322,6 +7496,7 @@ static void fts5DecodeFunction(
 ){
   i64 iRowid;                     /* Rowid for record being decoded */
   int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
+  int bTomb;
   const u8 *aBlob; int n;         /* Record to decode */
   u8 *a = 0;
   Fts5Buffer s;                   /* Build up text to return here */
@@ -7344,7 +7519,7 @@ static void fts5DecodeFunction(
   if( a==0 ) goto decode_out;
   if( n>0 ) memcpy(a, aBlob, n);
 
-  fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);
+  fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
 
   fts5DebugRowid(&rc, &s, iRowid);
   if( bDlidx ){
@@ -7363,6 +7538,9 @@ static void fts5DecodeFunction(
           " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
       );
     }
+  }else if( bTomb ){
+    u32 nElem  = fts5GetU32(&a[4]);
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
   }else if( iSegid==0 ){
     if( iRowid==FTS5_AVERAGES_ROWID ){
       fts5DecodeAverages(&rc, &s, a, n);
@@ -7731,10 +7909,10 @@ static int fts5structColumnMethod(
       sqlite3_result_int(ctx, pSeg->pgnoLast);
       break;
     case 6: /* loc1 */
-      sqlite3_result_int(ctx, pSeg->iLoc1);
+      sqlite3_result_int(ctx, pSeg->iOrigin1);
       break;
     case 7: /* loc2 */
-      sqlite3_result_int(ctx, pSeg->iLoc2);
+      sqlite3_result_int(ctx, pSeg->iOrigin2);
       break;
   }
   return SQLITE_OK;
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 3179b52e92..18f05da887 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -122,6 +122,7 @@ set nRow 1000
 do_execsql_test 4.0 {
   CREATE TABLE t1(x);
   CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 100);
 }
 do_test 4.1 {
   for {set ii 0} {$ii < $nRow} {incr ii} {
@@ -203,5 +204,48 @@ do_execsql_test 5.6 {
   SELECT rowid FROM ft('one');
 } {1 4 5 6}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
+  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
+}
+
+do_test 6.1 {
+  db eval { SELECT rowid FROM ft('one two') } {
+    if {$rowid==1} {
+      db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') }
+    }
+  }
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+
+set lRowid [list -450 0 1 2 42]
+
+do_test 7.1 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); }
+  }
+  execsql COMMIT
+} {}
+
+do_test 7.2 {
+  execsql BEGIN
+  foreach r $lRowid {
+    execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
+do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
+
 finish_test
 
diff --git a/manifest b/manifest
index 4c89bf87f9..0316fa86f1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sa\scase\sof\san\sinfinite\sloop\sin\sfts5\swhen\sdealing\swith\scorrupt\srecords.
-D 2023-07-12T20:24:23.049
+C Store\slarge\stombstone\shash\stables\sin\smultiple\sdatabase\srecords.\sEnsure\sthe\ssame\shash\stables\shandle\srowid\s0.
+D 2023-07-15T18:57:57.087
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 7b9dd923301f8649db2f13a0d383c9270511555fc779e727e69f92c8d2e97b99
+F ext/fts5/fts5_index.c c38e8892905e9e57e22ca4441cb6fdb519f188d786efe541e9d4df24c34e9197
 F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
 F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test feb27b077be771141c298b68798b93efd0e2d6fd3cb759201628b69576d69228
+F ext/fts5/test/fts5contentless.test 08af7954537cf89acbdef4fd79f39b3a36f382683a1eb98d5c827c187f51e113
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 74d7610a8e32ac62a608141ff16bfe517c8d2cdfd43f81ef3a5df2b2003e541a
-R b3f674c48ef86903b0134e340f9975df
+P 0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
+R 681d2015e734a53885da51bd1dfb0cfa
 U dan
-Z 4a7fc420576ceeaedd11216cbfd17eb6
+Z bb27d80790c5d5605515450ca29c8287
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index b56caf54da..8729b7fc92 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0e801f11cd2d50fc710a80c2b3b805c7801e660cff1bcc87be89133d43796524
\ No newline at end of file
+4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3
\ No newline at end of file

From 55e0fd4a9d81bde6452f276eb05532c60ad1d49e Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 17 Jul 2023 17:59:58 +0000
Subject: [PATCH 13/66] Do not allow the 'delete' command to be used on
 contentless_delete=1 fts5 tables.

FossilOrigin-Name: cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
---
 ext/fts5/fts5Int.h                 |  4 ++--
 ext/fts5/fts5_index.c              | 18 +++++++++---------
 ext/fts5/fts5_main.c               |  9 ++++++++-
 ext/fts5/fts5_storage.c            | 12 ++++++------
 ext/fts5/test/fts5contentless.test | 19 ++++++++++++++++---
 manifest                           | 24 ++++++++++++------------
 manifest.uuid                      |  2 +-
 7 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 43f78c6d99..da2f90f230 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -536,8 +536,8 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
-int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc);
-int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid);
+int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
 
 /*
 ** End of interface to code in fts5_index.c.
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 2ed9eee1fd..0bb204ea92 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -96,8 +96,8 @@
 **
 **      Then, for V2 structures only:
 **
-**         + lower location counter value,
-**         + upper location counter value
+**         + lower origin counter value,
+**         + upper origin counter value
 **
 ** 2. The Averages Record:
 **
@@ -337,7 +337,7 @@ struct Fts5DoclistIter {
 ** nOriginCntr:
 **   This value is set to non-zero for structure records created for
 **   contentlessdelete=1 tables only. In that case it represents the
-**   location value to apply to the next top-level segment created.
+**   origin value to apply to the next top-level segment created.
 */
 struct Fts5StructureSegment {
   int iSegid;                     /* Segment id */
@@ -4509,7 +4509,7 @@ static void fts5IndexMergeLevel(
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
 
-    /* Set the range of locations that will go into the output segment */
+    /* Set the range of origins that will go into the output segment */
     if( pStruct->nOriginCntr>0 ){
       pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
       pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
@@ -6411,11 +6411,11 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
-int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
+int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
-    *piLoc = pStruct->nOriginCntr;
+    *piOrigin = pStruct->nOriginCntr;
     fts5StructureRelease(pStruct);
   }
   return fts5IndexReturn(p);
@@ -6684,9 +6684,9 @@ static void fts5IndexTombstoneAdd(
 
 /*
 ** Add iRowid to the tombstone list of the segment or segments that contain
-** rows from location iLoc.
+** rows from origin iOrigin.
 */
-int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
@@ -6695,7 +6695,7 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
       int iSeg;
       for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
-        if( pSeg->iOrigin1<=(u64)iLoc && pSeg->iOrigin2>=(u64)iLoc ){
+        if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 844b5c57ef..b10da49c75 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1653,7 +1653,14 @@ static int fts5UpdateMethod(
     if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
     ){
-      rc = fts5SpecialDelete(pTab, apVal);
+      if( pConfig->bContentlessDelete ){
+        fts5SetVtabError(pTab, 
+            "'delete' may not be used with a contentless_delete=1 table"
+        );
+        rc = SQLITE_ERROR;
+      }else{
+        rc = fts5SpecialDelete(pTab, apVal);
+      }
     }else{
       rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
     }
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 35c30d06e6..f28e4d7dd2 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -136,7 +136,7 @@ static int fts5StorageGetStmt(
 
       case FTS5_STMT_LOOKUP_DOCSIZE: 
         zSql = sqlite3_mprintf(azStmt[eStmt], 
-            (pC->bContentlessDelete ? ",location" : ""),
+            (pC->bContentlessDelete ? ",origin" : ""),
             pC->zDb, pC->zName
         );
         break;
@@ -332,7 +332,7 @@ int sqlite3Fts5StorageOpen(
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
       const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
       if( pConfig->bContentlessDelete ){
-        zCols = "id INTEGER PRIMARY KEY, sz BLOB, location INTEGER";
+        zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
       }
       rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
     }
@@ -472,7 +472,7 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
   assert( p->pConfig->bContentlessDelete );
   assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
 
-  /* Look up the location of the document in the %_docsize table. Store
+  /* Look up the origin of the document in the %_docsize table. Store
   ** this in stack variable iLoc.  */
   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
   if( rc==SQLITE_OK ){
@@ -510,9 +510,9 @@ static int fts5StorageInsertDocsize(
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
       if( p->pConfig->bContentlessDelete ){
-        i64 iLoc = 0;
-        rc = sqlite3Fts5IndexGetLocation(p->pIndex, &iLoc);
-        sqlite3_bind_int64(pReplace, 3, iLoc);
+        i64 iOrigin = 0;
+        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
+        sqlite3_bind_int64(pReplace, 3, iOrigin);
       }
       if( rc==SQLITE_OK ){
         sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 18f05da887..181ccb177e 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -51,7 +51,7 @@ foreach {tn sql bError} {
   execsql { ROLLBACK }
 }
 
-# Check that if contentless_delete=1 is specified, then the "location" 
+# Check that if contentless_delete=1 is specified, then the "origin" 
 # column is added to the %_docsize table.
 reset_db
 do_execsql_test 3.0 {
@@ -62,7 +62,7 @@ do_execsql_test 3.1 {
   SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
 } {
   {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
-  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, location INTEGER)}
+  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)}
 }
 
 do_execsql_test 3.2.1 {
@@ -77,7 +77,7 @@ do_execsql_test 3.3 {
   INSERT INTO x2 VALUES('second text');
 }
 do_execsql_test 3.4 {
-  SELECT id, location FROM x2_docsize
+  SELECT id, origin FROM x2_docsize
 } {1 1 2 2}
 do_execsql_test 3.5 {
   SELECT level, segment, loc1, loc2 FROM fts5_structure(
@@ -247,5 +247,18 @@ do_test 7.2 {
 do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
 do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
 
+#-------------------------------------------------------------------------
+reset_db
+
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft VALUES('hello world');
+}
+
+do_catchsql_test 8.1 {
+  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
+} {1 {'delete' may not be used with a contentless_delete=1 table}}
+
 finish_test
 
diff --git a/manifest b/manifest
index 07be41e1ec..2bd8a2d3d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\schanges\sinto\sthis\sbranch.
-D 2023-07-17T11:47:42.362
+C Do\snot\sallow\sthe\s'delete'\scommand\sto\sbe\sused\son\scontentless_delete=1\sfts5\stables.
+D 2023-07-17T17:59:58.252
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,15 +86,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h 40a234875f9bddd43b4b4281d946303d227de943773b9e84505a0d6f0419c16a
+F ext/fts5/fts5Int.h fa9dd8ecbda6340f406c6f21b9b524b4666817aa89850e60a42937eea87cef6a
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c c38e8892905e9e57e22ca4441cb6fdb519f188d786efe541e9d4df24c34e9197
-F ext/fts5/fts5_main.c c036530bbd39935b4b91fddb9d1b9d456e95c3685509aa975c87fc35445a9722
-F ext/fts5/fts5_storage.c 9a84d28154cb570773b26eb9645f8670089dee45c95afebf7a041e414266b3c3
+F ext/fts5/fts5_index.c 8e5fd1f1eb9489f1ba2eff2d3667ffd78a8d936c1180d284ccc6c37f25953a8f
+F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
+F ext/fts5/fts5_storage.c ee69d6c8195d2bf584ac8c5433e5c685812f39b48c7464f8aa76d9ed8b489119
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 08af7954537cf89acbdef4fd79f39b3a36f382683a1eb98d5c827c187f51e113
+F ext/fts5/test/fts5contentless.test 063249b7d0e92dc7ea54b4a179c466828927d720ee054031cbf6e06009c972bf
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -535,8 +535,8 @@ F ext/wasm/index.html b768e8659b4fe311912e54d42906449d51c0f84b7f036cca47ec1f93bf
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10
-F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 w ext/wasm/scratchpad-wasmfs-main.html
-F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 w ext/wasm/scratchpad-wasmfs-main.js
+F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96
+F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
 F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4410e60d0c76e057ee962124f9239c6e17fd5ccafdbb4d9b703448eabd7781e3 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
-R 3a33d523b0a231ecfa8d84bbc59989ba
+P c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
+R 461e0bbee60dbc3aadc58386c3f130fe
 U dan
-Z cf32871cae7a0d476c03b4d509e36da6
+Z 81dd61cf0a290b6d176bc4273c289b56
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 8d642d98f8..7ecfbd954a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
\ No newline at end of file
+cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
\ No newline at end of file

From 2cccced14054fe03ebd5cb22af83416a70b14006 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 17 Jul 2023 18:40:39 +0000
Subject: [PATCH 14/66] Fix cases where a row is inserted into a
 contentless_delete=1 fts5 table and then deleted within the same transaction.

FossilOrigin-Name: d928856a226fb7f001e55ff7e8eb58a656b982f1efa811de46c382b8b7cd778c
---
 ext/fts5/fts5_storage.c            |  7 +++++--
 ext/fts5/test/fts5contentless.test | 11 +++++++++--
 manifest                           | 14 +++++++-------
 manifest.uuid                      |  2 +-
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index f28e4d7dd2..c2f0ca9e44 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -411,7 +411,7 @@ static int fts5StorageDeleteFromIndex(
 ){
   Fts5Config *pConfig = p->pConfig;
   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
-  int rc;                         /* Return code */
+  int rc = SQLITE_OK;             /* Return code */
   int rc2;                        /* sqlite3_reset() return code */
   int iCol;
   Fts5InsertCtx ctx;
@@ -427,7 +427,6 @@ static int fts5StorageDeleteFromIndex(
 
   ctx.pStorage = p;
   ctx.iCol = -1;
-  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
       const char *zText;
@@ -582,6 +581,10 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   rc = fts5StorageLoadTotals(p, 1);
 
   /* Delete the index records */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  }
+
   if( rc==SQLITE_OK ){
     if( p->pConfig->bContentlessDelete ){
       rc = fts5StorageContentlessDelete(p, iDel);
diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test
index 181ccb177e..41a78e9c8f 100644
--- a/ext/fts5/test/fts5contentless.test
+++ b/ext/fts5/test/fts5contentless.test
@@ -248,17 +248,24 @@ do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
 do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
 
 #-------------------------------------------------------------------------
-reset_db
-
 reset_db
 do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
   INSERT INTO ft VALUES('hello world');
+  INSERT INTO ft VALUES('one two three');
 }
 
 do_catchsql_test 8.1 {
   INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
 } {1 {'delete' may not be used with a contentless_delete=1 table}}
 
+do_execsql_test 8.2 {
+  BEGIN;
+    INSERT INTO ft(rowid, x) VALUES(3, 'four four four');
+    DELETE FROM ft WHERE rowid=3;
+  COMMIT;
+  SELECT rowid FROM ft('four');
+} {}
+
 finish_test
 
diff --git a/manifest b/manifest
index 2bd8a2d3d4..a45f1f64f3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sallow\sthe\s'delete'\scommand\sto\sbe\sused\son\scontentless_delete=1\sfts5\stables.
-D 2023-07-17T17:59:58.252
+C Fix\scases\swhere\sa\srow\sis\sinserted\sinto\sa\scontentless_delete=1\sfts5\stable\sand\sthen\sdeleted\swithin\sthe\ssame\stransaction.
+D 2023-07-17T18:40:39.967
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 8e5fd1f1eb9489f1ba2eff2d3667ffd78a8d936c1180d284ccc6c37f25953a8f
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
-F ext/fts5/fts5_storage.c ee69d6c8195d2bf584ac8c5433e5c685812f39b48c7464f8aa76d9ed8b489119
+F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
-F ext/fts5/test/fts5contentless.test 063249b7d0e92dc7ea54b4a179c466828927d720ee054031cbf6e06009c972bf
+F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c4fb2f2ea0afe638fd7cffd89fbdb0a91589577c6f8299c7bbc17ac121be518b
-R 461e0bbee60dbc3aadc58386c3f130fe
+P cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
+R e5c49e6a91d60244d934bac112d65223
 U dan
-Z 81dd61cf0a290b6d176bc4273c289b56
+Z 153b54340cf10dbebadd96c0477be869
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 7ecfbd954a..6b9ee4a956 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cc694b83408ccb5d42204cb624145c76e95329cbe1d1fe8815c70a7a00af231a
\ No newline at end of file
+d928856a226fb7f001e55ff7e8eb58a656b982f1efa811de46c382b8b7cd778c
\ No newline at end of file

From d1db37a2f32b06d0cd53e66b18ee47ffea91ca87 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 18 Jul 2023 17:29:05 +0000
Subject: [PATCH 15/66] Enhance the sqlite3_stmt_explain() interface so that
 avoids unnecessary reprepare operations.

FossilOrigin-Name: 050f773addd605f6690348c98e9000a9a3663bbc26288a71973fd7b40468e8ab
---
 manifest        | 23 +++++++----------
 manifest.uuid   |  2 +-
 src/select.c    |  7 -----
 src/sqlite.h.in | 30 +++++++++++----------
 src/vdbeInt.h   |  4 ++-
 src/vdbeapi.c   | 69 ++++++++++++++++++++++++++++++++++++++++++++-----
 src/vdbeaux.c   | 37 +++++++-------------------
 7 files changed, 103 insertions(+), 69 deletions(-)

diff --git a/manifest b/manifest
index 560436430f..0926a795e7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sexperimental\ssqlite3_stmt_explain(S,E)\sinterface.
-D 2023-07-15T16:48:14.281
+C Enhance\sthe\ssqlite3_stmt_explain()\sinterface\sso\sthat\savoids\sunnecessary\nreprepare\soperations.
+D 2023-07-18T17:29:05.187
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,9 +637,9 @@ F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
+F src/select.c 3328b8c758016b400a4fea053ab1f5927929083a49d0c3921802ce99eea73cde
 F src/shell.c.in 3f125426a25c717fbf8b84b3b75a8b60fa989bf7a039ed38926d455329f9dd0f
-F src/sqlite.h.in 13d5458cd23e7b4759cff4d978ad09591a457b4a2821993579953a9a4257ce0f
+F src/sqlite.h.in 7b95e5b11914726ebcf95c633be3341ad2cb3c8aac62ecf188a14f74827aa68a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
 F src/sqliteInt.h dcb1a885e8b6cb78df618944b89d44361a99d0fe33e1bba2c150a855f7dc5599
@@ -709,9 +709,9 @@ F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
 F src/vdbe.c 74282a947234513872a83b0bab1b8c644ece64b3e27b053ef17677c8ff9c81e0
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
-F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae
-F src/vdbeapi.c c528ef4fafc8be172cbe4f48d8f15c9526bebde9f90185917fcc43fad264bcb6
-F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
+F src/vdbeInt.h 7a2329efe451ef0d1280736419f321cbc261146e453cdd22b27a0785b8742b15
+F src/vdbeapi.c 97a9dfdff49183ad7f7911f9d919eb8eb72677c39cdee6249ee5a060d3ce515a
+F src/vdbeaux.c 499b68a5585663ef36c05b421d8c9c60448aa3b9f01bba5ae7accfc89ee6ac50
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
 F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
@@ -2043,11 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
-R d7676f37f2e59b0f85cc021dadb3d7bc
-T *branch * sqlite3_stmt_explain
-T *sym-sqlite3_stmt_explain *
-T -sym-trunk *
+P 5683743ddf0bb051f2fe5d137cc18407e000e77e9faa9010a22e3134b575638b
+R a56fcd1282b166331b87ac37d0175ebe
 U drh
-Z bc12630434faa3e2cbe8d73f4285c221
+Z 96b7009f78463b1b25a8594262247cfd
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 8363f6b593..ab125fc443 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5683743ddf0bb051f2fe5d137cc18407e000e77e9faa9010a22e3134b575638b
\ No newline at end of file
+050f773addd605f6690348c98e9000a9a3663bbc26288a71973fd7b40468e8ab
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 4af812fa06..098b745849 100644
--- a/src/select.c
+++ b/src/select.c
@@ -2101,13 +2101,6 @@ void sqlite3GenerateColumnNames(
   int fullName;    /* TABLE.COLUMN if no AS clause and is a direct table ref */
   int srcName;     /* COLUMN or TABLE.COLUMN if no AS clause and is direct */
 
-#ifndef SQLITE_OMIT_EXPLAIN
-  /* If this is an EXPLAIN, skip this step */
-  if( pParse->explain ){
-    return;
-  }
-#endif
-
   if( pParse->colNamesSet ) return;
   /* Column names are determined by the left-most term of a compound select */
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 9e165c1e55..668cfa54ba 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -4425,24 +4425,28 @@ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
 ** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
 ** METHOD: sqlite3_stmt
 **
-** ^The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
 ** setting for prepared statement S.  If E is zero, then S becomes
 ** a normal prepared statement.  If E is 1, then S behaves as if
 ** its SQL text began with "EXPLAIN".  If E is 2, then S behaves as if
 ** its SQL text began with "EXPLAIN QUERY PLAN".
 **
-** Calling sqlite3_stmt_explain(S,E) causes prepared statement S
-** to be reprepared.  A call to sqlite3_stmt_explain(S,E) will fail
-** with SQLITE_ERROR if S cannot be re-prepared because it was created
-** using sqlite3_prepare() instead of the newer sqlite_prepare_v2() or
-** sqlite3_prepare_v3() interfaces and hence has no saved SQL text with
-** which to reprepare. Changing the explain setting for a prepared
-** statement does not change the original SQL text for the statement.
-** Hence, if the SQL text originally began with EXPLAIN or EXPLAIN
-** QUERY PLAN, but sqlite3_stmt_explain(S,0) is called to convert the
-** statement into an ordinary statement, the EXPLAIN or EXPLAIN QUERY
-** PLAN keywords still appear in the sqlite3_sql(S) output, even though
-** the statement now acts like a normal SQL statement.
+** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared.
+** SQLite tries to avoid a reprepare, but a reprepare might be necessary
+** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode.
+**
+** Because of the potential need to reprepare, a call to
+** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be
+** reprepared because it was created using sqlite3_prepare() instead of
+** the newer sqlite_prepare_v2() or sqlite3_prepare_v3() interfaces and
+** hence has no saved SQL text with which to reprepare.
+**
+** Changing the explain setting for a prepared statement does not change
+** the original SQL text for the statement.  Hence, if the SQL text originally
+** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0)
+** is called to convert the statement into an ordinary statement, the EXPLAIN
+** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S)
+** output, even though the statement now acts like a normal SQL statement.
 **
 ** This routine returns SQLITE_OK if the explain mode is successfully
 ** changed, or an error code if the explain mode could not be changed.
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 4c3394716b..baed0527d6 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -465,16 +465,18 @@ struct Vdbe {
   u32 nWrite;             /* Number of write operations that have occurred */
 #endif
   u16 nResColumn;         /* Number of columns in one row of the result set */
+  u16 nResAlloc;          /* Column slots allocated to aColName[] */
   u8 errorAction;         /* Recovery action to do in case of an error */
   u8 minWriteFileFormat;  /* Minimum file format for writable database files */
   u8 prepFlags;           /* SQLITE_PREPARE_* flags */
   u8 eVdbeState;          /* On of the VDBE_*_STATE values */
   bft expired:2;          /* 1: recompile VM immediately  2: when convenient */
-  bft explain:2;          /* True if EXPLAIN present on SQL command */
+  bft explain:2;          /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */
   bft changeCntOn:1;      /* True to update the change-counter */
   bft usesStmtJournal:1;  /* True if uses a statement journal */
   bft readOnly:1;         /* True for statements that do not write */
   bft bIsReader:1;        /* True for statements that read */
+  bft haveEqpOps:1;       /* Bytecode supports EXPLAIN QUERY PLAN */
   yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   u32 aCounter[9];        /* Counters used by sqlite3_stmt_status() */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 885b17f762..3185191ed9 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -1126,7 +1126,8 @@ int sqlite3_aggregate_count(sqlite3_context *p){
 */
 int sqlite3_column_count(sqlite3_stmt *pStmt){
   Vdbe *pVm = (Vdbe *)pStmt;
-  return pVm ? pVm->nResColumn : 0;
+  if( pVm==0 ) return 0;
+  return pVm->nResColumn;
 }
 
 /*
@@ -1299,6 +1300,32 @@ int sqlite3_column_type(sqlite3_stmt *pStmt, int i){
   return iType;
 }
 
+/*
+** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN.
+*/
+static const char * const azExplainColNames8[] = {
+   "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",  /* EXPLAIN */
+   "id", "parent", "notused", "detail"                         /* EQP */
+};
+static const u16 azExplainColNames16data[] = {
+  /*   0 */  'a', 'd', 'd', 'r',                0,
+  /*   5 */  'o', 'p', 'c', 'o', 'd', 'e',      0,
+  /*  12 */  'p', '1',                          0, 
+  /*  15 */  'p', '2',                          0,
+  /*  18 */  'p', '3',                          0,
+  /*  21 */  'p', '4',                          0,
+  /*  24 */  'p', '5',                          0,
+  /*  27 */  'c', 'o', 'm', 'm', 'e', 'n', 't', 0,
+  /*  35 */  'i', 'd',                          0,
+  /*  38 */  'p', 'a', 'r', 'e', 'n', 't',      0,
+  /*  45 */  'n', 'o', 't', 'u', 's', 'e', 'd', 0,
+  /*  53 */  'd', 'e', 't', 'a', 'i', 'l',      0
+};
+static const u8 iExplainColNames16[] = {
+  0, 5, 12, 15, 18, 21, 24, 27,
+  35, 38, 45, 53
+};
+
 /*
 ** Convert the N-th element of pStmt->pColName[] into a string using
 ** xFunc() then return that string.  If N is out of range, return 0.
@@ -1331,15 +1358,29 @@ static const void *columnName(
     return 0;
   }
 #endif
+  if( N<0 ) return 0;
   ret = 0;
   p = (Vdbe *)pStmt;
   db = p->db;
   assert( db!=0 );
-  n = sqlite3_column_count(pStmt);
-  if( N<n && N>=0 ){
+  sqlite3_mutex_enter(db->mutex);
+
+  if( p->explain ){
+    if( useType>0 ) goto columnName_end;
+    n = p->explain==1 ? 8 : 4;
+    if( N>=n ) goto columnName_end;
+    if( useUtf16 ){
+      int i = iExplainColNames16[N + 8*p->explain - 8];
+      ret = (void*)&azExplainColNames16data[i];
+    }else{
+      ret = (void*)azExplainColNames8[N + 8*p->explain - 8];
+    }
+    goto columnName_end;
+  }
+  n = p->nResColumn;
+  if( N<n ){
     u8 prior_mallocFailed = db->mallocFailed;
     N += useType*n;
-    sqlite3_mutex_enter(db->mutex);
 #ifndef SQLITE_OMIT_UTF16
     if( useUtf16 ){
       ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]);
@@ -1356,8 +1397,9 @@ static const void *columnName(
       sqlite3OomClear(db);
       ret = 0;
     }
-    sqlite3_mutex_leave(db->mutex);
   }
+columnName_end:
+  sqlite3_mutex_leave(db->mutex);
   return ret;
 }
 
@@ -1824,8 +1866,21 @@ int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){
   if( v->explain==eMode ) return SQLITE_OK;
   if( v->zSql==0 || eMode<0 || eMode>2 ) return SQLITE_ERROR;
   sqlite3_mutex_enter(v->db->mutex);
-  v->explain = eMode;
-  rc = sqlite3Reprepare(v);
+  if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){
+    /* No reprepare necessary */
+    v->explain = eMode;
+    rc = SQLITE_OK;
+  }else{
+    int haveEqpOps = v->explain==2 || v->haveEqpOps;
+    v->explain = eMode;
+    rc = sqlite3Reprepare(v);
+    v->haveEqpOps = haveEqpOps!=0;
+  }
+  if( v->explain ){
+    v->nResColumn = 12 - 4*v->explain;
+  }else{
+    v->nResColumn = v->nResAlloc;
+  }
   sqlite3_mutex_leave(v->db->mutex);
   return rc;
 }
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index a0eff155dd..eec93a6b1a 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -2418,8 +2418,8 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetInt64(pMem, pOp->p1);
         sqlite3VdbeMemSetInt64(pMem+1, pOp->p2);
         sqlite3VdbeMemSetInt64(pMem+2, pOp->p3);
-        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);    
-        p->nResColumn = 4;
+        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);
+        assert( p->nResColumn==4 );
       }else{
         sqlite3VdbeMemSetInt64(pMem+0, i);
         sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode),
@@ -2438,7 +2438,7 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetNull(pMem+7);
 #endif
         sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free);
-        p->nResColumn = 8;
+        assert( p->nResColumn==8 );
       }
       p->pResultRow = pMem;
       if( db->mallocFailed ){
@@ -2652,26 +2652,9 @@ void sqlite3VdbeMakeReady(
   resolveP2Values(p, &nArg);
   p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
   if( pParse->explain ){
-    static const char * const azColName[] = {
-       "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
-       "id", "parent", "notused", "detail"
-    };
-    int iFirst, mx, i;
     if( nMem<10 ) nMem = 10;
     p->explain = pParse->explain;
-    if( pParse->explain==2 ){
-      sqlite3VdbeSetNumCols(p, 4);
-      iFirst = 8;
-      mx = 12;
-    }else{
-      sqlite3VdbeSetNumCols(p, 8);
-      iFirst = 0;
-      mx = 8;
-    }
-    for(i=iFirst; i<mx; i++){
-      sqlite3VdbeSetColName(p, i-iFirst, COLNAME_NAME,
-                            azColName[i], SQLITE_STATIC);
-    }
+    p->nResColumn = 12 - 4*p->explain;
   }
   p->expired = 0;
 
@@ -2824,12 +2807,12 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){
   int n;
   sqlite3 *db = p->db;
 
-  if( p->nResColumn ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+  if( p->nResAlloc ){
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbFree(db, p->aColName);
   }
   n = nResColumn*COLNAME_N;
-  p->nResColumn = (u16)nResColumn;
+  p->nResColumn = p->nResAlloc = (u16)nResColumn;
   p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n );
   if( p->aColName==0 ) return;
   initMemArray(p->aColName, n, db, MEM_Null);
@@ -2854,14 +2837,14 @@ int sqlite3VdbeSetColName(
 ){
   int rc;
   Mem *pColName;
-  assert( idx<p->nResColumn );
+  assert( idx<p->nResAlloc );
   assert( var<COLNAME_N );
   if( p->db->mallocFailed ){
     assert( !zName || xDel!=SQLITE_DYNAMIC );
     return SQLITE_NOMEM_BKPT;
   }
   assert( p->aColName!=0 );
-  pColName = &(p->aColName[idx+var*p->nResColumn]);
+  pColName = &(p->aColName[idx+var*p->nResAlloc]);
   rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel);
   assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 );
   return rc;
@@ -3685,7 +3668,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   assert( db!=0 );
   assert( p->db==0 || p->db==db );
   if( p->aColName ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbNNFreeNN(db, p->aColName);
   }
   for(pSub=p->pProgram; pSub; pSub=pNext){

From d05bf0fe617ef8e1245eb260b1a547cec0771212 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 18 Jul 2023 19:52:32 +0000
Subject: [PATCH 16/66] Fix various problems with fts5 contentless_delete=1
 tables.

FossilOrigin-Name: 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
---
 ext/fts5/fts5_index.c               |  51 ++++++++--
 ext/fts5/test/fts5contentless2.test | 152 ++++++++++++++++++++++++++++
 manifest                            |  15 +--
 manifest.uuid                       |   2 +-
 4 files changed, 203 insertions(+), 17 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless2.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ed919298e7..8804e94bb0 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3038,13 +3038,19 @@ static int fts5IndexTombstoneQuery(
   return 0;
 }
 
+/*
+** Return true if the iterator passed as the only argument points
+** to an segment entry for which there is a tombstone. Return false
+** if there is no tombstone or if the iterator is already at EOF.
+*/
 static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
   int iFirst = pIter->aFirst[1].iFirst;
   Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
 
-  if( pSeg->nTombstone ){
+  if( pSeg->pLeaf && pSeg->nTombstone ){
     /* Figure out which page the rowid might be present on. */
-    int iPg = pSeg->iRowid % pSeg->nTombstone;
+    int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone;
+    assert( iPg>=0 );
 
     if( pSeg->apTombstone[iPg]==0 ){
       pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
@@ -6428,7 +6434,12 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
 /*
 ** Buffer pPg contains a page of a tombstone hash table - one of nPg.
 */
-static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
+static int fts5IndexTombstoneAddToPage(
+  Fts5Data *pPg, 
+  int bForce,
+  int nPg, 
+  u64 iRowid
+){
   int szKey = TOMBSTONE_KEYSIZE(pPg);
   int nSlot = (pPg->nn - 8) / szKey;
   int iSlot = (iRowid / nPg) % nSlot;
@@ -6440,6 +6451,10 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
     return 0;
   }
 
+  if( bForce==0 && nElem>=(nSlot/2) ){
+    return 1;
+  }
+
   fts5PutU32(&pPg->p[4], nElem+1);
   if( szKey==4 ){
     u32 *aSlot = (u32*)&pPg->p[8];
@@ -6451,7 +6466,7 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
     fts5PutU64((u8*)&aSlot[iSlot], iRowid);
   }
 
-  return nElem >= (nSlot/2);
+  return 0;
 }
 
 /*
@@ -6489,7 +6504,7 @@ static int fts5IndexTombstoneRehash(
     }
 
     if( pData ){
-      int szKeyIn = pData->p[3] ? 8 : 4;
+      int szKeyIn = TOMBSTONE_KEYSIZE(pData);
       int nSlotIn = (pData->nn - 8) / szKeyIn;
       int iIn;
       for(iIn=0; iIn<nSlotIn; iIn++){
@@ -6507,7 +6522,7 @@ static int fts5IndexTombstoneRehash(
         /* If iVal is not 0 at this point, insert it into the new hash table */
         if( iVal ){
           Fts5Data *pPg = apOut[(iVal % nOut)];
-          res = fts5IndexTombstoneAddToPage(pPg, nOut, iVal);
+          res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal);
           if( res ) break;
         }
       }
@@ -6642,7 +6657,6 @@ static void fts5IndexTombstoneAdd(
   Fts5Data *pPg = 0;
   int iPg = -1;
   int szKey = 0;
-
   int nHash = 0;
   Fts5Data **apHash = 0;
 
@@ -6654,7 +6668,7 @@ static void fts5IndexTombstoneAdd(
       return;
     }
 
-    if( 0==fts5IndexTombstoneAddToPage(pPg, pSeg->nPgTombstone, iRowid) ){
+    if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){
       fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn);
       fts5DataRelease(pPg);
       return;
@@ -6673,7 +6687,7 @@ static void fts5IndexTombstoneAdd(
   ** table pages, then write them all out to disk. */
   if( nHash ){
     int ii = 0;
-    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid);
+    fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid);
     for(ii=0; ii<nHash; ii++){
       i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii);
       fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
@@ -7544,7 +7558,26 @@ static void fts5DecodeFunction(
     }
   }else if( bTomb ){
     u32 nElem  = fts5GetU32(&a[4]);
+    int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8;
+    int nSlot = (n - 8) / szKey;
+    int ii;
     sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
+    if( aBlob[1] ){
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0");
+    }
+    for(ii=0; ii<nSlot; ii++){
+      u64 iVal = 0;
+      if( szKey==4 ){
+        u32 *aSlot = (u32*)&aBlob[8];
+        if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]);
+      }else{
+        u64 *aSlot = (u64*)&aBlob[8];
+        if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]);
+      }
+      if( iVal!=0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal);
+      }
+    }
   }else if( iSegid==0 ){
     if( iRowid==FTS5_AVERAGES_ROWID ){
       fts5DecodeAverages(&rc, &s, a, n);
diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test
new file mode 100644
index 0000000000..c9e93c6600
--- /dev/null
+++ b/ext/fts5/test/fts5contentless2.test
@@ -0,0 +1,152 @@
+# 2023 July 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc vocab {} {
+  list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp
+}
+
+proc document {nToken} {
+  set doc [list]
+  set vocab [vocab]
+  for {set ii 0} {$ii < $nToken} {incr ii} {
+    lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set doc
+}
+db func document document
+
+proc contains {doc token} {
+  expr {[lsearch $doc $token]>=0}
+}
+db func contains contains
+
+proc do_compare_tables_test {tn} {
+  uplevel [list do_test $tn {
+    foreach v [vocab] {
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }]
+      set l2 [execsql { SELECT rowid FROM t2($v) }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 1]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set w "[string range $v 0 0]*"
+      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
+      set l2 [execsql { SELECT rowid FROM t2($w) }]
+      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
+
+      set l1 [execsql { 
+        SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC 
+      }]
+      set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }]
+      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
+    }
+    set {} {}
+  } {}]
+}
+
+proc lshuffle {in} {
+  set L [list]
+  set ret [list]
+  foreach elem $in { lappend L [list [expr rand()] $elem] }
+  foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] }
+  set ret
+}
+
+expr srand(0)
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      doc, prefix=2, content=, contentless_delete=1
+  );
+
+  CREATE TABLE t1(doc);
+  CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
+    DELETE FROM t2 WHERE rowid = old.rowid;
+  END;
+}
+
+set SMALLEST64 -9223372036854775808
+set LARGEST64   9223372036854775807
+
+foreach {tn r1 r2} {
+  1   0               50
+  2   $SMALLEST64     $SMALLEST64+50
+  3   $LARGEST64-50   $LARGEST64
+  4   -50             -1
+} {
+  set r1 [expr $r1]
+  set r2 [expr $r2]
+
+  do_test 1.1.$tn {
+    execsql BEGIN
+    for {set ii $r1} {$ii <= $r2} {incr ii} {
+      execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); }
+    }
+    execsql COMMIT
+  } {}
+}
+do_test 1.2 {
+  db eval { SELECT rowid, doc FROM t1 } {
+    execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) }
+  }
+} {}
+
+foreach {tn rowid} {
+  1  $SMALLEST64
+  2  0
+  3  -5
+  4  -30
+  5  $LARGEST64
+  6  $LARGEST64-1
+} {
+  set rowid [expr $rowid]
+  do_execsql_test 1.3.$tn.1 {
+    DELETE FROM t1 WHERE rowid=$rowid
+  }
+  do_compare_tables_test 1.3.$tn.2
+}
+
+set iTest 1
+foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] {
+  if {($iTest % 50)==0} {
+    execsql { INSERT INTO t2(t2) VALUES('optimize') }
+  }
+  if {($iTest % 5)==0} {
+    execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) }
+  }
+  do_execsql_test 1.4.$iTest.1($r) {
+    DELETE FROM t1 WHERE rowid=$r
+  }
+  do_compare_tables_test 1.4.$iTest.2
+  incr iTest
+}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t1
+} {}
+
+finish_test
+
diff --git a/manifest b/manifest
index b2e058d24d..53072385c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\sFTS5\smemory\sleak\sfix\sfrom\strunk.
-D 2023-07-18T17:43:47.072
+C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables.
+D 2023-07-18T19:52:32.733
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 78c234c47a4870dd8dad7289f457188b8b8c039ce6ecf0e1d8aea0d369857522
+F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -133,6 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
+F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2044,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5f66eb4e2603278dcc9dbfe4bf506cba1aa03180cfb492a0dfc3a8be32cc994b 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a
-R 6df4842f998a408f262f6f4830fd595f
-U drh
-Z 63584765e02f480fe2927dfa6f8c1662
+P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
+R 0b35403fc3d2abc67db6df3c073d3f31
+U dan
+Z c153ae75657e3c69ee3dd162b0a421d3
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 28852ffb55..aa3336730e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
\ No newline at end of file
+0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
\ No newline at end of file

From d1fbaa071bac376206cc009ecdce95b13e131b62 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 19 Jul 2023 18:47:02 +0000
Subject: [PATCH 17/66] Fix various issues with code added to this branch.

FossilOrigin-Name: 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
---
 ext/fts5/fts5_index.c     | 261 ++++++++++++++++++++++++++++----------
 ext/fts5/fts5_storage.c   |  16 ++-
 ext/fts5/test/fts5aa.test |   1 -
 manifest                  |  16 +--
 manifest.uuid             |   2 +-
 5 files changed, 213 insertions(+), 83 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 8804e94bb0..614d6d31c3 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,22 @@
 
 #define FTS5_MAX_LEVEL 64
 
+/*
+** There are two versions of the format used for the structure record:
+**
+**   1. the legacy format, that may be read by all fts5 versions, and
+**
+**   2. the V2 format, which is used by contentless_delete=1 databases.
+**
+** Both begin with a 4-byte "configuration cookie" value. Then, a legacy
+** format structure record contains a varint - the number of levels in
+** the structure. Whereas a V2 structure record contains the constant
+** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a
+** varint has to be at least 16256 to begin with "0xFF". And the default
+** maximum number of levels is 64. 
+**
+** See below for more on structure record formats.
+*/
 #define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01"
 
 /*
@@ -65,7 +81,7 @@
 **
 **     CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
 **
-** , contains the following 5 types of records. See the comments surrounding
+** , contains the following 6 types of records. See the comments surrounding
 ** the FTS5_*_ROWID macros below for a description of how %_data rowids are 
 ** assigned to each fo them.
 **
@@ -73,13 +89,13 @@
 **
 **   The set of segments that make up an index - the index structure - are
 **   recorded in a single record within the %_data table. The record consists
-**   of a single 32-bit configuration cookie value followed by a list of 
-**   SQLite varints. If the FTS table features more than one index (because
-**   there are one or more prefix indexes), it is guaranteed that all share
-**   the same cookie value.
+**   of a single 32-bit configuration cookie value followed by a list of
+**   SQLite varints. 
 **
-**   Immediately following the configuration cookie, the record begins with
-**   three varints:
+**   If the structure record is a V2 record, the configuration cookie is 
+**   followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. 
+**
+**   Next, the record continues with three varints:
 **
 **     + number of levels,
 **     + total number of segments on all levels,
@@ -97,7 +113,8 @@
 **      Then, for V2 structures only:
 **
 **         + lower origin counter value,
-**         + upper origin counter value
+**         + upper origin counter value,
+**         + the number of tombstone hash pages.
 **
 ** 2. The Averages Record:
 **
@@ -214,6 +231,38 @@
 **     * A list of delta-encoded varints - the first rowid on each subsequent
 **       child page. 
 **
+** 6. Tombstone Hash Page
+**
+**   These records are only ever present in contentless_delete=1 tables. 
+**   There are zero or more of these associated with each segment. They
+**   are used to store the tombstone rowids for rows contained in the
+**   associated segments.
+**
+**   The set of nHashPg tombstone hash pages associated with a single 
+**   segment together form a single hash table containing tombstone rowids.
+**   To find the page of the hash on which a key might be stored:
+**
+**       iPg = (rowid % nHashPg)
+**
+**   Then, within page iPg, which has nSlot slots:
+**
+**       iSlot = (rowid / nHashPg) % nSlot
+**
+**   Each tombstone hash page begins with an 8 byte header: 
+**
+**     1-byte:  Key-size (the size in bytes of each slot). Either 4 or 8.
+**     1-byte:  rowid-0-tombstone flag. This flag is only valid on the 
+**              first tombstone hash page for each segment (iPg=0). If set,
+**              the hash table contains rowid 0. If clear, it does not.
+**              Rowid 0 is handled specially.
+**     2-bytes: unused.
+**     4-bytes: Big-endian integer containing number of entries on page.
+**
+**   Following this are nSlot 4 or 8 byte slots (depending on the key-size
+**   in the first byte of the page header). The number of slots may be
+**   determined based on the size of the page record and the key-size:
+**
+**     nSlot = (nByte - 8) / key-size
 */
 
 /*
@@ -247,8 +296,7 @@
 
 #define FTS5_SEGMENT_ROWID(segid, pgno)       fts5_dri(segid, 0, 0, pgno)
 #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
-
-#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid + (1<<16), 0, 0, ipg)
+#define FTS5_TOMBSTONE_ROWID(segid,ipg)       fts5_dri(segid+(1<<16), 0, 0, ipg)
 
 #ifdef SQLITE_DEBUG
 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
@@ -304,7 +352,6 @@ struct Fts5Index {
 
   /* State used by the fts5DataXXX() functions. */
   sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
-  sqlite3_stmt *pReaderOpt;
   sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
@@ -357,7 +404,7 @@ struct Fts5StructureLevel {
 struct Fts5Structure {
   int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
-  u64 nOriginCntr;
+  u64 nOriginCntr;                /* Origin value for next top-level segment */
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[1];   /* Array of nLevel level objects */
@@ -446,6 +493,13 @@ struct Fts5CResult {
 **
 ** iTermIdx:
 **     Index of current term on iTermLeafPgno.
+**
+** apTombstone/nTombstone:
+**     These are used for contentless_delete=1 tables only. When the cursor
+**     is first allocated, the apTombstone[] array is allocated so that it
+**     is large enough for all tombstones hash pages associated with the
+**     segment. The pages themselves are loaded lazily from the database as
+**     they are required.
 */
 struct Fts5SegIter {
   Fts5StructureSegment *pSeg;     /* Segment to iterate through */
@@ -585,6 +639,11 @@ static u16 fts5GetU16(const u8 *aIn){
   return ((u16)aIn[0] << 8) + aIn[1];
 } 
 
+/*
+** The only argument points to a buffer at least 8 bytes in size. This
+** function interprets the first 8 bytes of the buffer as a 64-bit big-endian
+** unsigned integer and returns the result.
+*/
 static u64 fts5GetU64(u8 *a){
   return ((u64)a[0] << 56)
        + ((u64)a[1] << 48)
@@ -596,6 +655,22 @@ static u64 fts5GetU64(u8 *a){
        + ((u64)a[7] << 0);
 }
 
+/*
+** The only argument points to a buffer at least 4 bytes in size. This
+** function interprets the first 4 bytes of the buffer as a 32-bit big-endian
+** unsigned integer and returns the result.
+*/
+static u32 fts5GetU32(const u8 *a){
+  return ((u32)a[0] << 24)
+       + ((u32)a[1] << 16)
+       + ((u32)a[2] << 8)
+       + ((u32)a[3] << 0);
+} 
+
+/*
+** Write iVal, formated as a 64-bit big-endian unsigned integer, to the
+** buffer indicated by the first argument.
+*/
 static void fts5PutU64(u8 *a, u64 iVal){
   a[0] = ((iVal >> 56) & 0xFF);
   a[1] = ((iVal >> 48) & 0xFF);
@@ -607,12 +682,10 @@ static void fts5PutU64(u8 *a, u64 iVal){
   a[7] = ((iVal >>  0) & 0xFF);
 }
 
-static u32 fts5GetU32(const u8 *a){
-  return ((u32)a[0] << 24)
-       + ((u32)a[1] << 16)
-       + ((u32)a[2] << 8)
-       + ((u32)a[3] << 0);
-} 
+/*
+** Write iVal, formated as a 32-bit big-endian unsigned integer, to the
+** buffer indicated by the first argument.
+*/
 static void fts5PutU32(u8 *a, u32 iVal){
   a[0] = ((iVal >> 24) & 0xFF);
   a[1] = ((iVal >> 16) & 0xFF);
@@ -684,7 +757,6 @@ void sqlite3Fts5IndexCloseReader(Fts5Index *p){
   }
 }
 
-
 /*
 ** Retrieve a record from the %_data table.
 **
@@ -799,6 +871,7 @@ static int fts5IndexPrepareStmt(
   return p->rc;
 }
 
+
 /*
 ** INSERT OR REPLACE a record into the %_data table.
 */
@@ -969,12 +1042,13 @@ static int fts5StructureDecode(
   sqlite3_int64 nByte;            /* Bytes of space to allocate at pRet */
   Fts5Structure *pRet = 0;        /* Structure object to return */
   int bStructureV2 = 0;           /* True for FTS5_STRUCTURE_V2 */
-  u64 nOriginCntr = 0;
+  u64 nOriginCntr = 0;            /* Largest origin value seen so far */
 
   /* Grab the cookie value */
   if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
   i = 4;
 
+  /* Check if this is a V2 structure record. Set bStructureV2 if it is. */
   if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){
     i += 4;
     bStructureV2 = 1;
@@ -1819,7 +1893,12 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
-static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
+/*
+** Allocate a tombstone hash page array (pIter->apTombstone) for the 
+** iterator passed as the second argument. If an OOM error occurs, leave
+** an error in the Fts5Index object.
+*/
+static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){
   const int nTomb = pIter->pSeg->nPgTombstone;
   if( nTomb>0 ){
     Fts5Data **apTomb = 0;
@@ -1872,7 +1951,7 @@ static void fts5SegIterInit(
     pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
-    fts5SegIterLoadTombstone(p, pIter);
+    fts5SegIterAllocTombstone(p, pIter);
   }
 }
 
@@ -2574,7 +2653,7 @@ static void fts5SegIterSeekInit(
   }
 
   fts5SegIterSetNext(p, pIter);
-  fts5SegIterLoadTombstone(p, pIter);
+  fts5SegIterAllocTombstone(p, pIter);
 
   /* Either:
   **
@@ -2655,18 +2734,26 @@ static void fts5SegIterHashInit(
   fts5SegIterSetNext(p, pIter);
 }
 
+/*
+** Array ap[] contains n elements. Release each of these elements using
+** fts5DataRelease(). Then free the array itself using sqlite3_free().
+*/
+static void fts5IndexFreeArray(Fts5Data **ap, int n){
+  int ii;
+  for(ii=0; ii<n; ii++){
+    fts5DataRelease(ap[ii]);
+  }
+  sqlite3_free(ap);
+}
+
 /*
 ** Zero the iterator passed as the only argument.
 */
 static void fts5SegIterClear(Fts5SegIter *pIter){
-  int ii;
   fts5BufferFree(&pIter->term);
   fts5DataRelease(pIter->pLeaf);
   fts5DataRelease(pIter->pNextLeaf);
-  for(ii=0; ii<pIter->nTombstone; ii++){
-    fts5DataRelease(pIter->apTombstone[ii]);
-  }
-  sqlite3_free(pIter->apTombstone);
+  fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone);
   fts5DlidxIterFree(pIter->pDlidx);
   sqlite3_free(pIter->aRowidOffset);
   memset(pIter, 0, sizeof(Fts5SegIter));
@@ -3004,16 +3091,21 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
   pIter->iSwitchRowid = pSeg->iRowid;
 }
 
+/*
+** The argument to this macro must be an Fts5Data structure containing a
+** tombstone hash page. This macro returns the key-size of the hash-page.
+*/
 #define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
 /*
-** Query a single tombstone hash table for rowid iRowid. The tombstone hash
-** table is one of nHashTable tables.
+** Query a single tombstone hash table for rowid iRowid. Return true if
+** it is found or false otherwise. The tombstone hash table is one of
+** nHashTable tables.
 */
 static int fts5IndexTombstoneQuery(
-  Fts5Data *pHash,
-  int nHashTable,
-  u64 iRowid
+  Fts5Data *pHash,                /* Hash table page to query */
+  int nHashTable,                 /* Number of pages attached to segment */
+  u64 iRowid                      /* Rowid to query hash for */
 ){
   int szKey = TOMBSTONE_KEYSIZE(pHash);
   int nSlot = (pHash->nn - 8) / szKey;
@@ -3052,6 +3144,8 @@ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
     int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone;
     assert( iPg>=0 );
 
+    /* If tombstone hash page iPg has not yet been loaded from the 
+    ** database, load it now. */
     if( pSeg->apTombstone[iPg]==0 ){
       pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
           FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)
@@ -6088,7 +6182,6 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
     assert( p->pReader==0 );
     fts5StructureInvalidate(p);
     sqlite3_finalize(p->pWriter);
-    sqlite3_finalize(p->pReaderOpt);
     sqlite3_finalize(p->pDeleter);
     sqlite3_finalize(p->pIdxWriter);
     sqlite3_finalize(p->pIdxDeleter);
@@ -6421,6 +6514,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
   return fts5IndexReturn(p);
 }
 
+/*
+** Retrieve the origin value that will be used for the segment currently
+** being accumulated in the in-memory hash table when it is flushed to
+** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to
+** the queried value. Or, if an error occurs, an error code is returned
+** and the final value of (*piOrigin) is undefined.
+*/
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
@@ -6432,7 +6532,15 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
 }
 
 /*
-** Buffer pPg contains a page of a tombstone hash table - one of nPg.
+** Buffer pPg contains a page of a tombstone hash table - one of nPg pages
+** associated with the same segment. This function adds rowid iRowid to
+** the hash table. The caller is required to guarantee that there is at
+** least one free slot on the page.
+**
+** If parameter bForce is false and the hash table is deemed to be full
+** (more than half of the slots are occupied), then non-zero is returned
+** and iRowid not inserted. Or, if bForce is true or if the hash table page
+** is not full, iRowid is inserted and zero returned.
 */
 static int fts5IndexTombstoneAddToPage(
   Fts5Data *pPg, 
@@ -6470,9 +6578,16 @@ static int fts5IndexTombstoneAddToPage(
 }
 
 /*
-** Return 0 if the hash is successfully rebuilt using nOut pages. Or 
-** non-zero if it is not. In this case the caller should retry with a 
-** larger nOut parameter.
+** This function attempts to build a new hash containing all the keys 
+** currently in the tombstone hash table for segment pSeg. The new
+** hash will be stored in the nOut buffers passed in array apOut[].
+** All pages of the new hash use key-size szKey (4 or 8).
+**
+** Return 0 if the hash is successfully rebuilt into the nOut pages. 
+** Or non-zero if it is not (because one page became overfull). In this 
+** case the caller should retry with a larger nOut parameter.
+**
+** Parameter pData1 is page iPg1 of the hash table being rebuilt.
 */
 static int fts5IndexTombstoneRehash(
   Fts5Index *p,
@@ -6539,16 +6654,19 @@ static int fts5IndexTombstoneRehash(
   return res;
 }
 
-static void fts5IndexTombstoneFreeArray(Fts5Data **ap, int n){
-  int ii;
-  for(ii=0; ii<n; ii++){
-    fts5DataRelease(ap[ii]);
-  }
-  sqlite3_free(ap);
-}
-
 /*
 ** This is called to rebuild the hash table belonging to segment pSeg.
+** If parameter pData1 is not NULL, then one page of the existing hash table
+** has already been loaded - pData1, which is page iPg1. The key-size for
+** the new hash table is szKey (4 or 8).
+**
+** If successful, the new hash table is not written to disk. Instead, 
+** output parameter (*pnOut) is set to the number of pages in the new
+** hash table, and (*papOut) to point to an array of buffers containing
+** the new page data.
+**
+** If an error occurs, an error code is left in the Fts5Index object and
+** both output parameters set to 0 before returning.
 */
 static void fts5IndexTombstoneRebuild(
   Fts5Index *p,
@@ -6565,11 +6683,28 @@ static void fts5IndexTombstoneRebuild(
   int nOut = 0;
 
   /* Figure out how many output pages (nOut) and how many slots per 
-  ** page (nSlot).  */
+  ** page (nSlot).  There are three possibilities: 
+  **
+  **   1. The hash table does not yet exist. In this case the new hash
+  **      table will consist of a single page with MINSLOT slots.
+  **
+  **   2. The hash table exists but is currently a single page. In this
+  **      case an attempt is made to grow the page to accommodate the new
+  **      entry. The page is allowed to grow up to nSlotPerPage (see above)
+  **      slots.
+  **
+  **   3. The hash table already consists of more than one page, or of
+  **      a single page already so large that it cannot be grown. In this
+  **      case the new hash consists of (nPg*2+1) pages of nSlotPerPage
+  **      slots each, where nPg is the current number of pages in the 
+  **      hash table.
+  */
   if( pSeg->nPgTombstone==0 ){
+    /* Case 1. */
     nOut = 1;
     nSlot = MINSLOT;
   }else if( pSeg->nPgTombstone==1 ){
+    /* Case 2. */
     int nElem = (int)fts5GetU32(&pData1->p[4]);
     assert( pData1 && iPg1==0 );
 
@@ -6586,6 +6721,7 @@ static void fts5IndexTombstoneRebuild(
     }
   }
   if( nOut==0 ){
+    /* Case 3. */
     nOut = (pSeg->nPgTombstone * 2 + 1);
     nSlot = nSlotPerPage;
   }
@@ -6597,6 +6733,7 @@ static void fts5IndexTombstoneRebuild(
     int szPage = 0;
     Fts5Data **apOut = 0;
 
+    /* Allocate space for the new hash table */
     apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
     szPage = 8 + nSlot*szKey;
     for(ii=0; ii<nOut; ii++){
@@ -6610,10 +6747,11 @@ static void fts5IndexTombstoneRebuild(
       apOut[ii] = pNew;
     }
 
+    /* Rebuild the hash table. */
     res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
     if( res==0 ){
       if( p->rc ){
-        fts5IndexTombstoneFreeArray(apOut, nOut);
+        fts5IndexFreeArray(apOut, nOut);
         apOut = 0;
         nOut = 0;
       }
@@ -6621,9 +6759,11 @@ static void fts5IndexTombstoneRebuild(
       *papOut = apOut;
       break;
     }
+    
+    /* If control flows to here, it was not possible to rebuild the hash
+    ** table. Free all buffers and then try again with more pages. */
     assert( p->rc==SQLITE_OK );
-
-    fts5IndexTombstoneFreeArray(apOut, nOut);
+    fts5IndexFreeArray(apOut, nOut);
     nSlot = nSlotPerPage;
     nOut = nOut*2 + 1;
   }
@@ -6632,22 +6772,6 @@ static void fts5IndexTombstoneRebuild(
 
 /*
 ** Add a tombstone for rowid iRowid to segment pSeg.
-**
-** All tombstones for a single segment are stored in a blob formatted to 
-** contain a hash table. The format is:
-**
-**   * Key-size: 1 byte. Either 4 or 8.
-**   * rowid-0-flag: 1 byte. Either 0 or 1.
-**   * UNUSED: 2 bytes.
-**   * 32-bit big-endian integer. The number of entries currently in the hash
-**     table. This does not change when the rowid-0-flag is set - it only
-**     includes entries in the hash table.
-**
-** Then an array of entries. The number of entries can be calculated based
-** on the size of the blob in the database and the size of the keys as 
-** specified by the first 32-bit field of the hash table header.
-**
-** All values in the hash table are stored as big-endian integers.
 */
 static void fts5IndexTombstoneAdd(
   Fts5Index *p, 
@@ -6697,12 +6821,13 @@ static void fts5IndexTombstoneAdd(
   }
 
   fts5DataRelease(pPg);
-  fts5IndexTombstoneFreeArray(apHash, nHash);
+  fts5IndexFreeArray(apHash, nHash);
 }
 
 /*
 ** Add iRowid to the tombstone list of the segment or segments that contain
-** rows from origin iOrigin.
+** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite
+** error code otherwise.
 */
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index c2f0ca9e44..0a0af9d4b5 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -463,8 +463,14 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
+/*
+** This function is called to process a DELETE on a contentless_delete=1
+** table. It adds the tombstone required to delete the entry with rowid 
+** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
+** an SQLite error code.
+*/
 static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
-  i64 iLoc = 0;
+  i64 iOrigin = 0;
   sqlite3_stmt *pLookup = 0;
   int rc = SQLITE_OK;
 
@@ -472,18 +478,18 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
   assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
 
   /* Look up the origin of the document in the %_docsize table. Store
-  ** this in stack variable iLoc.  */
+  ** this in stack variable iOrigin.  */
   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
   if( rc==SQLITE_OK ){
     sqlite3_bind_int64(pLookup, 1, iDel);
     if( SQLITE_ROW==sqlite3_step(pLookup) ){
-      iLoc = sqlite3_column_int64(pLookup, 1);
+      iOrigin = sqlite3_column_int64(pLookup, 1);
     }
     rc = sqlite3_reset(pLookup);
   }
 
-  if( rc==SQLITE_OK && iLoc!=0 ){
-    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
+  if( rc==SQLITE_OK && iOrigin!=0 ){
+    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
   }
 
   return rc;
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 9ae8bab65f..59ce4f6a1f 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -50,7 +50,6 @@ do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
 }
 
-breakpoint
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
 } {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
diff --git a/manifest b/manifest
index 53072385c5..1b02848d2b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables.
-D 2023-07-18T19:52:32.733
+C Fix\svarious\sissues\swith\scode\sadded\sto\sthis\sbranch.
+D 2023-07-19T18:47:02.359
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,9 +92,9 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48
+F ext/fts5/fts5_index.c 67f3a9fc321cdbf43545980d3ae848f6e8f8b066359f0bc08a6b1e9753c9490f
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
-F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
+F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff
@@ -105,7 +105,7 @@ F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test 2106b14aa665cb7e9832cb40fec5b278c404236ed21fdab756484d5f8739b712
+F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
-R 0b35403fc3d2abc67db6df3c073d3f31
+P 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
+R 221d8a4262bccaa94e920ab3984953a0
 U dan
-Z c153ae75657e3c69ee3dd162b0a421d3
+Z c2c5b391f8fc6df06c3245812d127c81
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index aa3336730e..c6a3d16d89 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
\ No newline at end of file
+8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
\ No newline at end of file

From 0b3791b537dca9bee9cb73546b8657f22637526e Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 16:07:04 +0000
Subject: [PATCH 18/66] Add tests and fixes for the new code on this branch.

FossilOrigin-Name: 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
---
 ext/fts5/fts5_index.c               |  29 +++-----
 ext/fts5/test/fts5contentless2.test |  56 ++++++++++++++
 ext/fts5/test/fts5faultF.test       | 111 ++++++++++++++++++++++++++++
 manifest                            |  15 ++--
 manifest.uuid                       |   2 +-
 5 files changed, 188 insertions(+), 25 deletions(-)
 create mode 100644 ext/fts5/test/fts5faultF.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 614d6d31c3..c1acf4acd9 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2739,11 +2739,13 @@ static void fts5SegIterHashInit(
 ** fts5DataRelease(). Then free the array itself using sqlite3_free().
 */
 static void fts5IndexFreeArray(Fts5Data **ap, int n){
-  int ii;
-  for(ii=0; ii<n; ii++){
-    fts5DataRelease(ap[ii]);
+  if( ap ){
+    int ii;
+    for(ii=0; ii<n; ii++){
+      fts5DataRelease(ap[ii]);
+    }
+    sqlite3_free(ap);
   }
-  sqlite3_free(ap);
 }
 
 /*
@@ -6707,18 +6709,9 @@ static void fts5IndexTombstoneRebuild(
     /* Case 2. */
     int nElem = (int)fts5GetU32(&pData1->p[4]);
     assert( pData1 && iPg1==0 );
-
     nOut = 1;
-    while( nSlot<nElem*2 ){
-      nSlot = nSlot * 2;
-      if( nSlot>nSlotPerPage ){ 
-        nOut = 0; 
-        break;
-      }
-    }
-    if( nOut && nSlot>nSlotPerPage/2 ){
-      nSlot = nSlotPerPage;
-    }
+    nSlot = MAX(nElem*4, MINSLOT);
+    if( nSlot>nSlotPerPage ) nOut = 0; 
   }
   if( nOut==0 ){
     /* Case 3. */
@@ -6743,12 +6736,14 @@ static void fts5IndexTombstoneRebuild(
       if( pNew ){
         pNew->nn = szPage;
         pNew->p = (u8*)&pNew[1];
+        apOut[ii] = pNew;
       }
-      apOut[ii] = pNew;
     }
 
     /* Rebuild the hash table. */
-    res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    if( p->rc==SQLITE_OK ){
+      res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
+    }
     if( res==0 ){
       if( p->rc ){
         fts5IndexFreeArray(apOut, nOut);
diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test
index c9e93c6600..fbb857ab38 100644
--- a/ext/fts5/test/fts5contentless2.test
+++ b/ext/fts5/test/fts5contentless2.test
@@ -148,5 +148,61 @@ do_execsql_test 1.5 {
   SELECT * FROM t1
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    DELETE FROM t2 WHERE rowid=32;
+    DELETE FROM t2 WHERE rowid=64;
+    DELETE FROM t2 WHERE rowid=96;
+    DELETE FROM t2 WHERE rowid=128;
+    DELETE FROM t2 WHERE rowid=160;
+    DELETE FROM t2 WHERE rowid=192;
+  COMMIT;
+}
+
+do_execsql_test 2.2 {
+  SELECT * FROM t2('128');
+} {}
+
+#-------------------------------------------------------------------------
+
+foreach {tn step} {
+  1     3 
+  2     7
+  3     15
+} {
+  set step [expr $step]
+
+  reset_db
+  db func document document
+  do_execsql_test 3.$tn.0 {
+    CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
+    INSERT INTO t2(t2, rank) VALUES('pgsz', 100);
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
+  }
+  do_execsql_test 3.$tn.1 {
+    DELETE FROM t2 WHERE (rowid % $step)==0
+  }
+  do_execsql_test 3.$tn.2 {
+    SELECT * FROM t2( $step * 5 )
+  } {}
+}
+
+
+
 finish_test
 
diff --git a/ext/fts5/test/fts5faultF.test b/ext/fts5/test/fts5faultF.test
new file mode 100644
index 0000000000..96cc2b083f
--- /dev/null
+++ b/ext/fts5/test/fts5faultF.test
@@ -0,0 +1,111 @@
+# 2023 July 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors. Particularly those that may occur
+# when using contentless_delete=1 databases.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultF
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) 
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 
+}
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+  DELETE FROM t1 WHERE rowid IN (2, 4);
+}
+
+do_faultsim_test 2 -prep {
+  sqlite3 db test.db
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    SELECT rowid FROM t1('b c');
+  }
+} -test {
+  faultsim_test_result {0 {1 3}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
+    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
+  COMMIT;
+}
+
+faultsim_save_and_close
+do_faultsim_test 3 -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s;
+}
+
+do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 }
+
+faultsim_save_and_close
+do_faultsim_test 4 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT rowid FROM t1 }
+} -body {
+  execsql { 
+    DELETE FROM t1 WHERE rowid < 100
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 1b02848d2b..658e5239df 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sissues\swith\scode\sadded\sto\sthis\sbranch.
-D 2023-07-19T18:47:02.359
+C Add\stests\sand\sfixes\sfor\sthe\snew\scode\son\sthis\sbranch.
+D 2023-07-20T16:07:04.821
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 67f3a9fc321cdbf43545980d3ae848f6e8f8b066359f0bc08a6b1e9753c9490f
+F ext/fts5/fts5_index.c 69d44e0358de3db791165c1accbbabcb0e41b7942b12d644f3d515c9a5434386
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -133,7 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
-F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a
+F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -161,6 +161,7 @@ F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c
 F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5
 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
+F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -2045,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
-R 221d8a4262bccaa94e920ab3984953a0
+P 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
+R 8705dc9e0fc623331a29c540c6694db8
 U dan
-Z c2c5b391f8fc6df06c3245812d127c81
+Z 5d09d15bf916dffd1a9ed46964991145
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index c6a3d16d89..02e8325d20 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
\ No newline at end of file
+5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
\ No newline at end of file

From 51d7869096bbeefb4f5365f29c78f47287e822df Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 20 Jul 2023 17:45:09 +0000
Subject: [PATCH 19/66] Experimental framework upon which to build a better
 JSON parse structure that supports cached of modified JSON.  All of these
 changes are tentative and subject to change or removal.  Incremental
 check-in.

FossilOrigin-Name: e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 26 ++++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 140124f2f1..9bdb33ae9d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\simprovement\sto\sJSON\sparser\sperformance.
-D 2023-07-19T17:24:36.452
+C Experimental\sframework\supon\swhich\sto\sbuild\sa\sbetter\sJSON\sparse\sstructure\sthat\nsupports\scached\sof\smodified\sJSON.\s\sAll\sof\sthese\schanges\sare\stentative\sand\nsubject\sto\schange\sor\sremoval.\s\sIncremental\scheck-in.
+D 2023-07-20T17:45:09.052
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c e48136fce64e5004b1b8be76624cc9a4ac9ff6ae630a97df187c0ea4b9692d1b
+F src/json.c 888fcc121f7466bdf1518a8ae6adbfa7c4562438d046f1f21c942d625a208143
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd
-R ab82eb710f468841d4023aaae186be2d
+P 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6
+R 086ba0782e2494053078635aa92191a8
 U drh
-Z d9c18383c20eeea0feca70f3dd0775a9
+Z a6562dc595d378e643249c9e30c8bba6
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 4a7e995a3f..654058e2a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6
\ No newline at end of file
+e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 176dcbfdb9..2b309ee32e 100644
--- a/src/json.c
+++ b/src/json.c
@@ -59,6 +59,7 @@ static const char jsonIsSpace[] = {
 typedef struct JsonString JsonString;
 typedef struct JsonNode JsonNode;
 typedef struct JsonParse JsonParse;
+typedef struct JsonTask JsonTask;
 
 /* An instance of this object represents a JSON string
 ** under construction.  Really, this is a generic string accumulator
@@ -74,6 +75,15 @@ struct JsonString {
   char zSpace[100];        /* Initial static space */
 };
 
+/* A deferred cleanup task.  A list of JsonTask objects might be
+** run when the JsonParse object is destroyed.
+*/
+struct JsonTask {
+  JsonTask *pJTNext;       /* Next in a list */
+  void (*xOp)(void*);      /* Routine to run */
+  void *pArg;              /* Argument to xOp() */
+};
+
 /* JSON type values
 */
 #define JSON_NULL     0
@@ -123,6 +133,7 @@ struct JsonNode {
   } u;
 };
 
+
 /* A completely parsed JSON string
 */
 struct JsonParse {
@@ -131,10 +142,13 @@ struct JsonParse {
   JsonNode *aNode;   /* Array of nodes containing the parse */
   const char *zJson; /* Original JSON string */
   u32 *aUp;          /* Index of parent of each node */
+  JsonTask *pClean;  /* Cleanup operations prior to freeing this object */
   u16 iDepth;        /* Nesting depth */
   u8 nErr;           /* Number of errors seen */
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
+  u8 nJPRef;         /* Number of references to this object */
+  u8 bIgnoreEdits;   /* Ignore edit marks during search */
   int nJson;         /* Length of the zJson string in bytes */
   u32 iErr;          /* Error location in zJson[] */
   u32 iHold;         /* Replace cache line with the lowest iHold value */
@@ -531,6 +545,8 @@ static u32 jsonNodeSize(JsonNode *pNode){
 ** delete the JsonParse object itself.
 */
 static void jsonParseReset(JsonParse *pParse){
+  assert( pParse->pClean==0 );
+  assert( pParse->nJPRef<=1 );
   sqlite3_free(pParse->aNode);
   pParse->aNode = 0;
   pParse->nNode = 0;
@@ -543,6 +559,16 @@ static void jsonParseReset(JsonParse *pParse){
 ** Free a JsonParse object that was obtained from sqlite3_malloc().
 */
 static void jsonParseFree(JsonParse *pParse){
+  if( pParse->nJPRef>1 ){
+    pParse->nJPRef--;
+    return;
+  }
+  while( pParse->pClean ){
+    JsonTask *pTask = pParse->pClean;
+    pParse->pClean = pTask->pJTNext;
+    pTask->xOp(pTask->pArg);
+    sqlite3_free(pTask);
+  }
   jsonParseReset(pParse);
   sqlite3_free(pParse);
 }

From d6f5aa824ec77fe59ce9874dc5b26e595f2bfb4f Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 20:09:26 +0000
Subject: [PATCH 20/66] Avoid an infinite loop that could be entered when
 dealing with corrupt fts5 tombstone hash pages.

FossilOrigin-Name: 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
---
 ext/fts5/fts5_index.c               | 24 ++++++---
 ext/fts5/test/fts5contentless3.test | 76 +++++++++++++++++++++++++++++
 manifest                            | 13 ++---
 manifest.uuid                       |  2 +-
 4 files changed, 101 insertions(+), 14 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless3.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index c1acf4acd9..99ff4c3b5f 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3109,9 +3109,10 @@ static int fts5IndexTombstoneQuery(
   int nHashTable,                 /* Number of pages attached to segment */
   u64 iRowid                      /* Rowid to query hash for */
 ){
-  int szKey = TOMBSTONE_KEYSIZE(pHash);
-  int nSlot = (pHash->nn - 8) / szKey;
+  const int szKey = TOMBSTONE_KEYSIZE(pHash);
+  const int nSlot = (pHash->nn - 8) / szKey;
   int iSlot = (iRowid / nHashTable) % nSlot;
+  int nCollide = nSlot;
 
   if( iRowid==0 ){
     return pHash->p[1];
@@ -3119,12 +3120,14 @@ static int fts5IndexTombstoneQuery(
     u32 *aSlot = (u32*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      if( nCollide--==0 ) break;
       iSlot = (iSlot+1)%nSlot;
     }
   }else{
     u64 *aSlot = (u64*)&pHash->p[8];
     while( aSlot[iSlot] ){
       if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
+      if( nCollide--==0 ) break;
       iSlot = (iSlot+1)%nSlot;
     }
   }
@@ -6550,10 +6553,11 @@ static int fts5IndexTombstoneAddToPage(
   int nPg, 
   u64 iRowid
 ){
-  int szKey = TOMBSTONE_KEYSIZE(pPg);
-  int nSlot = (pPg->nn - 8) / szKey;
+  const int szKey = TOMBSTONE_KEYSIZE(pPg);
+  const int nSlot = (pPg->nn - 8) / szKey;
+  const int nElem = fts5GetU32(&pPg->p[4]);
   int iSlot = (iRowid / nPg) % nSlot;
-  int nElem = fts5GetU32(&pPg->p[4]);
+  int nCollide = nSlot;
 
   if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
   if( iRowid==0 ){
@@ -6568,11 +6572,17 @@ static int fts5IndexTombstoneAddToPage(
   fts5PutU32(&pPg->p[4], nElem+1);
   if( szKey==4 ){
     u32 *aSlot = (u32*)&pPg->p[8];
-    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    while( aSlot[iSlot] ){
+      iSlot = (iSlot + 1) % nSlot;
+      if( nCollide--==0 ) return 0;
+    }
     fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
   }else{
     u64 *aSlot = (u64*)&pPg->p[8];
-    while( aSlot[iSlot] ) iSlot = (iSlot + 1) % nSlot;
+    while( aSlot[iSlot] ){
+      iSlot = (iSlot + 1) % nSlot;
+      if( nCollide--==0 ) return 0;
+    }
     fts5PutU64((u8*)&aSlot[iSlot], iRowid);
   }
 
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
new file mode 100644
index 0000000000..34b7b23da4
--- /dev/null
+++ b/ext/fts5/test/fts5contentless3.test
@@ -0,0 +1,76 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO ft VALUES('one one one');
+    INSERT INTO ft VALUES('two two two');
+    INSERT INTO ft VALUES('three three three');
+    INSERT INTO ft VALUES('four four four');
+    INSERT INTO ft VALUES('five five five');
+  COMMIT;
+
+  DELETE FROM ft WHERE rowid=3;
+}
+
+proc myhex {hex} { binary decode hex $hex }
+db func myhex myhex
+
+do_execsql_test 1.1 {
+  UPDATE ft_data SET block = 
+    myhex('04000000 00000001'                   ||
+          '01020304 01020304 01020304 01020304' ||
+          '01020304 01020304 01020304 01020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM ft WHERE rowid=1
+}
+
+do_execsql_test 1.3 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.3 {
+  UPDATE ft_data SET block = 
+    myhex('08000000 00000001'                                               ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304' ||
+      '0000000001020304 0000000001020304 0000000001020304 0000000001020304'
+    )
+  WHERE id = (SELECT max(id) FROM ft_data);
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.5 {
+  DELETE FROM ft WHERE rowid=4
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 658e5239df..08dc29ea64 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sand\sfixes\sfor\sthe\snew\scode\son\sthis\sbranch.
-D 2023-07-20T16:07:04.821
+C Avoid\san\sinfinite\sloop\sthat\scould\sbe\sentered\swhen\sdealing\swith\scorrupt\sfts5\stombstone\shash\spages.
+D 2023-07-20T20:09:26.394
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 69d44e0358de3db791165c1accbbabcb0e41b7942b12d644f3d515c9a5434386
+F ext/fts5/fts5_index.c 5b5b9944ef97bf9aa0726fdbd82552d488b6fcee32e086e6d32ea9934e1809e2
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,6 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
+F ext/fts5/test/fts5contentless3.test db92625a2e3f9e5ba047ce8ded58c825affce45cc711ed6d7c96fe54b95894a1
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2046,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8d09011fa2c6ae9cc88e1766f9aad4578efbf9e0e311b8c6efdffe7a3f88f923
-R 8705dc9e0fc623331a29c540c6694db8
+P 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
+R a2bc7a639e489f4200810e70aeab413a
 U dan
-Z 5d09d15bf916dffd1a9ed46964991145
+Z 36cb6c12a15fce5364a14d77012ed267
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 02e8325d20..87b279a730 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
\ No newline at end of file
+69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
\ No newline at end of file

From 3a51f8c3078bacc8c8059f985d511042f8561da3 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 20 Jul 2023 20:29:56 +0000
Subject: [PATCH 21/66] Fix some divide-by-zero errors that could occur when
 handling corrupt tombstone hash records.

FossilOrigin-Name: 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
---
 ext/fts5/fts5_index.c               |  7 +++++--
 ext/fts5/test/fts5contentless3.test | 20 ++++++++++++++++++++
 manifest                            | 14 +++++++-------
 manifest.uuid                       |  2 +-
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 99ff4c3b5f..d5235b2496 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3099,6 +3099,9 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){
 */
 #define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8)
 
+#define TOMBSTONE_NSLOT(pPg)   \
+  ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1)
+
 /*
 ** Query a single tombstone hash table for rowid iRowid. Return true if
 ** it is found or false otherwise. The tombstone hash table is one of
@@ -3110,7 +3113,7 @@ static int fts5IndexTombstoneQuery(
   u64 iRowid                      /* Rowid to query hash for */
 ){
   const int szKey = TOMBSTONE_KEYSIZE(pHash);
-  const int nSlot = (pHash->nn - 8) / szKey;
+  const int nSlot = TOMBSTONE_NSLOT(pHash);
   int iSlot = (iRowid / nHashTable) % nSlot;
   int nCollide = nSlot;
 
@@ -6554,7 +6557,7 @@ static int fts5IndexTombstoneAddToPage(
   u64 iRowid
 ){
   const int szKey = TOMBSTONE_KEYSIZE(pPg);
-  const int nSlot = (pPg->nn - 8) / szKey;
+  const int nSlot = TOMBSTONE_NSLOT(pPg);
   const int nElem = fts5GetU32(&pPg->p[4]);
   int iSlot = (iRowid / nPg) % nSlot;
   int nCollide = nSlot;
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index 34b7b23da4..ec5477e676 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -29,6 +29,10 @@ do_execsql_test 1.0 {
     INSERT INTO ft VALUES('three three three');
     INSERT INTO ft VALUES('four four four');
     INSERT INTO ft VALUES('five five five');
+    INSERT INTO ft VALUES('six six six');
+    INSERT INTO ft VALUES('seven seven seven');
+    INSERT INTO ft VALUES('eight eight eight');
+    INSERT INTO ft VALUES('nine nine nine');
   COMMIT;
 
   DELETE FROM ft WHERE rowid=3;
@@ -71,6 +75,22 @@ do_execsql_test 1.5 {
   DELETE FROM ft WHERE rowid=4
 }
 
+do_execsql_test 1.6 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.7 {
+  SELECT rowid FROM ft('two');
+} {2}
+
+do_execsql_test 1.8 {
+  UPDATE ft_data SET block = myhex('04000000 00000000')
+    WHERE id = (SELECT max(id) FROM ft_data);
+}
+do_execsql_test 1.9 {
+  DELETE FROM ft WHERE rowid=8
+} {}
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 08dc29ea64..9e392f9f22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\san\sinfinite\sloop\sthat\scould\sbe\sentered\swhen\sdealing\swith\scorrupt\sfts5\stombstone\shash\spages.
-D 2023-07-20T20:09:26.394
+C Fix\ssome\sdivide-by-zero\serrors\sthat\scould\soccur\swhen\shandling\scorrupt\stombstone\shash\srecords.
+D 2023-07-20T20:29:56.954
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 5b5b9944ef97bf9aa0726fdbd82552d488b6fcee32e086e6d32ea9934e1809e2
+F ext/fts5/fts5_index.c db653198e9bb76d285df246df1f14ebf3f1cfcbca0605f2bf272026f52021011
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test db92625a2e3f9e5ba047ce8ded58c825affce45cc711ed6d7c96fe54b95894a1
+F ext/fts5/test/fts5contentless3.test b773267d7d5434d0a503b9d73de240f5769efb0e3576e133dd76eccb318116bc
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5aac50e92e956b15367c75c20c17bc1c75e84e2752bfffe4ad0a266cb9bd3b8a
-R a2bc7a639e489f4200810e70aeab413a
+P 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
+R 716993a5a4c359e4b4afccc9c55270f5
 U dan
-Z 36cb6c12a15fce5364a14d77012ed267
+Z f01327fd76f39e2a6bc2b85c23603eff
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 87b279a730..a6af9f94f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
\ No newline at end of file
+7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
\ No newline at end of file

From c8f4fec2dea3c2d89eaa38bf5eb7dd0de8b02e37 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 21 Jul 2023 11:09:53 +0000
Subject: [PATCH 22/66] More infrastructure changes towards improving JSON
 cache performance. Incremental check-in.

FossilOrigin-Name: 1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707
---
 manifest      | 12 +++++-----
 manifest.uuid |  2 +-
 src/json.c    | 61 +++++++++++++++++++++++++++++----------------------
 3 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index 9bdb33ae9d..715018c361 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\sframework\supon\swhich\sto\sbuild\sa\sbetter\sJSON\sparse\sstructure\sthat\nsupports\scached\sof\smodified\sJSON.\s\sAll\sof\sthese\schanges\sare\stentative\sand\nsubject\sto\schange\sor\sremoval.\s\sIncremental\scheck-in.
-D 2023-07-20T17:45:09.052
+C More\sinfrastructure\schanges\stowards\simproving\sJSON\scache\sperformance.\nIncremental\scheck-in.
+D 2023-07-21T11:09:53.138
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 888fcc121f7466bdf1518a8ae6adbfa7c4562438d046f1f21c942d625a208143
+F src/json.c 3bdc8916ee1acf47f63373278b7913befea89fc8f4bfc0299ae9a2b34b35ee07
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6
-R 086ba0782e2494053078635aa92191a8
+P e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d
+R 35c799f493c52a97264947f70c69a12e
 U drh
-Z a6562dc595d378e643249c9e30c8bba6
+Z 7b5e59d8c6f3a8847c09c081e8b2a5bc
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 654058e2a7..b1ca498e22 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d
\ No newline at end of file
+1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 2b309ee32e..b7c94a8a06 100644
--- a/src/json.c
+++ b/src/json.c
@@ -72,6 +72,7 @@ struct JsonString {
   u64 nUsed;               /* Bytes of zBuf[] currently used */
   u8 bStatic;              /* True if zBuf is static space */
   u8 bErr;                 /* True if an error has been encountered */
+  u8 bOrig;                /* Ignore edits when rendering JSON */
   char zSpace[100];        /* Initial static space */
 };
 
@@ -148,7 +149,7 @@ struct JsonParse {
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   u8 nJPRef;         /* Number of references to this object */
-  u8 bIgnoreEdits;   /* Ignore edit marks during search */
+  u8 bOrig;          /* Ignore edit marks during search */
   int nJson;         /* Length of the zJson string in bytes */
   u32 iErr;          /* Error location in zJson[] */
   u32 iHold;         /* Replace cache line with the lowest iHold value */
@@ -174,6 +175,7 @@ static void jsonZero(JsonString *p){
   p->nAlloc = sizeof(p->zSpace);
   p->nUsed = 0;
   p->bStatic = 1;
+  p->bOrig = 0;
 }
 
 /* Initialize the JsonString object
@@ -584,7 +586,7 @@ static void jsonRenderNode(
   sqlite3_value **aReplace       /* Replacement values */
 ){
   assert( pNode!=0 );
-  if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){
+  if( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) && !pOut->bOrig ){
     if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){
       assert( pNode->eU==4 );
       jsonAppendValue(pOut, aReplace[pNode->u.iReplace]);
@@ -650,13 +652,13 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '[');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){
+          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){
             jsonAppendSeparator(pOut);
             jsonRenderNode(&pNode[j], pOut, aReplace);
           }
           j += jsonNodeSize(&pNode[j]);
         }
-        if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
+        if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break;
         assert( pNode->eU==2 );
         pNode = &pNode[pNode->u.iAppend];
         j = 1;
@@ -669,7 +671,7 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '{');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){
+          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){
             jsonAppendSeparator(pOut);
             jsonRenderNode(&pNode[j], pOut, aReplace);
             jsonAppendChar(pOut, ':');
@@ -677,7 +679,7 @@ static void jsonRenderNode(
           }
           j += 1 + jsonNodeSize(&pNode[j+1]);
         }
-        if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
+        if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break;
         assert( pNode->eU==2 );
         pNode = &pNode[pNode->u.iAppend];
         j = 1;
@@ -694,10 +696,12 @@ static void jsonRenderNode(
 static void jsonReturnJson(
   JsonNode *pNode,            /* Node to return */
   sqlite3_context *pCtx,      /* Return value for this function */
-  sqlite3_value **aReplace    /* Array of replacement values */
+  sqlite3_value **aReplace,   /* Array of replacement values */
+  int bOrig                   /* Ignore edits if true */
 ){
   JsonString s;
   jsonInit(&s, pCtx);
+  s.bOrig = bOrig!=0;
   jsonRenderNode(pNode, &s, aReplace);
   jsonResult(&s);
   sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
@@ -740,7 +744,8 @@ static u32 jsonHexToInt4(const char *z){
 static void jsonReturn(
   JsonNode *pNode,            /* Node to return */
   sqlite3_context *pCtx,      /* Return value for this function */
-  sqlite3_value **aReplace    /* Array of replacement values */
+  sqlite3_value **aReplace,   /* Array of replacement values */
+  int bOrig                   /* Ignore edits if true */
 ){
   switch( pNode->eType ){
     default: {
@@ -887,7 +892,7 @@ static void jsonReturn(
     }
     case JSON_ARRAY:
     case JSON_OBJECT: {
-      jsonReturnJson(pNode, pCtx, aReplace);
+      jsonReturnJson(pNode, pCtx, aReplace, bOrig);
       break;
     }
   }
@@ -1797,7 +1802,7 @@ static JsonNode *jsonLookupStep(
   const char *zKey;
   JsonNode *pRoot = &pParse->aNode[iRoot];
   if( zPath[0]==0 ) return pRoot;
-  if( pRoot->jnFlags & JNODE_REPLACE ) return 0;
+  if( (pRoot->jnFlags & JNODE_REPLACE)!=0 && !pParse->bOrig ) return 0;
   if( zPath[0]=='.' ){
     if( pRoot->eType!=JSON_OBJECT ) return 0;
     zPath++;
@@ -1830,7 +1835,7 @@ static JsonNode *jsonLookupStep(
         j++;
         j += jsonNodeSize(&pRoot[j]);
       }
-      if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
+      if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
       assert( pRoot->eU==2 );
       iRoot += pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -1839,6 +1844,7 @@ static JsonNode *jsonLookupStep(
     if( pApnd ){
       u32 iStart, iLabel;
       JsonNode *pNode;
+      assert( !pParse->bOrig );
       iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
       iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
       zPath += i;
@@ -1868,10 +1874,10 @@ static JsonNode *jsonLookupStep(
         if( pRoot->eType!=JSON_ARRAY ) return 0;
         for(;;){
           while( j<=pBase->n ){
-            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++;
+            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i++;
             j += jsonNodeSize(&pBase[j]);
           }
-          if( (pBase->jnFlags & JNODE_APPEND)==0 ) break;
+          if( (pBase->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
           assert( pBase->eU==2 );
           iBase += pBase->u.iAppend;
           pBase = &pParse->aNode[iBase];
@@ -1901,11 +1907,13 @@ static JsonNode *jsonLookupStep(
     zPath += j + 1;
     j = 1;
     for(;;){
-      while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){
-        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--;
+      while( j<=pRoot->n
+          && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE) && !pParse->bOrig))
+      ){
+        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i--;
         j += jsonNodeSize(&pRoot[j]);
       }
-      if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
+      if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
       assert( pRoot->eU==2 );
       iRoot += pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -1917,6 +1925,7 @@ static JsonNode *jsonLookupStep(
     if( i==0 && pApnd ){
       u32 iStart;
       JsonNode *pNode;
+      assert( !pParse->bOrig );
       iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0);
       pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr);
       if( pParse->oom ) return 0;
@@ -2269,15 +2278,15 @@ static void jsonExtractFunc(
       }
       if( pNode ){
         if( flags & JSON_JSON ){
-          jsonReturnJson(pNode, ctx, 0);
+          jsonReturnJson(pNode, ctx, 0, 1);
         }else{
-          jsonReturn(pNode, ctx, 0);
+          jsonReturn(pNode, ctx, 0, 1);
           sqlite3_result_subtype(ctx, 0);
         }
       }
     }else{
       pNode = jsonLookup(p, zPath, 0, ctx);
-      if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0);
+      if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0, 1);
     }
   }else{
     /* Two or more PATH arguments results in a JSON array with each
@@ -2405,7 +2414,7 @@ static void jsonPatchFunc(
   pResult = jsonMergePatch(&x, 0, y.aNode);
   assert( pResult!=0 || x.oom );
   if( pResult ){
-    jsonReturnJson(pResult, ctx, 0);
+    jsonReturnJson(pResult, ctx, 0, 0);
   }else{
     sqlite3_result_error_nomem(ctx);
   }
@@ -2482,7 +2491,7 @@ static void jsonRemoveFunc(
     if( pNode ) pNode->jnFlags |= JNODE_REMOVE;
   }
   if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){
-    jsonReturnJson(x.aNode, ctx, 0);
+    jsonReturnJson(x.aNode, ctx, 0, 0);
   }
 remove_done:
   jsonParseReset(&x);
@@ -2527,7 +2536,7 @@ static void jsonReplaceFunc(
     assert( x.aNode[0].eU==4 );
     sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
   }else{
-    jsonReturnJson(x.aNode, ctx, argv);
+    jsonReturnJson(x.aNode, ctx, argv, 0);
   }
 replace_err:
   jsonParseReset(&x);
@@ -2586,7 +2595,7 @@ static void jsonSetFunc(
     assert( x.aNode[0].eU==4 );
     sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
   }else{
-    jsonReturnJson(x.aNode, ctx, argv);
+    jsonReturnJson(x.aNode, ctx, argv, 0);
   }
 jsonSetDone:
   jsonParseReset(&x);
@@ -3095,7 +3104,7 @@ static int jsonEachColumn(
     case JEACH_KEY: {
       if( p->i==0 ) break;
       if( p->eType==JSON_OBJECT ){
-        jsonReturn(pThis, ctx, 0);
+        jsonReturn(pThis, ctx, 0, 0);
       }else if( p->eType==JSON_ARRAY ){
         u32 iKey;
         if( p->bRecursive ){
@@ -3111,7 +3120,7 @@ static int jsonEachColumn(
     }
     case JEACH_VALUE: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
-      jsonReturn(pThis, ctx, 0);
+      jsonReturn(pThis, ctx, 0, 0);
       break;
     }
     case JEACH_TYPE: {
@@ -3122,7 +3131,7 @@ static int jsonEachColumn(
     case JEACH_ATOM: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
       if( pThis->eType>=JSON_ARRAY ) break;
-      jsonReturn(pThis, ctx, 0);
+      jsonReturn(pThis, ctx, 0, 0);
       break;
     }
     case JEACH_ID: {

From 4e0c157d02c1116006701982a6fde8ab2a502eaf Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 21 Jul 2023 19:33:35 +0000
Subject: [PATCH 23/66] Ensure the fts5 'optimize' command correctly rewrites
 any index that consists of a single segment and one or more tombstone hash
 pages.

FossilOrigin-Name: f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
---
 ext/fts5/fts5_index.c               | 19 +++++++-----
 ext/fts5/test/fts5contentless3.test | 47 +++++++++++++++++++++++++++++
 manifest                            | 14 ++++-----
 manifest.uuid                       |  2 +-
 4 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index d5235b2496..9488c687e5 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5483,17 +5483,22 @@ static Fts5Structure *fts5IndexOptimizeStruct(
   /* Figure out if this structure requires optimization. A structure does
   ** not require optimization if either:
   **
-  **  + it consists of fewer than two segments, or 
-  **  + all segments are on the same level, or
-  **  + all segments except one are currently inputs to a merge operation.
+  **  1. it consists of fewer than two segments, or 
+  **  2. all segments are on the same level, or
+  **  3. all segments except one are currently inputs to a merge operation.
   **
-  ** In the first case, return NULL. In the second, increment the ref-count
-  ** on *pStruct and return a copy of the pointer to it.
+  ** In the first case, if there are no tombstone hash pages, return NULL. In
+  ** the second, increment the ref-count on *pStruct and return a copy of the
+  ** pointer to it.
   */
-  if( nSeg<2 ) return 0;
+  if( nSeg==0 ) return 0;
   for(i=0; i<pStruct->nLevel; i++){
     int nThis = pStruct->aLevel[i].nSeg;
-    if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
+    int nMerge = pStruct->aLevel[i].nMerge;
+    if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){
+      if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){
+        return 0;
+      }
       fts5StructureRef(pStruct);
       return pStruct;
     }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index ec5477e676..a478f795f0 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -91,6 +91,53 @@ do_execsql_test 1.9 {
   DELETE FROM ft WHERE rowid=8
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+    INSERT INTO ft VALUES('one one one');
+    INSERT INTO ft VALUES('two two two');
+    INSERT INTO ft VALUES('three three three');
+    INSERT INTO ft VALUES('four four four');
+    INSERT INTO ft VALUES('five five five');
+    INSERT INTO ft VALUES('six six six');
+    INSERT INTO ft VALUES('seven seven seven');
+    INSERT INTO ft VALUES('eight eight eight');
+    INSERT INTO ft VALUES('nine nine nine');
+}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.2 {
+  SELECT count(*) FROM ft_data
+} {3}
+do_execsql_test 2.3 {
+  DELETE FROM ft WHERE rowid=5
+}
+do_execsql_test 2.4 {
+  SELECT count(*) FROM ft_data
+} {4}
+
+# Check that an 'optimize' works (rewrites the index) if there is a single
+# segment with one or more tombstone hash pages.
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+do_execsql_test 2.6 {
+  SELECT count(*) FROM ft_data
+} {3}
+
+# Check that an 'optimize' is a no-op if there is a single segment
+# and no tombstone hash pages.
+do_execsql_test 2.7 {
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+
+
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 9e392f9f22..7e0682938e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sdivide-by-zero\serrors\sthat\scould\soccur\swhen\shandling\scorrupt\stombstone\shash\srecords.
-D 2023-07-20T20:29:56.954
+C Ensure\sthe\sfts5\s'optimize'\scommand\scorrectly\srewrites\sany\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\sand\sone\sor\smore\stombstone\shash\spages.
+D 2023-07-21T19:33:35.722
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c db653198e9bb76d285df246df1f14ebf3f1cfcbca0605f2bf272026f52021011
+F ext/fts5/fts5_index.c 3817749456b9c4e59a172cfef8cbf8d7b02e3d11d03c85da749dce0edc1af0ca
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test b773267d7d5434d0a503b9d73de240f5769efb0e3576e133dd76eccb318116bc
+F ext/fts5/test/fts5contentless3.test bd1521137ecda248ffc27f126eb07442c03a9c29ed9aed01fc405ae86b5e62db
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 69ce2ce035279f2a00c2238187cf4d2a9092c3410f5900e4613fe4e46311169e
-R 716993a5a4c359e4b4afccc9c55270f5
+P 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
+R 6e42e7eceee582bdc020aa8182af6a3a
 U dan
-Z f01327fd76f39e2a6bc2b85c23603eff
+Z f1b6971f3b1dbfa08900d3843d873627
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a6af9f94f9..2830d8dc7f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
\ No newline at end of file
+f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
\ No newline at end of file

From 330e36c2c60ab97f297e5d6b2cca4eb998f4dcea Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Fri, 21 Jul 2023 21:10:33 +0000
Subject: [PATCH 24/66] Fix fts5 incremental optimization so that it too can
 handle an index that consists of a single segment with one or more tombstone
 hash pages.

FossilOrigin-Name: e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
---
 ext/fts5/fts5_index.c               | 18 +++++---
 ext/fts5/test/fts5contentless3.test | 72 +++++++++++++++++++++++++----
 manifest                            | 14 +++---
 manifest.uuid                       |  2 +-
 4 files changed, 83 insertions(+), 23 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9488c687e5..9bd23a41f4 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4759,6 +4759,8 @@ static int fts5IndexMerge(
     if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
       fts5StructurePromote(p, iBestLvl+1, pStruct);
     }
+
+    if( nMin==1 ) nMin = 2;
   }
   *ppStruct = pStruct;
   return bRet;
@@ -5582,7 +5584,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
       Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
       fts5StructureRelease(pStruct);
       pStruct = pNew;
-      nMin = 2;
+      nMin = 1;
       nMerge = nMerge*-1;
     }
     if( pStruct && pStruct->nLevel ){
@@ -6698,8 +6700,8 @@ static void fts5IndexTombstoneRebuild(
   Fts5Data ***papOut              /* OUT: Output hash pages */
 ){
   const int MINSLOT = 32;
-  int nSlotPerPage = (p->pConfig->pgsz - 8) / szKey;
-  int nSlot = MINSLOT;            /* Number of slots in each output page */
+  int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey);
+  int nSlot = 0;                  /* Number of slots in each output page */
   int nOut = 0;
 
   /* Figure out how many output pages (nOut) and how many slots per 
@@ -6745,6 +6747,7 @@ static void fts5IndexTombstoneRebuild(
     Fts5Data **apOut = 0;
 
     /* Allocate space for the new hash table */
+    assert( nSlot>=MINSLOT );
     apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
     szPage = 8 + nSlot*szKey;
     for(ii=0; ii<nOut; ii++){
@@ -7942,8 +7945,8 @@ static int fts5structConnectMethod(
 
   rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE xyz("
-          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2,"
-      "struct HIDDEN);"
+          "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
+          "npgtombstone, struct HIDDEN);"
   );
   if( rc==SQLITE_OK ){
     pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
@@ -7970,7 +7973,7 @@ static int fts5structBestIndexMethod(
   pIdxInfo->idxNum = 0;
   for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
     if( p->usable==0 ) continue;
-    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==8 ){
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==9 ){
       rc = SQLITE_OK;
       pIdxInfo->aConstraintUsage[i].omit = 1;
       pIdxInfo->aConstraintUsage[i].argvIndex = 1;
@@ -8089,6 +8092,9 @@ static int fts5structColumnMethod(
     case 7: /* loc2 */
       sqlite3_result_int(ctx, pSeg->iOrigin2);
       break;
+    case 8: /* npgtombstone */
+      sqlite3_result_int(ctx, pSeg->nPgTombstone);
+      break;
   }
   return SQLITE_OK;
 }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index a478f795f0..6316628c55 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -95,15 +95,15 @@ do_execsql_test 1.9 {
 reset_db
 do_execsql_test 2.0 {
   CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
-    INSERT INTO ft VALUES('one one one');
-    INSERT INTO ft VALUES('two two two');
-    INSERT INTO ft VALUES('three three three');
-    INSERT INTO ft VALUES('four four four');
-    INSERT INTO ft VALUES('five five five');
-    INSERT INTO ft VALUES('six six six');
-    INSERT INTO ft VALUES('seven seven seven');
-    INSERT INTO ft VALUES('eight eight eight');
-    INSERT INTO ft VALUES('nine nine nine');
+  INSERT INTO ft VALUES('one one one');
+  INSERT INTO ft VALUES('two two two');
+  INSERT INTO ft VALUES('three three three');
+  INSERT INTO ft VALUES('four four four');
+  INSERT INTO ft VALUES('five five five');
+  INSERT INTO ft VALUES('six six six');
+  INSERT INTO ft VALUES('seven seven seven');
+  INSERT INTO ft VALUES('eight eight eight');
+  INSERT INTO ft VALUES('nine nine nine');
 }
 
 do_execsql_test 2.1 {
@@ -135,7 +135,61 @@ do_execsql_test 2.7 {
   SELECT rowid FROM ft_data;
 } [db eval {SELECT rowid FROM ft_data}]
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
 
+do_execsql_test 3.1 {
+  SELECT count(*) FROM ft_data
+} {200}
+
+do_execsql_test 3.2 {
+  DELETE FROM ft WHERE (rowid % 50)==0;
+  SELECT count(*) FROM ft_data;
+} {203}
+
+do_execsql_test 3.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 500);
+  SELECT rowid FROM ft_data;
+} [db eval {SELECT rowid FROM ft_data}]
+
+do_execsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('merge', -1000);
+  SELECT count(*) FROM ft_data;
+} {197}
+
+do_execsql_test 3.5 {
+  DELETE FROM ft WHERE (rowid % 50)==1;
+  SELECT count(*) FROM ft_data;
+} {200}
+
+do_execsql_test 3.6 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {1 0 3}
+
+do_test 3.6 {
+  while 1 {
+    set nChange [db total_changes]
+    execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) }
+    if {([db total_changes] - $nChange)<2} break
+  }
+} {}
+
+do_execsql_test 3.7 {
+  SELECT level, segment, npgtombstone FROM fts5_structure(
+    (SELECT block FROM ft_data WHERE id=10)
+  )
+} {2 0 0}
 
 
 
diff --git a/manifest b/manifest
index 7e0682938e..3e4343c8ea 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthe\sfts5\s'optimize'\scommand\scorrectly\srewrites\sany\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\sand\sone\sor\smore\stombstone\shash\spages.
-D 2023-07-21T19:33:35.722
+C Fix\sfts5\sincremental\soptimization\sso\sthat\sit\stoo\scan\shandle\san\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\swith\sone\sor\smore\stombstone\shash\spages.
+D 2023-07-21T21:10:33.579
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 3817749456b9c4e59a172cfef8cbf8d7b02e3d11d03c85da749dce0edc1af0ca
+F ext/fts5/fts5_index.c e500a5d33ae312c2ebab91123f0ef5f9bbc3eb555252c7d40ba4d8688780c7ca
 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test bd1521137ecda248ffc27f126eb07442c03a9c29ed9aed01fc405ae86b5e62db
+F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7567ca0676f0d45026f5cd4f3fbcd09119c2eaab8ec1711499609c16c452b5e4
-R 6e42e7eceee582bdc020aa8182af6a3a
+P f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
+R 8117856750f55ecca6e7594df2bcd37e
 U dan
-Z f1b6971f3b1dbfa08900d3843d873627
+Z 3675210f196caeb196ef6ee2658c36e9
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 2830d8dc7f..92b4b794ac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
\ No newline at end of file
+e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
\ No newline at end of file

From 2159292ce07565d6791395ac2ff9205e14a7b16a Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Sat, 22 Jul 2023 19:47:46 +0000
Subject: [PATCH 25/66] Integrate contentless delete with auto-merge.

FossilOrigin-Name: 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
---
 ext/fts5/fts5Int.h                  |   6 +
 ext/fts5/fts5_hash.c                |   7 +
 ext/fts5/fts5_index.c               | 358 +++++++++++++++-------------
 ext/fts5/fts5_main.c                |   3 +-
 ext/fts5/test/fts5contentless4.test |  61 +++++
 manifest                            |  19 +-
 manifest.uuid                       |   2 +-
 7 files changed, 282 insertions(+), 174 deletions(-)
 create mode 100644 ext/fts5/test/fts5contentless4.test

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index da2f90f230..24417483c1 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -623,6 +623,11 @@ int sqlite3Fts5HashWrite(
 */
 void sqlite3Fts5HashClear(Fts5Hash*);
 
+/*
+** Return true if the hash is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash*);
+
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
   int nPre,
@@ -644,6 +649,7 @@ void sqlite3Fts5HashScanEntry(Fts5Hash *,
 );
 
 
+
 /*
 ** End of interface to code in fts5_hash.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index bc9244fc01..2010e4ff9c 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -529,6 +529,13 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
+/*
+** Return true if the hash table is empty, false otherwise.
+*/
+int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
+  return pHash->nEntry==0;
+}
+
 void sqlite3Fts5HashScanNext(Fts5Hash *p){
   assert( !sqlite3Fts5HashScanEof(p) );
   p->pScan = p->pScan->pScanNext;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9bd23a41f4..392218864c 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,6 +56,8 @@
 
 #define FTS5_MAX_LEVEL 64
 
+#define FTS5_MERGE_TOMBSTONE_WEIGHT 5
+
 /*
 ** There are two versions of the format used for the structure record:
 **
@@ -332,6 +334,12 @@ struct Fts5Data {
 
 /*
 ** One object per %_data table.
+**
+** nContentlessDelete:
+**   The number of contentless delete operations since the most recent
+**   call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked
+**   so that extra auto-merge work can be done by fts5IndexFlush() to
+**   account for the delete operations.
 */
 struct Fts5Index {
   Fts5Config *pConfig;            /* Virtual table configuration */
@@ -346,6 +354,7 @@ struct Fts5Index {
   int nPendingData;               /* Current bytes of pending data */
   i64 iWriteRowid;                /* Rowid for current doc being written */
   int bDelete;                    /* Current write is a delete */
+  int nContentlessDelete;         /* Number of contentless delete ops */
 
   /* Error state. */
   int rc;                         /* Current error code */
@@ -3980,6 +3989,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
     sqlite3Fts5HashClear(p->pHash);
     p->nPendingData = 0;
   }
+  p->nContentlessDelete = 0;
 }
 
 /*
@@ -4722,6 +4732,7 @@ static int fts5IndexMerge(
   int nRem = nPg;
   int bRet = 0;
   Fts5Structure *pStruct = *ppStruct;
+  int bTombstone = 0;
   while( nRem>0 && p->rc==SQLITE_OK ){
     int iLvl;                   /* To iterate through levels */
     int iBestLvl = 0;           /* Level offering the most input segments */
@@ -4731,6 +4742,7 @@ static int fts5IndexMerge(
     assert( pStruct->nLevel>0 );
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
       Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
+      int nThisSeg = 0;
       if( pLvl->nMerge ){
         if( pLvl->nMerge>nBest ){
           iBestLvl = iLvl;
@@ -4738,8 +4750,20 @@ static int fts5IndexMerge(
         }
         break;
       }
-      if( pLvl->nSeg>nBest ){
-        nBest = pLvl->nSeg;
+      nThisSeg = pLvl->nSeg;
+      if( bTombstone && nThisSeg ){
+        int iSeg;
+        int nPg = 0;
+        int nTomb = 0;
+        for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+          nPg += pSeg->pgnoLast;
+          nTomb += pSeg->nPgTombstone;
+        }
+        nThisSeg += ((nTomb*FTS5_MERGE_TOMBSTONE_WEIGHT) / nPg);
+      }
+      if( nThisSeg>nBest ){
+        nBest = nThisSeg;
         iBestLvl = iLvl;
       }
     }
@@ -4752,7 +4776,12 @@ static int fts5IndexMerge(
 #endif
 
     if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
-      break;
+      if( bTombstone || p->pConfig->bContentlessDelete==0 ){
+        break;
+      }else{
+        bTombstone = 1;
+        continue;
+      }
     }
     bRet = 1;
     fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
@@ -5273,192 +5302,195 @@ static void fts5FlushOneHash(Fts5Index *p){
   /* Obtain a reference to the index structure and allocate a new segment-id
   ** for the new level-0 segment.  */
   pStruct = fts5StructureRead(p);
-  iSegid = fts5AllocateSegid(p, pStruct);
   fts5StructureInvalidate(p);
 
-  if( iSegid ){
-    const int pgsz = p->pConfig->pgsz;
-    int eDetail = p->pConfig->eDetail;
-    int bSecureDelete = p->pConfig->bSecureDelete;
-    Fts5StructureSegment *pSeg;   /* New segment within pStruct */
-    Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
-    Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
-
-    Fts5SegWriter writer;
-    fts5WriteInit(p, &writer, iSegid);
-
-    pBuf = &writer.writer.buf;
-    pPgidx = &writer.writer.pgidx;
-
-    /* fts5WriteInit() should have initialized the buffers to (most likely)
-    ** the maximum space required. */
-    assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) );
-    assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) );
-
-    /* Begin scanning through hash table entries. This loop runs once for each
-    ** term/doclist currently stored within the hash table. */
-    if( p->rc==SQLITE_OK ){
-      p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
-    }
-    while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
-      const char *zTerm;          /* Buffer containing term */
-      int nTerm;                  /* Size of zTerm in bytes */
-      const u8 *pDoclist;         /* Pointer to doclist for this term */
-      int nDoclist;               /* Size of doclist in bytes */
-
-      /* Get the term and doclist for this entry. */
-      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
-      nTerm = (int)strlen(zTerm);
-      if( bSecureDelete==0 ){
-        fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
-        if( p->rc!=SQLITE_OK ) break;
-        assert( writer.bFirstRowidInPage==0 );
+  if( sqlite3Fts5HashIsEmpty(pHash)==0 ){
+    iSegid = fts5AllocateSegid(p, pStruct);
+    if( iSegid ){
+      const int pgsz = p->pConfig->pgsz;
+      int eDetail = p->pConfig->eDetail;
+      int bSecureDelete = p->pConfig->bSecureDelete;
+      Fts5StructureSegment *pSeg; /* New segment within pStruct */
+      Fts5Buffer *pBuf;           /* Buffer in which to assemble leaf page */
+      Fts5Buffer *pPgidx;         /* Buffer in which to assemble pgidx */
+  
+      Fts5SegWriter writer;
+      fts5WriteInit(p, &writer, iSegid);
+  
+      pBuf = &writer.writer.buf;
+      pPgidx = &writer.writer.pgidx;
+  
+      /* fts5WriteInit() should have initialized the buffers to (most likely)
+      ** the maximum space required. */
+      assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) );
+      assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) );
+  
+      /* Begin scanning through hash table entries. This loop runs once for each
+      ** term/doclist currently stored within the hash table. */
+      if( p->rc==SQLITE_OK ){
+        p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
       }
-
-      if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
-        /* The entire doclist will fit on the current leaf. */
-        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
-      }else{
-        int bTermWritten = !bSecureDelete;
-        i64 iRowid = 0;
-        i64 iPrev = 0;
-        int iOff = 0;
-
-        /* The entire doclist will not fit on this leaf. The following 
-        ** loop iterates through the poslists that make up the current 
-        ** doclist.  */
-        while( p->rc==SQLITE_OK && iOff<nDoclist ){
-          u64 iDelta = 0;
-          iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
-          iRowid += iDelta;
-
-          /* If in secure delete mode, and if this entry in the poslist is
-          ** in fact a delete, then edit the existing segments directly
-          ** using fts5FlushSecureDelete().  */
-          if( bSecureDelete ){
-            if( eDetail==FTS5_DETAIL_NONE ){
-              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
-                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
-                iOff++;
+      while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
+        const char *zTerm;        /* Buffer containing term */
+        int nTerm;                /* Size of zTerm in bytes */
+        const u8 *pDoclist;       /* Pointer to doclist for this term */
+        int nDoclist;             /* Size of doclist in bytes */
+  
+        /* Get the term and doclist for this entry. */
+        sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
+        nTerm = (int)strlen(zTerm);
+        if( bSecureDelete==0 ){
+          fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+          if( p->rc!=SQLITE_OK ) break;
+          assert( writer.bFirstRowidInPage==0 );
+        }
+  
+        if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
+          /* The entire doclist will fit on the current leaf. */
+          fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
+        }else{
+          int bTermWritten = !bSecureDelete;
+          i64 iRowid = 0;
+          i64 iPrev = 0;
+          int iOff = 0;
+  
+          /* The entire doclist will not fit on this leaf. The following 
+          ** loop iterates through the poslists that make up the current 
+          ** doclist.  */
+          while( p->rc==SQLITE_OK && iOff<nDoclist ){
+            u64 iDelta = 0;
+            iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
+            iRowid += iDelta;
+  
+            /* If in secure delete mode, and if this entry in the poslist is
+            ** in fact a delete, then edit the existing segments directly
+            ** using fts5FlushSecureDelete().  */
+            if( bSecureDelete ){
+              if( eDetail==FTS5_DETAIL_NONE ){
                 if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                  iOff++;
+                  if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                    iOff++;
+                    nDoclist = 0;
+                  }else{
+                    continue;
+                  }
+                }
+              }else if( (pDoclist[iOff] & 0x01) ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
                   iOff++;
-                  nDoclist = 0;
-                }else{
                   continue;
                 }
               }
-            }else if( (pDoclist[iOff] & 0x01) ){
-              fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
-              if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
-                iOff++;
-                continue;
-              }
             }
-          }
-
-          if( p->rc==SQLITE_OK && bTermWritten==0 ){
-            fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
-            bTermWritten = 1;
-            assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
-          }
-          
-          if( writer.bFirstRowidInPage ){
-            fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
-            writer.bFirstRowidInPage = 0;
-            fts5WriteDlidxAppend(p, &writer, iRowid);
-          }else{
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
-          }
-          if( p->rc!=SQLITE_OK ) break;
-          assert( pBuf->n<=pBuf->nSpace );
-          iPrev = iRowid;
-
-          if( eDetail==FTS5_DETAIL_NONE ){
-            if( iOff<nDoclist && pDoclist[iOff]==0 ){
-              pBuf->p[pBuf->n++] = 0;
-              iOff++;
+  
+            if( p->rc==SQLITE_OK && bTermWritten==0 ){
+              fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+              bTermWritten = 1;
+              assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
+            }
+            
+            if( writer.bFirstRowidInPage ){
+              fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
+              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
+              writer.bFirstRowidInPage = 0;
+              fts5WriteDlidxAppend(p, &writer, iRowid);
+            }else{
+              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
+            }
+            if( p->rc!=SQLITE_OK ) break;
+            assert( pBuf->n<=pBuf->nSpace );
+            iPrev = iRowid;
+  
+            if( eDetail==FTS5_DETAIL_NONE ){
               if( iOff<nDoclist && pDoclist[iOff]==0 ){
                 pBuf->p[pBuf->n++] = 0;
                 iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0 ){
+                  pBuf->p[pBuf->n++] = 0;
+                  iOff++;
+                }
+              }
+              if( (pBuf->n + pPgidx->n)>=pgsz ){
+                fts5WriteFlushLeaf(p, &writer);
               }
-            }
-            if( (pBuf->n + pPgidx->n)>=pgsz ){
-              fts5WriteFlushLeaf(p, &writer);
-            }
-          }else{
-            int bDummy;
-            int nPos;
-            int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
-            nCopy += nPos;
-            if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
-              /* The entire poslist will fit on the current leaf. So copy
-              ** it in one go. */
-              fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
             }else{
-              /* The entire poslist will not fit on this leaf. So it needs
-              ** to be broken into sections. The only qualification being
-              ** that each varint must be stored contiguously.  */
-              const u8 *pPoslist = &pDoclist[iOff];
-              int iPos = 0;
-              while( p->rc==SQLITE_OK ){
-                int nSpace = pgsz - pBuf->n - pPgidx->n;
-                int n = 0;
-                if( (nCopy - iPos)<=nSpace ){
-                  n = nCopy - iPos;
-                }else{
-                  n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
+              int bDummy;
+              int nPos;
+              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
+              nCopy += nPos;
+              if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
+                /* The entire poslist will fit on the current leaf. So copy
+                ** it in one go. */
+                fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
+              }else{
+                /* The entire poslist will not fit on this leaf. So it needs
+                ** to be broken into sections. The only qualification being
+                ** that each varint must be stored contiguously.  */
+                const u8 *pPoslist = &pDoclist[iOff];
+                int iPos = 0;
+                while( p->rc==SQLITE_OK ){
+                  int nSpace = pgsz - pBuf->n - pPgidx->n;
+                  int n = 0;
+                  if( (nCopy - iPos)<=nSpace ){
+                    n = nCopy - iPos;
+                  }else{
+                    n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
+                  }
+                  assert( n>0 );
+                  fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
+                  iPos += n;
+                  if( (pBuf->n + pPgidx->n)>=pgsz ){
+                    fts5WriteFlushLeaf(p, &writer);
+                  }
+                  if( iPos>=nCopy ) break;
                 }
-                assert( n>0 );
-                fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
-                iPos += n;
-                if( (pBuf->n + pPgidx->n)>=pgsz ){
-                  fts5WriteFlushLeaf(p, &writer);
-                }
-                if( iPos>=nCopy ) break;
               }
+              iOff += nCopy;
             }
-            iOff += nCopy;
           }
         }
+  
+        /* TODO2: Doclist terminator written here. */
+        /* pBuf->p[pBuf->n++] = '\0'; */
+        assert( pBuf->n<=pBuf->nSpace );
+        if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
       }
-
-      /* TODO2: Doclist terminator written here. */
-      /* pBuf->p[pBuf->n++] = '\0'; */
-      assert( pBuf->n<=pBuf->nSpace );
-      if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
-    }
-    sqlite3Fts5HashClear(pHash);
-    fts5WriteFinish(p, &writer, &pgnoLast);
-
-    assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
-    if( pgnoLast>0 ){
-      /* Update the Fts5Structure. It is written back to the database by the
-      ** fts5StructureRelease() call below.  */
-      if( pStruct->nLevel==0 ){
-        fts5StructureAddLevel(&p->rc, &pStruct);
-      }
-      fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
-      if( p->rc==SQLITE_OK ){
-        pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
-        pSeg->iSegid = iSegid;
-        pSeg->pgnoFirst = 1;
-        pSeg->pgnoLast = pgnoLast;
-        if( pStruct->nOriginCntr>0 ){
-          pSeg->iOrigin1 = pStruct->nOriginCntr;
-          pSeg->iOrigin2 = pStruct->nOriginCntr;
-          pStruct->nOriginCntr++;
+      sqlite3Fts5HashClear(pHash);
+      fts5WriteFinish(p, &writer, &pgnoLast);
+  
+      assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
+      if( pgnoLast>0 ){
+        /* Update the Fts5Structure. It is written back to the database by the
+        ** fts5StructureRelease() call below.  */
+        if( pStruct->nLevel==0 ){
+          fts5StructureAddLevel(&p->rc, &pStruct);
         }
-        pStruct->nSegment++;
+        fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
+        if( p->rc==SQLITE_OK ){
+          pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
+          pSeg->iSegid = iSegid;
+          pSeg->pgnoFirst = 1;
+          pSeg->pgnoLast = pgnoLast;
+          if( pStruct->nOriginCntr>0 ){
+            pSeg->iOrigin1 = pStruct->nOriginCntr;
+            pSeg->iOrigin2 = pStruct->nOriginCntr;
+            pStruct->nOriginCntr++;
+          }
+          pStruct->nSegment++;
+        }
+        fts5StructurePromote(p, 0, pStruct);
       }
-      fts5StructurePromote(p, 0, pStruct);
     }
   }
 
-  fts5IndexAutomerge(p, &pStruct, pgnoLast);
+  fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete);
   fts5IndexCrisismerge(p, &pStruct);
   fts5StructureWrite(p, pStruct);
   fts5StructureRelease(pStruct);
+  p->nContentlessDelete = 0;
 }
 
 /*
@@ -5466,7 +5498,7 @@ static void fts5FlushOneHash(Fts5Index *p){
 */
 static void fts5IndexFlush(Fts5Index *p){
   /* Unless it is empty, flush the hash table to disk */
-  if( p->nPendingData ){
+  if( p->nPendingData || (p->nContentlessDelete && p->pConfig->nAutomerge>0) ){
     assert( p->pHash );
     p->nPendingData = 0;
     fts5FlushOneHash(p);
@@ -6800,6 +6832,8 @@ static void fts5IndexTombstoneAdd(
   int nHash = 0;
   Fts5Data **apHash = 0;
 
+  p->nContentlessDelete++;
+
   if( pSeg->nPgTombstone>0 ){
     iPg = iRowid % pSeg->nPgTombstone;
     pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg));
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index b10da49c75..d1c79d3604 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1777,8 +1777,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
-  fts5TripCursors(pTab);
-  rc = sqlite3Fts5StorageSync(pTab->pStorage);
+  rc = sqlite3Fts5FlushToDisk(&pTab->p);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
new file mode 100644
index 0000000000..21eba71b1a
--- /dev/null
+++ b/ext/fts5/test/fts5contentless4.test
@@ -0,0 +1,61 @@
+# 2023 July 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc document {n} {
+  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+  set ret [list]
+  for {set ii 0} {$ii < $n} {incr ii} {
+    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+  }
+  set ret
+}
+db func document document
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 240);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+do_execsql_test 1.1 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM ft WHERE rowid < 1000
+}
+
+execsql_pp {
+  SELECT * FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+}
+
+finish_test
+
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 3e4343c8ea..f9de77a7fc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sfts5\sincremental\soptimization\sso\sthat\sit\stoo\scan\shandle\san\sindex\sthat\sconsists\sof\sa\ssingle\ssegment\swith\sone\sor\smore\stombstone\shash\spages.
-D 2023-07-21T21:10:33.579
+C Integrate\scontentless\sdelete\swith\sauto-merge.
+D 2023-07-22T19:47:46.359
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,14 +86,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h fa9dd8ecbda6340f406c6f21b9b524b4666817aa89850e60a42937eea87cef6a
+F ext/fts5/fts5Int.h f59c14f725ad0fcb8a81b9bf012e5021c6501bf43e73aa00b00d728e2ac7efaf
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
-F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c e500a5d33ae312c2ebab91123f0ef5f9bbc3eb555252c7d40ba4d8688780c7ca
-F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
+F ext/fts5/fts5_hash.c 60224220ccfb2846b741b6dbb1b8872094ec6d87b3118c04244dafc83e6f9c40
+F ext/fts5/fts5_index.c 31b8c8dd6913d76d6d7755342e36816495e5ad177d253f6bf39e0efdb9dc31e0
+F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -135,6 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
+F ext/fts5/test/fts5contentless4.test 3b11ccbbe928d45eb8f985c0137a8fe2c69b70b940b10de31540040de5674311
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2047,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f4926006b371d9a1439a25384bd50a50c2f1c03f75a7c2c3134ae72abb971c91
-R 8117856750f55ecca6e7594df2bcd37e
+P e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
+R 223fbfa40ff414b8abee5011e5d1533a
 U dan
-Z 3675210f196caeb196ef6ee2658c36e9
+Z 8cb202caf797f67b895f2a9a207e2618
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 92b4b794ac..13263786f5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
\ No newline at end of file
+85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
\ No newline at end of file

From c76a8862a73fdf90ec5ab1aa088b8b600bba2365 Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Mon, 24 Jul 2023 15:41:58 +0000
Subject: [PATCH 26/66] Resolve an ES6 module export construct which is
 incompatible with node.js, as reported in [forum:b9680fa9ad|forum post
 b9680fa9ad].

FossilOrigin-Name: 80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615
---
 ext/wasm/api/extern-post-js.c-pp.js |  3 ++-
 ext/wasm/demo-123.js                |  4 ++--
 manifest                            | 14 +++++++-------
 manifest.uuid                       |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js
index fac00370dd..00cb3fab90 100644
--- a/ext/wasm/api/extern-post-js.c-pp.js
+++ b/ext/wasm/api/extern-post-js.c-pp.js
@@ -119,5 +119,6 @@ const toExportForESM =
   return globalThis.sqlite3InitModule /* required for ESM */;
 })();
 //#if target=es6-module
-export { toExportForESM as default, toExportForESM as sqlite3InitModule }
+sqlite3InitModule = toExportForESM;
+export default sqlite3InitModule;
 //#endif
diff --git a/ext/wasm/demo-123.js b/ext/wasm/demo-123.js
index 311afcc827..8e03ee80fa 100644
--- a/ext/wasm/demo-123.js
+++ b/ext/wasm/demo-123.js
@@ -235,10 +235,10 @@
 
        - get change count (total or statement-local, 32- or 64-bit)
        - get a DB's file name
-    
+
        Misc. Stmt features:
 
-       - Various forms of bind() 
+       - Various forms of bind()
        - clearBindings()
        - reset()
        - Various forms of step()
diff --git a/manifest b/manifest
index ef9ea0e9d6..8d61da8af1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\swriting\sthe\sfilename\sheader\sin\sa\sopfs-sahpool\sfile,\sensure\sthat\sall\sremaining\sbytes\sin\sthat\spart\sof\sthe\sheader\sare\szeroed\sout\sto\savoid\sthe\sdownstream\sproblems\sreported\sin\s[forum:d50ec48a293988a5|forum\spost\sd50ec48a293988a5].
-D 2023-07-23T22:14:41.081
+C Resolve\san\sES6\smodule\sexport\sconstruct\swhich\sis\sincompatible\swith\snode.js,\sas\sreported\sin\s[forum:b9680fa9ad|forum\spost\sb9680fa9ad].
+D 2023-07-24T15:41:58.448
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -489,7 +489,7 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134
-F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe
+F ext/wasm/api/extern-post-js.c-pp.js e7257ea56d4077d0773d7537fcb8eade59aad0c118dab92497c27939edca41cb
 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41
 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1
 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62
@@ -516,7 +516,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7
 F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6
+F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d2e602cda44bf35e76167143262b4f91826d25780d0e095e680a31d5dedb2018
-R 47af828f435c00e0fb124001796bea21
+P c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3
+R 1ab3847cf4557b99f62130a0f45e24bd
 U stephan
-Z 3f8f8cf2fbd99e80728947340dd883ad
+Z 4a3a3c6a8d937dc54de34852d2276ed0
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 9921927a14..649aa580be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3
\ No newline at end of file
+80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615
\ No newline at end of file

From 9125d5ab6c043dca59ac5ff10f23558820701a01 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 24 Jul 2023 17:59:25 +0000
Subject: [PATCH 27/66] Incremental progress toward improved caching of parsed
 JSON.

FossilOrigin-Name: f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85
---
 manifest      |  12 +-
 manifest.uuid |   2 +-
 src/json.c    | 334 +++++++++++++++++++++++++++++++++++++-------------
 3 files changed, 256 insertions(+), 92 deletions(-)

diff --git a/manifest b/manifest
index 552d7e205e..e99cb3d132 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sjson-opt\sbranch.
-D 2023-07-24T12:37:23.834
+C Incremental\sprogress\stoward\simproved\scaching\sof\sparsed\sJSON.
+D 2023-07-24T17:59:25.604
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c d34d10ea1520bb71ee69c500deb4a20847a57b4c59c1ecb807c935f53c870885
+F src/json.c 3584f5fb0ff48009ebabb34704328c2086d924fc2403fdf0e48afce5b9f22777
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707 c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3
-R 2ab9518ef262a1aca9fe7f6a2ac934e0
+P 00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456
+R 831d8993ebdb0e60bc4ab13513e5d282
 U drh
-Z 40e6676e6b6212e46c825865d3333a87
+Z 8cedb8d1881aa5ce4f4f5c5fea73c221
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 37810abc34..5be0ea0fd7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456
\ No newline at end of file
+f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index bf7c2be3ae..97ebb6af65 100644
--- a/src/json.c
+++ b/src/json.c
@@ -72,7 +72,6 @@ struct JsonString {
   u64 nUsed;               /* Bytes of zBuf[] currently used */
   u8 bStatic;              /* True if zBuf is static space */
   u8 bErr;                 /* True if an error has been encountered */
-  u8 bOrig;                /* Ignore edits when rendering JSON */
   char zSpace[100];        /* Initial static space */
 };
 
@@ -87,14 +86,16 @@ struct JsonTask {
 
 /* JSON type values
 */
-#define JSON_NULL     0
-#define JSON_TRUE     1
-#define JSON_FALSE    2
-#define JSON_INT      3
-#define JSON_REAL     4
-#define JSON_STRING   5
-#define JSON_ARRAY    6
-#define JSON_OBJECT   7
+#define JSON_SUBST    0    /* Special edit node.  Uses u.iPrev */
+#define JSON_PATCH    1    /* Special edit node.  Uses u.pPatch */
+#define JSON_NULL     2
+#define JSON_TRUE     3
+#define JSON_FALSE    4
+#define JSON_INT      5
+#define JSON_REAL     6
+#define JSON_STRING   7
+#define JSON_ARRAY    8
+#define JSON_OBJECT   9
 
 /* The "subtype" set for JSON values */
 #define JSON_SUBTYPE  74    /* Ascii for "J" */
@@ -103,6 +104,7 @@ struct JsonTask {
 ** Names of the various JSON types:
 */
 static const char * const jsonType[] = {
+  "subst", "patch",
   "null", "true", "false", "integer", "real", "text", "array", "object"
 };
 
@@ -124,13 +126,15 @@ struct JsonNode {
   u8 eType;              /* One of the JSON_ type values */
   u8 jnFlags;            /* JNODE flags */
   u8 eU;                 /* Which union element to use */
-  u32 n;                 /* Bytes of content, or number of sub-nodes */
+  u32 n;                 /* Bytes of content for INT, REAL or STRING
+                         ** Number of sub-nodes for ARRAY and OBJECT
+                         ** Node that SUBST applies to */
   union {
     const char *zJContent; /* 1: Content for INT, REAL, and STRING */
     u32 iAppend;           /* 2: More terms for ARRAY and OBJECT */
     u32 iKey;              /* 3: Key for ARRAY objects in json_tree() */
-    u32 iReplace;          /* 4: Replacement content for JNODE_REPLACE */
     JsonNode *pPatch;      /* 5: Node chain of patch for JNODE_PATCH */
+    u32 iPrev;             /* 6: Previous SUBST node, or 0 */
   } u;
 };
 
@@ -149,9 +153,9 @@ struct JsonParse {
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   u8 nJPRef;         /* Number of references to this object */
-  u8 bOrig;          /* Ignore edit marks during search */
   int nJson;         /* Length of the zJson string in bytes */
   u32 iErr;          /* Error location in zJson[] */
+  u32 iSubst;        /* Last known JSON_SUBST node */
   u32 iHold;         /* Replace cache line with the lowest iHold value */
 };
 
@@ -175,7 +179,6 @@ static void jsonZero(JsonString *p){
   p->nAlloc = sizeof(p->zSpace);
   p->nUsed = 0;
   p->bStatic = 1;
-  p->bOrig = 0;
 }
 
 /* Initialize the JsonString object
@@ -548,7 +551,12 @@ static u32 jsonNodeSize(JsonNode *pNode){
 ** delete the JsonParse object itself.
 */
 static void jsonParseReset(JsonParse *pParse){
-  assert( pParse->pClean==0 );
+  while( pParse->pClean ){
+    JsonTask *pTask = pParse->pClean;
+    pParse->pClean = pTask->pJTNext;
+    pTask->xOp(pTask->pArg);
+    sqlite3_free(pTask);
+  }
   assert( pParse->nJPRef<=1 );
   sqlite3_free(pParse->aNode);
   pParse->aNode = 0;
@@ -566,35 +574,63 @@ static void jsonParseFree(JsonParse *pParse){
     pParse->nJPRef--;
     return;
   }
-  while( pParse->pClean ){
-    JsonTask *pTask = pParse->pClean;
-    pParse->pClean = pTask->pJTNext;
-    pTask->xOp(pTask->pArg);
-    sqlite3_free(pTask);
-  }
   jsonParseReset(pParse);
   sqlite3_free(pParse);
 }
 
+/*
+** Add a cleanup task to the JsonParse object.
+**
+** If an OOM occurs, the cleanup operation happens immediately
+** and this function returns SQLITE_NOMEM.
+*/
+static int jsonParseAddCleanup(
+  JsonParse *pParse,          /* Add the cleanup task to this parser */
+  void(*xOp)(void*),          /* The cleanup task */
+  void *pArg                  /* Argument to the cleanup */
+){
+  JsonTask *pTask = sqlite3_malloc64( sizeof(*pTask) );
+  if( pTask==0 ){
+    pParse->oom = 1;
+    xOp(pArg);
+    return SQLITE_ERROR;
+  }
+  pTask->pJTNext = pParse->pClean;
+  pParse->pClean = pTask;
+  pTask->xOp = xOp;
+  pTask->pArg = pArg;
+  return SQLITE_OK;
+}
+
 /*
 ** Convert the JsonNode pNode into a pure JSON string and
 ** append to pOut.  Subsubstructure is also included.  Return
 ** the number of JsonNode objects that are encoded.
 */
 static void jsonRenderNode(
+  JsonParse *pParse,             /* the complete parse of the JSON */
   JsonNode *pNode,               /* The node to render */
-  JsonString *pOut,              /* Write JSON here */
-  sqlite3_value **aReplace       /* Replacement values */
+  JsonString *pOut               /* Write JSON here */
 ){
   assert( pNode!=0 );
-  if( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) && !pOut->bOrig ){
-    if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){
-      assert( pNode->eU==4 );
-      jsonAppendValue(pOut, aReplace[pNode->u.iReplace]);
-      return;
+  while( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) ){
+    if( (pNode->jnFlags & JNODE_REPLACE)!=0 ){
+      u32 idx = (u32)(pNode - pParse->aNode);
+      u32 i = pParse->iSubst;
+      while( 1 /*exit-by-break*/ ){
+        assert( i<pParse->nNode );
+        assert( pParse->aNode[i].eType==JSON_SUBST );
+        assert( pParse->aNode[i].eU==6 );
+        assert( pParse->aNode[i].u.iPrev<i );
+        if( pParse->aNode[i].n==idx ){
+          pNode = &pParse->aNode[i+1];
+          break;
+        }
+        i = pParse->aNode[i].u.iPrev;
+      }
+    }else{
+      pNode = pNode->u.pPatch;
     }
-    assert( pNode->eU==5 );
-    pNode = pNode->u.pPatch;
   }
   switch( pNode->eType ){
     default: {
@@ -653,13 +689,13 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '[');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){
+          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){
             jsonAppendSeparator(pOut);
-            jsonRenderNode(&pNode[j], pOut, aReplace);
+            jsonRenderNode(pParse, &pNode[j], pOut);
           }
           j += jsonNodeSize(&pNode[j]);
         }
-        if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break;
+        if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
         assert( pNode->eU==2 );
         pNode = &pNode[pNode->u.iAppend];
         j = 1;
@@ -672,15 +708,15 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '{');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){
+          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){
             jsonAppendSeparator(pOut);
-            jsonRenderNode(&pNode[j], pOut, aReplace);
+            jsonRenderNode(pParse, &pNode[j], pOut);
             jsonAppendChar(pOut, ':');
-            jsonRenderNode(&pNode[j+1], pOut, aReplace);
+            jsonRenderNode(pParse, &pNode[j+1], pOut);
           }
           j += 1 + jsonNodeSize(&pNode[j+1]);
         }
-        if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break;
+        if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
         assert( pNode->eU==2 );
         pNode = &pNode[pNode->u.iAppend];
         j = 1;
@@ -695,15 +731,13 @@ static void jsonRenderNode(
 ** Return a JsonNode and all its descendants as a JSON string.
 */
 static void jsonReturnJson(
+  JsonParse *pParse,          /* The complete JSON */
   JsonNode *pNode,            /* Node to return */
-  sqlite3_context *pCtx,      /* Return value for this function */
-  sqlite3_value **aReplace,   /* Array of replacement values */
-  int bOrig                   /* Ignore edits if true */
+  sqlite3_context *pCtx       /* Return value for this function */
 ){
   JsonString s;
   jsonInit(&s, pCtx);
-  s.bOrig = bOrig!=0;
-  jsonRenderNode(pNode, &s, aReplace);
+  jsonRenderNode(pParse, pNode, &s);
   jsonResult(&s);
   sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
 }
@@ -743,10 +777,9 @@ static u32 jsonHexToInt4(const char *z){
 ** Make the JsonNode the return value of the function.
 */
 static void jsonReturn(
+  JsonParse *pParse,          /* Complete JSON parse tree */
   JsonNode *pNode,            /* Node to return */
-  sqlite3_context *pCtx,      /* Return value for this function */
-  sqlite3_value **aReplace,   /* Array of replacement values */
-  int bOrig                   /* Ignore edits if true */
+  sqlite3_context *pCtx       /* Return value for this function */
 ){
   switch( pNode->eType ){
     default: {
@@ -893,7 +926,7 @@ static void jsonReturn(
     }
     case JSON_ARRAY:
     case JSON_OBJECT: {
-      jsonReturnJson(pNode, pCtx, aReplace, bOrig);
+      jsonReturnJson(pParse, pNode, pCtx);
       break;
     }
   }
@@ -962,6 +995,22 @@ static int jsonParseAddNode(
   return pParse->nNode++;
 }
 
+/*
+** Add a new JSON_SUBST node.  The node immediately following
+** this new node will be the substitute content for iNode.
+*/
+static int jsonParseAddSubstNode(
+  JsonParse *pParse,       /* Add the JSON_SUBST here */
+  u32 iNode                /* References this node */
+){
+  int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0);
+  if( idx<=0 ) return idx;
+  pParse->aNode[idx].eU = 6;
+  pParse->aNode[idx].u.iPrev = pParse->iSubst;
+  pParse->iSubst = idx;
+  return idx;
+}
+
 /*
 ** Return true if z[] begins with 2 (or more) hexadecimal digits
 */
@@ -1749,6 +1798,7 @@ static JsonParse *jsonParseCached(
     sqlite3_free(p);
     return 0;
   }
+  p->nJPRef = 1;
   p->nJson = nJson;
   p->iHold = iMaxHold+1;
   sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
@@ -1803,7 +1853,6 @@ static JsonNode *jsonLookupStep(
   const char *zKey;
   JsonNode *pRoot = &pParse->aNode[iRoot];
   if( zPath[0]==0 ) return pRoot;
-  if( (pRoot->jnFlags & JNODE_REPLACE)!=0 && !pParse->bOrig ) return 0;
   if( zPath[0]=='.' ){
     if( pRoot->eType!=JSON_OBJECT ) return 0;
     zPath++;
@@ -1836,7 +1885,7 @@ static JsonNode *jsonLookupStep(
         j++;
         j += jsonNodeSize(&pRoot[j]);
       }
-      if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
+      if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
       assert( pRoot->eU==2 );
       iRoot += pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -1845,7 +1894,6 @@ static JsonNode *jsonLookupStep(
     if( pApnd ){
       u32 iStart, iLabel;
       JsonNode *pNode;
-      assert( !pParse->bOrig );
       iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
       iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
       zPath += i;
@@ -1875,10 +1923,10 @@ static JsonNode *jsonLookupStep(
         if( pRoot->eType!=JSON_ARRAY ) return 0;
         for(;;){
           while( j<=pBase->n ){
-            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i++;
+            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++;
             j += jsonNodeSize(&pBase[j]);
           }
-          if( (pBase->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
+          if( (pBase->jnFlags & JNODE_APPEND)==0 ) break;
           assert( pBase->eU==2 );
           iBase += pBase->u.iAppend;
           pBase = &pParse->aNode[iBase];
@@ -1908,13 +1956,11 @@ static JsonNode *jsonLookupStep(
     zPath += j + 1;
     j = 1;
     for(;;){
-      while( j<=pRoot->n
-          && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE) && !pParse->bOrig))
-      ){
-        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i--;
+      while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){
+        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--;
         j += jsonNodeSize(&pRoot[j]);
       }
-      if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break;
+      if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
       assert( pRoot->eU==2 );
       iRoot += pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -1926,7 +1972,6 @@ static JsonNode *jsonLookupStep(
     if( i==0 && pApnd ){
       u32 iStart;
       JsonNode *pNode;
-      assert( !pParse->bOrig );
       iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0);
       pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr);
       if( pParse->oom ) return 0;
@@ -2060,6 +2105,60 @@ static void jsonRemoveAllNulls(JsonNode *pNode){
 ** SQL functions used for testing and debugging
 ****************************************************************************/
 
+#if 0  /* One for debugging.  zero normally */
+/*
+** Print N node entries.
+*/
+static void jsonDebugPrintNodeEntries(
+  JsonNode *aNode,  /* First node entry to print */
+  int N,            /* Number of node entries to print */
+  int ofst,         /* Node number for p */
+  int nDent         /* Indent by this many spaces */
+){
+  int i;
+  for(i=0; i<N; i++){
+    const char *zType;
+    if( aNode[i].jnFlags & JNODE_LABEL ){
+      zType = "label";
+    }else{
+      zType = jsonType[aNode[i].eType];
+    }
+    if( nDent>0 ) printf("%*s", nDent, "");
+    if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){
+      printf("node %4u: %-7s %02x n=%-5d",
+             i+ofst, zType, aNode[i].jnFlags, aNode[i].n);
+    }else{
+      printf("node %4u: %-7s    n=%-5d",
+             i+ofst, zType, aNode[i].n);
+    }
+    switch( aNode[i].eU ){
+      case 1:  printf(" zJContent=[%.*s]\n",
+                      aNode[i].n, aNode[i].u.zJContent);           break;
+      case 2:  printf(" iAppend=%u\n", aNode[i].u.iAppend);        break;
+      case 3:  printf(" iKey=%u\n", aNode[i].u.iKey);              break;
+      case 5: {
+        JsonNode *pX = aNode[i].u.pPatch;
+        printf(" pPatch=...\n");
+        jsonDebugPrintNodeEntries(pX, jsonNodeSize(pX), 0, nDent+3);
+        break;
+      }
+      case 6:  printf(" iPrev=%u\n", aNode[i].u.iPrev);      break;
+      default: printf("\n");
+    }
+  }
+}
+static void jsonDebugPrintParse(JsonParse *p){
+  jsonDebugPrintNodeEntries(p->aNode, p->nNode, 0, 0);
+}
+static void jsonDebugPrintNode(JsonNode *pNode){
+  jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode), 0, 0);
+}
+#else
+   /* The usual case */
+# define jsonDebugPrintNode(X)
+# define jsonDebugPrintParse(X)
+#endif
+
 #ifdef SQLITE_DEBUG
 /*
 ** The json_parse(JSON) function returns a string which describes
@@ -2279,15 +2378,15 @@ static void jsonExtractFunc(
       }
       if( pNode ){
         if( flags & JSON_JSON ){
-          jsonReturnJson(pNode, ctx, 0, 1);
+          jsonReturnJson(p, pNode, ctx);
         }else{
-          jsonReturn(pNode, ctx, 0, 1);
+          jsonReturn(p, pNode, ctx);
           sqlite3_result_subtype(ctx, 0);
         }
       }
     }else{
       pNode = jsonLookup(p, zPath, 0, ctx);
-      if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0, 1);
+      if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx);
     }
   }else{
     /* Two or more PATH arguments results in a JSON array with each
@@ -2301,7 +2400,7 @@ static void jsonExtractFunc(
       if( p->nErr ) break;
       jsonAppendSeparator(&jx);
       if( pNode ){
-        jsonRenderNode(pNode, &jx, 0);
+        jsonRenderNode(p, pNode, &jx);
       }else{
         jsonAppendRawNZ(&jx, "null", 4);
       }
@@ -2415,7 +2514,8 @@ static void jsonPatchFunc(
   pResult = jsonMergePatch(&x, 0, y.aNode);
   assert( pResult!=0 || x.oom );
   if( pResult ){
-    jsonReturnJson(pResult, ctx, 0, 0);
+    jsonDebugPrintNode(pResult);
+    jsonReturnJson(&x, pResult, ctx);
   }else{
     sqlite3_result_error_nomem(ctx);
   }
@@ -2492,12 +2592,89 @@ static void jsonRemoveFunc(
     if( pNode ) pNode->jnFlags |= JNODE_REMOVE;
   }
   if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){
-    jsonReturnJson(x.aNode, ctx, 0, 0);
+    jsonReturnJson(&x, x.aNode, ctx);
   }
 remove_done:
+  jsonDebugPrintParse(&x);
   jsonParseReset(&x);
 }
 
+/*
+** Substitute the value at iNode with the pValue parameter.
+*/
+static void jsonReplaceNode(
+  sqlite3_context *pCtx,
+  JsonParse *p,
+  int iNode,
+  sqlite3_value *pValue
+){
+  int idx = jsonParseAddSubstNode(p, iNode);
+  if( idx<=0 ){
+    assert( p->oom );
+    return;
+  }
+  switch( sqlite3_value_type(pValue) ){
+    case SQLITE_NULL: {
+      jsonParseAddNode(p, JSON_NULL, 0, 0);
+      break;
+    }
+    case SQLITE_FLOAT: {
+      char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue));
+      int n;
+      if( z==0 ){
+        p->oom = 1;
+        break;
+      }
+      n = sqlite3Strlen30(z);
+      jsonParseAddNode(p, JSON_REAL, n, z);
+      jsonParseAddCleanup(p, sqlite3_free, z);
+      break;
+    }
+    case SQLITE_INTEGER: {
+      char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue));
+      int n;
+      if( z==0 ){
+        p->oom = 1;
+        break;
+      }
+      n = sqlite3Strlen30(z);
+      jsonParseAddNode(p, JSON_INT, n, z);
+      jsonParseAddCleanup(p, sqlite3_free, z);
+
+      break;
+    }
+    case SQLITE_TEXT: {
+      const char *z = (const char*)sqlite3_value_text(pValue);
+      u32 n = (u32)sqlite3_value_bytes(pValue);
+      if( z==0 ){
+         p->oom = 1;
+         break;
+      }
+      if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){
+        int k = jsonParseAddNode(p, JSON_STRING, n, z);
+        if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW;
+        jsonParseAddCleanup(p, sqlite3_free, sqlite3DbStrDup(0,z));
+      }else{
+        int k= jsonParseAddNode(p, JSON_PATCH, 0, 0);
+        if( k>0 ){
+          JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx);
+          if( pPatch==0 ){
+            p->oom = 1;
+            break;
+          }
+          assert( pPatch->nJPRef>=1 );
+          pPatch->nJPRef++;
+          p->aNode[k].jnFlags |= JNODE_PATCH;
+          p->aNode[k].eU = 5;
+          p->aNode[k].u.pPatch = pPatch->aNode;
+          jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch);
+        }
+      }
+      break;
+    }
+  }
+}
+  
 /*
 ** json_replace(JSON, PATH, VALUE, ...)
 **
@@ -2526,20 +2703,13 @@ static void jsonReplaceFunc(
     pNode = jsonLookup(&x, zPath, 0, ctx);
     if( x.nErr ) goto replace_err;
     if( pNode ){
-      assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 );
-      testcase( pNode->eU!=0 && pNode->eU!=1 );
       pNode->jnFlags |= (u8)JNODE_REPLACE;
-      VVA( pNode->eU =  4 );
-      pNode->u.iReplace = i + 1;
+      jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
     }
   }
-  if( x.aNode[0].jnFlags & JNODE_REPLACE ){
-    assert( x.aNode[0].eU==4 );
-    sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
-  }else{
-    jsonReturnJson(x.aNode, ctx, argv, 0);
-  }
+  jsonReturnJson(&x, x.aNode, ctx);
 replace_err:
+  jsonDebugPrintParse(&x);
   jsonParseReset(&x);
 }
 
@@ -2585,19 +2755,13 @@ static void jsonSetFunc(
     }else if( x.nErr ){
       goto jsonSetDone;
     }else if( pNode && (bApnd || bIsSet) ){
-      testcase( pNode->eU!=0 && pNode->eU!=1 );
-      assert( pNode->eU!=3 && pNode->eU!=5 );
-      VVA( pNode->eU = 4 );
       pNode->jnFlags |= (u8)JNODE_REPLACE;
-      pNode->u.iReplace = i + 1;
+      jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
     }
   }
-  if( x.aNode[0].jnFlags & JNODE_REPLACE ){
-    assert( x.aNode[0].eU==4 );
-    sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
-  }else{
-    jsonReturnJson(x.aNode, ctx, argv, 0);
-  }
+  jsonDebugPrintParse(&x);
+  jsonReturnJson(&x, x.aNode, ctx);
+
 jsonSetDone:
   jsonParseReset(&x);
 }
@@ -3105,7 +3269,7 @@ static int jsonEachColumn(
     case JEACH_KEY: {
       if( p->i==0 ) break;
       if( p->eType==JSON_OBJECT ){
-        jsonReturn(pThis, ctx, 0, 0);
+        jsonReturn(&p->sParse, pThis, ctx);
       }else if( p->eType==JSON_ARRAY ){
         u32 iKey;
         if( p->bRecursive ){
@@ -3121,7 +3285,7 @@ static int jsonEachColumn(
     }
     case JEACH_VALUE: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
-      jsonReturn(pThis, ctx, 0, 0);
+      jsonReturn(&p->sParse, pThis, ctx);
       break;
     }
     case JEACH_TYPE: {
@@ -3132,7 +3296,7 @@ static int jsonEachColumn(
     case JEACH_ATOM: {
       if( pThis->jnFlags & JNODE_LABEL ) pThis++;
       if( pThis->eType>=JSON_ARRAY ) break;
-      jsonReturn(pThis, ctx, 0, 0);
+      jsonReturn(&p->sParse, pThis, ctx);
       break;
     }
     case JEACH_ID: {

From 24730de8d14efb28c0be0b109401e131dc60fc61 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Mon, 24 Jul 2023 19:13:06 +0000
Subject: [PATCH 28/66] Add the fts5 'delete-automerge' integer option. A level
 is eligible for auto-merging if it has a greater than or equal percentage of
 its entries deleted by tombstones than the 'delete-automerge' option. Default
 value is 10.

FossilOrigin-Name: b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
---
 ext/fts5/fts5Int.h                  |   1 +
 ext/fts5/fts5_config.c              |  15 +++
 ext/fts5/fts5_hash.c                |  16 ++-
 ext/fts5/fts5_index.c               | 148 ++++++++++++++++++----------
 ext/fts5/test/fts5contentless3.test |   1 -
 ext/fts5/test/fts5contentless4.test |  23 ++++-
 manifest                            |  22 ++---
 manifest.uuid                       |   2 +-
 8 files changed, 159 insertions(+), 69 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 24417483c1..89b553161b 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -214,6 +214,7 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
+  int nDeleteAutomerge;           /* 'delete-automerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index e0c85bbd3f..af2ef40bac 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -22,6 +22,8 @@
 #define FTS5_DEFAULT_CRISISMERGE   16
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
 
+#define FTS5_DEFAULT_DELETE_AUTOMERGE 10      /* default 10% */
+
 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (64*1024)
 
@@ -922,6 +924,18 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
+  else if( 0==sqlite3_stricmp(zKey, "delete-automerge") ){
+    int nVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nVal = sqlite3_value_int(pVal);
+    }else{
+      *pbBadkey = 1;
+    }
+    if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
+    if( nVal>100 ) nVal = 0;
+    pConfig->nDeleteAutomerge = nVal;
+  }
+
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
     const char *zIn = (const char*)sqlite3_value_text(pVal);
     char *zRank;
@@ -970,6 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
+  pConfig->nDeleteAutomerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index 2010e4ff9c..7e50c36608 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -475,7 +475,6 @@ static int fts5HashEntrySort(
     pList = fts5HashEntryMerge(pList, ap[i]);
   }
 
-  pHash->nEntry = 0;
   sqlite3_free(ap);
   *ppSorted = pList;
   return SQLITE_OK;
@@ -529,10 +528,25 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
+#ifdef SQLITE_DEBUG
+static int fts5HashCount(Fts5Hash *pHash){
+  int nEntry = 0;
+  int ii;
+  for(ii=0; ii<pHash->nSlot; ii++){
+    Fts5HashEntry *p = 0;
+    for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
+      nEntry++;
+    }
+  }
+  return nEntry;
+}
+#endif
+
 /*
 ** Return true if the hash table is empty, false otherwise.
 */
 int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
+  assert( pHash->nEntry==fts5HashCount(pHash) );
   return pHash->nEntry==0;
 }
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 392218864c..ce52825533 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -56,8 +56,6 @@
 
 #define FTS5_MAX_LEVEL 64
 
-#define FTS5_MERGE_TOMBSTONE_WEIGHT 5
-
 /*
 ** There are two versions of the format used for the structure record:
 **
@@ -355,6 +353,7 @@ struct Fts5Index {
   i64 iWriteRowid;                /* Rowid for current doc being written */
   int bDelete;                    /* Current write is a delete */
   int nContentlessDelete;         /* Number of contentless delete ops */
+  int nPendingRow;                /* Number of INSERT in hash table */
 
   /* Error state. */
   int rc;                         /* Current error code */
@@ -404,6 +403,8 @@ struct Fts5StructureSegment {
   u64 iOrigin1;
   u64 iOrigin2;
   int nPgTombstone;               /* Number of tombstone hash table pages */
+  i64 nEntryTombstone;            /* Number of tombstone entries that "count" */
+  i64 nEntry;                     /* Number of rows in this segment */
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -1117,6 +1118,8 @@ static int fts5StructureDecode(
             i += fts5GetVarint(&pData[i], &pSeg->iOrigin1);
             i += fts5GetVarint(&pData[i], &pSeg->iOrigin2);
             i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone);
+            i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone);
+            i += fts5GetVarint(&pData[i], &pSeg->nEntry);
             nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2);
           }
           if( pSeg->pgnoLast<pSeg->pgnoFirst ){
@@ -1372,13 +1375,16 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
       assert( pLvl->nMerge<=pLvl->nSeg );
 
       for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
-        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid);
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst);
+        fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast);
         if( pStruct->nOriginCntr>0 ){
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin1);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin2);
-          fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nPgTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone);
+          fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry);
         }
       }
     }
@@ -3988,6 +3994,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
   if( p->pHash ){
     sqlite3Fts5HashClear(p->pHash);
     p->nPendingData = 0;
+    p->nPendingRow = 0;
   }
   p->nContentlessDelete = 0;
 }
@@ -4627,7 +4634,7 @@ static void fts5IndexMergeLevel(
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
 
-    /* Set the range of origins that will go into the output segment */
+    /* Set the range of origins that will go into the output segment. */
     if( pStruct->nOriginCntr>0 ){
       pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
       pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
@@ -4691,8 +4698,11 @@ static void fts5IndexMergeLevel(
     int i;
 
     /* Remove the redundant segments from the %_data table */
+    assert( pSeg->nEntry==0 );
     for(i=0; i<nInput; i++){
-      fts5DataRemoveSegment(p, &pLvl->aSeg[i]);
+      Fts5StructureSegment *pOld = &pLvl->aSeg[i];
+      pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone);
+      fts5DataRemoveSegment(p, pOld);
     }
 
     /* Remove the redundant segments from the input level */
@@ -4718,6 +4728,43 @@ static void fts5IndexMergeLevel(
   if( pnRem ) *pnRem -= writer.nLeafWritten;
 }
 
+/*
+** If this is not a contentless_delete=1 table, or if the 'delete-automerge'
+** configuration option is set to 0, then this function always returns -1.
+** Otherwise, it searches the structure object passed as the second argument
+** for a level suitable for merging due to having a large number of 
+** tombstones in the tombstone hash. If one is found, its index is returned.
+** Otherwise, if there is no suitable level, -1.
+*/
+static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
+  Fts5Config *pConfig = p->pConfig;
+  int iRet = -1;
+  if( pConfig->bContentlessDelete && pConfig->nDeleteAutomerge>0 ){
+    int ii;
+    int nBest = 0;
+
+    for(ii=0; ii<pStruct->nLevel; ii++){
+      Fts5StructureLevel *pLvl = &pStruct->aLevel[ii];
+      i64 nEntry = 0;
+      i64 nTomb = 0;
+      int iSeg;
+      for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+        nEntry += pLvl->aSeg[iSeg].nEntry;
+        nTomb += pLvl->aSeg[iSeg].nEntryTombstone;
+      }
+      assert( nEntry>0 || pLvl->nSeg==0 );
+      if( nEntry>0 ){
+        int nPercent = (nTomb * 100) / nEntry;
+        if( nPercent>=pConfig->nDeleteAutomerge && nPercent>nBest ){
+          iRet = ii;
+          nBest = nPercent;
+        }
+      }
+    }
+  }
+  return iRet;
+}
+
 /*
 ** Do up to nPg pages of automerge work on the index.
 **
@@ -4738,51 +4785,28 @@ static int fts5IndexMerge(
     int iBestLvl = 0;           /* Level offering the most input segments */
     int nBest = 0;              /* Number of input segments on best level */
 
-    /* Set iBestLvl to the level to read input segments from. */
+    /* Set iBestLvl to the level to read input segments from. Or to -1 if
+    ** there is no level suitable to merge segments from.  */
     assert( pStruct->nLevel>0 );
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
       Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
-      int nThisSeg = 0;
       if( pLvl->nMerge ){
         if( pLvl->nMerge>nBest ){
           iBestLvl = iLvl;
-          nBest = pLvl->nMerge;
+          nBest = nMin;
         }
         break;
       }
-      nThisSeg = pLvl->nSeg;
-      if( bTombstone && nThisSeg ){
-        int iSeg;
-        int nPg = 0;
-        int nTomb = 0;
-        for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-          nPg += pSeg->pgnoLast;
-          nTomb += pSeg->nPgTombstone;
-        }
-        nThisSeg += ((nTomb*FTS5_MERGE_TOMBSTONE_WEIGHT) / nPg);
-      }
-      if( nThisSeg>nBest ){
-        nBest = nThisSeg;
+      if( pLvl->nSeg>nBest ){
+        nBest = pLvl->nSeg;
         iBestLvl = iLvl;
       }
     }
-
-    /* If nBest is still 0, then the index must be empty. */
-#ifdef SQLITE_DEBUG
-    for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
-      assert( pStruct->aLevel[iLvl].nSeg==0 );
+    if( nBest<nMin ){
+      iBestLvl = fts5IndexFindDeleteMerge(p, pStruct);
     }
-#endif
 
-    if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
-      if( bTombstone || p->pConfig->bContentlessDelete==0 ){
-        break;
-      }else{
-        bTombstone = 1;
-        continue;
-      }
-    }
+    if( iBestLvl<0 ) break;
     bRet = 1;
     fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
     if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
@@ -5477,6 +5501,7 @@ static void fts5FlushOneHash(Fts5Index *p){
           if( pStruct->nOriginCntr>0 ){
             pSeg->iOrigin1 = pStruct->nOriginCntr;
             pSeg->iOrigin2 = pStruct->nOriginCntr;
+            pSeg->nEntry = p->nPendingRow;
             pStruct->nOriginCntr++;
           }
           pStruct->nSegment++;
@@ -5498,10 +5523,11 @@ static void fts5FlushOneHash(Fts5Index *p){
 */
 static void fts5IndexFlush(Fts5Index *p){
   /* Unless it is empty, flush the hash table to disk */
-  if( p->nPendingData || (p->nContentlessDelete && p->pConfig->nAutomerge>0) ){
+  if( p->nPendingData || p->nContentlessDelete ){
     assert( p->pHash );
-    p->nPendingData = 0;
     fts5FlushOneHash(p);
+    p->nPendingData = 0;
+    p->nPendingRow = 0;
   }
 }
 
@@ -5579,6 +5605,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
 
   assert( p->rc==SQLITE_OK );
   fts5IndexFlush(p);
+  assert( p->nContentlessDelete==0 );
   pStruct = fts5StructureRead(p);
   fts5StructureInvalidate(p);
 
@@ -5608,7 +5635,10 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
 ** INSERT command.
 */
 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
-  Fts5Structure *pStruct = fts5StructureRead(p);
+  Fts5Structure *pStruct = 0;
+
+  fts5IndexFlush(p);
+  pStruct = fts5StructureRead(p);
   if( pStruct ){
     int nMin = p->pConfig->nUsermerge;
     fts5StructureInvalidate(p);
@@ -6130,6 +6160,9 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
 
   p->iWriteRowid = iRowid;
   p->bDelete = bDelete;
+  if( bDelete==0 ){
+    p->nPendingRow++;
+  }
   return fts5IndexReturn(p);
 }
 
@@ -6883,12 +6916,17 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
   Fts5Structure *pStruct;
   pStruct = fts5StructureRead(p);
   if( pStruct ){
+    int bFound = 0;               /* True after pSeg->nEntryTombstone incr. */
     int iLvl;
-    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+    for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
       int iSeg;
-      for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+      for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
         Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
         if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
+          if( bFound==0 ){
+            pSeg->nEntryTombstone++;
+            bFound = 1;
+          }
           fts5IndexTombstoneAdd(p, pSeg, iRowid);
         }
       }
@@ -7980,7 +8018,7 @@ static int fts5structConnectMethod(
   rc = sqlite3_declare_vtab(db, 
       "CREATE TABLE xyz("
           "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
-          "npgtombstone, struct HIDDEN);"
+          "npgtombstone, nentrytombstone, nentry, struct HIDDEN);"
   );
   if( rc==SQLITE_OK ){
     pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
@@ -8007,7 +8045,7 @@ static int fts5structBestIndexMethod(
   pIdxInfo->idxNum = 0;
   for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
     if( p->usable==0 ) continue;
-    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==9 ){
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){
       rc = SQLITE_OK;
       pIdxInfo->aConstraintUsage[i].omit = 1;
       pIdxInfo->aConstraintUsage[i].argvIndex = 1;
@@ -8120,15 +8158,21 @@ static int fts5structColumnMethod(
     case 5: /* leaf2 */
       sqlite3_result_int(ctx, pSeg->pgnoLast);
       break;
-    case 6: /* loc1 */
-      sqlite3_result_int(ctx, pSeg->iOrigin1);
+    case 6: /* origin1 */
+      sqlite3_result_int64(ctx, pSeg->iOrigin1);
       break;
-    case 7: /* loc2 */
-      sqlite3_result_int(ctx, pSeg->iOrigin2);
+    case 7: /* origin2 */
+      sqlite3_result_int64(ctx, pSeg->iOrigin2);
       break;
     case 8: /* npgtombstone */
       sqlite3_result_int(ctx, pSeg->nPgTombstone);
       break;
+    case 9: /* nentrytombstone */
+      sqlite3_result_int64(ctx, pSeg->nEntryTombstone);
+      break;
+    case 10: /* nentry */
+      sqlite3_result_int64(ctx, pSeg->nEntry);
+      break;
   }
   return SQLITE_OK;
 }
diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test
index 6316628c55..a44311e45a 100644
--- a/ext/fts5/test/fts5contentless3.test
+++ b/ext/fts5/test/fts5contentless3.test
@@ -192,6 +192,5 @@ do_execsql_test 3.7 {
 } {2 0 0}
 
 
-
 finish_test
 
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 21eba71b1a..9eb1900ee3 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -39,19 +39,36 @@ do_execsql_test 1.0 {
   )
   INSERT INTO ft SELECT document(12) FROM s;
 }
+
 do_execsql_test 1.1 {
   INSERT INTO ft(ft) VALUES('optimize');
 }
 
 do_execsql_test 1.2 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 0}
+
+do_execsql_test 1.3 {
+  DELETE FROM ft WHERE rowid < 50
+}
+
+do_execsql_test 1.4 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 0 1000 49}
+
+do_execsql_test 1.5 {
   DELETE FROM ft WHERE rowid < 1000
 }
 
-execsql_pp {
-  SELECT * FROM fts5_structure((
+do_execsql_test 1.6 {
+  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
     SELECT block FROM ft_data WHERE id=10
   ))
-}
+} {1 0 1 0}
 
 finish_test
 
diff --git a/manifest b/manifest
index f9de77a7fc..8a6f302dd1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Integrate\scontentless\sdelete\swith\sauto-merge.
-D 2023-07-22T19:47:46.359
+C Add\sthe\sfts5\s'delete-automerge'\sinteger\soption.\sA\slevel\sis\seligible\sfor\sauto-merging\sif\sit\shas\sa\sgreater\sthan\sor\sequal\spercentage\sof\sits\sentries\sdeleted\sby\stombstones\sthan\sthe\s'delete-automerge'\soption.\sDefault\svalue\sis\s10.
+D 2023-07-24T19:13:06.235
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,13 +86,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h f59c14f725ad0fcb8a81b9bf012e5021c6501bf43e73aa00b00d728e2ac7efaf
+F ext/fts5/fts5Int.h 7decc306406187d1826c5eba9b8e8e6661b85580e5da1203760c0c2de9bc4a5e
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
+F ext/fts5/fts5_config.c c35f3433586f9152af2f444356020bf5c0f2e1d0fae29e67ab45de81277d07c9
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
-F ext/fts5/fts5_hash.c 60224220ccfb2846b741b6dbb1b8872094ec6d87b3118c04244dafc83e6f9c40
-F ext/fts5/fts5_index.c 31b8c8dd6913d76d6d7755342e36816495e5ad177d253f6bf39e0efdb9dc31e0
+F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
+F ext/fts5/fts5_index.c f5d50e218db3d32dd12c83b3f700bf79ed7f24e3f60f43f5b56e62146e2c31b5
 F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -134,8 +134,8 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
-F ext/fts5/test/fts5contentless4.test 3b11ccbbe928d45eb8f985c0137a8fe2c69b70b940b10de31540040de5674311
+F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
+F ext/fts5/test/fts5contentless4.test 4403cbbbb5021b36b24d914addb22a782699d1a03a98fede705793f7184dbcd8
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
-R 223fbfa40ff414b8abee5011e5d1533a
+P 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
+R 067f23316c8bda4fd7b0c9766119547d
 U dan
-Z 8cb202caf797f67b895f2a9a207e2618
+Z b4aabcdd1189dc9cd9c674268a107cec
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 13263786f5..d3ab0c925e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
\ No newline at end of file
+b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
\ No newline at end of file

From 27553579c0c15fa12d07f85da6ff12ffc1425b35 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 24 Jul 2023 22:34:26 +0000
Subject: [PATCH 29/66] Rework the JSON parse structure to facilitate better
 caching.  Passes all tests.

FossilOrigin-Name: ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5
---
 manifest      |  12 +--
 manifest.uuid |   2 +-
 src/json.c    | 224 ++++++++++++++++++++++++++++----------------------
 3 files changed, 135 insertions(+), 103 deletions(-)

diff --git a/manifest b/manifest
index e99cb3d132..6a75e96be4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Incremental\sprogress\stoward\simproved\scaching\sof\sparsed\sJSON.
-D 2023-07-24T17:59:25.604
+C Rework\sthe\sJSON\sparse\sstructure\sto\sfacilitate\sbetter\scaching.\s\sPasses\sall\ntests.
+D 2023-07-24T22:34:26.093
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 3584f5fb0ff48009ebabb34704328c2086d924fc2403fdf0e48afce5b9f22777
+F src/json.c e743eb83ad6d581bdfd355aee1d0f2c3025e955ec1799706ec4542b46e3af7f2
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456
-R 831d8993ebdb0e60bc4ab13513e5d282
+P f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85
+R fee4083d53f0bb74692c3aad54ce5646
 U drh
-Z 8cedb8d1881aa5ce4f4f5c5fea73c221
+Z 5a89ce2654bac729f780024f319554e3
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 5be0ea0fd7..0cc1f0a629 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85
\ No newline at end of file
+ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 97ebb6af65..b109a8759d 100644
--- a/src/json.c
+++ b/src/json.c
@@ -87,15 +87,14 @@ struct JsonTask {
 /* JSON type values
 */
 #define JSON_SUBST    0    /* Special edit node.  Uses u.iPrev */
-#define JSON_PATCH    1    /* Special edit node.  Uses u.pPatch */
-#define JSON_NULL     2
-#define JSON_TRUE     3
-#define JSON_FALSE    4
-#define JSON_INT      5
-#define JSON_REAL     6
-#define JSON_STRING   7
-#define JSON_ARRAY    8
-#define JSON_OBJECT   9
+#define JSON_NULL     1
+#define JSON_TRUE     2
+#define JSON_FALSE    3
+#define JSON_INT      4
+#define JSON_REAL     5
+#define JSON_STRING   6
+#define JSON_ARRAY    7
+#define JSON_OBJECT   8
 
 /* The "subtype" set for JSON values */
 #define JSON_SUBTYPE  74    /* Ascii for "J" */
@@ -104,7 +103,7 @@ struct JsonTask {
 ** Names of the various JSON types:
 */
 static const char * const jsonType[] = {
-  "subst", "patch",
+  "subst",
   "null", "true", "false", "integer", "real", "text", "array", "object"
 };
 
@@ -113,8 +112,8 @@ static const char * const jsonType[] = {
 #define JNODE_RAW     0x01         /* Content is raw, not JSON encoded */
 #define JNODE_ESCAPE  0x02         /* Content is text with \ escapes */
 #define JNODE_REMOVE  0x04         /* Do not output */
-#define JNODE_REPLACE 0x08         /* Replace with JsonNode.u.iReplace */
-#define JNODE_PATCH   0x10         /* Patch with JsonNode.u.pPatch */
+#define JNODE_REPLACE 0x08         /* Target of a JSON_SUBST node */
+                  /*  0x10         Available for reuse */
 #define JNODE_APPEND  0x20         /* More ARRAY/OBJECT entries at u.iAppend */
 #define JNODE_LABEL   0x40         /* Is a label of an object */
 #define JNODE_JSON5   0x80         /* Node contains JSON5 enhancements */
@@ -133,8 +132,7 @@ struct JsonNode {
     const char *zJContent; /* 1: Content for INT, REAL, and STRING */
     u32 iAppend;           /* 2: More terms for ARRAY and OBJECT */
     u32 iKey;              /* 3: Key for ARRAY objects in json_tree() */
-    JsonNode *pPatch;      /* 5: Node chain of patch for JNODE_PATCH */
-    u32 iPrev;             /* 6: Previous SUBST node, or 0 */
+    u32 iPrev;             /* 4: Previous SUBST node, or 0 */
   } u;
 };
 
@@ -613,23 +611,19 @@ static void jsonRenderNode(
   JsonString *pOut               /* Write JSON here */
 ){
   assert( pNode!=0 );
-  while( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) ){
-    if( (pNode->jnFlags & JNODE_REPLACE)!=0 ){
-      u32 idx = (u32)(pNode - pParse->aNode);
-      u32 i = pParse->iSubst;
-      while( 1 /*exit-by-break*/ ){
-        assert( i<pParse->nNode );
-        assert( pParse->aNode[i].eType==JSON_SUBST );
-        assert( pParse->aNode[i].eU==6 );
-        assert( pParse->aNode[i].u.iPrev<i );
-        if( pParse->aNode[i].n==idx ){
-          pNode = &pParse->aNode[i+1];
-          break;
-        }
-        i = pParse->aNode[i].u.iPrev;
+  while( (pNode->jnFlags & JNODE_REPLACE)!=0 ){
+    u32 idx = (u32)(pNode - pParse->aNode);
+    u32 i = pParse->iSubst;
+    while( 1 /*exit-by-break*/ ){
+      assert( i<pParse->nNode );
+      assert( pParse->aNode[i].eType==JSON_SUBST );
+      assert( pParse->aNode[i].eU==4 );
+      assert( pParse->aNode[i].u.iPrev<i );
+      if( pParse->aNode[i].n==idx ){
+        pNode = &pParse->aNode[i+1];
+        break;
       }
-    }else{
-      pNode = pNode->u.pPatch;
+      i = pParse->aNode[i].u.iPrev;
     }
   }
   switch( pNode->eType ){
@@ -697,7 +691,7 @@ static void jsonRenderNode(
         }
         if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
         assert( pNode->eU==2 );
-        pNode = &pNode[pNode->u.iAppend];
+        pNode = &pParse->aNode[pNode->u.iAppend];
         j = 1;
       }
       jsonAppendChar(pOut, ']');
@@ -718,7 +712,7 @@ static void jsonRenderNode(
         }
         if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
         assert( pNode->eU==2 );
-        pNode = &pNode[pNode->u.iAppend];
+        pNode = &pParse->aNode[pNode->u.iAppend];
         j = 1;
       }
       jsonAppendChar(pOut, '}');
@@ -736,6 +730,10 @@ static void jsonReturnJson(
   sqlite3_context *pCtx       /* Return value for this function */
 ){
   JsonString s;
+  if( pParse->oom ){
+    sqlite3_result_error_nomem(pCtx);
+    return;
+  }
   jsonInit(&s, pCtx);
   jsonRenderNode(pParse, pNode, &s);
   jsonResult(&s);
@@ -948,6 +946,12 @@ static int jsonParseAddNode(JsonParse*,u32,u32,const char*);
 #endif
 
 
+/*
+** Add a single node to pParse->aNode after first expanding the
+** size of the aNode array.  Return the index of the new node.
+**
+** If an OOM error occurs, set pParse->oom and return -1.
+*/
 static JSON_NOINLINE int jsonParseAddNodeExpand(
   JsonParse *pParse,        /* Append the node to this object */
   u32 eType,                /* Node type */
@@ -964,7 +968,7 @@ static JSON_NOINLINE int jsonParseAddNodeExpand(
     pParse->oom = 1;
     return -1;
   }
-  pParse->nAlloc = nNew;
+  pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode);
   pParse->aNode = pNew;
   assert( pParse->nNode<pParse->nAlloc );
   return jsonParseAddNode(pParse, eType, n, zContent);
@@ -995,6 +999,31 @@ static int jsonParseAddNode(
   return pParse->nNode++;
 }
 
+/*
+** Add an array of new nodes to the current pParse->aNode array.
+** Return the index of the first node added.
+**
+** If an OOM error occurs, set pParse->oom.
+*/
+static void jsonParseAddNodeArray(
+  JsonParse *pParse,        /* Append the node to this object */
+  JsonNode *aNode,          /* Array of nodes to add */
+  u32 nNode                 /* Number of elements in aNew */
+){
+  if( pParse->nNode + nNode > pParse->nAlloc ){
+    u32 nNew = pParse->nNode + nNode;
+    JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode));
+    if( aNew==0 ){
+      pParse->oom = 1;
+      return;
+    }
+    pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode);
+    pParse->aNode = aNew;
+  }
+  memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode));
+  pParse->nNode += nNode;
+}
+
 /*
 ** Add a new JSON_SUBST node.  The node immediately following
 ** this new node will be the substitute content for iNode.
@@ -1004,8 +1033,9 @@ static int jsonParseAddSubstNode(
   u32 iNode                /* References this node */
 ){
   int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0);
-  if( idx<=0 ) return idx;
-  pParse->aNode[idx].eU = 6;
+  if( pParse->oom ) return -1;
+  pParse->aNode[iNode].jnFlags |= JNODE_REPLACE;
+  pParse->aNode[idx].eU = 4;
   pParse->aNode[idx].u.iPrev = pParse->iSubst;
   pParse->iSubst = idx;
   return idx;
@@ -1852,6 +1882,22 @@ static JsonNode *jsonLookupStep(
   u32 i, j, nKey;
   const char *zKey;
   JsonNode *pRoot = &pParse->aNode[iRoot];
+  while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){
+    u32 idx = (u32)(pRoot - pParse->aNode);
+    u32 i = pParse->iSubst;
+    while( 1 /*exit-by-break*/ ){
+      assert( i<pParse->nNode );
+      assert( pParse->aNode[i].eType==JSON_SUBST );
+      assert( pParse->aNode[i].eU==4 );
+      assert( pParse->aNode[i].u.iPrev<i );
+      if( pParse->aNode[i].n==idx ){
+        pRoot = &pParse->aNode[i+1];
+        iRoot = i+1;
+        break;
+      }
+      i = pParse->aNode[i].u.iPrev;
+    }
+  }
   if( zPath[0]==0 ) return pRoot;
   if( zPath[0]=='.' ){
     if( pRoot->eType!=JSON_OBJECT ) return 0;
@@ -1887,7 +1933,7 @@ static JsonNode *jsonLookupStep(
       }
       if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
       assert( pRoot->eU==2 );
-      iRoot += pRoot->u.iAppend;
+      iRoot = pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
       j = 1;
     }
@@ -1902,7 +1948,7 @@ static JsonNode *jsonLookupStep(
       if( pNode ){
         pRoot = &pParse->aNode[iRoot];
         assert( pRoot->eU==0 );
-        pRoot->u.iAppend = iStart - iRoot;
+        pRoot->u.iAppend = iStart;
         pRoot->jnFlags |= JNODE_APPEND;
         VVA( pRoot->eU = 2 );
         pParse->aNode[iLabel].jnFlags |= JNODE_RAW;
@@ -1928,7 +1974,7 @@ static JsonNode *jsonLookupStep(
           }
           if( (pBase->jnFlags & JNODE_APPEND)==0 ) break;
           assert( pBase->eU==2 );
-          iBase += pBase->u.iAppend;
+          iBase = pBase->u.iAppend;
           pBase = &pParse->aNode[iBase];
           j = 1;
         }
@@ -1962,7 +2008,7 @@ static JsonNode *jsonLookupStep(
       }
       if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
       assert( pRoot->eU==2 );
-      iRoot += pRoot->u.iAppend;
+      iRoot = pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
       j = 1;
     }
@@ -1978,7 +2024,7 @@ static JsonNode *jsonLookupStep(
       if( pNode ){
         pRoot = &pParse->aNode[iRoot];
         assert( pRoot->eU==0 );
-        pRoot->u.iAppend = iStart - iRoot;
+        pRoot->u.iAppend = iStart;
         pRoot->jnFlags |= JNODE_APPEND;
         VVA( pRoot->eU = 2 );
       }
@@ -2111,9 +2157,7 @@ static void jsonRemoveAllNulls(JsonNode *pNode){
 */
 static void jsonDebugPrintNodeEntries(
   JsonNode *aNode,  /* First node entry to print */
-  int N,            /* Number of node entries to print */
-  int ofst,         /* Node number for p */
-  int nDent         /* Indent by this many spaces */
+  int N             /* Number of node entries to print */
 ){
   int i;
   for(i=0; i<N; i++){
@@ -2123,35 +2167,28 @@ static void jsonDebugPrintNodeEntries(
     }else{
       zType = jsonType[aNode[i].eType];
     }
-    if( nDent>0 ) printf("%*s", nDent, "");
     if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){
       printf("node %4u: %-7s %02x n=%-5d",
-             i+ofst, zType, aNode[i].jnFlags, aNode[i].n);
+             i, zType, aNode[i].jnFlags, aNode[i].n);
     }else{
       printf("node %4u: %-7s    n=%-5d",
-             i+ofst, zType, aNode[i].n);
+             i, zType, aNode[i].n);
     }
     switch( aNode[i].eU ){
       case 1:  printf(" zJContent=[%.*s]\n",
                       aNode[i].n, aNode[i].u.zJContent);           break;
       case 2:  printf(" iAppend=%u\n", aNode[i].u.iAppend);        break;
       case 3:  printf(" iKey=%u\n", aNode[i].u.iKey);              break;
-      case 5: {
-        JsonNode *pX = aNode[i].u.pPatch;
-        printf(" pPatch=...\n");
-        jsonDebugPrintNodeEntries(pX, jsonNodeSize(pX), 0, nDent+3);
-        break;
-      }
-      case 6:  printf(" iPrev=%u\n", aNode[i].u.iPrev);      break;
+      case 4:  printf(" iPrev=%u\n", aNode[i].u.iPrev);            break;
       default: printf("\n");
     }
   }
 }
 static void jsonDebugPrintParse(JsonParse *p){
-  jsonDebugPrintNodeEntries(p->aNode, p->nNode, 0, 0);
+  jsonDebugPrintNodeEntries(p->aNode, p->nNode);
 }
 static void jsonDebugPrintNode(JsonNode *pNode){
-  jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode), 0, 0);
+  jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode));
 }
 #else
    /* The usual case */
@@ -2447,45 +2484,38 @@ static JsonNode *jsonMergePatch(
       assert( pTarget[j].eType==JSON_STRING );
       assert( pTarget[j].jnFlags & JNODE_LABEL );
       if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){
-        if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break;
+        if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break;
         if( pPatch[i+1].eType==JSON_NULL ){
           pTarget[j+1].jnFlags |= JNODE_REMOVE;
         }else{
           JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]);
           if( pNew==0 ) return 0;
-          pTarget = &pParse->aNode[iTarget];
-          if( pNew!=&pTarget[j+1] ){
-            assert( pTarget[j+1].eU==0
-                 || pTarget[j+1].eU==1
-                 || pTarget[j+1].eU==2 );
-            testcase( pTarget[j+1].eU==1 );
-            testcase( pTarget[j+1].eU==2 );
-            VVA( pTarget[j+1].eU = 5 );
-            pTarget[j+1].u.pPatch = pNew;
-            pTarget[j+1].jnFlags |= JNODE_PATCH;
+          if( pNew!=&pParse->aNode[iTarget+j+1] ){
+            jsonParseAddSubstNode(pParse, iTarget+j+1);
+            jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew));
           }
+          pTarget = &pParse->aNode[iTarget];
         }
         break;
       }
     }
     if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){
-      int iStart, iPatch;
-      iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
+      int iStart;
+      JsonNode *pApnd;
+      u32 nApnd;
+      iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0);
       jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
-      iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
+      pApnd = &pPatch[i+1];
+      if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd);
+      nApnd = jsonNodeSize(pApnd);
+      jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd));
       if( pParse->oom ) return 0;
-      jsonRemoveAllNulls(pPatch);
-      pTarget = &pParse->aNode[iTarget];
-      assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 );
-      testcase( pParse->aNode[iRoot].eU==2 );
+      pParse->aNode[iStart].n = 1+nApnd;
       pParse->aNode[iRoot].jnFlags |= JNODE_APPEND;
+      pParse->aNode[iRoot].u.iAppend = iStart;
       VVA( pParse->aNode[iRoot].eU = 2 );
-      pParse->aNode[iRoot].u.iAppend = iStart - iRoot;
       iRoot = iStart;
-      assert( pParse->aNode[iPatch].eU==0 );
-      VVA( pParse->aNode[iPatch].eU = 5 );
-      pParse->aNode[iPatch].jnFlags |= JNODE_PATCH;
-      pParse->aNode[iPatch].u.pPatch = &pPatch[i+1];
+      pTarget = &pParse->aNode[iTarget];
     }
   }
   return pTarget;
@@ -2513,7 +2543,8 @@ static void jsonPatchFunc(
   }
   pResult = jsonMergePatch(&x, 0, y.aNode);
   assert( pResult!=0 || x.oom );
-  if( pResult ){
+  if( pResult && x.oom==0 ){
+    jsonDebugPrintParse(&x);
     jsonDebugPrintNode(pResult);
     jsonReturnJson(&x, pResult, ctx);
   }else{
@@ -2652,23 +2683,26 @@ static void jsonReplaceNode(
       }
       if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){
         int k = jsonParseAddNode(p, JSON_STRING, n, z);
+        char *zCopy = sqlite3DbStrDup(0, z);
         if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW;
-        jsonParseAddCleanup(p, sqlite3_free, sqlite3DbStrDup(0,z));
-      }else{
-        int k= jsonParseAddNode(p, JSON_PATCH, 0, 0);
-        if( k>0 ){
-          JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx);
-          if( pPatch==0 ){
-            p->oom = 1;
-            break;
-          }
-          assert( pPatch->nJPRef>=1 );
-          pPatch->nJPRef++;
-          p->aNode[k].jnFlags |= JNODE_PATCH;
-          p->aNode[k].eU = 5;
-          p->aNode[k].u.pPatch = pPatch->aNode;
-          jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch);
+        if( zCopy ){
+          jsonParseAddCleanup(p, sqlite3_free, zCopy);
+        }else{
+          sqlite3_result_error_nomem(pCtx);
         }
+      }else{
+        JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx);
+        if( pPatch==0 ){
+          p->oom = 1;
+          break;
+        }
+        jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode);
+        /* The nodes copied out of pPatch and into p likely contain
+        ** u.zJContent pointers into pPatch->zJson.  So preserve the
+        ** content of pPatch until p is destroyed. */
+        assert( pPatch->nJPRef>=1 );
+        pPatch->nJPRef++;
+        jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch);
       }
       break;
     }
@@ -2703,7 +2737,6 @@ static void jsonReplaceFunc(
     pNode = jsonLookup(&x, zPath, 0, ctx);
     if( x.nErr ) goto replace_err;
     if( pNode ){
-      pNode->jnFlags |= (u8)JNODE_REPLACE;
       jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
     }
   }
@@ -2755,7 +2788,6 @@ static void jsonSetFunc(
     }else if( x.nErr ){
       goto jsonSetDone;
     }else if( pNode && (bApnd || bIsSet) ){
-      pNode->jnFlags |= (u8)JNODE_REPLACE;
       jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
     }
   }

From 3d2a559ecb5087f554c36344ca790fb8f4502f2e Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 24 Jul 2023 22:45:59 +0000
Subject: [PATCH 30/66] Clean up some #defines in json.

FossilOrigin-Name: 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 15 +++++++--------
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 6a75e96be4..7f3392f540 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rework\sthe\sJSON\sparse\sstructure\sto\sfacilitate\sbetter\scaching.\s\sPasses\sall\ntests.
-D 2023-07-24T22:34:26.093
+C Clean\sup\ssome\s#defines\sin\sjson.
+D 2023-07-24T22:45:59.601
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c e743eb83ad6d581bdfd355aee1d0f2c3025e955ec1799706ec4542b46e3af7f2
+F src/json.c a12c5517137b9ab77183b4a15084da264c32190f834fd3d45fda1eb77d9e327a
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85
-R fee4083d53f0bb74692c3aad54ce5646
+P ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5
+R c1e40d83b9a43b4c127658e4fcf06fd6
 U drh
-Z 5a89ce2654bac729f780024f319554e3
+Z 6091135e309dfa309be3fb90139aabae
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 0cc1f0a629..91a2b6e391 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5
\ No newline at end of file
+327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index b109a8759d..217c1070c9 100644
--- a/src/json.c
+++ b/src/json.c
@@ -109,14 +109,13 @@ static const char * const jsonType[] = {
 
 /* Bit values for the JsonNode.jnFlag field
 */
-#define JNODE_RAW     0x01         /* Content is raw, not JSON encoded */
-#define JNODE_ESCAPE  0x02         /* Content is text with \ escapes */
-#define JNODE_REMOVE  0x04         /* Do not output */
-#define JNODE_REPLACE 0x08         /* Target of a JSON_SUBST node */
-                  /*  0x10         Available for reuse */
-#define JNODE_APPEND  0x20         /* More ARRAY/OBJECT entries at u.iAppend */
-#define JNODE_LABEL   0x40         /* Is a label of an object */
-#define JNODE_JSON5   0x80         /* Node contains JSON5 enhancements */
+#define JNODE_RAW     0x01  /* Content is raw, not JSON encoded */
+#define JNODE_ESCAPE  0x02  /* Content is text with \ escapes */
+#define JNODE_REMOVE  0x04  /* Do not output */
+#define JNODE_REPLACE 0x08  /* Target of a JSON_SUBST node */
+#define JNODE_APPEND  0x10  /* More ARRAY/OBJECT entries at u.iAppend */
+#define JNODE_LABEL   0x20  /* Is a label of an object */
+#define JNODE_JSON5   0x40  /* Node contains JSON5 enhancements */
 
 
 /* A single node of parsed JSON

From b715fe9d80b3656fd0afb82847932e6592306609 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 24 Jul 2023 23:27:05 +0000
Subject: [PATCH 31/66] It is an error to try to insert a BLOB value into JSON.

FossilOrigin-Name: 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 15 +++++++++++----
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 7f3392f540..192ed6f6b3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clean\sup\ssome\s#defines\sin\sjson.
-D 2023-07-24T22:45:59.601
+C It\sis\san\serror\sto\stry\sto\sinsert\sa\sBLOB\svalue\sinto\sJSON.
+D 2023-07-24T23:27:05.772
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c a12c5517137b9ab77183b4a15084da264c32190f834fd3d45fda1eb77d9e327a
+F src/json.c fc5b67025d4ca026d5c0696cfd5c265c1cd0fa2f669fbe78d268e955f853e75c
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5
-R c1e40d83b9a43b4c127658e4fcf06fd6
+P 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2
+R bdfb959c0be321608fe703b9663eda08
 U drh
-Z 6091135e309dfa309be3fb90139aabae
+Z 1ec3b6ff4cb1459575429b1da04d690b
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 91a2b6e391..9eac82e0e2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2
\ No newline at end of file
+28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 217c1070c9..0ce68f1fcb 100644
--- a/src/json.c
+++ b/src/json.c
@@ -733,10 +733,12 @@ static void jsonReturnJson(
     sqlite3_result_error_nomem(pCtx);
     return;
   }
-  jsonInit(&s, pCtx);
-  jsonRenderNode(pParse, pNode, &s);
-  jsonResult(&s);
-  sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
+  if( pParse->nErr==0 ){
+    jsonInit(&s, pCtx);
+    jsonRenderNode(pParse, pNode, &s);
+    jsonResult(&s);
+    sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
+  }
 }
 
 /*
@@ -2705,6 +2707,11 @@ static void jsonReplaceNode(
       }
       break;
     }
+    case SQLITE_BLOB: {
+      sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1);
+      p->nErr++;
+      break;
+    }
   }
 }
   

From 322a3257033d2b7e1a1a136909ea1e73dc7d9291 Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Tue, 25 Jul 2023 12:26:05 +0000
Subject: [PATCH 32/66] Reformulate a (sed -i) construct in
 ext/wasm/GNUmakefile to account for Mac's sed -i being different than GNU's.

FossilOrigin-Name: 907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c
---
 ext/wasm/GNUmakefile |  7 ++++++-
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile
index 8fedaee7b6..2346773337 100644
--- a/ext/wasm/GNUmakefile
+++ b/ext/wasm/GNUmakefile
@@ -677,10 +677,15 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles)
 #
 # Upstream RFE:
 # https://github.com/emscripten-core/emscripten/issues/18237
+#
+# Maintenance reminder: Mac sed -i works differently than GNU sed, so
+# don't use that flag here.
 define SQLITE3.xJS.ESM-EXPORT-DEFAULT
 if [ x1 = x$(1) ]; then \
 		echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \
-		sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \
+		{\
+			sed -e '0,/^export default/{/^export default/d;}' $@ > $@.tmp && mv $@.tmp $@; \
+		} || exit $$?; \
 		if [ x != x$(2) ]; then \
 			if ! grep -q '^export default' $@; then \
 				echo "Cannot find export default." 1>&2; \
diff --git a/manifest b/manifest
index 8d61da8af1..55ef9f2d35 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Resolve\san\sES6\smodule\sexport\sconstruct\swhich\sis\sincompatible\swith\snode.js,\sas\sreported\sin\s[forum:b9680fa9ad|forum\spost\sb9680fa9ad].
-D 2023-07-24T15:41:58.448
+C Reformulate\sa\s(sed\s-i)\sconstruct\sin\sext/wasm/GNUmakefile\sto\saccount\sfor\sMac's\ssed\s-i\sbeing\sdifferent\sthan\sGNU's.
+D 2023-07-25T12:26:05.338
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 4e8260d05c52d9924b853efbdfe052bd483cfe42f055567c1bbf29d274794b22
+F ext/wasm/GNUmakefile a3e316bf51d8915a49d4728e0d41f19ec4c3ead3409d1171989a4fe163ec2214
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3
-R 1ab3847cf4557b99f62130a0f45e24bd
+P 80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615
+R 6a5bd7ff24557139b19e78c4a48cb406
 U stephan
-Z 4a3a3c6a8d937dc54de34852d2276ed0
+Z 524f840828c91fa63ac22024dbea11bc
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 649aa580be..d8bafbc0f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615
\ No newline at end of file
+907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c
\ No newline at end of file

From 039d494d5fe9ab68d458eb64241ec356da89f1f8 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 13:53:42 +0000
Subject: [PATCH 33/66] Further tests for 'delete-automerge'.

FossilOrigin-Name: ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
---
 ext/fts5/test/fts5contentless4.test | 57 ++++++++++++++++++++++++++++-
 manifest                            | 12 +++---
 manifest.uuid                       |  2 +-
 3 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 9eb1900ee3..d707003c6f 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -70,9 +70,64 @@ do_execsql_test 1.6 {
   ))
 } {1 0 1 0}
 
-finish_test
+#--------------------------------------------------------------------------
+reset_db
+db func document document
 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
 
+do_test 2.1 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {5000}
+
+for {set ii 5000} {$ii >= 0} {incr ii -100} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid > $ii
+  }
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } $ii
+}
+
+execsql_pp {
+  SELECT * FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+}
+
+do_test 2.4 {
+  for {set ii 0} {$ii < 5000} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document(12) ) }
+  }
+} {}
+
+for {set ii 1} {$ii <= 5000} {incr ii 10} {
+  do_execsql_test 2.3.$ii {
+    DELETE FROM ft WHERE rowid = $ii;
+    INSERT INTO ft VALUES( document(12) );
+    INSERT INTO ft(ft, rank) VALUES('merge', -10);
+  }
+
+  do_execsql_test 2.3.$ii.2 {
+    SELECT 
+      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
+    FROM 
+      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
+  } 5000
+}
 
 finish_test
 
diff --git a/manifest b/manifest
index 8a6f302dd1..c125c285b2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sfts5\s'delete-automerge'\sinteger\soption.\sA\slevel\sis\seligible\sfor\sauto-merging\sif\sit\shas\sa\sgreater\sthan\sor\sequal\spercentage\sof\sits\sentries\sdeleted\sby\stombstones\sthan\sthe\s'delete-automerge'\soption.\sDefault\svalue\sis\s10.
-D 2023-07-24T19:13:06.235
+C Further\stests\sfor\s'delete-automerge'.
+D 2023-07-25T13:53:42.704
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test 4403cbbbb5021b36b24d914addb22a782699d1a03a98fede705793f7184dbcd8
+F ext/fts5/test/fts5contentless4.test b42b76b4b4c7d6c8dcc4d681fe98f804241bed22e31d1c125f1dbab3104cadab
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
-R 067f23316c8bda4fd7b0c9766119547d
+P b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
+R 57589268405704f94631ba59ff450fcb
 U dan
-Z b4aabcdd1189dc9cd9c674268a107cec
+Z 4728fe3315208b53f9df48d307acc67e
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d3ab0c925e..f1f62f3312 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
\ No newline at end of file
+ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
\ No newline at end of file

From f02cc9a3248b07095847f9a5e93e092e4fa6e116 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 25 Jul 2023 15:08:18 +0000
Subject: [PATCH 34/66] Create the new RCStr class of strings and try to use
 them for JSON storage.

FossilOrigin-Name: c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c
---
 manifest        |  21 ++++---
 manifest.uuid   |   2 +-
 src/json.c      |  42 +++++++++-----
 src/printf.c    | 143 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/sqliteInt.h |  52 ++++++++++++++++++
 src/vdbemem.c   |  23 ++++++++
 6 files changed, 258 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index 192ed6f6b3..b4a115361a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C It\sis\san\serror\sto\stry\sto\sinsert\sa\sBLOB\svalue\sinto\sJSON.
-D 2023-07-24T23:27:05.772
+C Create\sthe\snew\sRCStr\sclass\sof\sstrings\sand\stry\sto\suse\sthem\sfor\sJSON\sstorage.
+D 2023-07-25T15:08:18.543
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c fc5b67025d4ca026d5c0696cfd5c265c1cd0fa2f669fbe78d268e955f853e75c
+F src/json.c 3add12eb29f09a99dc260bcde427ea925f600cf0bb0d9abb0618a03d7e50eead
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
-F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f
+F src/printf.c 21e410b0a3904dddb39ef1b245cfb991302022a6f0fc16f0a8a13539d6d2f713
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
@@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81
 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h dcb1a885e8b6cb78df618944b89d44361a99d0fe33e1bba2c150a855f7dc5599
+F src/sqliteInt.h 59b755dec944aa3b068e962ef53264de6fde3d6b6df2d5869ea3afdb7facdf60
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c 40afb83ed848e235848ffdd3ba25adca4ba602111b8ed3b05ae3b1b12e0eacee
+F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
@@ -2044,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2
-R bdfb959c0be321608fe703b9663eda08
+P 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042
+R 4c5edeff0b59e42b46536f2b483d14f6
+T *branch * json-opt-rcstr
+T *sym-json-opt-rcstr *
+T -sym-json-opt *
 U drh
-Z 1ec3b6ff4cb1459575429b1da04d690b
+Z 3f6f4e5856b228a13fb14e603baeb559
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 9eac82e0e2..59c5808d17 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042
\ No newline at end of file
+c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 0ce68f1fcb..3bfa0c7484 100644
--- a/src/json.c
+++ b/src/json.c
@@ -191,7 +191,7 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){
 ** initial state.
 */
 static void jsonReset(JsonString *p){
-  if( !p->bStatic ) sqlite3_free(p->zBuf);
+  if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf);
   jsonZero(p);
 }
 
@@ -212,7 +212,7 @@ static int jsonGrow(JsonString *p, u32 N){
   char *zNew;
   if( p->bStatic ){
     if( p->bErr ) return 1;
-    zNew = sqlite3_malloc64(nTotal);
+    zNew = sqlite3RCStrNew(nTotal);
     if( zNew==0 ){
       jsonOom(p);
       return SQLITE_NOMEM;
@@ -221,12 +221,12 @@ static int jsonGrow(JsonString *p, u32 N){
     p->zBuf = zNew;
     p->bStatic = 0;
   }else{
-    zNew = sqlite3_realloc64(p->zBuf, nTotal);
-    if( zNew==0 ){
-      jsonOom(p);
+    p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal);
+    if( p->zBuf==0 ){
+      p->bErr = 1;
+      jsonZero(p);
       return SQLITE_NOMEM;
     }
-    p->zBuf = zNew;
   }
   p->nAlloc = nTotal;
   return SQLITE_OK;
@@ -514,16 +514,26 @@ static void jsonAppendValue(
 
 
 /* Make the JSON in p the result of the SQL function.
+**
+** The JSON string is reset.
 */
 static void jsonResult(JsonString *p){
   if( p->bErr==0 ){
-    jsonAppendChar(p, 0);
-    sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1,
-                          p->bStatic ? SQLITE_TRANSIENT : sqlite3_free,
-                          SQLITE_UTF8);
-    jsonZero(p);
+    if( p->bStatic ){
+      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT,
+                            SQLITE_UTF8);
+    }else{
+      jsonAppendChar(p, 0);
+      p->nUsed--;
+      sqlite3RCStrRef(p->zBuf);
+      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
+                            (void(*)(void*))sqlite3RCStrUnref,
+                            SQLITE_UTF8);
+    }
+  }else{
+    sqlite3_result_error_nomem(p->pCtx);
   }
-  assert( p->bStatic );
+  jsonReset(p);
 }
 
 /**************************************************************************
@@ -1885,7 +1895,7 @@ static JsonNode *jsonLookupStep(
   JsonNode *pRoot = &pParse->aNode[iRoot];
   while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){
     u32 idx = (u32)(pRoot - pParse->aNode);
-    u32 i = pParse->iSubst;
+    i = pParse->iSubst;
     while( 1 /*exit-by-break*/ ){
       assert( i<pParse->nNode );
       assert( pParse->aNode[i].eType==JSON_SUBST );
@@ -2950,7 +2960,8 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
       assert( pStr->bStatic );
     }else if( isFinal ){
       sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
-                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
+                          pStr->bStatic ? SQLITE_TRANSIENT :
+                              (void(*)(void*))sqlite3RCStrUnref);
       pStr->bStatic = 1;
     }else{
       sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
@@ -3058,7 +3069,8 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
       assert( pStr->bStatic );
     }else if( isFinal ){
       sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
-                          pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
+                          pStr->bStatic ? SQLITE_TRANSIENT :
+                          (void(*)(void*))sqlite3RCStrUnref);
       pStr->bStatic = 1;
     }else{
       sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
diff --git a/src/printf.c b/src/printf.c
index 3fb1a322a0..7b84288168 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1366,3 +1366,146 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){
   sqlite3_str_vappendf(p, zFormat, ap);
   va_end(ap);
 }
+
+
+/*****************************************************************************
+** Reference counted string storage
+*****************************************************************************/
+
+/*
+** Increase the reference count of the string by one.
+**
+** The input parameter is returned.
+*/
+char *sqlite3RCStrRef(char *z){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  p->nRCRef++;
+  return z;
+}
+
+/*
+** Decrease the reference count by one.  Free the string when the
+** reference count reaches zero.
+*/
+void sqlite3RCStrUnref(char *z){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  if( p->nRCRef>=2 ){
+    p->nRCRef--;
+  }else{
+    if( p->xFree ) p->xFree(p->pAttach);
+#ifdef SQLITE_DEBUG
+    p->uMagic = 0;
+#endif
+    sqlite3_free(p);
+  }
+}
+
+/*
+** Return true if the reference count on the string is exactly one, meaning
+** that the string can be modified.  Return false if the reference count
+** is greater than one.
+*/
+int sqlite3RCStrIsWriteable(char *z){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  return p->nRCRef==1;
+}
+
+/*
+** Create a new string that is capable of holding N bytes of text, not counting
+** the zero byte at the end.  The string is uninitialized.
+**
+** The reference count is initially 1.  Call sqlite3RCStrUnref() to free the
+** newly allocated string.
+**
+** This routine returns 0 on an OOM.
+*/
+char *sqlite3RCStrNew(u64 N){
+  RCStr *p = sqlite3_malloc64( N + sizeof(*p) );
+  if( p==0 ) return 0;
+  p->nRCRef = 1;
+  p->xFree = 0;
+  p->pAttach = 0;
+#ifdef SQLITE_DEBUG
+  p->uMagic = SQLITE_RCSTR_MAGIC;
+#endif
+  return (char*)&p[1];
+}
+
+/*
+** Return the number of bytes allocated to the string.  The value returned
+** does not include the space for the zero-terminator at the end.
+*/
+u64 sqlite3RCStrSize(char *z){
+  RCStr *p = (RCStr*)z;
+  u64 N;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  N = sqlite3_msize(p);
+  N -= sizeof(p) + 1;
+  return N;
+}
+
+/*
+** Change the size of the string so that it is able to hold N bytes.
+** The string might be reallocated, so return the new allocation.
+*/
+char *sqlite3RCStrResize(char *z, u64 N){
+  RCStr *p = (RCStr*)z;
+  RCStr *pNew;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef==1 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1);
+  if( pNew==0 ){
+    sqlite3_free(p);
+    return 0;
+  }else{
+    return (char*)&pNew[1];
+  }
+}
+
+/*
+** Add a new attachment to the string.
+**
+** A string may have no more than one attachment.  When a new attachment
+** is added, any prior attachment is destroyed.  Remove an attachment
+** by adding a zero-attachment.
+*/
+void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  if( p->xFree ) p->xFree(p->pAttach);
+  p->xFree = xFree;
+  p->pAttach = pAttach;
+}
+  
+/*
+** Return the attachment associated with a string if the attachment
+** has the destructure specified in the second argument.  If the
+** string has no attachment or if the destructor does not match,
+** then return a NULL pointr.
+*/
+void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){
+  RCStr *p = (RCStr*)z;
+  assert( p!=0 );
+  p--;
+  assert( p->nRCRef>0 );
+  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
+  return p->xFree==xFree ? p->pAttach : 0;
+}
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f214862f74..adf4d34dec 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1281,6 +1281,7 @@ typedef struct Parse Parse;
 typedef struct ParseCleanup ParseCleanup;
 typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
+typedef struct RCStr RCStr;
 typedef struct RenameToken RenameToken;
 typedef struct Returning Returning;
 typedef struct RowSet RowSet;
@@ -4061,6 +4062,47 @@ struct sqlite3_str {
 
 #define isMalloced(X)  (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0)
 
+/*
+** The following object is the header for an "RCStr" or "reference-counted
+** string".  An RCStr is passed around and used like any other char*
+** that has been dynamically allocated.  The important interface
+** difference is that it uses sqlite3RCStrUnref() as its destructor
+** rather than sqlite3_free().  Other than that the two are interchangeable.
+**
+** Thus to return an RCStr object as the result of an SQL function use:
+** 
+**    sqlite3_result_text64(ctx,z,sz,sqlite3RCStrUnref,SQLITE_UTF8)
+**                                   ^^^^^^^^^^^^^^^^^
+**                                   Instead of sqlite3_free() or similar
+**
+** An SQL function can check its arguments to see if they are RCStr
+** strings using the sqlite3ValueIsOfClass() function:
+**
+**    sqlite3ValueIsOfClass(argv[i], sqlite3RCStrUnref);
+**
+** An RCStr string might be better than an ordinary string in some cases
+** because:
+**
+**    (1)  You can duplicate it using sqlite3RCStrRef(x).
+**
+**    (2)  You can also add an associated object to the string.  For
+**         example, if the string is JSON, perhaps the associated object
+**         is a parse of that JSON.
+**
+** Methods for an RCStr string begin with "sqlite3RCStr...".
+*/
+struct RCStr {
+  u32 nRCRef;            /* Number of references */
+#ifdef SQLITE_DEBUG
+  u32 uMagic;            /* Magic number for sanity checking */
+#endif
+  void *pAttach;         /* Attachment to this string */
+  void (*xFree)(void*);  /* Destructor for the attachment */
+};
+
+/* The Magic number used by RCStr for sanity checking.  SQLITE_DEBUG only. */
+#define SQLITE_RCSTR_MAGIC 0x3dc05d54
+
 
 /*
 ** A pointer to this structure is used to communicate information
@@ -5180,6 +5222,7 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
+int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
@@ -5287,6 +5330,15 @@ void sqlite3OomClear(sqlite3*);
 int sqlite3ApiExit(sqlite3 *db, int);
 int sqlite3OpenTempDatabase(Parse *);
 
+char *sqlite3RCStrRef(char*);
+void sqlite3RCStrUnref(char*);
+char *sqlite3RCStrNew(u64);
+u64 sqlite3RCStrSize(char*);
+char *sqlite3RCStrResize(char*,u64);
+int sqlite3RCStrIsWriteable(char*);
+void sqlite3RCStrAttach(char*, void*, void(*)(void*));
+void *sqlite3RCStrGetAttachment(char*,void(*)(void*));
+
 void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int);
 int sqlite3StrAccumEnlarge(StrAccum*, i64);
 char *sqlite3StrAccumFinish(StrAccum*);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index b5a794ae8f..87dfbbebd8 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -333,6 +333,11 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){
       pMem->flags |= MEM_Term;
       return;
     }
+    if( pMem->xDel==(void(*)(void*))sqlite3RCStrUnref ){
+      /* Blindly assume that all RCStr objects are zero-terminated */
+      pMem->flags |= MEM_Term;
+      return;
+    }
   }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){
     pMem->z[pMem->n] = 0;
     pMem->flags |= MEM_Term;
@@ -1363,6 +1368,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
+/* Return true if sqlit3_value object pVal is a string or blob value
+** that uses the destructor specified in the second argument.
+**
+** TODO:  Maybe someday promote this interface into a published API so
+** that third-party extensions can get access to it?
+*/
+int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
+  if( ALWAYS(pVal!=0)
+   && (pVal->flags & (MEM_Str|MEM_Blob))!=0
+   && (pVal->flags & MEM_Dyn)!=0
+   && pVal->xDel==xFree
+  ){
+    return 1;
+  }else{
+    return 0;
+  }
+}
+
 /*
 ** Create a new sqlite3_value object.
 */

From 4bca1248455c6b8c4c79ac953461f4726429ed02 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 25 Jul 2023 15:43:01 +0000
Subject: [PATCH 35/66] Fix a minor problem with error reporting in JSON.

FossilOrigin-Name: c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4
---
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 src/json.c    |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index b4a115361a..f3393df05d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Create\sthe\snew\sRCStr\sclass\sof\sstrings\sand\stry\sto\suse\sthem\sfor\sJSON\sstorage.
-D 2023-07-25T15:08:18.543
+C Fix\sa\sminor\sproblem\swith\serror\sreporting\sin\sJSON.
+D 2023-07-25T15:43:01.908
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 3add12eb29f09a99dc260bcde427ea925f600cf0bb0d9abb0618a03d7e50eead
+F src/json.c 8f44f1aadfe1967af70dc9e6390823806f5a42cc301f11df435a1c3d4950308d
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,11 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042
-R 4c5edeff0b59e42b46536f2b483d14f6
-T *branch * json-opt-rcstr
-T *sym-json-opt-rcstr *
-T -sym-json-opt *
+P c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c
+R 625da2283c68c7b3c0e2eb949802be15
 U drh
-Z 3f6f4e5856b228a13fb14e603baeb559
+Z 238deba72513ff58266287df7bec917f
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 59c5808d17..70df72c921 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c
\ No newline at end of file
+c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 3bfa0c7484..18c4721455 100644
--- a/src/json.c
+++ b/src/json.c
@@ -530,7 +530,7 @@ static void jsonResult(JsonString *p){
                             (void(*)(void*))sqlite3RCStrUnref,
                             SQLITE_UTF8);
     }
-  }else{
+  }else if( p->bErr==1 ){
     sqlite3_result_error_nomem(p->pCtx);
   }
   jsonReset(p);

From 3f874b58fbfd05a1cf5881a365812fe4869be0ac Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 15:48:58 +0000
Subject: [PATCH 36/66] Change the name of the fts5 'delete-automerge' option
 to 'deletemerge'. And add tests for it.

FossilOrigin-Name: 1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
---
 ext/fts5/fts5Int.h                  |  2 +-
 ext/fts5/fts5_config.c              |  6 +--
 ext/fts5/fts5_index.c               |  6 +--
 ext/fts5/test/fts5contentless4.test | 66 +++++++++++++++++++++++++++++
 manifest                            | 18 ++++----
 manifest.uuid                       |  2 +-
 6 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 89b553161b..8bbafbaaf4 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -214,7 +214,7 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
-  int nDeleteAutomerge;           /* 'delete-automerge' */
+  int nDeleteMerge;           /* 'deletemerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index af2ef40bac..5d0770502e 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -924,7 +924,7 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
-  else if( 0==sqlite3_stricmp(zKey, "delete-automerge") ){
+  else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
     int nVal = -1;
     if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
       nVal = sqlite3_value_int(pVal);
@@ -933,7 +933,7 @@ int sqlite3Fts5ConfigSetValue(
     }
     if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
     if( nVal>100 ) nVal = 0;
-    pConfig->nDeleteAutomerge = nVal;
+    pConfig->nDeleteMerge = nVal;
   }
 
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
@@ -984,7 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
-  pConfig->nDeleteAutomerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
+  pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ce52825533..5e20da2b04 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4729,7 +4729,7 @@ static void fts5IndexMergeLevel(
 }
 
 /*
-** If this is not a contentless_delete=1 table, or if the 'delete-automerge'
+** If this is not a contentless_delete=1 table, or if the 'deletemerge'
 ** configuration option is set to 0, then this function always returns -1.
 ** Otherwise, it searches the structure object passed as the second argument
 ** for a level suitable for merging due to having a large number of 
@@ -4739,7 +4739,7 @@ static void fts5IndexMergeLevel(
 static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
   Fts5Config *pConfig = p->pConfig;
   int iRet = -1;
-  if( pConfig->bContentlessDelete && pConfig->nDeleteAutomerge>0 ){
+  if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){
     int ii;
     int nBest = 0;
 
@@ -4755,7 +4755,7 @@ static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
       assert( nEntry>0 || pLvl->nSeg==0 );
       if( nEntry>0 ){
         int nPercent = (nTomb * 100) / nEntry;
-        if( nPercent>=pConfig->nDeleteAutomerge && nPercent>nBest ){
+        if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){
           iRet = ii;
           nBest = nPercent;
         }
diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index d707003c6f..6463b18d00 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -129,5 +129,71 @@ for {set ii 1} {$ii <= 5000} {incr ii 10} {
   } 5000
 }
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text');
+} {1 {SQL logic error}}
+do_catchsql_test 3.2 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+} {0 {}}
+do_execsql_test 3.3 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 50}
+do_catchsql_test 3.4 {
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 101);
+} {0 {}}
+do_execsql_test 3.5 {
+  SELECT * FROM ft_config WHERE k='deletemerge'
+} {deletemerge 101}
+
+do_execsql_test 3.6 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.7 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {94 100}
+
+do_execsql_test 3.8 {
+  DELETE FROM ft WHERE rowid=95
+}
+
+do_execsql_test 3.9 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {95 100}
+
+do_execsql_test 3.10 {
+  DELETE FROM ft;
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ft SELECT document(12) FROM s;
+  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
+}
+
+do_execsql_test 3.11 {
+  DELETE FROM ft WHERE rowid<95
+}
+
+do_execsql_test 3.12 {
+  SELECT nentrytombstone, nentry FROM fts5_structure((
+    SELECT block FROM ft_data WHERE id=10
+  ))
+} {0 6}
+
 finish_test
 
diff --git a/manifest b/manifest
index c125c285b2..3cacac4565 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\stests\sfor\s'delete-automerge'.
-D 2023-07-25T13:53:42.704
+C Change\sthe\sname\sof\sthe\sfts5\s'delete-automerge'\soption\sto\s'deletemerge'.\sAnd\sadd\stests\sfor\sit.
+D 2023-07-25T15:48:58.767
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -86,13 +86,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h 7decc306406187d1826c5eba9b8e8e6661b85580e5da1203760c0c2de9bc4a5e
+F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c c35f3433586f9152af2f444356020bf5c0f2e1d0fae29e67ab45de81277d07c9
+F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c f5d50e218db3d32dd12c83b3f700bf79ed7f24e3f60f43f5b56e62146e2c31b5
+F ext/fts5/fts5_index.c 182cf576bae17682adf2fe8cefb8bc3b46bea27e6a227e444859b3c065322687
 F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test b42b76b4b4c7d6c8dcc4d681fe98f804241bed22e31d1c125f1dbab3104cadab
+F ext/fts5/test/fts5contentless4.test 52aad02fe9eb1bfa9272ee35ac95f6dc31eab57ba470c2df0ed63f0bfbe95bc2
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b314be66b9ac0190b5373b3b6baec012382bc588c2d86c2edab796669a4303c3
-R 57589268405704f94631ba59ff450fcb
+P ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
+R 4c7400d78b496d9cb420c06c5a4d6bad
 U dan
-Z 4728fe3315208b53f9df48d307acc67e
+Z a03778de7e1285f066cc1d73e2bb1632
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index f1f62f3312..736b30b29f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
\ No newline at end of file
+1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
\ No newline at end of file

From 5326953e5799542071373922c1da43e96f65335b Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 16:48:54 +0000
Subject: [PATCH 37/66] Add extra test for 'deletemerge'.

FossilOrigin-Name: bc33cff4203cef12518e0f43d380a06d53d67c725fb96cfe6e934b7dc97a7efd
---
 ext/fts5/test/fts5contentless4.test | 49 +++++++++++++++++++++++++++++
 manifest                            | 12 +++----
 manifest.uuid                       |  2 +-
 3 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test
index 6463b18d00..1c2666dcf8 100644
--- a/ext/fts5/test/fts5contentless4.test
+++ b/ext/fts5/test/fts5contentless4.test
@@ -195,5 +195,54 @@ do_execsql_test 3.12 {
   ))
 } {0 6}
 
+#-------------------------------------------------------------------------
+reset_db
+db func document document
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1);
+  INSERT INTO x1(x1, rank) VALUES('usermerge', 16);
+  INSERT INTO x1(x1, rank) VALUES('deletemerge', 40);
+  INSERT INTO x1 VALUES('one');
+  INSERT INTO x1 VALUES('two');
+  INSERT INTO x1 VALUES('three');
+  INSERT INTO x1 VALUES('four');
+  INSERT INTO x1 VALUES('five');
+  INSERT INTO x1 VALUES('six');
+  INSERT INTO x1 VALUES('seven');
+  INSERT INTO x1 VALUES('eight');
+  INSERT INTO x1 VALUES('nine');
+  INSERT INTO x1 VALUES('ten');
+}
+
+do_execsql_test 4.1 {
+  SELECT level, segment FROM fts5_structure((
+    SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+}
+
+for {set ii 1} {$ii < 4} {incr ii} {
+  do_execsql_test 4.2.$ii {
+    DELETE FROM x1 WHERE rowid = $ii;
+    INSERT INTO x1(x1, rank) VALUES('merge', 5);
+    SELECT level, segment FROM fts5_structure((
+          SELECT block FROM x1_data WHERE id=10
+    ))
+  } {
+    0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
+  }
+}
+
+do_execsql_test 4.3 {
+  DELETE FROM x1 WHERE rowid = $ii;
+  INSERT INTO x1(x1, rank) VALUES('merge', 5);
+  SELECT level, segment, nentry FROM fts5_structure((
+        SELECT block FROM x1_data WHERE id=10
+  ))
+} {
+  1 0   6
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index 3cacac4565..34b86f367f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sthe\sfts5\s'delete-automerge'\soption\sto\s'deletemerge'.\sAnd\sadd\stests\sfor\sit.
-D 2023-07-25T15:48:58.767
+C Add\sextra\stest\sfor\s'deletemerge'.
+D 2023-07-25T16:48:54.280
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed
 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
 F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
 F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
-F ext/fts5/test/fts5contentless4.test 52aad02fe9eb1bfa9272ee35ac95f6dc31eab57ba470c2df0ed63f0bfbe95bc2
+F ext/fts5/test/fts5contentless4.test 0f43ededc2874f65d7da99b641a82239854d98d3fa43db729f284b723f23b69f
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
@@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ca26c7a37a7e680be633f43be28f8877bdf9917448ea51c3bedc9b2352a00601
-R 4c7400d78b496d9cb420c06c5a4d6bad
+P 1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
+R 5e50972831b0957645434d1b66948061
 U dan
-Z a03778de7e1285f066cc1d73e2bb1632
+Z 17e8bfe851f3a2d36e6b484a74a5eecf
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 736b30b29f..20b6bcd047 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1079300db2a7d1fbc86a01c215c234a3af64889c5396e6da63ff4f3c7efae4c5
\ No newline at end of file
+bc33cff4203cef12518e0f43d380a06d53d67c725fb96cfe6e934b7dc97a7efd
\ No newline at end of file

From 0c39b13e468e44b3c0231c7199c4d7bf9bca5a0a Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Tue, 25 Jul 2023 17:54:25 +0000
Subject: [PATCH 38/66] Fix a harmless "set-but-not-used" compiler warning in
 sqlite3session.c.

FossilOrigin-Name: 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
---
 ext/session/sqlite3session.c |  1 +
 manifest                     | 14 +++++++-------
 manifest.uuid                |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index 6794088494..9f862f2465 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -891,6 +891,7 @@ static int sessionPreupdateEqual(
         rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal);
       }
       assert( rc==SQLITE_OK );
+      (void)rc;                   /* Suppress warning about unused variable */
       if( sqlite3_value_type(pVal)!=eType ) return 0;
 
       /* A SessionChange object never has a NULL value in a PK column */
diff --git a/manifest b/manifest
index 55ef9f2d35..f9ef9ad44c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reformulate\sa\s(sed\s-i)\sconstruct\sin\sext/wasm/GNUmakefile\sto\saccount\sfor\sMac's\ssed\s-i\sbeing\sdifferent\sthan\sGNU's.
-D 2023-07-25T12:26:05.338
+C Fix\sa\sharmless\s"set-but-not-used"\scompiler\swarning\sin\ssqlite3session.c.
+D 2023-07-25T17:54:25.000
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -475,7 +475,7 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a
 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795
 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c
 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc
-F ext/session/sqlite3session.c e50a9218ee360db0a25298adc6614162d80ebe65d3f6a5b0a021e0902f6536a1
+F ext/session/sqlite3session.c 1971b61ca45babf0d9e4bb669a65b0903135e9828af2fcd4f0c8f1b7acf36b6f
 F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb
 F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615
-R 6a5bd7ff24557139b19e78c4a48cb406
-U stephan
-Z 524f840828c91fa63ac22024dbea11bc
+P 907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c
+R 262b3aa3dc9bbec239621d23dc374114
+U dan
+Z f47917699640df15bd88b843742372bf
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d8bafbc0f9..7561758d66 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c
\ No newline at end of file
+54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
\ No newline at end of file

From 440e696d5e0b31a2542a01b3fba9eb356fea52ad Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 25 Jul 2023 18:28:03 +0000
Subject: [PATCH 39/66] Incremental improvements to JSON parsing - trying to
 fold in the RCStr object.

FossilOrigin-Name: 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09
---
 manifest      |  12 ++--
 manifest.uuid |   2 +-
 src/json.c    | 184 +++++++++++++++++++++++++++++++++++++++++++-------
 3 files changed, 166 insertions(+), 32 deletions(-)

diff --git a/manifest b/manifest
index f3393df05d..09e5a7cfff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sminor\sproblem\swith\serror\sreporting\sin\sJSON.
-D 2023-07-25T15:43:01.908
+C Incremental\simprovements\sto\sJSON\sparsing\s-\strying\sto\sfold\sin\sthe\sRCStr\sobject.
+D 2023-07-25T18:28:03.341
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 8f44f1aadfe1967af70dc9e6390823806f5a42cc301f11df435a1c3d4950308d
+F src/json.c 28fe8ed9e63293fd5708d129d4edd4a634887aabb0364685195974e35ee10762
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c
-R 625da2283c68c7b3c0e2eb949802be15
+P c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4
+R 2b5df60a4958d2a70ce11e3569ae2bab
 U drh
-Z 238deba72513ff58266287df7bec917f
+Z d10e127820cbbcdc4a84e1b86eb42adb
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 70df72c921..a811037656 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4
\ No newline at end of file
+4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 18c4721455..194d0421eb 100644
--- a/src/json.c
+++ b/src/json.c
@@ -142,7 +142,7 @@ struct JsonParse {
   u32 nNode;         /* Number of slots of aNode[] used */
   u32 nAlloc;        /* Number of slots of aNode[] allocated */
   JsonNode *aNode;   /* Array of nodes containing the parse */
-  const char *zJson; /* Original JSON string */
+  char *zJson;       /* Original JSON string */
   u32 *aUp;          /* Index of parent of each node */
   JsonTask *pClean;  /* Cleanup operations prior to freeing this object */
   u16 iDepth;        /* Nesting depth */
@@ -150,6 +150,8 @@ struct JsonParse {
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   u8 nJPRef;         /* Number of references to this object */
+  u8 bOwnsJson;      /* This object owns zJson and response for freeing it */
+  u8 isMod;          /* aNode contains edits from the original zJson */
   int nJson;         /* Length of the zJson string in bytes */
   u32 iErr;          /* Error location in zJson[] */
   u32 iSubst;        /* Last known JSON_SUBST node */
@@ -565,24 +567,41 @@ static void jsonParseReset(JsonParse *pParse){
     sqlite3_free(pTask);
   }
   assert( pParse->nJPRef<=1 );
-  sqlite3_free(pParse->aNode);
-  pParse->aNode = 0;
+  if( pParse->aNode ){
+    sqlite3_free(pParse->aNode);
+    pParse->aNode = 0;
+  }
   pParse->nNode = 0;
   pParse->nAlloc = 0;
-  sqlite3_free(pParse->aUp);
-  pParse->aUp = 0;
+  if( pParse->aUp ){
+    sqlite3_free(pParse->aUp);
+    pParse->aUp = 0;
+  }
+  if( pParse->bOwnsJson ){
+    /* Order operations so that if the destructor for pParse->zJson
+    ** invokes jsonParseFree(), the recursion will terminate harmlessly */
+    char *z = pParse->zJson;
+    pParse->zJson = 0;
+    pParse->bOwnsJson = 0;
+    sqlite3RCStrUnref(z);
+  }
 }
 
 /*
 ** Free a JsonParse object that was obtained from sqlite3_malloc().
+**
+** Note that destroying JsonParse might call sqlite3RCStrUnref() to
+** destroy the zJson value.  The RCStr object might recursively invoke
+** JsonParse to destroy this pParse object again.  Take care to ensure
+** that this recursive destructor sequence terminates harmlessly.
 */
 static void jsonParseFree(JsonParse *pParse){
   if( pParse->nJPRef>1 ){
     pParse->nJPRef--;
-    return;
+  }else{
+    jsonParseReset(pParse);
+    sqlite3_free(pParse);
   }
-  jsonParseReset(pParse);
-  sqlite3_free(pParse);
 }
 
 /*
@@ -1218,7 +1237,7 @@ static const struct NanInfName {
 **
 ** Special return values:
 **
-**      0    End if input
+**      0    End of input
 **     -1    Syntax error
 **     -2    '}' seen
 **     -3    ']' seen
@@ -1685,7 +1704,7 @@ json_parse_restart:
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
   sqlite3_context *pCtx,       /* Report errors here */
-  const char *zJson            /* Input JSON text to be parsed */
+  char *zJson                  /* Input JSON text to be parsed */
 ){
   int i;
   memset(pParse, 0, sizeof(*pParse));
@@ -1719,6 +1738,121 @@ static int jsonParse(
   return 0;
 }
 
+#if 0
+/*
+** This is a destructor for JSON strings.  We make it a separate function
+** so that the sqlite3ValueIsOfClass() function can be used to unambiguously
+** identify sqlite3_value objects that are known JSON strings.
+*/
+static void jsonClass(void *p){
+  sqlite3RCStrUnref((char*)p);
+}
+#endif
+
+#if 0
+/*
+** Process SQL function argument pJson as a JSON string.  Return a pointer
+** to its parse.
+**
+** If any error is encountered, return a NULL pointer and leave an error
+** message in pCtx.
+*/
+static JsonParse *jsonParseFromFunctionArg(
+  sqlite3_value *pJson,       /* An SQL function argument containing JSON */
+  sqlite3_context *pCtx,      /* For accessing the cache */
+  sqlite3_context *pErrCtx,   /* For reporting errors */
+  int bUnchng                 /* Only accept cached parse that are unchanged */
+){
+  JsonParse *pParse;
+  char *zJson;
+  int nJson;
+  JsonParse *pMatch = 0;
+  int iKey;
+  int iMinKey = 0;
+  u32 iMinHold = 0xffffffff;
+  u32 iMaxHold = 0;
+
+  char *zJson = (char*)sqlite3_value_text(pJson);
+  int nJson = sqlite3_value_bytes(pJson);
+  if( zJson==0 ) goto json_parse_value_nomem;
+  if( sqlite3ValueIsOfClass(pJson, jsonClass) ){
+    assert( zJson!=0 );
+    (void)sqlite3RCStrRef(zJson);
+    pParse = sqlite3RCStrGetAttachment(zJson, (void(*)(void*)jsonParseFree);
+    if( pParse ){
+      return pParse;
+    }
+  }else{
+    char *z = zJson;
+    zJson = sqlite3RCStrNew( nJson );
+    if( zJson==0 ) goto json_parse_value_nomem;
+    assert( strlen(z)==nJson );
+    memcpy(zJson, z, nJson+1);
+  }
+
+  /* At this point zJson is an RCStr object that does not yet have a
+  ** parse.  This procedure is holding its own reference to that zJson
+  ** and needs to release it, or hand it off, prior to returning.
+  **
+  ** The next step is to try to find a parse of the JSON that already
+  ** exists in cache.
+  */
+  for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
+    JsonParse *p;
+    p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
+    if( p==0 ){
+      iMinKey = iKey;
+      break;
+    }
+    if( pMatch==0
+     && p->nJson==nJson
+     && (p->isMod==0 || bUnchng==0)
+     && memcmp(p->zJson,zJson,nJson)==0
+    ){
+      p->nErr = 0;
+      pMatch = p;
+    }else if( p->iHold<iMinHold ){
+      iMinHold = p->iHold;
+      iMinKey = iKey;
+    }
+    if( p->iHold>iMaxHold ){
+      iMaxHold = p->iHold;
+    }
+  }
+
+  if( pMatch ){
+    pParse = pMatch;
+    pParse->iHold = iMaxHold+1;
+  }else{
+    /* No parse of zJson could be found in cache.  So parse it afresh.
+    */
+    pParse = sqlite3_malloc64( sizeof(JsonParse) );
+    if( pParse==0 ) goto json_parse_value_nomem;
+    if( jsonParse(pParse, pCtx, zJson) ){
+      sqlite3_free(pParse);
+      sqlite3RCStrUnref(zJson);
+      return 0;
+    }
+    pParse->bOwnsJson = 1;
+    pParse->nJson = nJson;
+    pParse->nJPRef = 1;
+    pParse->iHold = iMaxHold+1;
+    sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, pParse,
+                        (void(*)(void*))jsonParseFree);
+    pParse = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
+  }
+  if( pParse ){
+    pParse->nJPRef++;  /* The caller will own this object */
+  }
+  return pParse;
+
+json_parse_value_nomem:
+  if( zJson ) sqlite3RCStrUnref(zJson);
+  if( pErrCtx ) sqlite3_result_error_nomem(pCtx);
+  return 0;
+}
+#endif
+
 /* Mark node i of pParse as being a child of iParent.  Call recursively
 ** to fill in all the descendants of node i.
 */
@@ -1786,11 +1920,11 @@ static int jsonParseFindParents(JsonParse *pParse){
 */
 static JsonParse *jsonParseCached(
   sqlite3_context *pCtx,
-  sqlite3_value **argv,
+  sqlite3_value *pJson,
   sqlite3_context *pErrCtx
 ){
-  const char *zJson = (const char*)sqlite3_value_text(argv[0]);
-  int nJson = sqlite3_value_bytes(argv[0]);
+  char *zJson = (char*)sqlite3_value_text(pJson);
+  int nJson = sqlite3_value_bytes(pJson);
   JsonParse *p;
   JsonParse *pMatch = 0;
   int iKey;
@@ -2223,7 +2357,7 @@ static void jsonParseFunc(
   u32 i;
 
   assert( argc==1 );
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
   jsonParseFindParents(&x);
   jsonInit(&s, ctx);
   for(i=0; i<x.nNode; i++){
@@ -2331,7 +2465,7 @@ static void jsonArrayLengthFunc(
   u32 i;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv, ctx);
+  p = jsonParseCached(ctx, argv[0], ctx);
   if( p==0 ) return;
   assert( p->nNode );
   if( argc==2 ){
@@ -2393,7 +2527,7 @@ static void jsonExtractFunc(
   JsonString jx;
 
   if( argc<2 ) return;
-  p = jsonParseCached(ctx, argv, ctx);
+  p = jsonParseCached(ctx, argv[0], ctx);
   if( p==0 ) return;
   if( argc==2 ){
     /* With a single PATH argument */
@@ -2547,8 +2681,8 @@ static void jsonPatchFunc(
   JsonNode *pResult;   /* The result of the merge */
 
   UNUSED_PARAMETER(argc);
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
-  if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){
     jsonParseReset(&x);
     return;
   }
@@ -2624,7 +2758,7 @@ static void jsonRemoveFunc(
   u32 i;
 
   if( argc<1 ) return;
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2702,7 +2836,7 @@ static void jsonReplaceNode(
           sqlite3_result_error_nomem(pCtx);
         }
       }else{
-        JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx);
+        JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx);
         if( pPatch==0 ){
           p->oom = 1;
           break;
@@ -2746,7 +2880,7 @@ static void jsonReplaceFunc(
     jsonWrongNumArgs(ctx, "replace");
     return;
   }
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2792,7 +2926,7 @@ static void jsonSetFunc(
     jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
     return;
   }
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2830,7 +2964,7 @@ static void jsonTypeFunc(
   const char *zPath;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv, ctx);
+  p = jsonParseCached(ctx, argv[0], ctx);
   if( p==0 ) return;
   if( argc==2 ){
     zPath = (const char*)sqlite3_value_text(argv[1]);
@@ -2857,7 +2991,7 @@ static void jsonValidFunc(
   JsonParse *p;          /* The parse */
   UNUSED_PARAMETER(argc);
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
-  p = jsonParseCached(ctx, argv, 0);
+  p = jsonParseCached(ctx, argv[0], 0);
   if( p==0 || p->oom ){
     sqlite3_result_error_nomem(ctx);
     sqlite3_free(p);
@@ -2903,7 +3037,7 @@ static void jsonErrorFunc(
   JsonParse *p;          /* The parse */
   UNUSED_PARAMETER(argc);
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
-  p = jsonParseCached(ctx, argv, 0);
+  p = jsonParseCached(ctx, argv[0], 0);
   if( p==0 || p->oom ){
     sqlite3_result_error_nomem(ctx);
     sqlite3_free(p);

From 59b8e666f669a97e7d62fd1a6e24daed446f027e Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 25 Jul 2023 20:26:47 +0000
Subject: [PATCH 40/66] Clarify ownership of the various objects involved in
 parsing JSON.

FossilOrigin-Name: afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567
---
 manifest      |  14 ++--
 manifest.uuid |   2 +-
 src/json.c    | 205 ++++++++++++++++----------------------------------
 src/printf.c  |   2 +-
 4 files changed, 75 insertions(+), 148 deletions(-)

diff --git a/manifest b/manifest
index 09e5a7cfff..437038d380 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Incremental\simprovements\sto\sJSON\sparsing\s-\strying\sto\sfold\sin\sthe\sRCStr\sobject.
-D 2023-07-25T18:28:03.341
+C Clarify\sownership\sof\sthe\svarious\sobjects\sinvolved\sin\sparsing\sJSON.
+D 2023-07-25T20:26:47.054
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 28fe8ed9e63293fd5708d129d4edd4a634887aabb0364685195974e35ee10762
+F src/json.c 97f8c20c1cfefbc54e2b8b2143a24aae4993c12de6e74764af443e033b01b013
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
-F src/printf.c 21e410b0a3904dddb39ef1b245cfb991302022a6f0fc16f0a8a13539d6d2f713
+F src/printf.c 1406ade1451adfa4374d8e9bbb8606109742c1216f5dbc95e011bf721fd91365
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4
-R 2b5df60a4958d2a70ce11e3569ae2bab
+P 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09
+R cbb913300d23d0635674a5fded1ff5ab
 U drh
-Z d10e127820cbbcdc4a84e1b86eb42adb
+Z f943de02bc9e737c29b8eb0aa52b28c4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a811037656..f222cbf323 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09
\ No newline at end of file
+afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 194d0421eb..6cb429f407 100644
--- a/src/json.c
+++ b/src/json.c
@@ -171,6 +171,17 @@ struct JsonParse {
 ** Utility routines for dealing with JsonString objects
 **************************************************************************/
 
+#if 0
+/*
+** This is a destructor for JSON strings.  We make it a separate function
+** so that the sqlite3ValueIsOfClass() function can be used to unambiguously
+** identify sqlite3_value objects that are known JSON strings.
+*/
+static void jsonStringClass(void *p){
+  sqlite3RCStrUnref((char*)p);
+}
+#endif
+
 /* Set the JsonString object to an empty string
 */
 static void jsonZero(JsonString *p){
@@ -1700,16 +1711,27 @@ json_parse_restart:
 ** pParse.
 **
 ** pParse is uninitialized when this routine is called.
+**
+** pParse->nJPRef set to 1.  The caller becomes the owner of the 
+** the JsonParse object.
+**
+** pParse->bOwnsJson is to bTakeJson.  If bTakeJson is 1, the newly initialized
+** JsonParse object will become the own the zJson input string.  If bTakeJson
+** is 0, then the caller is responsible for preserving zJson for the lifetime
+** of the JsonParse object.
 */
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
   sqlite3_context *pCtx,       /* Report errors here */
-  char *zJson                  /* Input JSON text to be parsed */
+  char *zJson,                 /* Input JSON text to be parsed */
+  int bTakeJson                /* Assume ownership of zJson if true */
 ){
   int i;
   memset(pParse, 0, sizeof(*pParse));
   if( zJson==0 ) return 1;
   pParse->zJson = zJson;
+  pParse->bOwnsJson = bTakeJson;
+  pParse->nJPRef = 1;
   i = jsonParseValue(pParse, 0);
   if( pParse->oom ) i = -1;
   if( i>0 ){
@@ -1738,120 +1760,6 @@ static int jsonParse(
   return 0;
 }
 
-#if 0
-/*
-** This is a destructor for JSON strings.  We make it a separate function
-** so that the sqlite3ValueIsOfClass() function can be used to unambiguously
-** identify sqlite3_value objects that are known JSON strings.
-*/
-static void jsonClass(void *p){
-  sqlite3RCStrUnref((char*)p);
-}
-#endif
-
-#if 0
-/*
-** Process SQL function argument pJson as a JSON string.  Return a pointer
-** to its parse.
-**
-** If any error is encountered, return a NULL pointer and leave an error
-** message in pCtx.
-*/
-static JsonParse *jsonParseFromFunctionArg(
-  sqlite3_value *pJson,       /* An SQL function argument containing JSON */
-  sqlite3_context *pCtx,      /* For accessing the cache */
-  sqlite3_context *pErrCtx,   /* For reporting errors */
-  int bUnchng                 /* Only accept cached parse that are unchanged */
-){
-  JsonParse *pParse;
-  char *zJson;
-  int nJson;
-  JsonParse *pMatch = 0;
-  int iKey;
-  int iMinKey = 0;
-  u32 iMinHold = 0xffffffff;
-  u32 iMaxHold = 0;
-
-  char *zJson = (char*)sqlite3_value_text(pJson);
-  int nJson = sqlite3_value_bytes(pJson);
-  if( zJson==0 ) goto json_parse_value_nomem;
-  if( sqlite3ValueIsOfClass(pJson, jsonClass) ){
-    assert( zJson!=0 );
-    (void)sqlite3RCStrRef(zJson);
-    pParse = sqlite3RCStrGetAttachment(zJson, (void(*)(void*)jsonParseFree);
-    if( pParse ){
-      return pParse;
-    }
-  }else{
-    char *z = zJson;
-    zJson = sqlite3RCStrNew( nJson );
-    if( zJson==0 ) goto json_parse_value_nomem;
-    assert( strlen(z)==nJson );
-    memcpy(zJson, z, nJson+1);
-  }
-
-  /* At this point zJson is an RCStr object that does not yet have a
-  ** parse.  This procedure is holding its own reference to that zJson
-  ** and needs to release it, or hand it off, prior to returning.
-  **
-  ** The next step is to try to find a parse of the JSON that already
-  ** exists in cache.
-  */
-  for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
-    JsonParse *p;
-    p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
-    if( p==0 ){
-      iMinKey = iKey;
-      break;
-    }
-    if( pMatch==0
-     && p->nJson==nJson
-     && (p->isMod==0 || bUnchng==0)
-     && memcmp(p->zJson,zJson,nJson)==0
-    ){
-      p->nErr = 0;
-      pMatch = p;
-    }else if( p->iHold<iMinHold ){
-      iMinHold = p->iHold;
-      iMinKey = iKey;
-    }
-    if( p->iHold>iMaxHold ){
-      iMaxHold = p->iHold;
-    }
-  }
-
-  if( pMatch ){
-    pParse = pMatch;
-    pParse->iHold = iMaxHold+1;
-  }else{
-    /* No parse of zJson could be found in cache.  So parse it afresh.
-    */
-    pParse = sqlite3_malloc64( sizeof(JsonParse) );
-    if( pParse==0 ) goto json_parse_value_nomem;
-    if( jsonParse(pParse, pCtx, zJson) ){
-      sqlite3_free(pParse);
-      sqlite3RCStrUnref(zJson);
-      return 0;
-    }
-    pParse->bOwnsJson = 1;
-    pParse->nJson = nJson;
-    pParse->nJPRef = 1;
-    pParse->iHold = iMaxHold+1;
-    sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, pParse,
-                        (void(*)(void*))jsonParseFree);
-    pParse = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
-  }
-  if( pParse ){
-    pParse->nJPRef++;  /* The caller will own this object */
-  }
-  return pParse;
-
-json_parse_value_nomem:
-  if( zJson ) sqlite3RCStrUnref(zJson);
-  if( pErrCtx ) sqlite3_result_error_nomem(pCtx);
-  return 0;
-}
-#endif
 
 /* Mark node i of pParse as being a child of iParent.  Call recursively
 ** to fill in all the descendants of node i.
@@ -1905,18 +1813,28 @@ static int jsonParseFindParents(JsonParse *pParse){
 ** Obtain a complete parse of the JSON found in the first argument
 ** of the argv array.  Use the sqlite3_get_auxdata() cache for this
 ** parse if it is available.  If the cache is not available or if it
-** is no longer valid, parse the JSON again and return the new parse,
-** and also register the new parse so that it will be available for
+** is no longer valid, parse the JSON again and return the new parse.
+** Also register the new parse so that it will be available for
 ** future sqlite3_get_auxdata() calls.
 **
 ** If an error occurs and pErrCtx!=0 then report the error on pErrCtx
 ** and return NULL.
 **
-** If an error occurs and pErrCtx==0 then return the Parse object with
-** JsonParse.nErr non-zero.  If the caller invokes this routine with
-** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller
-** is responsible for invoking jsonParseFree() on the returned value.
-** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0.
+** The returned pointer (if it is not NULL) is owned by the cache in
+** most cases, not the caller.  The caller does NOT need to invoke
+** jsonParseFree(), in most cases.
+**
+** Except, if an error occurs and pErrCtx==0 then return the JsonParse
+** object with JsonParse.nErr non-zero and the caller will own the JsonParse
+** object.  In that case, it will be the responsibility of the caller to
+** invoke jsonParseFree().  To summarize:
+**
+**   pErrCtx!=0 || p->nErr==0      ==>   Return value p is owned by the
+**                                       cache.  Call does not need to
+**                                       free it.
+**
+**   pErrCtx==0 && p->nErr!=0      ==>   Return value is owned by the caller
+**                                       and so the caller must free it.
 */
 static JsonParse *jsonParseCached(
   sqlite3_context *pCtx,
@@ -1931,6 +1849,7 @@ static JsonParse *jsonParseCached(
   int iMinKey = 0;
   u32 iMinHold = 0xffffffff;
   u32 iMaxHold = 0;
+
   if( zJson==0 ) return 0;
   for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
     p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
@@ -1955,27 +1874,35 @@ static JsonParse *jsonParseCached(
   if( pMatch ){
     pMatch->nErr = 0;
     pMatch->iHold = iMaxHold+1;
+    assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */
     return pMatch;
   }
-  p = sqlite3_malloc64( sizeof(*p) + nJson + 1 );
+  p = sqlite3_malloc64( sizeof(*p) );
   if( p==0 ){
     sqlite3_result_error_nomem(pCtx);
     return 0;
   }
   memset(p, 0, sizeof(*p));
-  p->zJson = (char*)&p[1];
-  memcpy((char*)p->zJson, zJson, nJson+1);
-  if( jsonParse(p, pErrCtx, p->zJson) ){
-    if( pErrCtx==0 ){
-      p->nErr = 1;
-      return p;
-    }
+  p->zJson = sqlite3RCStrNew( nJson );
+  if( p->zJson==0 ){
     sqlite3_free(p);
+    sqlite3_result_error_nomem(pCtx);
+    return 0;
+  }
+  memcpy(p->zJson, zJson, nJson);
+  p->zJson[nJson] = 0;
+  if( jsonParse(p, pErrCtx, p->zJson, 1) ){
+    if( pErrCtx==0 ){
+      p->nErr = 1;
+      assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */
+      return p;
+    }
+    jsonParseFree(p);
     return 0;
   }
-  p->nJPRef = 1;
   p->nJson = nJson;
   p->iHold = iMaxHold+1;
+  /* Transfer ownership of the new JsonParse to the cache */
   sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
                       (void(*)(void*))jsonParseFree);
   return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
@@ -2357,7 +2284,7 @@ static void jsonParseFunc(
   u32 i;
 
   assert( argc==1 );
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
   jsonParseFindParents(&x);
   jsonInit(&s, ctx);
   for(i=0; i<x.nNode; i++){
@@ -2681,8 +2608,8 @@ static void jsonPatchFunc(
   JsonNode *pResult;   /* The result of the merge */
 
   UNUSED_PARAMETER(argc);
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
-  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
+  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1]), 0) ){
     jsonParseReset(&x);
     return;
   }
@@ -2758,7 +2685,7 @@ static void jsonRemoveFunc(
   u32 i;
 
   if( argc<1 ) return;
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2880,7 +2807,7 @@ static void jsonReplaceFunc(
     jsonWrongNumArgs(ctx, "replace");
     return;
   }
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2926,7 +2853,7 @@ static void jsonSetFunc(
     jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
     return;
   }
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -3046,7 +2973,7 @@ static void jsonErrorFunc(
   }else{
     int n = 1;
     u32 i;
-    const char *z = p->zJson;
+    const char *z = (const char*)sqlite3_value_text(argv[0]);
     for(i=0; i<p->iErr && ALWAYS(z[i]); i++){
       if( (z[i]&0xc0)!=0x80 ) n++;
     }
@@ -3639,7 +3566,7 @@ static int jsonEachFilter(
   p->zJson = sqlite3_malloc64( n+1 );
   if( p->zJson==0 ) return SQLITE_NOMEM;
   memcpy(p->zJson, z, (size_t)n+1);
-  if( jsonParse(&p->sParse, 0, p->zJson) ){
+  if( jsonParse(&p->sParse, 0, p->zJson, 0) ){
     int rc = SQLITE_NOMEM;
     if( p->sParse.oom==0 ){
       sqlite3_free(cur->pVtab->zErrMsg);
diff --git a/src/printf.c b/src/printf.c
index 7b84288168..222e70be29 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1430,7 +1430,7 @@ int sqlite3RCStrIsWriteable(char *z){
 ** This routine returns 0 on an OOM.
 */
 char *sqlite3RCStrNew(u64 N){
-  RCStr *p = sqlite3_malloc64( N + sizeof(*p) );
+  RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 );
   if( p==0 ) return 0;
   p->nRCRef = 1;
   p->xFree = 0;

From 0f200bc580eda625f48866d0311ce6f57bfa531a Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 00:48:45 +0000
Subject: [PATCH 41/66] Add the JsonParse.zAlt field to old revised JSON text
 after a change. Demonstrate that this elminates the need for reparsing after
 a change by using it in the json_remove() function.  This is an incremental
 check-in containing lots of cruft.

FossilOrigin-Name: f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738
---
 manifest        |  18 +++---
 manifest.uuid   |   2 +-
 src/json.c      | 146 +++++++++++++++++++++++++++++++++++-------------
 src/printf.c    |  10 +++-
 src/sqliteInt.h |  10 ++--
 src/vdbemem.c   |   2 +
 6 files changed, 134 insertions(+), 54 deletions(-)

diff --git a/manifest b/manifest
index 437038d380..875c317328 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clarify\sownership\sof\sthe\svarious\sobjects\sinvolved\sin\sparsing\sJSON.
-D 2023-07-25T20:26:47.054
+C Add\sthe\sJsonParse.zAlt\sfield\sto\sold\srevised\sJSON\stext\safter\sa\schange.\nDemonstrate\sthat\sthis\selminates\sthe\sneed\sfor\sreparsing\safter\sa\schange\nby\susing\sit\sin\sthe\sjson_remove()\sfunction.\s\sThis\sis\san\sincremental\scheck-in\ncontaining\slots\sof\scruft.
+D 2023-07-26T00:48:45.768
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 97f8c20c1cfefbc54e2b8b2143a24aae4993c12de6e74764af443e033b01b013
+F src/json.c a7823b0593eac42a608f8439e6b1f28022ac5fce9aa81a82856e48dd07ab601d
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
-F src/printf.c 1406ade1451adfa4374d8e9bbb8606109742c1216f5dbc95e011bf721fd91365
+F src/printf.c 56e362bbefbe61b5b211f1886e3cdb9b25d8ba817c12df3a15f11b9bdc152efd
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
@@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81
 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h 59b755dec944aa3b068e962ef53264de6fde3d6b6df2d5869ea3afdb7facdf60
+F src/sqliteInt.h 29c8e6d0b6dfa9c2c356f23ab8349d7fac3679d5f7cfc01d4ee45c9171db2960
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35
+F src/vdbemem.c 46ea371b10c3573cf9287d385411fbcc6cd42099ad3505d20e33f2f3ed91537d
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09
-R cbb913300d23d0635674a5fded1ff5ab
+P afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567
+R 33c717c250039ffbe53dc8d7b2094929
 U drh
-Z f943de02bc9e737c29b8eb0aa52b28c4
+Z 96ad60bdf5cbde0ec2b901493a583f6e
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index f222cbf323..dce1119850 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567
\ No newline at end of file
+f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 6cb429f407..d7d464442b 100644
--- a/src/json.c
+++ b/src/json.c
@@ -142,7 +142,8 @@ struct JsonParse {
   u32 nNode;         /* Number of slots of aNode[] used */
   u32 nAlloc;        /* Number of slots of aNode[] allocated */
   JsonNode *aNode;   /* Array of nodes containing the parse */
-  char *zJson;       /* Original JSON string */
+  char *zJson;       /* Original JSON string (before edits) */
+  char *zAlt;        /* Revised JSON after edits applied.  Maybe NULL */
   u32 *aUp;          /* Index of parent of each node */
   JsonTask *pClean;  /* Cleanup operations prior to freeing this object */
   u16 iDepth;        /* Nesting depth */
@@ -151,8 +152,10 @@ struct JsonParse {
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   u8 nJPRef;         /* Number of references to this object */
   u8 bOwnsJson;      /* This object owns zJson and response for freeing it */
-  u8 isMod;          /* aNode contains edits from the original zJson */
+  u8 useMod;         /* Actually use the edits contain inside aNode */
+  u8 hasMod;         /* aNode contains edits from the original zJson */
   int nJson;         /* Length of the zJson string in bytes */
+  int nAlt;          /* Length of alternative JSON string zAlt, in bytes */
   u32 iErr;          /* Error location in zJson[] */
   u32 iSubst;        /* Last known JSON_SUBST node */
   u32 iHold;         /* Replace cache line with the lowest iHold value */
@@ -245,6 +248,16 @@ static int jsonGrow(JsonString *p, u32 N){
   return SQLITE_OK;
 }
 
+/* Try to force the string to be an RCStr string, rather than a
+** static string.  Return true on success.  The only reason this
+** might fail is due to an OOM fault.
+*/
+static int jsonForceRCStr(JsonString *p){
+  if( p->bStatic==0 ) return 1;
+  jsonGrow(p, p->nAlloc+1);
+  return p->bStatic==0;
+}
+
 /* Append N bytes from zIn onto the end of the JsonString string.
 */
 static SQLITE_NOINLINE void jsonAppendExpand(
@@ -588,6 +601,11 @@ static void jsonParseReset(JsonParse *pParse){
     sqlite3_free(pParse->aUp);
     pParse->aUp = 0;
   }
+  if( pParse->zAlt ){
+    char *z = pParse->zAlt;
+    pParse->zAlt = 0;
+    sqlite3RCStrUnref(z);
+  }
   if( pParse->bOwnsJson ){
     /* Order operations so that if the destructor for pParse->zJson
     ** invokes jsonParseFree(), the recursion will terminate harmlessly */
@@ -650,7 +668,7 @@ static void jsonRenderNode(
   JsonString *pOut               /* Write JSON here */
 ){
   assert( pNode!=0 );
-  while( (pNode->jnFlags & JNODE_REPLACE)!=0 ){
+  while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){
     u32 idx = (u32)(pNode - pParse->aNode);
     u32 i = pParse->iSubst;
     while( 1 /*exit-by-break*/ ){
@@ -722,13 +740,14 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '[');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){
+          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){
             jsonAppendSeparator(pOut);
             jsonRenderNode(pParse, &pNode[j], pOut);
           }
           j += jsonNodeSize(&pNode[j]);
         }
         if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
+        if( pParse->useMod==0 ) break;
         assert( pNode->eU==2 );
         pNode = &pParse->aNode[pNode->u.iAppend];
         j = 1;
@@ -741,7 +760,7 @@ static void jsonRenderNode(
       jsonAppendChar(pOut, '{');
       for(;;){
         while( j<=pNode->n ){
-          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){
+          if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){
             jsonAppendSeparator(pOut);
             jsonRenderNode(pParse, &pNode[j], pOut);
             jsonAppendChar(pOut, ':');
@@ -750,6 +769,7 @@ static void jsonRenderNode(
           j += 1 + jsonNodeSize(&pNode[j+1]);
         }
         if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
+        if( pParse->useMod==0 ) break;
         assert( pNode->eU==2 );
         pNode = &pParse->aNode[pNode->u.iAppend];
         j = 1;
@@ -766,7 +786,8 @@ static void jsonRenderNode(
 static void jsonReturnJson(
   JsonParse *pParse,          /* The complete JSON */
   JsonNode *pNode,            /* Node to return */
-  sqlite3_context *pCtx       /* Return value for this function */
+  sqlite3_context *pCtx,      /* Return value for this function */
+  int bGenerateAlt            /* Also store the rendered text in zAlt */
 ){
   JsonString s;
   if( pParse->oom ){
@@ -776,6 +797,12 @@ static void jsonReturnJson(
   if( pParse->nErr==0 ){
     jsonInit(&s, pCtx);
     jsonRenderNode(pParse, pNode, &s);
+    if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){
+      jsonAppendChar(&s, 0);
+      s.nUsed--;
+      pParse->zAlt = sqlite3RCStrRef(s.zBuf);
+      pParse->nAlt = s.nUsed;
+    }
     jsonResult(&s);
     sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
   }
@@ -965,7 +992,7 @@ static void jsonReturn(
     }
     case JSON_ARRAY:
     case JSON_OBJECT: {
-      jsonReturnJson(pParse, pNode, pCtx);
+      jsonReturnJson(pParse, pNode, pCtx, 0);
       break;
     }
   }
@@ -1079,6 +1106,8 @@ static int jsonParseAddSubstNode(
   pParse->aNode[idx].eU = 4;
   pParse->aNode[idx].u.iPrev = pParse->iSubst;
   pParse->iSubst = idx;
+  pParse->hasMod = 1;
+  pParse->useMod = 1;
   return idx;
 }
 
@@ -1729,6 +1758,7 @@ static int jsonParse(
   int i;
   memset(pParse, 0, sizeof(*pParse));
   if( zJson==0 ) return 1;
+//printf("PARSE %s\n", zJson);
   pParse->zJson = zJson;
   pParse->bOwnsJson = bTakeJson;
   pParse->nJPRef = 1;
@@ -1837,9 +1867,10 @@ static int jsonParseFindParents(JsonParse *pParse){
 **                                       and so the caller must free it.
 */
 static JsonParse *jsonParseCached(
-  sqlite3_context *pCtx,
-  sqlite3_value *pJson,
-  sqlite3_context *pErrCtx
+  sqlite3_context *pCtx,         /* Context to use for cache search */
+  sqlite3_value *pJson,          /* Function param containing JSON text */
+  sqlite3_context *pErrCtx,      /* Write parse errors here if not NULL */
+  int bUnedited                  /* No prior edits allowed */
 ){
   char *zJson = (char*)sqlite3_value_text(pJson);
   int nJson = sqlite3_value_bytes(pJson);
@@ -1859,10 +1890,24 @@ static JsonParse *jsonParseCached(
     }
     if( pMatch==0
      && p->nJson==nJson
+     && (p->hasMod==0 || bUnedited==0)
      && memcmp(p->zJson,zJson,nJson)==0
     ){
       p->nErr = 0;
+      p->useMod = 0;
       pMatch = p;
+//printf("HIT %s at %d\n", zJson, iKey);
+    }else
+    if( pMatch==0
+     && p->zAlt!=0
+     && bUnedited==0
+     && p->nAlt==nJson
+     && memcmp(p->zAlt, zJson, nJson)==0
+    ){
+      p->nErr = 0;
+      p->useMod = 1;
+      pMatch = p;
+//printf("HIT %s at %d-alt\n", zJson, iKey);
     }else if( p->iHold<iMinHold ){
       iMinHold = p->iHold;
       iMinKey = iKey;
@@ -1872,11 +1917,18 @@ static JsonParse *jsonParseCached(
     }
   }
   if( pMatch ){
+    /* The input JSON text was found in the cache.  Use the preexisting
+    ** parse of this JSON */
     pMatch->nErr = 0;
     pMatch->iHold = iMaxHold+1;
     assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */
     return pMatch;
   }
+
+  /* The input JSON was not found anywhere in the cache.  We will need
+  ** to parse it ourselves and generate a new JsonParse object.
+  */
+//printf("MISS %s\n", zJson);
   p = sqlite3_malloc64( sizeof(*p) );
   if( p==0 ){
     sqlite3_result_error_nomem(pCtx);
@@ -1954,7 +2006,7 @@ static JsonNode *jsonLookupStep(
   u32 i, j, nKey;
   const char *zKey;
   JsonNode *pRoot = &pParse->aNode[iRoot];
-  while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){
+  while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){
     u32 idx = (u32)(pRoot - pParse->aNode);
     i = pParse->iSubst;
     while( 1 /*exit-by-break*/ ){
@@ -2004,6 +2056,7 @@ static JsonNode *jsonLookupStep(
         j += jsonNodeSize(&pRoot[j]);
       }
       if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
+      if( pParse->useMod==0 ) break;
       assert( pRoot->eU==2 );
       iRoot = pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -2012,6 +2065,7 @@ static JsonNode *jsonLookupStep(
     if( pApnd ){
       u32 iStart, iLabel;
       JsonNode *pNode;
+      assert( pParse->useMod );
       iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
       iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
       zPath += i;
@@ -2041,10 +2095,11 @@ static JsonNode *jsonLookupStep(
         if( pRoot->eType!=JSON_ARRAY ) return 0;
         for(;;){
           while( j<=pBase->n ){
-            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++;
+            if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++;
             j += jsonNodeSize(&pBase[j]);
           }
           if( (pBase->jnFlags & JNODE_APPEND)==0 ) break;
+          if( pParse->useMod==0 ) break;
           assert( pBase->eU==2 );
           iBase = pBase->u.iAppend;
           pBase = &pParse->aNode[iBase];
@@ -2074,11 +2129,14 @@ static JsonNode *jsonLookupStep(
     zPath += j + 1;
     j = 1;
     for(;;){
-      while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){
-        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--;
+      while( j<=pRoot->n 
+         && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod))
+      ){
+        if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--;
         j += jsonNodeSize(&pRoot[j]);
       }
       if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
+      if( pParse->useMod==0 ) break;
       assert( pRoot->eU==2 );
       iRoot = pRoot->u.iAppend;
       pRoot = &pParse->aNode[iRoot];
@@ -2090,6 +2148,7 @@ static JsonNode *jsonLookupStep(
     if( i==0 && pApnd ){
       u32 iStart;
       JsonNode *pNode;
+      assert( pParse->useMod );
       iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0);
       pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr);
       if( pParse->oom ) return 0;
@@ -2392,7 +2451,7 @@ static void jsonArrayLengthFunc(
   u32 i;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv[0], ctx);
+  p = jsonParseCached(ctx, argv[0], ctx, 0);
   if( p==0 ) return;
   assert( p->nNode );
   if( argc==2 ){
@@ -2405,9 +2464,14 @@ static void jsonArrayLengthFunc(
     return;
   }
   if( pNode->eType==JSON_ARRAY ){
-    assert( (pNode->jnFlags & JNODE_APPEND)==0 );
-    for(i=1; i<=pNode->n; n++){
-      i += jsonNodeSize(&pNode[i]);
+    while( 1 /*exit-by-break*/ ){
+      for(i=1; i<=pNode->n; n++){
+        i += jsonNodeSize(&pNode[i]);
+      }
+      if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
+      if( p->useMod==0 ) break;
+      assert( pNode->eU==2 );
+      pNode = &p->aNode[pNode->u.iAppend];
     }
   }
   sqlite3_result_int64(ctx, n);
@@ -2454,7 +2518,7 @@ static void jsonExtractFunc(
   JsonString jx;
 
   if( argc<2 ) return;
-  p = jsonParseCached(ctx, argv[0], ctx);
+  p = jsonParseCached(ctx, argv[0], ctx, 0);
   if( p==0 ) return;
   if( argc==2 ){
     /* With a single PATH argument */
@@ -2487,7 +2551,7 @@ static void jsonExtractFunc(
       }
       if( pNode ){
         if( flags & JSON_JSON ){
-          jsonReturnJson(p, pNode, ctx);
+          jsonReturnJson(p, pNode, ctx, 0);
         }else{
           jsonReturn(p, pNode, ctx);
           sqlite3_result_subtype(ctx, 0);
@@ -2613,12 +2677,14 @@ static void jsonPatchFunc(
     jsonParseReset(&x);
     return;
   }
+  x.useMod = 1;
+  y.useMod = 1;
   pResult = jsonMergePatch(&x, 0, y.aNode);
   assert( pResult!=0 || x.oom );
   if( pResult && x.oom==0 ){
     jsonDebugPrintParse(&x);
     jsonDebugPrintNode(pResult);
-    jsonReturnJson(&x, pResult, ctx);
+    jsonReturnJson(&x, pResult, ctx, 0);
   }else{
     sqlite3_result_error_nomem(ctx);
   }
@@ -2679,27 +2745,30 @@ static void jsonRemoveFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *pParse;          /* The parse */
   JsonNode *pNode;
   const char *zPath;
   u32 i;
 
   if( argc<1 ) return;
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
-  assert( x.nNode );
+  pParse = jsonParseCached(ctx, argv[0], ctx, argc>1);
+  if( pParse==0 ) return;  
   for(i=1; i<(u32)argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
     if( zPath==0 ) goto remove_done;
-    pNode = jsonLookup(&x, zPath, 0, ctx);
-    if( x.nErr ) goto remove_done;
-    if( pNode ) pNode->jnFlags |= JNODE_REMOVE;
+    pNode = jsonLookup(pParse, zPath, 0, ctx);
+    if( pParse->nErr ) goto remove_done;
+    if( pNode ){
+      pNode->jnFlags |= JNODE_REMOVE;
+      pParse->hasMod = 1;
+      pParse->useMod = 1;
+    }
   }
-  if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){
-    jsonReturnJson(&x, x.aNode, ctx);
+  if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){
+    jsonReturnJson(pParse, pParse->aNode, ctx, 1);
   }
 remove_done:
-  jsonDebugPrintParse(&x);
-  jsonParseReset(&x);
+  jsonDebugPrintParse(p);
 }
 
 /*
@@ -2763,7 +2832,7 @@ static void jsonReplaceNode(
           sqlite3_result_error_nomem(pCtx);
         }
       }else{
-        JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx);
+        JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1);
         if( pPatch==0 ){
           p->oom = 1;
           break;
@@ -2817,7 +2886,7 @@ static void jsonReplaceFunc(
       jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
     }
   }
-  jsonReturnJson(&x, x.aNode, ctx);
+  jsonReturnJson(&x, x.aNode, ctx, 0);
 replace_err:
   jsonDebugPrintParse(&x);
   jsonParseReset(&x);
@@ -2858,6 +2927,7 @@ static void jsonSetFunc(
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
     bApnd = 0;
+    x.useMod = 1;
     pNode = jsonLookup(&x, zPath, &bApnd, ctx);
     if( x.oom ){
       sqlite3_result_error_nomem(ctx);
@@ -2869,7 +2939,7 @@ static void jsonSetFunc(
     }
   }
   jsonDebugPrintParse(&x);
-  jsonReturnJson(&x, x.aNode, ctx);
+  jsonReturnJson(&x, x.aNode, ctx, 0);
 
 jsonSetDone:
   jsonParseReset(&x);
@@ -2891,7 +2961,7 @@ static void jsonTypeFunc(
   const char *zPath;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv[0], ctx);
+  p = jsonParseCached(ctx, argv[0], ctx, 0);
   if( p==0 ) return;
   if( argc==2 ){
     zPath = (const char*)sqlite3_value_text(argv[1]);
@@ -2918,12 +2988,12 @@ static void jsonValidFunc(
   JsonParse *p;          /* The parse */
   UNUSED_PARAMETER(argc);
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
-  p = jsonParseCached(ctx, argv[0], 0);
+  p = jsonParseCached(ctx, argv[0], 0, 0);
   if( p==0 || p->oom ){
     sqlite3_result_error_nomem(ctx);
     sqlite3_free(p);
   }else{
-    sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0);
+    sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod));
     if( p->nErr ) jsonParseFree(p);
   }
 }
@@ -2964,7 +3034,7 @@ static void jsonErrorFunc(
   JsonParse *p;          /* The parse */
   UNUSED_PARAMETER(argc);
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
-  p = jsonParseCached(ctx, argv[0], 0);
+  p = jsonParseCached(ctx, argv[0], 0, 0);
   if( p==0 || p->oom ){
     sqlite3_result_error_nomem(ctx);
     sqlite3_free(p);
diff --git a/src/printf.c b/src/printf.c
index 222e70be29..a55bd2b85b 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1406,6 +1406,7 @@ void sqlite3RCStrUnref(char *z){
   }
 }
 
+#if 0
 /*
 ** Return true if the reference count on the string is exactly one, meaning
 ** that the string can be modified.  Return false if the reference count
@@ -1419,6 +1420,7 @@ int sqlite3RCStrIsWriteable(char *z){
   assert( p->uMagic==SQLITE_RCSTR_MAGIC );
   return p->nRCRef==1;
 }
+#endif
 
 /*
 ** Create a new string that is capable of holding N bytes of text, not counting
@@ -1441,6 +1443,7 @@ char *sqlite3RCStrNew(u64 N){
   return (char*)&p[1];
 }
 
+#if 0
 /*
 ** Return the number of bytes allocated to the string.  The value returned
 ** does not include the space for the zero-terminator at the end.
@@ -1456,6 +1459,7 @@ u64 sqlite3RCStrSize(char *z){
   N -= sizeof(p) + 1;
   return N;
 }
+#endif
 
 /*
 ** Change the size of the string so that it is able to hold N bytes.
@@ -1477,6 +1481,7 @@ char *sqlite3RCStrResize(char *z, u64 N){
   }
 }
 
+#if 0
 /*
 ** Add a new attachment to the string.
 **
@@ -1494,7 +1499,9 @@ void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){
   p->xFree = xFree;
   p->pAttach = pAttach;
 }
-  
+#endif
+
+#if 0
 /*
 ** Return the attachment associated with a string if the attachment
 ** has the destructure specified in the second argument.  If the
@@ -1509,3 +1516,4 @@ void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){
   assert( p->uMagic==SQLITE_RCSTR_MAGIC );
   return p->xFree==xFree ? p->pAttach : 0;
 }
+#endif
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index adf4d34dec..358063534f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -5222,7 +5222,7 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
-int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
+//int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
@@ -5333,11 +5333,11 @@ int sqlite3OpenTempDatabase(Parse *);
 char *sqlite3RCStrRef(char*);
 void sqlite3RCStrUnref(char*);
 char *sqlite3RCStrNew(u64);
-u64 sqlite3RCStrSize(char*);
+//u64 sqlite3RCStrSize(char*);
 char *sqlite3RCStrResize(char*,u64);
-int sqlite3RCStrIsWriteable(char*);
-void sqlite3RCStrAttach(char*, void*, void(*)(void*));
-void *sqlite3RCStrGetAttachment(char*,void(*)(void*));
+//int sqlite3RCStrIsWriteable(char*);
+//void sqlite3RCStrAttach(char*, void*, void(*)(void*));
+//void *sqlite3RCStrGetAttachment(char*,void(*)(void*));
 
 void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int);
 int sqlite3StrAccumEnlarge(StrAccum*, i64);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 87dfbbebd8..d44a41716f 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1368,6 +1368,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
+#if 0
 /* Return true if sqlit3_value object pVal is a string or blob value
 ** that uses the destructor specified in the second argument.
 **
@@ -1385,6 +1386,7 @@ int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
     return 0;
   }
 }
+#endif
 
 /*
 ** Create a new sqlite3_value object.

From 44f53b96472a660e42f4c4f33e01f0fc9c691440 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 01:05:08 +0000
Subject: [PATCH 42/66] Extend the enhancement to json_set() and
 json_replace().  Clean up cruft.

FossilOrigin-Name: 2dbb22c75e86f2e3ced38ac14b4943570d5c2f86cd5e37e875bf0c863be28836
---
 manifest        | 18 +++++------
 manifest.uuid   |  2 +-
 src/json.c      | 44 ++++++++++++--------------
 src/printf.c    | 82 -------------------------------------------------
 src/sqliteInt.h | 45 ++++++---------------------
 src/vdbemem.c   | 20 ------------
 6 files changed, 39 insertions(+), 172 deletions(-)

diff --git a/manifest b/manifest
index 875c317328..b712bbf97f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sJsonParse.zAlt\sfield\sto\sold\srevised\sJSON\stext\safter\sa\schange.\nDemonstrate\sthat\sthis\selminates\sthe\sneed\sfor\sreparsing\safter\sa\schange\nby\susing\sit\sin\sthe\sjson_remove()\sfunction.\s\sThis\sis\san\sincremental\scheck-in\ncontaining\slots\sof\scruft.
-D 2023-07-26T00:48:45.768
+C Extend\sthe\senhancement\sto\sjson_set()\sand\sjson_replace().\s\sClean\sup\scruft.
+D 2023-07-26T01:05:08.003
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c a7823b0593eac42a608f8439e6b1f28022ac5fce9aa81a82856e48dd07ab601d
+F src/json.c 790ca70b1f8884610fe5401ac6f260d955d47d9c14c9f3d2aa6541fd881faf07
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1
-F src/printf.c 56e362bbefbe61b5b211f1886e3cdb9b25d8ba817c12df3a15f11b9bdc152efd
+F src/printf.c e3ba080e2f409f9bfcc8d34724e6fc160e9c718dc92d0548f6b71b8b6f860ce2
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
@@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81
 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h 29c8e6d0b6dfa9c2c356f23ab8349d7fac3679d5f7cfc01d4ee45c9171db2960
+F src/sqliteInt.h 30d7b0d586a4d03a384dcb60088c81b6fc6f74ce85cc3a0b3242eedc3cc24dbd
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c 46ea371b10c3573cf9287d385411fbcc6cd42099ad3505d20e33f2f3ed91537d
+F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567
-R 33c717c250039ffbe53dc8d7b2094929
+P f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738
+R 80113afdab5484e027b5180529a678b7
 U drh
-Z 96ad60bdf5cbde0ec2b901493a583f6e
+Z dc859019c8db0487fd3e03bdb875d59d
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index dce1119850..4bbeaca0db 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738
\ No newline at end of file
+2dbb22c75e86f2e3ced38ac14b4943570d5c2f86cd5e37e875bf0c863be28836
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index d7d464442b..a4a3e65c92 100644
--- a/src/json.c
+++ b/src/json.c
@@ -1758,7 +1758,6 @@ static int jsonParse(
   int i;
   memset(pParse, 0, sizeof(*pParse));
   if( zJson==0 ) return 1;
-//printf("PARSE %s\n", zJson);
   pParse->zJson = zJson;
   pParse->bOwnsJson = bTakeJson;
   pParse->nJPRef = 1;
@@ -1896,7 +1895,6 @@ static JsonParse *jsonParseCached(
       p->nErr = 0;
       p->useMod = 0;
       pMatch = p;
-//printf("HIT %s at %d\n", zJson, iKey);
     }else
     if( pMatch==0
      && p->zAlt!=0
@@ -1907,7 +1905,6 @@ static JsonParse *jsonParseCached(
       p->nErr = 0;
       p->useMod = 1;
       pMatch = p;
-//printf("HIT %s at %d-alt\n", zJson, iKey);
     }else if( p->iHold<iMinHold ){
       iMinHold = p->iHold;
       iMinKey = iKey;
@@ -1928,7 +1925,6 @@ static JsonParse *jsonParseCached(
   /* The input JSON was not found anywhere in the cache.  We will need
   ** to parse it ourselves and generate a new JsonParse object.
   */
-//printf("MISS %s\n", zJson);
   p = sqlite3_malloc64( sizeof(*p) );
   if( p==0 ){
     sqlite3_result_error_nomem(pCtx);
@@ -2866,7 +2862,7 @@ static void jsonReplaceFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *pParse;          /* The parse */
   JsonNode *pNode;
   const char *zPath;
   u32 i;
@@ -2876,20 +2872,20 @@ static void jsonReplaceFunc(
     jsonWrongNumArgs(ctx, "replace");
     return;
   }
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
-  assert( x.nNode );
+  pParse = jsonParseCached(ctx, argv[0], ctx, argc>1);
+  if( pParse==0 ) return;
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
-    if( x.nErr ) goto replace_err;
+    pParse->useMod = 1;
+    pNode = jsonLookup(pParse, zPath, 0, ctx);
+    if( pParse->nErr ) goto replace_err;
     if( pNode ){
-      jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
+      jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]);
     }
   }
-  jsonReturnJson(&x, x.aNode, ctx, 0);
+  jsonReturnJson(pParse, pParse->aNode, ctx, 1);
 replace_err:
-  jsonDebugPrintParse(&x);
-  jsonParseReset(&x);
+  jsonDebugPrintParse(pParse);
 }
 
 
@@ -2910,7 +2906,7 @@ static void jsonSetFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *pParse;       /* The parse */
   JsonNode *pNode;
   const char *zPath;
   u32 i;
@@ -2922,27 +2918,27 @@ static void jsonSetFunc(
     jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
     return;
   }
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
-  assert( x.nNode );
+  pParse = jsonParseCached(ctx, argv[0], ctx, argc>1);
+  if( pParse==0 ) return;
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
     bApnd = 0;
-    x.useMod = 1;
-    pNode = jsonLookup(&x, zPath, &bApnd, ctx);
-    if( x.oom ){
+    pParse->useMod = 1;
+    pNode = jsonLookup(pParse, zPath, &bApnd, ctx);
+    if( pParse->oom ){
       sqlite3_result_error_nomem(ctx);
       goto jsonSetDone;
-    }else if( x.nErr ){
+    }else if( pParse->nErr ){
       goto jsonSetDone;
     }else if( pNode && (bApnd || bIsSet) ){
-      jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]);
+      jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]);
     }
   }
-  jsonDebugPrintParse(&x);
-  jsonReturnJson(&x, x.aNode, ctx, 0);
+  jsonDebugPrintParse(pParse);
+  jsonReturnJson(pParse, pParse->aNode, ctx, 1);
 
 jsonSetDone:
-  jsonParseReset(&x);
+  /* no cleanup required */;
 }
 
 /*
diff --git a/src/printf.c b/src/printf.c
index a55bd2b85b..87ad91f795 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1394,34 +1394,13 @@ void sqlite3RCStrUnref(char *z){
   assert( p!=0 );
   p--;
   assert( p->nRCRef>0 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
   if( p->nRCRef>=2 ){
     p->nRCRef--;
   }else{
-    if( p->xFree ) p->xFree(p->pAttach);
-#ifdef SQLITE_DEBUG
-    p->uMagic = 0;
-#endif
     sqlite3_free(p);
   }
 }
 
-#if 0
-/*
-** Return true if the reference count on the string is exactly one, meaning
-** that the string can be modified.  Return false if the reference count
-** is greater than one.
-*/
-int sqlite3RCStrIsWriteable(char *z){
-  RCStr *p = (RCStr*)z;
-  assert( p!=0 );
-  p--;
-  assert( p->nRCRef>0 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
-  return p->nRCRef==1;
-}
-#endif
-
 /*
 ** Create a new string that is capable of holding N bytes of text, not counting
 ** the zero byte at the end.  The string is uninitialized.
@@ -1435,32 +1414,9 @@ char *sqlite3RCStrNew(u64 N){
   RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 );
   if( p==0 ) return 0;
   p->nRCRef = 1;
-  p->xFree = 0;
-  p->pAttach = 0;
-#ifdef SQLITE_DEBUG
-  p->uMagic = SQLITE_RCSTR_MAGIC;
-#endif
   return (char*)&p[1];
 }
 
-#if 0
-/*
-** Return the number of bytes allocated to the string.  The value returned
-** does not include the space for the zero-terminator at the end.
-*/
-u64 sqlite3RCStrSize(char *z){
-  RCStr *p = (RCStr*)z;
-  u64 N;
-  assert( p!=0 );
-  p--;
-  assert( p->nRCRef>0 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
-  N = sqlite3_msize(p);
-  N -= sizeof(p) + 1;
-  return N;
-}
-#endif
-
 /*
 ** Change the size of the string so that it is able to hold N bytes.
 ** The string might be reallocated, so return the new allocation.
@@ -1471,7 +1427,6 @@ char *sqlite3RCStrResize(char *z, u64 N){
   assert( p!=0 );
   p--;
   assert( p->nRCRef==1 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
   pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1);
   if( pNew==0 ){
     sqlite3_free(p);
@@ -1480,40 +1435,3 @@ char *sqlite3RCStrResize(char *z, u64 N){
     return (char*)&pNew[1];
   }
 }
-
-#if 0
-/*
-** Add a new attachment to the string.
-**
-** A string may have no more than one attachment.  When a new attachment
-** is added, any prior attachment is destroyed.  Remove an attachment
-** by adding a zero-attachment.
-*/
-void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){
-  RCStr *p = (RCStr*)z;
-  assert( p!=0 );
-  p--;
-  assert( p->nRCRef>0 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
-  if( p->xFree ) p->xFree(p->pAttach);
-  p->xFree = xFree;
-  p->pAttach = pAttach;
-}
-#endif
-
-#if 0
-/*
-** Return the attachment associated with a string if the attachment
-** has the destructure specified in the second argument.  If the
-** string has no attachment or if the destructor does not match,
-** then return a NULL pointr.
-*/
-void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){
-  RCStr *p = (RCStr*)z;
-  assert( p!=0 );
-  p--;
-  assert( p->nRCRef>0 );
-  assert( p->uMagic==SQLITE_RCSTR_MAGIC );
-  return p->xFree==xFree ? p->pAttach : 0;
-}
-#endif
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 358063534f..a19a16d50d 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4066,44 +4066,22 @@ struct sqlite3_str {
 ** The following object is the header for an "RCStr" or "reference-counted
 ** string".  An RCStr is passed around and used like any other char*
 ** that has been dynamically allocated.  The important interface
-** difference is that it uses sqlite3RCStrUnref() as its destructor
-** rather than sqlite3_free().  Other than that the two are interchangeable.
+** differences:
 **
-** Thus to return an RCStr object as the result of an SQL function use:
-** 
-**    sqlite3_result_text64(ctx,z,sz,sqlite3RCStrUnref,SQLITE_UTF8)
-**                                   ^^^^^^^^^^^^^^^^^
-**                                   Instead of sqlite3_free() or similar
+**   1.  RCStr strings are reference counted.  They are deallocated
+**       when the reference count reaches zero.
 **
-** An SQL function can check its arguments to see if they are RCStr
-** strings using the sqlite3ValueIsOfClass() function:
+**   2.  Use sqlite3RCStrUnref() to free an RCStr string rather than
+**       sqlite3_free()
 **
-**    sqlite3ValueIsOfClass(argv[i], sqlite3RCStrUnref);
-**
-** An RCStr string might be better than an ordinary string in some cases
-** because:
-**
-**    (1)  You can duplicate it using sqlite3RCStrRef(x).
-**
-**    (2)  You can also add an associated object to the string.  For
-**         example, if the string is JSON, perhaps the associated object
-**         is a parse of that JSON.
-**
-** Methods for an RCStr string begin with "sqlite3RCStr...".
+**   3.  Make a (read-only) copy of a read-only RCStr string using
+**       sqlite3RCStrRef().
 */
 struct RCStr {
-  u32 nRCRef;            /* Number of references */
-#ifdef SQLITE_DEBUG
-  u32 uMagic;            /* Magic number for sanity checking */
-#endif
-  void *pAttach;         /* Attachment to this string */
-  void (*xFree)(void*);  /* Destructor for the attachment */
+  u64 nRCRef;            /* Number of references */
+  /* Total structure size should be a multiple of 8 bytes for alignment */
 };
 
-/* The Magic number used by RCStr for sanity checking.  SQLITE_DEBUG only. */
-#define SQLITE_RCSTR_MAGIC 0x3dc05d54
-
-
 /*
 ** A pointer to this structure is used to communicate information
 ** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback.
@@ -5222,7 +5200,6 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
-//int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
@@ -5333,11 +5310,7 @@ int sqlite3OpenTempDatabase(Parse *);
 char *sqlite3RCStrRef(char*);
 void sqlite3RCStrUnref(char*);
 char *sqlite3RCStrNew(u64);
-//u64 sqlite3RCStrSize(char*);
 char *sqlite3RCStrResize(char*,u64);
-//int sqlite3RCStrIsWriteable(char*);
-//void sqlite3RCStrAttach(char*, void*, void(*)(void*));
-//void *sqlite3RCStrGetAttachment(char*,void(*)(void*));
 
 void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int);
 int sqlite3StrAccumEnlarge(StrAccum*, i64);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index d44a41716f..c73e59c362 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1368,26 +1368,6 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
-#if 0
-/* Return true if sqlit3_value object pVal is a string or blob value
-** that uses the destructor specified in the second argument.
-**
-** TODO:  Maybe someday promote this interface into a published API so
-** that third-party extensions can get access to it?
-*/
-int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
-  if( ALWAYS(pVal!=0)
-   && (pVal->flags & (MEM_Str|MEM_Blob))!=0
-   && (pVal->flags & MEM_Dyn)!=0
-   && pVal->xDel==xFree
-  ){
-    return 1;
-  }else{
-    return 0;
-  }
-}
-#endif
-
 /*
 ** Create a new sqlite3_value object.
 */

From 3ba691412109e192b5e052e5006337f9ba907fbe Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Wed, 26 Jul 2023 07:57:55 +0000
Subject: [PATCH 43/66] Reformulate [907dfc4a7aa1] using awk instead of sed for
 better cross-platform portability.

FossilOrigin-Name: 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da
---
 ext/wasm/GNUmakefile |  6 +++---
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile
index 2346773337..45790bd701 100644
--- a/ext/wasm/GNUmakefile
+++ b/ext/wasm/GNUmakefile
@@ -678,13 +678,13 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles)
 # Upstream RFE:
 # https://github.com/emscripten-core/emscripten/issues/18237
 #
-# Maintenance reminder: Mac sed -i works differently than GNU sed, so
-# don't use that flag here.
+# Maintenance reminder: Mac sed works differently than GNU sed, so
+# don't use sed for this.
 define SQLITE3.xJS.ESM-EXPORT-DEFAULT
 if [ x1 = x$(1) ]; then \
 		echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \
 		{\
-			sed -e '0,/^export default/{/^export default/d;}' $@ > $@.tmp && mv $@.tmp $@; \
+			awk '/^export default/ && !f{f=1; next} 1' $@ > $@.tmp && mv $@.tmp $@; \
 		} || exit $$?; \
 		if [ x != x$(2) ]; then \
 			if ! grep -q '^export default' $@; then \
diff --git a/manifest b/manifest
index f9ef9ad44c..5f921c0e86 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\s"set-but-not-used"\scompiler\swarning\sin\ssqlite3session.c.
-D 2023-07-25T17:54:25.000
+C Reformulate\s[907dfc4a7aa1]\susing\sawk\sinstead\sof\ssed\sfor\sbetter\scross-platform\sportability.
+D 2023-07-26T07:57:55.835
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile a3e316bf51d8915a49d4728e0d41f19ec4c3ead3409d1171989a4fe163ec2214
+F ext/wasm/GNUmakefile f1aab41e2afaad68907ff8e8eea32ccca97ab8c216e33dbea817f9e263d4b502
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c
-R 262b3aa3dc9bbec239621d23dc374114
-U dan
-Z f47917699640df15bd88b843742372bf
+P 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
+R bd64b4635e08e21fec1b91b485560c00
+U stephan
+Z 9b053abd49b0547974c7dd9e0baf8df0
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 7561758d66..9d9c089cd8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
\ No newline at end of file
+82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da
\ No newline at end of file

From 9a184daad3991239595b007e1bb2b1167aeb2cac Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 11:00:47 +0000
Subject: [PATCH 44/66] Fix jsonForceRCStr() to also add the NULL terminator.

FossilOrigin-Name: 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 28 ++++++++++++++++------------
 3 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index 44f7e7b86b..e2bd372d36 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\slatest\strunk\sfixes\sinto\sthe\sjson-opt\sbranch.
-D 2023-07-26T01:15:34.571
+C Fix\sjsonForceRCStr()\sto\salso\sadd\sthe\sNULL\sterminator.
+D 2023-07-26T11:00:47.983
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 790ca70b1f8884610fe5401ac6f260d955d47d9c14c9f3d2aa6541fd881faf07
+F src/json.c b3f20f01dc6e11071b0b5a8fda73a16c23fd7e65356ec715c0082e904f3fb284
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a4c1af616e672a0d4d04f2652e645617758231598fb7161b956883512303ae87 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
-R 31a2d8c82324ae8c8feb0c4037e19a24
+P ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a
+R d6f55ea1e7a815bfa0f96bc129102acf
 U drh
-Z df17bf2cf88062fccaa06a3f8068b84c
+Z d5c0e742c545064eb8b7fe7e17a2bad7
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 80d363ba5c..02d918d3db 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a
\ No newline at end of file
+134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index a4a3e65c92..3ad5af659c 100644
--- a/src/json.c
+++ b/src/json.c
@@ -248,16 +248,6 @@ static int jsonGrow(JsonString *p, u32 N){
   return SQLITE_OK;
 }
 
-/* Try to force the string to be an RCStr string, rather than a
-** static string.  Return true on success.  The only reason this
-** might fail is due to an OOM fault.
-*/
-static int jsonForceRCStr(JsonString *p){
-  if( p->bStatic==0 ) return 1;
-  jsonGrow(p, p->nAlloc+1);
-  return p->bStatic==0;
-}
-
 /* Append N bytes from zIn onto the end of the JsonString string.
 */
 static SQLITE_NOINLINE void jsonAppendExpand(
@@ -315,6 +305,22 @@ static void jsonAppendChar(JsonString *p, char c){
   }
 }
 
+/* Try to force the string to be a zero-terminated RCStr string.
+**
+** Return true on success.  Return false if an OOM prevents this
+** from happening.
+*/
+static int jsonForceRCStr(JsonString *p){
+  jsonAppendChar(p, 0);
+  if( p->bErr ) return 0;
+  p->nUsed--;
+  if( p->bStatic==0 ) return 1;
+  p->nAlloc = 0;
+  jsonGrow(p, p->nUsed);
+  return p->bStatic==0;
+}
+
+
 /* Append a comma separator to the output buffer, if the previous
 ** character is not '[' or '{'.
 */
@@ -798,8 +804,6 @@ static void jsonReturnJson(
     jsonInit(&s, pCtx);
     jsonRenderNode(pParse, pNode, &s);
     if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){
-      jsonAppendChar(&s, 0);
-      s.nUsed--;
       pParse->zAlt = sqlite3RCStrRef(s.zBuf);
       pParse->nAlt = s.nUsed;
     }

From aef4fbf2032894a8b3096953b9aed54763b12e7f Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Wed, 26 Jul 2023 11:11:39 +0000
Subject: [PATCH 45/66] Add SAHPoolUtil.getFileNames() method, and tests for
 it, per [forum:a3da1e34d8|forum feedback]. Add a test to demonstrate that two
 SAH pools can coexist so long as they have different names.

FossilOrigin-Name: 72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e
---
 ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 15 +++++++++++++++
 ext/wasm/tester1.c-pp.js                      | 18 +++++++++++++++---
 manifest                                      | 14 +++++++-------
 manifest.uuid                                 |  2 +-
 4 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
index 9bf1421a9f..ee8d1a5d68 100644
--- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
@@ -501,6 +501,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     /* Current number of in-use files from pool. */
     getFileCount(){return this.#mapFilenameToSAH.size}
 
+    /* Returns an array of the names of all
+       currently-opened client-specified filenames. */
+    getFileNames(){
+      const rc = [];
+      const iter = this.#mapFilenameToSAH.keys();
+      for(const n of iter) rc.push(n);
+      return rc;
+    }
+
 //    #createFileObject(sah,clientName,opaqueName){
 //      const f = Object.assign(Object.create(null),{
 //        clientName, opaqueName
@@ -901,6 +910,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     getCapacity(){ return this.#p.getCapacity(this.#p) }
 
     getFileCount(){ return this.#p.getFileCount() }
+    getFileNames(){ return this.#p.getFileNames() }
 
     async reserveMinimumCapacity(min){
       const c = this.#p.getCapacity();
@@ -1059,6 +1069,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      Returns the number of files from the pool currently allocated to
      slots. This is not the same as the files being "opened".
 
+     - array getFileNames()
+
+     Returns an array of the names of the files currently allocated to
+     slots. This list is the same length as getFileCount().
+
      - void importDb(name, byteArray)
 
      Imports the contents of an SQLite database, provided as a byte
diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js
index 905340b234..db4b50f4b6 100644
--- a/ext/wasm/tester1.c-pp.js
+++ b/ext/wasm/tester1.c-pp.js
@@ -3072,10 +3072,23 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
         db = new u2.OpfsSAHPoolDb(dbName);
         T.assert(1 === u1.getFileCount());
         db.close();
-        T.assert(1 === u1.getFileCount())
+        const fileNames = u1.getFileNames();
+        T.assert(1 === fileNames.length)
+          .assert(dbName === fileNames[0])
+          .assert(1 === u1.getFileCount())
           .assert(true === u1.unlink(dbName))
           .assert(false === u1.unlink(dbName))
-          .assert(0 === u1.getFileCount());
+          .assert(0 === u1.getFileCount())
+          .assert(0 === u1.getFileNames().length);
+
+        // Demonstrate that two SAH pools can coexist so long as
+        // they have different names.
+        const conf2 = JSON.parse(JSON.stringify(sahPoolConfig));
+        conf2.name += '-test2';
+        const POther = await inst(conf2);
+        log("Installed second SAH instance as",conf2.name);
+        T.assert(0 === POther.getFileCount())
+          .assert(true === await POther.removeVfs());
         if(0){
            /* Enable this block to inspect vfs's contents via the dev
               console or OPFS Explorer browser extension.  The
@@ -3087,7 +3100,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
           .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name));
 
         let cErr, u3;
-        const conf2 = JSON.parse(JSON.stringify(sahPoolConfig));
         conf2.$testThrowInInit = new Error("Testing throwing during init.");
         conf2.name = sahPoolConfig.name+'-err';
         const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e);
diff --git a/manifest b/manifest
index 5f921c0e86..5f4f8afbf0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reformulate\s[907dfc4a7aa1]\susing\sawk\sinstead\sof\ssed\sfor\sbetter\scross-platform\sportability.
-D 2023-07-26T07:57:55.835
+C Add\sSAHPoolUtil.getFileNames()\smethod,\sand\stests\sfor\sit,\sper\s[forum:a3da1e34d8|forum\sfeedback].\sAdd\sa\stest\sto\sdemonstrate\sthat\stwo\sSAH\spools\scan\scoexist\sso\slong\sas\sthey\shave\sdifferent\snames.
+D 2023-07-26T11:11:39.057
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
-F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 7931b50b63246a3d6b46a4c703c28820aa10c5b1ae7c0718e1f58dae2cf6db85
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 44740935f6eeab3a0c6dc9f0ca93aebb6339f8d8365f03ea6d7808c134dbfe46
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d
 F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
@@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js f835c9f703b562142f23a3607fa4a34cb6aece5fb5d674ea5bd7d37b0e47e104
+F ext/wasm/tester1.c-pp.js b9a493a764f2113f6a8205d4e5573a25731d6f965338005460920389ac0693e0
 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1
-R bd64b4635e08e21fec1b91b485560c00
+P 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da
+R 49d65d8d7213fb16653e94d68282a21d
 U stephan
-Z 9b053abd49b0547974c7dd9e0baf8df0
+Z ce157938736081078b3928b891966201
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 9d9c089cd8..f9d3b51fcc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da
\ No newline at end of file
+72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e
\ No newline at end of file

From 8c7ee455ffc3d84dde5e970857931ddf1b8ee0b8 Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Wed, 26 Jul 2023 11:41:41 +0000
Subject: [PATCH 46/66] Remove the batch SQL runner from the JS build's 'all'
 target, as it's long-since unused and adds noticable build time. Add makefile
 comments about JSPI.

FossilOrigin-Name: b8f708e35d4fa027d12089ac7c5589c36da5f68b98cf9792bb11276bf233a860
---
 ext/wasm/GNUmakefile |  9 ++++++++-
 ext/wasm/index.html  |  3 ++-
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile
index 45790bd701..3cd1d930af 100644
--- a/ext/wasm/GNUmakefile
+++ b/ext/wasm/GNUmakefile
@@ -451,6 +451,13 @@ emcc.exportedRuntimeMethods := \
 emcc.jsflags += $(emcc.exportedRuntimeMethods)
 emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
 emcc.jsflags += -sIMPORTED_MEMORY
+#emcc.jsflags += -sASYNCIFY=2
+# ^^^ ASYNCIFY=2 is for experimental JSPI support
+# (https://v8.dev/blog/jspi), but enabling it causes the lib-level
+# init code to throw inexplicable complaints about C-level function
+# signatures not matching what we expect them to be. JSPI requires, as of
+# this writing, requires an experimental Chrome flag:
+# chrome://flags/#enable-experimental-webassembly-stack-switching
 emcc.jsflags += -sSTRICT_JS=0
 # STRICT_JS disabled due to:
 #   https://github.com/emscripten-core/emscripten/issues/18610
@@ -863,7 +870,7 @@ clean-batch:
 # a regular basis with different -Ox flags and rebuilding the batch
 # pieces each time is an unnecessary time sink.
 batch: batch-runner.list
-all: batch
+#all: batch
 # end batch-runner.js
 ########################################################################
 # Wasmified speedtest1 is our primary benchmarking tool.
diff --git a/ext/wasm/index.html b/ext/wasm/index.html
index d91dca6cb4..70ce0441e0 100644
--- a/ext/wasm/index.html
+++ b/ext/wasm/index.html
@@ -107,7 +107,8 @@
             <li><a href='module-symbols.html'>module-symbols</a> gives
               a high-level overview of the symbols exposed by the JS
               module.</li>
-            <li><a href='batch-runner.html'>batch-runner</a>: runs batches of SQL exported from speedtest1.</li>
+            <!--li><a href='batch-runner.html'>batch-runner</a>: runs batches of
+                SQL exported from speedtest1.</li-->
             <li><a href='test-opfs-vfs.html'>test-opfs-vfs</a>
               (<a href='test-opfs-vfs.html?opfs-sanity-check&opfs-verbose'>same
               with verbose output and sanity-checking tests</a>) is an
diff --git a/manifest b/manifest
index 5f4f8afbf0..9e1a1f3698 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSAHPoolUtil.getFileNames()\smethod,\sand\stests\sfor\sit,\sper\s[forum:a3da1e34d8|forum\sfeedback].\sAdd\sa\stest\sto\sdemonstrate\sthat\stwo\sSAH\spools\scan\scoexist\sso\slong\sas\sthey\shave\sdifferent\snames.
-D 2023-07-26T11:11:39.057
+C Remove\sthe\sbatch\sSQL\srunner\sfrom\sthe\sJS\sbuild's\s'all'\starget,\sas\sit's\slong-since\sunused\sand\sadds\snoticable\sbuild\stime.\sAdd\smakefile\scomments\sabout\sJSPI.
+D 2023-07-26T11:41:41.032
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile f1aab41e2afaad68907ff8e8eea32ccca97ab8c216e33dbea817f9e263d4b502
+F ext/wasm/GNUmakefile ddf1aede4275e404c7eda782462c33b6406fcd2dd327241f6b22c0f7b80938e4
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
@@ -531,7 +531,7 @@ F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa0
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html b4e55de741be9fb7656445ea55085f703a784aebde620e1c4852fa21c1ac1c5b
+F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da
-R 49d65d8d7213fb16653e94d68282a21d
+P 72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e
+R 9fd280b1c194f8e5991f01cbec9ff41b
 U stephan
-Z ce157938736081078b3928b891966201
+Z 790924414d34e6bddbbcb85cdb1bbe6d
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index f9d3b51fcc..70d28d9e10 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e
\ No newline at end of file
+b8f708e35d4fa027d12089ac7c5589c36da5f68b98cf9792bb11276bf233a860
\ No newline at end of file

From 7725370ff42e5be18aa97e480e9b8379167da692 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 11:43:39 +0000
Subject: [PATCH 47/66] Improved comments and other cleanup for the changes on
 this branch.

FossilOrigin-Name: bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900
---
 manifest      |  12 ++--
 manifest.uuid |   2 +-
 src/json.c    | 174 +++++++++++++++++++++++++-------------------------
 3 files changed, 93 insertions(+), 95 deletions(-)

diff --git a/manifest b/manifest
index e2bd372d36..ad979b08e4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sjsonForceRCStr()\sto\salso\sadd\sthe\sNULL\sterminator.
-D 2023-07-26T11:00:47.983
+C Improved\scomments\sand\sother\scleanup\sfor\sthe\schanges\son\sthis\sbranch.
+D 2023-07-26T11:43:39.965
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c b3f20f01dc6e11071b0b5a8fda73a16c23fd7e65356ec715c0082e904f3fb284
+F src/json.c 15ca3c2dfd5a525c225da5592f66d4ef6b789a83814f417be0a0af617071bc38
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a
-R d6f55ea1e7a815bfa0f96bc129102acf
+P 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2
+R 9df4f7348f2973215055bf0d567d169d
 U drh
-Z d5c0e742c545064eb8b7fe7e17a2bad7
+Z b486171855fdc62c1515867c409e89e4
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 02d918d3db..989660b794 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2
\ No newline at end of file
+bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 3ad5af659c..8fe3116bcf 100644
--- a/src/json.c
+++ b/src/json.c
@@ -59,7 +59,7 @@ static const char jsonIsSpace[] = {
 typedef struct JsonString JsonString;
 typedef struct JsonNode JsonNode;
 typedef struct JsonParse JsonParse;
-typedef struct JsonTask JsonTask;
+typedef struct JsonCleanup JsonCleanup;
 
 /* An instance of this object represents a JSON string
 ** under construction.  Really, this is a generic string accumulator
@@ -75,11 +75,11 @@ struct JsonString {
   char zSpace[100];        /* Initial static space */
 };
 
-/* A deferred cleanup task.  A list of JsonTask objects might be
+/* A deferred cleanup task.  A list of JsonCleanup objects might be
 ** run when the JsonParse object is destroyed.
 */
-struct JsonTask {
-  JsonTask *pJTNext;       /* Next in a list */
+struct JsonCleanup {
+  JsonCleanup *pJCNext;    /* Next in a list */
   void (*xOp)(void*);      /* Routine to run */
   void *pArg;              /* Argument to xOp() */
 };
@@ -136,29 +136,50 @@ struct JsonNode {
 };
 
 
-/* A completely parsed JSON string
+/* A parsed and possibly edited JSON string.  Lifecycle:
+**
+**   1.  JSON comes in and is parsed into an array aNode[].  The original
+**       JSON text is stored in zJson.  This object may or may not be the
+**       owner of the input JSON - the bOwnsJson variables determines which.
+**
+**   2.  Zero or more changes are made (via json_remove() or json_replace()
+**       or similar) to the aNode[] array.
+**
+**   3.  A new, edited and mimified JSON string is generated from aNode
+**       and stored in zAlt.  The JsonParse object always owns zAlt.
+**
+** Step 1 always happens.  Step 2 and 3 may or may not happen, depending
+** on the operation.
+**
+** aNode[].u.zJContent entries typically point into zJson.  Hence zJson
+** must remain valid for the lifespan of the parse.  For edits,
+** aNode[].u.zJContent might point to malloced space other than zJson.
+** Entries in pClup are responsible for freeing that extra malloced space.
+**
+** When walking the parse tree in aNode[], edits are ignored if useMod is
+** false.
 */
 struct JsonParse {
   u32 nNode;         /* Number of slots of aNode[] used */
   u32 nAlloc;        /* Number of slots of aNode[] allocated */
   JsonNode *aNode;   /* Array of nodes containing the parse */
   char *zJson;       /* Original JSON string (before edits) */
-  char *zAlt;        /* Revised JSON after edits applied.  Maybe NULL */
+  char *zAlt;        /* Revised and/or mimified JSON */
   u32 *aUp;          /* Index of parent of each node */
-  JsonTask *pClean;  /* Cleanup operations prior to freeing this object */
+  JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */
   u16 iDepth;        /* Nesting depth */
   u8 nErr;           /* Number of errors seen */
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
-  u8 nJPRef;         /* Number of references to this object */
   u8 bOwnsJson;      /* This object owns zJson and response for freeing it */
   u8 useMod;         /* Actually use the edits contain inside aNode */
   u8 hasMod;         /* aNode contains edits from the original zJson */
+  u32 nJPRef;        /* Number of references to this object */
   int nJson;         /* Length of the zJson string in bytes */
   int nAlt;          /* Length of alternative JSON string zAlt, in bytes */
   u32 iErr;          /* Error location in zJson[] */
-  u32 iSubst;        /* Last known JSON_SUBST node */
-  u32 iHold;         /* Replace cache line with the lowest iHold value */
+  u32 iSubst;        /* Last JSON_SUBST entry in aNode[] */
+  u32 iHold;         /* Age of this entry in the cache for LRU replacement */
 };
 
 /*
@@ -174,17 +195,6 @@ struct JsonParse {
 ** Utility routines for dealing with JsonString objects
 **************************************************************************/
 
-#if 0
-/*
-** This is a destructor for JSON strings.  We make it a separate function
-** so that the sqlite3ValueIsOfClass() function can be used to unambiguously
-** identify sqlite3_value objects that are known JSON strings.
-*/
-static void jsonStringClass(void *p){
-  sqlite3RCStrUnref((char*)p);
-}
-#endif
-
 /* Set the JsonString object to an empty string
 */
 static void jsonZero(JsonString *p){
@@ -202,7 +212,6 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){
   jsonZero(p);
 }
 
-
 /* Free all allocated memory and reset the JsonString object back to its
 ** initial state.
 */
@@ -211,8 +220,7 @@ static void jsonReset(JsonString *p){
   jsonZero(p);
 }
 
-
-/* Report an out-of-memory (OOM) condition 
+/* Report an out-of-memory (OOM) condition
 */
 static void jsonOom(JsonString *p){
   p->bErr = 1;
@@ -316,7 +324,9 @@ static int jsonForceRCStr(JsonString *p){
   p->nUsed--;
   if( p->bStatic==0 ) return 1;
   p->nAlloc = 0;
+  p->nUsed++;
   jsonGrow(p, p->nUsed);
+  p->nUsed--;
   return p->bStatic==0;
 }
 
@@ -392,7 +402,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){
       jsonAppendRawNZ(p, zIn, i);
       zIn += i;
       N -= i;
-      if( N==0 ) break;     
+      if( N==0 ) break;
     }
     assert( zIn[0]=='\\' );
     switch( (u8)zIn[1] ){
@@ -501,7 +511,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){
 
 
 /*
-** Append a function parameter value to the JSON string under 
+** Append a function parameter value to the JSON string under
 ** construction.
 */
 static void jsonAppendValue(
@@ -550,18 +560,11 @@ static void jsonAppendValue(
 ** The JSON string is reset.
 */
 static void jsonResult(JsonString *p){
-  if( p->bErr==0 ){
-    if( p->bStatic ){
-      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT,
-                            SQLITE_UTF8);
-    }else{
-      jsonAppendChar(p, 0);
-      p->nUsed--;
-      sqlite3RCStrRef(p->zBuf);
-      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
-                            (void(*)(void*))sqlite3RCStrUnref,
-                            SQLITE_UTF8);
-    }
+  if( p->bErr==0 && jsonForceRCStr(p) ){
+    sqlite3RCStrRef(p->zBuf);
+    sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
+                          (void(*)(void*))sqlite3RCStrUnref,
+                          SQLITE_UTF8);
   }else if( p->bErr==1 ){
     sqlite3_result_error_nomem(p->pCtx);
   }
@@ -590,9 +593,9 @@ static u32 jsonNodeSize(JsonNode *pNode){
 ** delete the JsonParse object itself.
 */
 static void jsonParseReset(JsonParse *pParse){
-  while( pParse->pClean ){
-    JsonTask *pTask = pParse->pClean;
-    pParse->pClean = pTask->pJTNext;
+  while( pParse->pClup ){
+    JsonCleanup *pTask = pParse->pClup;
+    pParse->pClup = pTask->pJCNext;
     pTask->xOp(pTask->pArg);
     sqlite3_free(pTask);
   }
@@ -608,17 +611,13 @@ static void jsonParseReset(JsonParse *pParse){
     pParse->aUp = 0;
   }
   if( pParse->zAlt ){
-    char *z = pParse->zAlt;
+    sqlite3RCStrUnref(pParse->zAlt);
     pParse->zAlt = 0;
-    sqlite3RCStrUnref(z);
   }
   if( pParse->bOwnsJson ){
-    /* Order operations so that if the destructor for pParse->zJson
-    ** invokes jsonParseFree(), the recursion will terminate harmlessly */
-    char *z = pParse->zJson;
-    pParse->zJson = 0;
     pParse->bOwnsJson = 0;
-    sqlite3RCStrUnref(z);
+    sqlite3RCStrUnref(pParse->zJson);
+    pParse->zJson = 0;
   }
 }
 
@@ -650,14 +649,14 @@ static int jsonParseAddCleanup(
   void(*xOp)(void*),          /* The cleanup task */
   void *pArg                  /* Argument to the cleanup */
 ){
-  JsonTask *pTask = sqlite3_malloc64( sizeof(*pTask) );
+  JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) );
   if( pTask==0 ){
     pParse->oom = 1;
     xOp(pArg);
     return SQLITE_ERROR;
   }
-  pTask->pJTNext = pParse->pClean;
-  pParse->pClean = pTask;
+  pTask->pJCNext = pParse->pClup;
+  pParse->pClup = pTask;
   pTask->xOp = xOp;
   pTask->pArg = pArg;
   return SQLITE_OK;
@@ -870,8 +869,7 @@ static void jsonReturn(
       int rc;
       int bNeg = 0;
       const char *z;
-     
-   
+
       assert( pNode->eU==1 );
       z = pNode->u.zJContent;
       if( z[0]=='-' ){ z++; bNeg = 1; }
@@ -1458,23 +1456,23 @@ json_parse_restart:
     cDelim = z[i];
     for(j=i+1; 1; j++){
       static const char aOk[256] = {
-      	  0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
-      	  0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
-      	  1, 1, 0, 1, 1, 1, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 0, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
+          0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
+          1, 1, 0, 1, 1, 1, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 0, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
 
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
-      	  1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
       };
       if( aOk[(unsigned char)z[j]] ) continue;
       c = z[j];
@@ -1740,18 +1738,18 @@ json_parse_restart:
 
 /*
 ** Parse a complete JSON string.  Return 0 on success or non-zero if there
-** are any errors.  If an error occurs, free all memory associated with
-** pParse.
+** are any errors.  If an error occurs, free all memory held by pParse,
+** but not pParse itself.
 **
 ** pParse is uninitialized when this routine is called.
 **
-** pParse->nJPRef set to 1.  The caller becomes the owner of the 
+** pParse->nJPRef set to 1.  The caller becomes the owner of the
 ** the JsonParse object.
 **
-** pParse->bOwnsJson is to bTakeJson.  If bTakeJson is 1, the newly initialized
-** JsonParse object will become the own the zJson input string.  If bTakeJson
-** is 0, then the caller is responsible for preserving zJson for the lifetime
-** of the JsonParse object.
+** pParse->bOwnsJson is set to bTakeJson.  If bTakeJson is 1, the newly
+** initialized JsonParse object will become the owner of the zJson input
+** string.  If bTakeJson is 0, then the caller is responsible for
+** preserving zJson for the lifetime of the JsonParse object.
 */
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
@@ -2129,7 +2127,7 @@ static JsonNode *jsonLookupStep(
     zPath += j + 1;
     j = 1;
     for(;;){
-      while( j<=pRoot->n 
+      while( j<=pRoot->n
          && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod))
       ){
         if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--;
@@ -2255,7 +2253,7 @@ static void jsonWrongNumArgs(
   char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments",
                                zFuncName);
   sqlite3_result_error(pCtx, zMsg, -1);
-  sqlite3_free(zMsg);     
+  sqlite3_free(zMsg);
 }
 
 /*
@@ -2391,7 +2389,7 @@ static void jsonTest1Func(
 
 /*
 ** Implementation of the json_QUOTE(VALUE) function.  Return a JSON value
-** corresponding to the SQL value input.  Mostly this means putting 
+** corresponding to the SQL value input.  Mostly this means putting
 ** double-quotes around strings and returning the unquoted string "null"
 ** when given a NULL input.
 */
@@ -2438,7 +2436,7 @@ static void jsonArrayFunc(
 ** json_array_length(JSON)
 ** json_array_length(JSON, PATH)
 **
-** Return the number of elements in the top-level JSON array.  
+** Return the number of elements in the top-level JSON array.
 ** Return 0 if the input is not a well-formed JSON array.
 */
 static void jsonArrayLengthFunc(
@@ -2752,7 +2750,7 @@ static void jsonRemoveFunc(
 
   if( argc<1 ) return;
   pParse = jsonParseCached(ctx, argv[0], ctx, argc>1);
-  if( pParse==0 ) return;  
+  if( pParse==0 ) return;
   for(i=1; i<(u32)argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
     if( zPath==0 ) goto remove_done;
@@ -2854,7 +2852,7 @@ static void jsonReplaceNode(
     }
   }
 }
-  
+
 /*
 ** json_replace(JSON, PATH, VALUE, ...)
 **
@@ -3269,7 +3267,7 @@ static int jsonEachConnect(
   UNUSED_PARAMETER(argv);
   UNUSED_PARAMETER(argc);
   UNUSED_PARAMETER(pAux);
-  rc = sqlite3_declare_vtab(db, 
+  rc = sqlite3_declare_vtab(db,
      "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path,"
                     "json HIDDEN,root HIDDEN)");
   if( rc==SQLITE_OK ){
@@ -3481,7 +3479,7 @@ static int jsonEachColumn(
       break;
     }
     case JEACH_ID: {
-      sqlite3_result_int64(ctx, 
+      sqlite3_result_int64(ctx,
          (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0));
       break;
     }
@@ -3581,8 +3579,8 @@ static int jsonEachBestIndex(
       idxMask |= iMask;
     }
   }
-  if( pIdxInfo->nOrderBy>0 
-   && pIdxInfo->aOrderBy[0].iColumn<0 
+  if( pIdxInfo->nOrderBy>0
+   && pIdxInfo->aOrderBy[0].iColumn<0
    && pIdxInfo->aOrderBy[0].desc==0
   ){
     pIdxInfo->orderByConsumed = 1;
@@ -3782,10 +3780,10 @@ void sqlite3RegisterJsonFunctions(void){
     JFUNCTION(json_parse,         1, 0,  jsonParseFunc),
     JFUNCTION(json_test1,         1, 0,  jsonTest1Func),
 #endif
-    WAGGREGATE(json_group_array,  1, 0, 0, 
+    WAGGREGATE(json_group_array,  1, 0, 0,
        jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse,
        SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC),
-    WAGGREGATE(json_group_object, 2, 0, 0, 
+    WAGGREGATE(json_group_object, 2, 0, 0,
        jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse,
        SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC)
   };

From 2b5719706647a6831e2d5fadd56d04f67fc63837 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 11:53:14 +0000
Subject: [PATCH 48/66] More comment improvements in json.c.  Do not run
 jsonLookup() following an OOM error.

FossilOrigin-Name: cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 11 +++++++----
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index ad979b08e4..490099b0ad 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\scomments\sand\sother\scleanup\sfor\sthe\schanges\son\sthis\sbranch.
-D 2023-07-26T11:43:39.965
+C More\scomment\simprovements\sin\sjson.c.\s\sDo\snot\srun\sjsonLookup()\sfollowing\nan\sOOM\serror.
+D 2023-07-26T11:53:14.698
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 15ca3c2dfd5a525c225da5592f66d4ef6b789a83814f417be0a0af617071bc38
+F src/json.c c784b923a8aaa5cb514c135a74d77aec1b5c9e1cbd63764adb10d44cc678bb6e
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2
-R 9df4f7348f2973215055bf0d567d169d
+P bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900
+R 968cb2bde9a2ab88d10bb801ca2ea188
 U drh
-Z b486171855fdc62c1515867c409e89e4
+Z 8edd1e3712b3e1a50841b71b7e4dc709
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 989660b794..979f3c9801 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900
\ No newline at end of file
+cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 8fe3116bcf..5deeb6acba 100644
--- a/src/json.c
+++ b/src/json.c
@@ -1841,9 +1841,10 @@ static int jsonParseFindParents(JsonParse *pParse){
 #define JSON_CACHE_SZ  4          /* Max number of cache entries */
 
 /*
-** Obtain a complete parse of the JSON found in the first argument
-** of the argv array.  Use the sqlite3_get_auxdata() cache for this
-** parse if it is available.  If the cache is not available or if it
+** Obtain a complete parse of the JSON found in the pJson argument
+**
+** Use the sqlite3_get_auxdata() cache to find a preexisting parse
+** if it is available.  If the cache is not available or if it
 ** is no longer valid, parse the JSON again and return the new parse.
 ** Also register the new parse so that it will be available for
 ** future sqlite3_get_auxdata() calls.
@@ -2003,7 +2004,9 @@ static JsonNode *jsonLookupStep(
 ){
   u32 i, j, nKey;
   const char *zKey;
-  JsonNode *pRoot = &pParse->aNode[iRoot];
+  JsonNode *pRoot;
+  if( pParse->oom ) return 0;
+  pRoot = &pParse->aNode[iRoot];
   while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){
     u32 idx = (u32)(pRoot - pParse->aNode);
     i = pParse->iSubst;

From 7286c598963bcec929f44c3a75ff0b257acaf1cd Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 13:17:43 +0000
Subject: [PATCH 49/66] Change the debugging "json_parse(X)" function so that
 it shows a more complete description of the JsonParse object on standard
 output and returns the mimified JSON.  Former behavior was to return the text
 of a decode of the aNode array.

FossilOrigin-Name: 1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9
---
 manifest          |  14 ++++----
 manifest.uuid     |   2 +-
 src/json.c        |  80 ++++++++++++++++++++++++----------------------
 test/fuzzdata6.db | Bin 1785856 -> 1785856 bytes
 4 files changed, 50 insertions(+), 46 deletions(-)

diff --git a/manifest b/manifest
index 490099b0ad..f0b7ab99fd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C More\scomment\simprovements\sin\sjson.c.\s\sDo\snot\srun\sjsonLookup()\sfollowing\nan\sOOM\serror.
-D 2023-07-26T11:53:14.698
+C Change\sthe\sdebugging\s"json_parse(X)"\sfunction\sso\sthat\sit\sshows\sa\smore\ncomplete\sdescription\sof\sthe\sJsonParse\sobject\son\sstandard\soutput\sand\sreturns\nthe\smimified\sJSON.\s\sFormer\sbehavior\swas\sto\sreturn\sthe\stext\sof\sa\sdecode\nof\sthe\saNode\sarray.
+D 2023-07-26T13:17:43.116
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c c784b923a8aaa5cb514c135a74d77aec1b5c9e1cbd63764adb10d44cc678bb6e
+F src/json.c 512bc389b42c68b34571ca532afb0f4bb235b1fc11ea5c9a4f6850c64fa12ab4
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -1131,7 +1131,7 @@ F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5b
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
 F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e42ed2
 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5
-F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7
+F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0
 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2
 F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c3493f80
 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900
-R 968cb2bde9a2ab88d10bb801ca2ea188
+P cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17
+R ecfa3d900e8af899b83f6d4944c0b5ca
 U drh
-Z 8edd1e3712b3e1a50841b71b7e4dc709
+Z 621bef24d346a4651dc554beab6ec135
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 979f3c9801..fa90aa429f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17
\ No newline at end of file
+1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 5deeb6acba..9eb2302684 100644
--- a/src/json.c
+++ b/src/json.c
@@ -118,7 +118,12 @@ static const char * const jsonType[] = {
 #define JNODE_JSON5   0x40  /* Node contains JSON5 enhancements */
 
 
-/* A single node of parsed JSON
+/* A single node of parsed JSON.  An array of these nodes describes
+** a parse of JSON + edits.
+**
+** Use the json_parse() SQL function (available when compiled with
+** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including
+** a complete listing and decoding of the array of JsonNodes.
 */
 struct JsonNode {
   u8 eType;              /* One of the JSON_ type values */
@@ -2283,7 +2288,7 @@ static void jsonRemoveAllNulls(JsonNode *pNode){
 ** SQL functions used for testing and debugging
 ****************************************************************************/
 
-#if 0  /* One for debugging.  zero normally */
+#if SQLITE_DEBUG
 /*
 ** Print N node entries.
 */
@@ -2299,12 +2304,15 @@ static void jsonDebugPrintNodeEntries(
     }else{
       zType = jsonType[aNode[i].eType];
     }
+    printf("node %4u: %-7s n=%-5d", i, zType, aNode[i].n);
     if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){
-      printf("node %4u: %-7s %02x n=%-5d",
-             i, zType, aNode[i].jnFlags, aNode[i].n);
-    }else{
-      printf("node %4u: %-7s    n=%-5d",
-             i, zType, aNode[i].n);
+      u8 f = aNode[i].jnFlags;
+      if( f & JNODE_RAW )     printf(" RAW");
+      if( f & JNODE_ESCAPE )  printf(" ESCAPE");
+      if( f & JNODE_REMOVE )  printf(" REMOVE");
+      if( f & JNODE_REPLACE ) printf(" REPLACE");
+      if( f & JNODE_APPEND )  printf(" APPEND");
+      if( f & JNODE_JSON5 )   printf(" JSON5");
     }
     switch( aNode[i].eU ){
       case 1:  printf(" zJContent=[%.*s]\n",
@@ -2316,6 +2324,10 @@ static void jsonDebugPrintNodeEntries(
     }
   }
 }
+#endif /* SQLITE_DEBUG */
+
+
+#if 0  /* 1 for debugging.  0 normally.  Requires -DSQLITE_DEBUG too */
 static void jsonDebugPrintParse(JsonParse *p){
   jsonDebugPrintNodeEntries(p->aNode, p->nNode);
 }
@@ -2330,45 +2342,37 @@ static void jsonDebugPrintNode(JsonNode *pNode){
 
 #ifdef SQLITE_DEBUG
 /*
-** The json_parse(JSON) function returns a string which describes
-** a parse of the JSON provided.  Or it returns NULL if JSON is not
-** well-formed.
+** SQL function:   json_parse(JSON)
+**
+** Parse JSON using jsonParseCached().  Then print a dump of that
+** parse on standard output.  Return the mimified JSON result, just
+** like the json() function.
 */
 static void jsonParseFunc(
   sqlite3_context *ctx,
   int argc,
   sqlite3_value **argv
 ){
-  JsonString s;       /* Output string - not real JSON */
-  JsonParse x;        /* The parse */
-  u32 i;
+  JsonParse *p;        /* The parse */
 
   assert( argc==1 );
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
-  jsonParseFindParents(&x);
-  jsonInit(&s, ctx);
-  for(i=0; i<x.nNode; i++){
-    const char *zType;
-    if( x.aNode[i].jnFlags & JNODE_LABEL ){
-      assert( x.aNode[i].eType==JSON_STRING );
-      zType = "label";
-    }else{
-      zType = jsonType[x.aNode[i].eType];
-    }
-    jsonPrintf(100, &s,"node %3u: %7s n=%-4d up=%-4d",
-               i, zType, x.aNode[i].n, x.aUp[i]);
-    assert( x.aNode[i].eU==0 || x.aNode[i].eU==1 );
-    if( x.aNode[i].u.zJContent!=0 ){
-      assert( x.aNode[i].eU==1 );
-      jsonAppendChar(&s, ' ');
-      jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n);
-    }else{
-      assert( x.aNode[i].eU==0 );
-    }
-    jsonAppendChar(&s, '\n');
-  }
-  jsonParseReset(&x);
-  jsonResult(&s);
+  p = jsonParseCached(ctx, argv[0], ctx, 0);
+  if( p==0 ) return;
+  printf("nNode     = %u\n", p->nNode);
+  printf("nAlloc    = %u\n", p->nAlloc);
+  printf("nJson     = %d\n", p->nJson);
+  printf("nAlt      = %d\n", p->nAlt);
+  printf("nErr      = %u\n", p->nErr);
+  printf("oom       = %u\n", p->oom);
+  printf("hasNonstd = %u\n", p->hasNonstd);
+  printf("bOwnsJson = %u\n", p->bOwnsJson);
+  printf("useMod    = %u\n", p->useMod);
+  printf("hasMod    = %u\n", p->hasMod);
+  printf("nJPRef    = %u\n", p->nJPRef);
+  printf("iSubst    = %u\n", p->iSubst);
+  printf("iHold     = %u\n", p->iHold);
+  jsonDebugPrintNodeEntries(p->aNode, p->nNode);
+  jsonReturnJson(p, p->aNode, ctx, 1);
 }
 
 /*
diff --git a/test/fuzzdata6.db b/test/fuzzdata6.db
index b1424c21e4194f352629d4d08b7b88d16ea0d1ee..67076d0026aca74aa3984260ecd3371006966501 100644
GIT binary patch
delta 2927
zcmaJ@Yj6|S72dmAy;l3sO4e(IWVtndfUVlbp*##&1{?#Kj0vQZX#%xD0|Cp#U~rpE
zhRVik5}C?Y82d1W7(srd8Ik}Ya7&sH!wexI)7GR++mJ93)1*wBbjBG<(hPmw)e3uX
zSNz#O-*>)y&iT&0XXEkqc)WekkJ!|VrAC$tZ~IkuREZ)bN-I%=5@nRAQHh$As9A|x
zPDiaT9<~M7Q4-xj748Ub#9Z3CI>=KLWn!ovo6^Nly>K!~M_^1rC2+VMm0ChuJD=KC
zR$UHPhAf*P{-K4#JKnVpjL`+R!4PLF&>dVAXD=48no76Sw-d%j*ir}$u?#~3E1~<2
z#R=^rZ1djK5Q|eowp}ZOC#XP7zUqy*UUmBI3Gp+rM(D8}v)nRO89JyF`-?Kqz-Y$i
zFe`P5I)+L>Yl@o#qkp!2i?;{44g_B{@*z`QV&T2m!Ov}j$h;)6fT8*gLlLe<3u(P+
zOM7|G+Z0F5rvi;~pXcgnfqUJ<ket0xbdf7ylyG9n@C9cLwEV^Pjd8YG^vkQe1Oz`X
z<Tk*>)$D1D?>+69pBqGmxrs$^Zk(GtncYr=zbxe3EjIZU#iRdFflm25%*DBza<0?8
zU5Td{>UlUn&W*x<Q=DnC$rzopUqZO^MK0MBuzo_>kU|CK$z7f|3vN3e5ZkyT)*EIg
zvkEDviZYuKR*Lu^S#qjMU4L~hzfqVd<`3W<bNF6_k5=$MIc1XvDH|<OfhKtva~Zx@
z&iSQ;``fd1cLapnrkPo{F)2KXAbyhz;Z^_Uf8F5qQ-PiG+n#^AY8`FjDeext)ZEYf
zp7!JFwDk)Fh6cgQD7E`Mxag=*hG6h>n^6_nfMDAKF$h(RU@_G0b5DbRjNs$X93^*Q
z*Z;%zl|q=V-G{f&lFBt%AhN%KYh~O*NWWqGSvD5`>mkADw8;rlq9a#gi?Ca^6MH$*
zlJ=#KERQJReZFj=&AWs<YI3~@d8eJY5kc@a7f?mbeQ_bki?3*1i^$J=uDE=TR`EFZ
zH9Ozj&3r&hYVo%amU_ek{A4$OL92)0%w_xdRKj7X8W5k|8+}2<(HHDTUz1~0AS%D_
zEpjQ2_v`}S%pR~jXB;;Kso4JNcs;b#OB|WVWU-d;1?d}O))=*TKYk-3ZAON0^832z
z)@9NmzjrMacvwE<`I~E=qfflbe~;}jpEFwNwQA1)K;YOQHCUAD<PS)9HsdS3YIjv_
zDBL8Oaoa{|BQjSf!_!bq(5>ppmAWTids54*LKj`RO<I9qLga!(V0J!Awy;_G(59?E
zg+!oSm-1GpmO+^=I_qgkL@<9uc#H^?=A&dYD9MLTWiV5hGO}BWA@7>$-H(~E(MYe+
zCI+M)sX?`Ofu016|5CEz*dA%U+UfcEDaCz+LR^|R%_U9X)90jCWU5L2kf4xSU*N`}
z20=ik#Z!X>EiXwHJls#rCMV6+J&91vcy&^0Q7I4TQnDjsO=*<c{FL+iHOid)l&6O^
zN=<&sUk^$V1b=Vj)hVdHFE5*O)qN>MD)x*sI6TIiyv>?v8CTfxtazFCvYX6r8fm&&
zHEjZca8xsGxt<^<eaa5bvV3Sgg5Q42m6D{F>Qad7S%C&URI@a7Oq+>_9tGFGt_jrZ
zp_%|XsR?L{u6f$Ca^jRGuvkw36ZCEg^OB2jeL||#o(vp1D{WV&qfRewa+*z(H*ev6
zc}ue92JO5nM4n!G@*-jTm}Ga|v#^sZrj!<|3tN5F$==GnF-=qG<jnhU*~iirHQNVu
z(3$wFPo#A!+Ngv2Rg<+REYO#c&SV^>xVKncHwn*2F4?hGJj65X3UiYA16}+w85$ab
z8=NL`(FeAkx!^28jG7}Hd7m$bmOajE>c;}V`;>nKsy^b&@t>m3a=*5x4cW5EKZCP(
z`Vt$65v?+$?E|%7va<$7^)?K&O+y~fj$BD!+S!*b=u5lKq}?b)TVDdhn7a(Uf-A<{
zKeY<E-?k@*W;4{DM<H_2ozM~uec&#I(d+I)2o1XJc<8b_wu@#+biJp7L|4^&I+@(s
z+k=MvkU8tEg!l>%K-+T9Sv;%J^T+Qh*)J(45~$$18E^SLzTKWL+)V|?oubqwUKbwZ
z4zXWbmzk5s8%!O25((t5FN1}M_qz42ZYewg>4&@@!aJ<jy!X;7FTS+Ow_q)eR)Qhz
zd)UxSZAnF3uqExgkqha5eB?3cuJY}M^l=}XtpsO|_=?s$<@3}`>MT`Q9Bc}7&H(?Y
ueCc$7{My|gq3$&YXTyzJ5;)<OO>{*Fl8b!R&|>uk_8#`j_^{vKTKqp;w#Og<

delta 3034
zcmai$X>e0j6vy99UbDR|H?PS{UPzjbLR%<I3n@FLEfp%DRz*i0fmDOaQb7vnpw=`c
z0g9w8o%R4io7PbV5d;yg>li<%xG&=<qKxT`$_Hm~6diSNyDt$)+oXQUm-D;l{PREO
zzI#HU#!#rS!;Q?HW9H?VnMuCgEx||-MS{9WP#+1hkzigVXov)jk)Y{B(7eBoV;BbO
z)-eU^y-NunH{a-G!#J`Djt!VrL0=;(5`C+h*51E-UBmjO6~*Nx@JYAn9tgc>;>r5g
z&Fu*t!+<_yDMi1MvXJG?d{c`O-M}*8DbPQ`=7x1$7QuY!>_lj}Xp*6Eucdx#XSaoP
zc5@pSdV3kq8uc@0wf$MyZEcgjktT~_?vUw%VZ1)b^mY{{H$!idvw>g~iy@)%zZueE
z$lxH?f%G+Pv)~_tICV&<|646|P1{UZd(*H`!#OVQ6zIR^nHh{{rn*6S^@PYDQP_yT
zZ7vkP$)_^?+~eZ&0H5db!P+kQNc<ePM5|RfvKDdkDH(G{hN!}I$a2L$@<Rdd2xyiI
z=x@f;q+TF=h(D*YTOMg!+w%&4V3^qiRq}5m(zKtCg+25Xha!w;y1GeuA^%rfg|wRQ
zH-BZ6*+nRFyfAq$B9TJjRudbk*-=iYCF2mpX9-ef{qm{83IrXJP(TJsh2_UJDVz>7
zo;tOI@JV5dN^RKDND2qUBE*iRtq18m=SX{y7%MD6<l4W4*J_-4#<M|vRr%fSx2=|r
z^A|01jE}Kz==7vKZvGB|zD9JCz<$w=dghC4hL1FdAHk7Yah!hU;|}PT<T~(O<dX2u
zHDadUlq*HBY=Jn5a`yk@91BxieohXOhqyjCQ_RnT`19ODP{xWT^5<ePPfj~vV>~r#
z8{q?Dv#O8yA16P(FO{GyqNXBvY%}&!QO=yj2o+y}pzKlX(Fz%(A+g6N8JyJSYIVEv
ziQQ#ukPh-cS!Nhp*mrbT^Xvj5I4tcJsQo*2q_9G&M__$RIzGg{JzE=gNu*(y^{K;Z
zfbj&?Q_ezr#P*g|6zVOzP22K5);pO%S9z!k0#%qNB?Dps)e*&p%N-ENi`DpEr1!_J
zN;i3UcvUUl=XO#*=tL-|?LOOm(oz0*OO<gO`+?4;N&N+Z?JiscfqD4HjJ)x?@hYUB
z7Mnz@VLCj+@&XMT$({G$yS3UAv)A@LfF%So_KLUbVx>8eIs}0Of=?HlFdEXl85ht>
z<8v}86H0Q1s}Own8(*x86=z5GHeb<NloO#@0xEA#&x}7dZd7nRE~1LZ=493&ZG?vQ
zWp?HW0S=GoOV2@UCJ!&>>EK#x1+zAlnlCf5R!PBkMDa4Mk<+s_GKKKlaRbs%k4*z>
zo9#NNKg^vO)^CcLNMc92-BWWFGm?!l9MC4BA}d0Co#|oqXAP%^>dUTliYYl6Gnu;w
z8!k=O<k30$-~F-j(V4la+pYOn_P^ko&9b*AAs7tee_z?S!0Ny{faxV2a)VNq-TE}1
zK6GYkvsSnSFW1!0%Zi*lnD(wFYm_E1lUELDt@g*Nb0dvM(nwWqq~KT@nVTDFJ&C8$
z0enWF7pihpjOwY3%^`#D3-tLq+Juc)J~1iD6))lYG|97bXQ9)6x!JQu#nP9;qFqmq
z)+390`F;^=0$c6$Er=0ZV=YE(W#!1nkm{(*R)8b^FY!uE=7MZth5Y;#UaE!XXA29L
znwy@4anU3`nY0-~POqk}P5IQW+LlQB1lDq`F~)wb^B$mIWgU3$k`0i5i&O@_o$@ar
zeb49Rr8Y0<KP8kvpjAGj{g~jRuZ5eS>{FqHoDRw*?(_>tAN14?5R$iOFJL?L-ylta
zj$QJfU^~yrmp1_VupK4z>)XCs+m$AdAY}Pw6phQ#cz!f)KN)u*Uur{4IEvAeq%`4p
z(9E{n-#r0t3@baijMWl+?>LH}_pGA;d>sxe>3-L-W@GATl~PKNmQ^WDEZb`AaYJX9
z;|{25a;$(KPdP16e?d8#x#NtVuYB>xM7r3>NMyWA$Gq%TT+PaNj=KEkWC?GQ&Wa28
zeU=~0wZ>T9AUj)k42g8fOcJr$dDd)fod|cbOcVxBD-}>b$=RNfdTU~llOz_os+XjM
z#a-9y7cwh5tL?Bd?iyrU8us1>54`5umT`n6`(1_0WU6K?^Ab~VwYSdGGzQ$g>KiBW
m)x)Z(<@P5guuSVTKrS`RXZTSJgUznGt$l8l^ts&)SN{tsuf1ge


From a1016b4275a473586647e9cd545409f709297f25 Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Wed, 26 Jul 2023 16:41:23 +0000
Subject: [PATCH 50/66] Improve the output when ".scanstats vm" is enabled.

FossilOrigin-Name: 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22
---
 manifest       |  12 ++--
 manifest.uuid  |   2 +-
 src/shell.c.in | 181 ++++++++++++++++++++++++++-----------------------
 3 files changed, 104 insertions(+), 91 deletions(-)

diff --git a/manifest b/manifest
index f50f10ae04..963fde8d6a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sexperimental\s".scanstats\svm"\scommand\sto\sthe\sshell\stool.
-D 2023-06-30T19:14:35.552
+C Improve\sthe\soutput\swhen\s".scanstats\svm"\sis\senabled.
+D 2023-07-26T16:41:23.325
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -638,7 +638,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
-F src/shell.c.in 6266eac6a51f1b931d72b3ea95b9f688f390cad7b43bbdb465f216b39eaaa0d3
+F src/shell.c.in 921831fe4acca388f66472f6e4076c4531edf17b1b0a64d85843606f339d00d3
 F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
@@ -2041,8 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2
-R 1fc4f436f0d8b18e387b07b97d0465bf
+P e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d
+R cd10daa978b5e0a84d84b4bb41300995
 U dan
-Z 39b6ffde042c1822fbacb7543ed0038f
+Z 2a26d9e4014ba821912c8a2a17121583
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index df83ac764a..88f96d904b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d
\ No newline at end of file
+7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 04407e54ae..57d16e8736 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1607,6 +1607,7 @@ static ShellState shellState;
 #define MODE_Box     16  /* Unicode box-drawing characters */
 #define MODE_Count   17  /* Output only a count of the rows of output */
 #define MODE_Off     18  /* No query output shown */
+#define MODE_ScanExp 19  /* Like MODE_Explain, but for ".scanstats vm" */
 
 static const char *modeDescr[] = {
   "line",
@@ -2520,38 +2521,58 @@ static int shell_callback(
       }
       break;
     }
+    case MODE_ScanExp:
     case MODE_Explain: {
-      static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13};
-      if( nArg>ArraySize(aExplainWidth) ){
-        nArg = ArraySize(aExplainWidth);
+      static const int aExplainWidth[] = {4,       13, 4, 4, 4, 13, 2, 13};
+      static const int aExplainMap[] =   {0,       1,  2, 3, 4, 5,  6, 7 };
+      static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13};
+      static const int aScanExpMap[] =   {0, 9, 8, 1,  2, 3, 4, 5,  6, 7 };
+
+      const int *aWidth = aExplainWidth;
+      const int *aMap = aExplainMap;
+      int nWidth = ArraySize(aExplainWidth);
+      int iIndent = 1;
+
+      if( p->cMode==MODE_ScanExp ){
+        aWidth = aScanExpWidth;
+        aMap = aScanExpMap;
+        nWidth = ArraySize(aScanExpWidth);
+        iIndent = 3;
       }
+      if( nArg>nWidth ) nArg = nWidth;
+
+      /* If this is the first row seen, print out the headers */
       if( p->cnt++==0 ){
         for(i=0; i<nArg; i++){
-          int w = aExplainWidth[i];
-          utf8_width_print(p->out, w, azCol[i]);
+          utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]);
           fputs(i==nArg-1 ? "\n" : "  ", p->out);
         }
         for(i=0; i<nArg; i++){
-          int w = aExplainWidth[i];
-          print_dashes(p->out, w);
+          print_dashes(p->out, aWidth[i]);
           fputs(i==nArg-1 ? "\n" : "  ", p->out);
         }
       }
+
+      /* If there is no data, exit early. */
       if( azArg==0 ) break;
+
       for(i=0; i<nArg; i++){
-        int w = aExplainWidth[i];
+        const char *zSep = "  ";
+        int w = aWidth[i];
+        const char *zVal = azArg[ aMap[i] ];
         if( i==nArg-1 ) w = 0;
-        if( azArg[i] && strlenChar(azArg[i])>w ){
-          w = strlenChar(azArg[i]);
+        if( zVal && strlenChar(zVal)>w ){
+          w = strlenChar(zVal);
+          zSep = " ";
         }
-        if( i==1 && p->aiIndent && p->pStmt ){
+        if( i==iIndent && p->aiIndent && p->pStmt ){
           if( p->iIndent<p->nIndent ){
             utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
           }
           p->iIndent++;
         }
-        utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue);
-        fputs(i==nArg-1 ? "\n" : "  ", p->out);
+        utf8_width_print(p->out, w, zVal ? zVal : p->nullValue);
+        fputs(i==nArg-1 ? "\n" : zSep, p->out);
       }
       break;
     }
@@ -3411,36 +3432,6 @@ static void display_explain_scanstats(
 }
 #endif
 
-static void exec_prepared_stmt(ShellState*, sqlite3_stmt*);
-
-/*
-** Display scan stats.
-*/
-static void display_scanstats(
-  sqlite3 *db,                    /* Database to query */
-  ShellState *pArg                /* Pointer to ShellState */
-){
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-  UNUSED_PARAMETER(db);
-  UNUSED_PARAMETER(pArg);
-#else
-  if( pArg->scanstatsOn==3 ){
-    int rc = SQLITE_OK;
-    sqlite3_stmt *pStmt = 0;
-    rc = sqlite3_prepare_v2(db, "SELECT * FROM bytecode(?)", -1, &pStmt, 0);
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pSave = pArg->pStmt;
-      pArg->pStmt = pStmt;
-      sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0);
-      exec_prepared_stmt(pArg, pStmt);
-      sqlite3_finalize(pStmt);
-      pArg->pStmt = pSave;
-    }
-  }else{
-    display_explain_scanstats(db, pArg);
-  }
-#endif
-}
 
 /*
 ** Parameter azArray points to a zero-terminated array of strings. zStr
@@ -3478,8 +3469,6 @@ static int str_in_array(const char *zStr, const char **azArray){
 **       and "Goto" by 2 spaces.
 */
 static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
-  const char *zSql;               /* The text of the SQL statement */
-  const char *z;                  /* Used to check if this is an EXPLAIN */
   int *abYield = 0;               /* True if op is an OP_Yield */
   int nAlloc = 0;                 /* Allocated size of p->aiIndent[], abYield */
   int iOp;                        /* Index of operation in p->aiIndent[] */
@@ -3490,65 +3479,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
                             "Rewind", 0 };
   const char *azGoto[] = { "Goto", 0 };
 
-  /* Try to figure out if this is really an EXPLAIN statement. If this
-  ** cannot be verified, return early.  */
-  if( sqlite3_column_count(pSql)!=8 ){
-    p->cMode = p->mode;
-    return;
-  }
-  zSql = sqlite3_sql(pSql);
-  if( zSql==0 ) return;
-  for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++);
-  if( sqlite3_strnicmp(z, "explain", 7) ){
-    p->cMode = p->mode;
-    return;
-  }
+  /* The caller guarantees that the leftmost 4 columns of the statement
+  ** passed to this function are equivalent to the leftmost 4 columns
+  ** of EXPLAIN statement output. In practice the statement may be
+  ** an EXPLAIN, or it may be a query on the bytecode() virtual table.  */
+  assert( sqlite3_column_count(pSql)>=4 );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) );
+  assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) );
 
   for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){
     int i;
     int iAddr = sqlite3_column_int(pSql, 0);
     const char *zOp = (const char*)sqlite3_column_text(pSql, 1);
-
-    /* Set p2 to the P2 field of the current opcode. Then, assuming that
-    ** p2 is an instruction address, set variable p2op to the index of that
-    ** instruction in the aiIndent[] array. p2 and p2op may be different if
-    ** the current instruction is part of a sub-program generated by an
-    ** SQL trigger or foreign key.  */
+    int p1 = sqlite3_column_int(pSql, 2);
     int p2 = sqlite3_column_int(pSql, 3);
+
+    /* Assuming that p2 is an instruction address, set variable p2op to the
+    ** index of that instruction in the aiIndent[] array. p2 and p2op may be
+    ** different if the current instruction is part of a sub-program generated
+    ** by an SQL trigger or foreign key.  */
     int p2op = (p2 + (iOp-iAddr));
 
     /* Grow the p->aiIndent array as required */
     if( iOp>=nAlloc ){
-      if( iOp==0 ){
-        /* Do further verification that this is explain output.  Abort if
-        ** it is not */
-        static const char *explainCols[] = {
-           "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" };
-        int jj;
-        for(jj=0; jj<ArraySize(explainCols); jj++){
-          if( cli_strcmp(sqlite3_column_name(pSql,jj),explainCols[jj])!=0 ){
-            p->cMode = p->mode;
-            sqlite3_reset(pSql);
-            return;
-          }
-        }
-      }
       nAlloc += 100;
       p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int));
       shell_check_oom(p->aiIndent);
       abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int));
       shell_check_oom(abYield);
     }
+
     abYield[iOp] = str_in_array(zOp, azYield);
     p->aiIndent[iOp] = 0;
     p->nIndent = iOp+1;
-
     if( str_in_array(zOp, azNext) && p2op>0 ){
       for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
     }
-    if( str_in_array(zOp, azGoto) && p2op<p->nIndent
-     && (abYield[p2op] || sqlite3_column_int(pSql, 2))
-    ){
+    if( str_in_array(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){
       for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
     }
   }
@@ -3568,6 +3537,48 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
+static void exec_prepared_stmt(ShellState*, sqlite3_stmt*);
+
+/*
+** Display scan stats.
+*/
+static void display_scanstats(
+  sqlite3 *db,                    /* Database to query */
+  ShellState *pArg                /* Pointer to ShellState */
+){
+#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
+  UNUSED_PARAMETER(db);
+  UNUSED_PARAMETER(pArg);
+#else
+  if( pArg->scanstatsOn==3 ){
+    const char *zSql = 
+      "  SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec,"
+      "   round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles"
+      "   FROM bytecode(?)";
+
+    int rc = SQLITE_OK;
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pSave = pArg->pStmt;
+      pArg->pStmt = pStmt;
+      sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0);
+
+      pArg->cnt = 0;
+      pArg->cMode = MODE_ScanExp;
+      explain_data_prepare(pArg, pStmt);
+      exec_prepared_stmt(pArg, pStmt);
+      explain_data_delete(pArg);
+
+      sqlite3_finalize(pStmt);
+      pArg->pStmt = pSave;
+    }
+  }else{
+    display_explain_scanstats(db, pArg);
+  }
+#endif
+}
+
 /*
 ** Disable and restore .wheretrace and .treetrace/.selecttrace settings.
 */
@@ -4387,6 +4398,7 @@ static int shell_exec(
           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
           if( rc==SQLITE_OK ){
             pArg->cMode = MODE_Explain;
+            assert( sqlite3_stmt_isexplain(pExplain)==1 );
             explain_data_prepare(pArg, pExplain);
             exec_prepared_stmt(pArg, pExplain);
             explain_data_delete(pArg);
@@ -4405,9 +4417,10 @@ static int shell_exec(
       }
 
       if( pArg ){
+        int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1);
         pArg->cMode = pArg->mode;
         if( pArg->autoExplain ){
-          if( sqlite3_stmt_isexplain(pStmt)==1 ){
+          if( bIsExplain ){
             pArg->cMode = MODE_Explain;
           }
           if( sqlite3_stmt_isexplain(pStmt)==2 ){
@@ -4417,7 +4430,7 @@ static int shell_exec(
 
         /* If the shell is currently in ".explain" mode, gather the extra
         ** data required to add indents to the output.*/
-        if( pArg->cMode==MODE_Explain ){
+        if( pArg->cMode==MODE_Explain && bIsExplain ){
           explain_data_prepare(pArg, pStmt);
         }
       }

From beb3fb690469c90bb77b00b041ff474cc6e76cf2 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 19:11:47 +0000
Subject: [PATCH 51/66] Make sure jsonReplaceNode() always leaves the JsonParse
 in a consistent state even if an error is encountered.

FossilOrigin-Name: 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b
---
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 src/json.c    |  1 +
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 96c91cdd84..5c7bcfa676 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".scanstats\svm"\scommand\sto\sthe\sshell\stool.\sFor\sprofiling\sVM\scode\sin\sSQLITE_ENABLE_STMT_SCANSTATUS\sbuilds.
-D 2023-07-26T17:51:05.557
+C Make\ssure\sjsonReplaceNode()\salways\sleaves\sthe\sJsonParse\sin\sa\sconsistent\sstate\seven\sif\san\serror\sis\sencountered.
+D 2023-07-26T19:11:47.116
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 512bc389b42c68b34571ca532afb0f4bb235b1fc11ea5c9a4f6850c64fa12ab4
+F src/json.c 00ee51e8365de51d76473900c181edb3265cde636badc779068bbb425e401211
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,9 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P df099ad713011b67b09cb1e5f0fa2e6b45f9cee0ce9d3c118c5dbca3563d20a5 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22
-R 41e3bcc8b2de77a8519996fee810e7fd
-T +closed 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22
-U dan
-Z 4c8141df90dc53dee045e3726a5dbff7
+P 0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d
+R f441bd92e9087b2cb305790bb3fcf419
+U drh
+Z 5cac453cc99ba9487a886bdc07348b7a
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a215484c0d..4831e7a0a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d
\ No newline at end of file
+01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 9eb2302684..7fb72aecb3 100644
--- a/src/json.c
+++ b/src/json.c
@@ -2853,6 +2853,7 @@ static void jsonReplaceNode(
       break;
     }
     case SQLITE_BLOB: {
+      jsonParseAddNode(p, JSON_NULL, 0, 0);
       sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1);
       p->nErr++;
       break;

From 3f3dc2f5d8c8fb5fad4d94f5d1dc05926cb34794 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 19:22:43 +0000
Subject: [PATCH 52/66] Change a switch() case to default for coverage.

FossilOrigin-Name: 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 5c7bcfa676..8b99a3700b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sjsonReplaceNode()\salways\sleaves\sthe\sJsonParse\sin\sa\sconsistent\sstate\seven\sif\san\serror\sis\sencountered.
-D 2023-07-26T19:11:47.116
+C Change\sa\sswitch()\scase\sto\sdefault\sfor\scoverage.
+D 2023-07-26T19:22:43.180
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 00ee51e8365de51d76473900c181edb3265cde636badc779068bbb425e401211
+F src/json.c e3675df7edfd0874bed04915de9a4d91300e6745a1359457e617b50049435b0d
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d
-R f441bd92e9087b2cb305790bb3fcf419
+P 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b
+R 54cc47c29344f6b15ce5cc5de43ab981
 U drh
-Z 5cac453cc99ba9487a886bdc07348b7a
+Z b1ee9ef830f4f85e2fbcc03f50cf7801
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 4831e7a0a7..6845fa3cbe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b
\ No newline at end of file
+04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 7fb72aecb3..26ce90c516 100644
--- a/src/json.c
+++ b/src/json.c
@@ -2852,7 +2852,7 @@ static void jsonReplaceNode(
       }
       break;
     }
-    case SQLITE_BLOB: {
+    default: {
       jsonParseAddNode(p, JSON_NULL, 0, 0);
       sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1);
       p->nErr++;

From f7a164f34518825ae0351fd1c4453df8f30c3afe Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 21:53:09 +0000
Subject: [PATCH 53/66] Minor changes to make coverage testing easier.

FossilOrigin-Name: ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    |  9 ++++++---
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 8b99a3700b..a2ffc6aed4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sa\sswitch()\scase\sto\sdefault\sfor\scoverage.
-D 2023-07-26T19:22:43.180
+C Minor\schanges\sto\smake\scoverage\stesting\seasier.
+D 2023-07-26T21:53:09.569
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c e3675df7edfd0874bed04915de9a4d91300e6745a1359457e617b50049435b0d
+F src/json.c 8c21f66368929975ed41e6f5aff141b5ff2706bc0f817b0a99dfea9cb6b1b7cd
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b
-R 54cc47c29344f6b15ce5cc5de43ab981
+P 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7
+R b26ff6d575d69d55cdae9f68086f2a13
 U drh
-Z b1ee9ef830f4f85e2fbcc03f50cf7801
+Z a6ad53b5bf3028b66640cc0d3b3e396d
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 6845fa3cbe..53efddff64 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7
\ No newline at end of file
+ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 26ce90c516..7fb860a6ef 100644
--- a/src/json.c
+++ b/src/json.c
@@ -2828,14 +2828,17 @@ static void jsonReplaceNode(
          break;
       }
       if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){
-        int k = jsonParseAddNode(p, JSON_STRING, n, z);
         char *zCopy = sqlite3DbStrDup(0, z);
-        if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW;
+        int k;
         if( zCopy ){
           jsonParseAddCleanup(p, sqlite3_free, zCopy);
-        }else{
+       }else{
+          p->oom = 1;
           sqlite3_result_error_nomem(pCtx);
         }
+        k = jsonParseAddNode(p, JSON_STRING, n, zCopy);
+        assert( k>0 || p->oom );
+        if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW;
       }else{
         JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1);
         if( pPatch==0 ){

From e94e132994e5144c39bbc68aa4a6180a3e60fd7a Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 26 Jul 2023 23:22:32 +0000
Subject: [PATCH 54/66] Reduce the number of memory allocations when parsing
 JSON.

FossilOrigin-Name: 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 31 ++++++++-----------------------
 3 files changed, 15 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index a2ffc6aed4..0f8c113090 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\schanges\sto\smake\scoverage\stesting\seasier.
-D 2023-07-26T21:53:09.569
+C Reduce\sthe\snumber\sof\smemory\sallocations\swhen\sparsing\sJSON.
+D 2023-07-26T23:22:32.813
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 8c21f66368929975ed41e6f5aff141b5ff2706bc0f817b0a99dfea9cb6b1b7cd
+F src/json.c bea467f5bb75311759f2a5f8a7bbf517384f6489bcbe85a41b7c39e8faeea698
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7
-R b26ff6d575d69d55cdae9f68086f2a13
+P ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764
+R 60fa67ed7b82b4d2c0db2475f139f2f9
 U drh
-Z a6ad53b5bf3028b66640cc0d3b3e396d
+Z 0d5cf9ec935b3fe18206d9fca33e2d88
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 53efddff64..46453a3ccb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764
\ No newline at end of file
+9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 7fb860a6ef..24b3613e14 100644
--- a/src/json.c
+++ b/src/json.c
@@ -144,8 +144,7 @@ struct JsonNode {
 /* A parsed and possibly edited JSON string.  Lifecycle:
 **
 **   1.  JSON comes in and is parsed into an array aNode[].  The original
-**       JSON text is stored in zJson.  This object may or may not be the
-**       owner of the input JSON - the bOwnsJson variables determines which.
+**       JSON text is stored in zJson.
 **
 **   2.  Zero or more changes are made (via json_remove() or json_replace()
 **       or similar) to the aNode[] array.
@@ -176,7 +175,6 @@ struct JsonParse {
   u8 nErr;           /* Number of errors seen */
   u8 oom;            /* Set to true if out of memory */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
-  u8 bOwnsJson;      /* This object owns zJson and response for freeing it */
   u8 useMod;         /* Actually use the edits contain inside aNode */
   u8 hasMod;         /* aNode contains edits from the original zJson */
   u32 nJPRef;        /* Number of references to this object */
@@ -619,11 +617,6 @@ static void jsonParseReset(JsonParse *pParse){
     sqlite3RCStrUnref(pParse->zAlt);
     pParse->zAlt = 0;
   }
-  if( pParse->bOwnsJson ){
-    pParse->bOwnsJson = 0;
-    sqlite3RCStrUnref(pParse->zJson);
-    pParse->zJson = 0;
-  }
 }
 
 /*
@@ -1759,14 +1752,12 @@ json_parse_restart:
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
   sqlite3_context *pCtx,       /* Report errors here */
-  char *zJson,                 /* Input JSON text to be parsed */
-  int bTakeJson                /* Assume ownership of zJson if true */
+  char *zJson                  /* Input JSON text to be parsed */
 ){
   int i;
   memset(pParse, 0, sizeof(*pParse));
   if( zJson==0 ) return 1;
   pParse->zJson = zJson;
-  pParse->bOwnsJson = bTakeJson;
   pParse->nJPRef = 1;
   i = jsonParseValue(pParse, 0);
   if( pParse->oom ) i = -1;
@@ -1933,21 +1924,16 @@ static JsonParse *jsonParseCached(
   /* The input JSON was not found anywhere in the cache.  We will need
   ** to parse it ourselves and generate a new JsonParse object.
   */
-  p = sqlite3_malloc64( sizeof(*p) );
+  p = sqlite3_malloc64( sizeof(*p) + nJson + 1 );
   if( p==0 ){
     sqlite3_result_error_nomem(pCtx);
     return 0;
   }
   memset(p, 0, sizeof(*p));
-  p->zJson = sqlite3RCStrNew( nJson );
-  if( p->zJson==0 ){
-    sqlite3_free(p);
-    sqlite3_result_error_nomem(pCtx);
-    return 0;
-  }
+  p->zJson = (char*)&p[1];
   memcpy(p->zJson, zJson, nJson);
   p->zJson[nJson] = 0;
-  if( jsonParse(p, pErrCtx, p->zJson, 1) ){
+  if( jsonParse(p, pErrCtx, p->zJson) ){
     if( pErrCtx==0 ){
       p->nErr = 1;
       assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */
@@ -2365,7 +2351,6 @@ static void jsonParseFunc(
   printf("nErr      = %u\n", p->nErr);
   printf("oom       = %u\n", p->oom);
   printf("hasNonstd = %u\n", p->hasNonstd);
-  printf("bOwnsJson = %u\n", p->bOwnsJson);
   printf("useMod    = %u\n", p->useMod);
   printf("hasMod    = %u\n", p->hasMod);
   printf("nJPRef    = %u\n", p->nJPRef);
@@ -2677,8 +2662,8 @@ static void jsonPatchFunc(
   JsonNode *pResult;   /* The result of the merge */
 
   UNUSED_PARAMETER(argc);
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return;
-  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1]), 0) ){
+  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){
     jsonParseReset(&x);
     return;
   }
@@ -3645,7 +3630,7 @@ static int jsonEachFilter(
   p->zJson = sqlite3_malloc64( n+1 );
   if( p->zJson==0 ) return SQLITE_NOMEM;
   memcpy(p->zJson, z, (size_t)n+1);
-  if( jsonParse(&p->sParse, 0, p->zJson, 0) ){
+  if( jsonParse(&p->sParse, 0, p->zJson) ){
     int rc = SQLITE_NOMEM;
     if( p->sParse.oom==0 ){
       sqlite3_free(cur->pVtab->zErrMsg);

From 93853a48465e5eb14ab347eecc53f2b58a45ee02 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 27 Jul 2023 00:21:59 +0000
Subject: [PATCH 55/66] Fix a performance regression in JSON associated with
 generating small snippets of JSON from a larger JSON string.

FossilOrigin-Name: 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/json.c    | 18 ++++++++++++------
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 0f8c113090..b9a6b41406 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reduce\sthe\snumber\sof\smemory\sallocations\swhen\sparsing\sJSON.
-D 2023-07-26T23:22:32.813
+C Fix\sa\sperformance\sregression\sin\sJSON\sassociated\swith\sgenerating\ssmall\nsnippets\sof\sJSON\sfrom\sa\slarger\sJSON\sstring.
+D 2023-07-27T00:21:59.567
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c bea467f5bb75311759f2a5f8a7bbf517384f6489bcbe85a41b7c39e8faeea698
+F src/json.c c992d2a87a6888bf16a5c7270ef523eb838cf49e6643420c3cf917124f41f97f
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764
-R 60fa67ed7b82b4d2c0db2475f139f2f9
+P 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3
+R 2d6be27c61998a8da52821b68136a1e9
 U drh
-Z 0d5cf9ec935b3fe18206d9fca33e2d88
+Z 453c25d31204436e7cf8e11e8d84051c
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 46453a3ccb..d9fba749f0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3
\ No newline at end of file
+837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 24b3613e14..493a1b99c4 100644
--- a/src/json.c
+++ b/src/json.c
@@ -563,12 +563,18 @@ static void jsonAppendValue(
 ** The JSON string is reset.
 */
 static void jsonResult(JsonString *p){
-  if( p->bErr==0 && jsonForceRCStr(p) ){
-    sqlite3RCStrRef(p->zBuf);
-    sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
-                          (void(*)(void*))sqlite3RCStrUnref,
-                          SQLITE_UTF8);
-  }else if( p->bErr==1 ){
+  if( p->bErr==0 ){
+    if( p->bStatic ){
+      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
+                            SQLITE_TRANSIENT, SQLITE_UTF8);
+    }else if( jsonForceRCStr(p) ){
+      sqlite3RCStrRef(p->zBuf);
+      sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
+                            (void(*)(void*))sqlite3RCStrUnref,
+                            SQLITE_UTF8);
+    }
+  }
+  if( p->bErr==1 ){
     sqlite3_result_error_nomem(p->pCtx);
   }
   jsonReset(p);

From 4cfd54256dca63f95717428416033237e3bad607 Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Thu, 27 Jul 2023 01:38:19 +0000
Subject: [PATCH 56/66] Accommodate a breaking change in emcc 3.1.44.

FossilOrigin-Name: 2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d
---
 ext/wasm/api/sqlite3-api-cleanup.js |  4 +++-
 ext/wasm/common/whwasmutil.js       |  3 ++-
 manifest                            | 17 +++++++++--------
 manifest.uuid                       |  2 +-
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js
index 3661464e1b..65dbb4eb64 100644
--- a/ext/wasm/api/sqlite3-api-cleanup.js
+++ b/ext/wasm/api/sqlite3-api-cleanup.js
@@ -22,7 +22,9 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build
   */
   const SABC = Object.assign(
     Object.create(null), {
-      exports: Module['asm'],
+      exports: ('undefined'===typeof wasmExports)
+        ? Module['asm']/* emscripten <=3.1.43 */
+        : wasmExports  /* emscripten >=3.1.44 */,
       memory: Module.wasmMemory /* gets set if built with -sIMPORTED_MEMORY */
     },
     globalThis.sqlite3ApiConfig || {}
diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js
index f48e8a7d21..ee7ea20c34 100644
--- a/ext/wasm/common/whwasmutil.js
+++ b/ext/wasm/common/whwasmutil.js
@@ -111,7 +111,8 @@
      of `target.instance` (a WebAssembly.Module instance) and it must
      contain the symbols exported by the WASM module associated with
      this code. In an Enscripten environment it must be set to
-     `Module['asm']`. The exports object must contain a minimum of the
+     `Module['asm']` (versions <=3.1.43) or `wasmExports` (versions
+     >=3.1.44). The exports object must contain a minimum of the
      following symbols:
 
      - `memory`: a WebAssembly.Memory object representing the WASM
diff --git a/manifest b/manifest
index b9a6b41406..f38df6f97b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sperformance\sregression\sin\sJSON\sassociated\swith\sgenerating\ssmall\nsnippets\sof\sJSON\sfrom\sa\slarger\sJSON\sstring.
-D 2023-07-27T00:21:59.567
+C Accommodate\sa\sbreaking\schange\sin\semcc\s3.1.44.
+D 2023-07-27T01:38:19.863
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -494,7 +494,7 @@ F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e
 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1
 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62
 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219
-F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf
+F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
 F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803
 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a
@@ -513,7 +513,7 @@ F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f
-F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3
+F ext/wasm/common/whwasmutil.js db6368ee57af90ee6691b6fb3ca97ee8064d12482e06a29113127c67b08f956e
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
 F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
@@ -2044,8 +2044,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3
-R 2d6be27c61998a8da52821b68136a1e9
-U drh
-Z 453c25d31204436e7cf8e11e8d84051c
+P 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4
+Q +4ce386030092b8bafe860350c66ef5516b6aea75bd1b4467ac184875109d66e7
+R d8cc9478d62bd3b1eb88b1d0f9b38e0e
+U stephan
+Z 31e0335b3afe627b111db5001ad6debf
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index d9fba749f0..81aebb401a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4
\ No newline at end of file
+2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d
\ No newline at end of file

From 9fcca3d27c18693eddf65a0edc0f4e609a3a24fc Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Thu, 27 Jul 2023 17:50:10 +0000
Subject: [PATCH 57/66] Dynamically determine whether the wasm.xWrap() argc
 check can be applied, depending on how the wasm environment exposes its
 exports.

FossilOrigin-Name: fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c
---
 ext/wasm/api/sqlite3-api-glue.js |  9 +++++++++
 manifest                         | 14 +++++++-------
 manifest.uuid                    |  2 +-
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js
index f444ec975c..572efeed5c 100644
--- a/ext/wasm/api/sqlite3-api-glue.js
+++ b/ext/wasm/api/sqlite3-api-glue.js
@@ -728,6 +728,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
        Populate api object with sqlite3_...() by binding the "raw" wasm
        exports into type-converting proxies using wasm.xWrap().
     */
+    if(0 === wasm.exports.sqlite3_step.length){
+      /* This environment wraps exports in nullary functions, which means
+         we must disable the arg-count validation we otherwise perform
+         on the wrappers. */
+      wasm.xWrap.doArgcCheck = false;
+      sqlite3.config.warn(
+        "Disabling sqlite3.wasm.xWrap.doArgcCheck due to environmental quirks."
+      );
+    }
     for(const e of wasm.bindingSignatures){
       capi[e[0]] = wasm.xWrap.apply(null, e);
     }
diff --git a/manifest b/manifest
index f38df6f97b..87e6fa0b52 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Accommodate\sa\sbreaking\schange\sin\semcc\s3.1.44.
-D 2023-07-27T01:38:19.863
+C Dynamically\sdetermine\swhether\sthe\swasm.xWrap()\sargc\scheck\scan\sbe\sapplied,\sdepending\son\show\sthe\swasm\senvironment\sexposes\sits\sexports.
+D 2023-07-27T17:50:10.521
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -495,7 +495,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08
 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62
 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219
 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
-F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803
+F ext/wasm/api/sqlite3-api-glue.js cc6b0bb093bdb6279d4af259200b7b9e150e3796a8a3a4cd09a4928c43d25e56
 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a
 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec
@@ -2044,9 +2044,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4
-Q +4ce386030092b8bafe860350c66ef5516b6aea75bd1b4467ac184875109d66e7
-R d8cc9478d62bd3b1eb88b1d0f9b38e0e
+P 2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d
+Q +86bb464f31b77ac7a89ee9a50c3d988a186f84fa0bbee16b15fcbbac33523fa1
+R 26c73044c10199d992b00944e358a20b
 U stephan
-Z 31e0335b3afe627b111db5001ad6debf
+Z 7df747b854c1185863732d1df816e30a
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 81aebb401a..7ece24bb69 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d
\ No newline at end of file
+fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c
\ No newline at end of file

From 594f5e24c4c3ef1389c24e0e5d1fd8f4bb5d68bb Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 27 Jul 2023 19:39:53 +0000
Subject: [PATCH 58/66] The OP_Column opcode caches large column values coming
 from overflow pages.

FossilOrigin-Name: ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888
---
 manifest      | 20 +++++------
 manifest.uuid |  2 +-
 src/btree.c   |  2 --
 src/btree.h   |  2 --
 src/vdbe.c    | 94 +++++++++++++++++++++++++++++++++++++++++++++++----
 src/vdbeInt.h | 18 +++++++++-
 src/vdbeaux.c | 17 +++++++++-
 7 files changed, 131 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index 37e9c1ea94..3ed2cf51e8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\senhancements\sinto\sthe\sjson-opt\sbranch.
-D 2023-07-27T18:19:46.300
+C The\sOP_Column\sopcode\scaches\slarge\scolumn\svalues\scoming\sfrom\soverflow\spages.
+D 2023-07-27T19:39:53.698
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -577,8 +577,8 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645
 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
-F src/btree.c 2281facb0531d53fb42c03d1f32bc1b5903564d782ec5ff4ffc63171d960e2aa
-F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc
+F src/btree.c 7a37bdf09f338561880860681cb03499a60c3bb0869e539c58bc1d2cdd705ff2
+F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
 F src/btreeInt.h 3b4eff7155c0cea6971dc51f62e3529934a15a6640ec607dd42a767e379cb3a9
 F src/build.c a8ae3b32d9aa9bbd2c0e97d7c0dd80def9fbca408425de1608f57ee6f47f45f4
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
@@ -708,11 +708,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12
+F src/vdbe.c 2465f86f43892f173be761f05a4e6ba381119d6e8f5df7d172cc9ab123c9037a
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
-F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7
+F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
-F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
+F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
 F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9 fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c
-R 26c73044c10199d992b00944e358a20b
+P 5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144
+R 84bfc9450f26b6c2dd62e6fd9e08f614
 U drh
-Z d6bca8891c39ac6c04dd8ce7fe687700
+Z 46d23d17a391909f5bac31a5a474dfd0
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index e0906cc4dc..1d4873e515 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144
\ No newline at end of file
+ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 8817efc71b..c23f86e1d2 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4826,7 +4826,6 @@ void sqlite3BtreeCursorUnpin(BtCursor *pCur){
   pCur->curFlags &= ~BTCF_Pinned;
 }
 
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
 /*
 ** Return the offset into the database file for the start of the
 ** payload to which the cursor is pointing.
@@ -4838,7 +4837,6 @@ i64 sqlite3BtreeOffset(BtCursor *pCur){
   return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) +
          (i64)(pCur->info.pPayload - pCur->pPage->aData);
 }
-#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */
 
 /*
 ** Return the number of bytes of payload for the entry that pCur is
diff --git a/src/btree.h b/src/btree.h
index b9078f9010..b45ace7e1d 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -321,9 +321,7 @@ int sqlite3BtreePrevious(BtCursor*, int flags);
 i64 sqlite3BtreeIntegerKey(BtCursor*);
 void sqlite3BtreeCursorPin(BtCursor*);
 void sqlite3BtreeCursorUnpin(BtCursor*);
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
 i64 sqlite3BtreeOffset(BtCursor*);
-#endif
 int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*);
 const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
 u32 sqlite3BtreePayloadSize(BtCursor*);
diff --git a/src/vdbe.c b/src/vdbe.c
index 075a632117..f479440dab 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -695,6 +695,83 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){
   return h;
 }
 
+
+/*
+** For OP_Column, factor out the case where content is loaded from
+** overflow pages, so that the code to implement this case is separate
+** the common case where all content fits on the page.  Factoring out
+** the code reduces register pressure and helps the common case
+** to run faster.
+*/
+static SQLITE_NOINLINE int vdbeColumnFromOverflow(
+  VdbeCursor *pC,       /* The BTree cursor from which we are reading */
+  int iCol,             /* The column to read */
+  int t,                /* The serial-type code for the column value */
+  i64 iOffset,          /* Offset to the start of the content value */
+  Mem *pDest            /* Store the value into this register. */
+){
+  int rc;
+  sqlite3 *db = pDest->db;
+  int encoding = pDest->enc;
+  int len = sqlite3VdbeSerialTypeLen(t);
+  assert( pC->eCurType==CURTYPE_BTREE );
+  if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG;
+  if( len > 4000 ){
+    /* Cache large column values that are on overflow pages using
+    ** an RCStr (reference counted string) so that if they are reloaded,
+    ** that do not have to be copied a second time.  The overhead of
+    ** creating and managing the cache is such that this is only
+    ** profitable for larger TEXT and BLOB values.
+    */
+    VdbeTxtBlbCache *pCache;
+    char *pBuf;
+    if( pC->colCache==0 ){
+      pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) );
+      if( pC->pCache==0 ) return SQLITE_NOMEM;
+      pC->colCache = 1;
+    }
+    pCache = pC->pCache;
+    if( pCache->pCValue==0
+     || pCache->iCol!=iCol
+     || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor)
+    ){
+      if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue);
+      pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 );
+      if( pBuf==0 ) return SQLITE_NOMEM;
+      rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf);
+      if( rc ) return rc;
+      pBuf[len] = 0;
+      pBuf[len+1] = 0;
+      pBuf[len+2] = 0;
+      pCache->iCol = iCol;
+      pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor);
+    }else{
+      pBuf = pCache->pCValue;
+    }
+    assert( t>=12 );
+    sqlite3RCStrRef(pBuf);
+    if( t&1 ){
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding,
+                                (void(*)(void*))sqlite3RCStrUnref);
+      pDest->flags |= MEM_Term;
+    }else{
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0,
+                                (void(*)(void*))sqlite3RCStrUnref);
+    }
+  }else{
+    rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest);
+    if( rc ) return rc;
+    sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
+    if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
+      pDest->z[len] = 0;
+      pDest->flags |= MEM_Term;
+    }
+  }
+  pDest->flags &= ~MEM_Ephem;
+  return rc;
+}
+
+
 /*
 ** Return the symbolic name for the data type of a pMem
 */
@@ -2902,6 +2979,10 @@ op_column_restart:
       ** dynamically allocated. */
       pC->aRow = 0;
       pC->szRow = 0;
+      if( pC->colCache && pC->pCache && pC->pCache->pCValue ){
+        sqlite3RCStrUnref(pC->pCache->pCValue);
+        pC->pCache->pCValue = 0;
+      }
 
       /* Make sure a corrupt database has not given us an oversize header.
       ** Do this now to avoid an oversize memory allocation.
@@ -3061,6 +3142,7 @@ op_column_restart:
   }else{
     u8 p5;
     pDest->enc = encoding;
+    assert( pDest->db==db );
     /* This branch happens only when content is on overflow pages */
     if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0
           && (p5==OPFLAG_TYPEOFARG
@@ -3084,14 +3166,12 @@ op_column_restart:
       */
       sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest);
     }else{
-      if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big;
-      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
-      if( rc!=SQLITE_OK ) goto abort_due_to_error;
-      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
-      if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
-        pDest->flags |= MEM_Term;
+      rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], pDest);
+      if( rc ){
+        if( rc==SQLITE_NOMEM ) goto no_mem;
+        if( rc==SQLITE_TOOBIG ) goto too_big;
+        goto abort_due_to_error;
       }
-      pDest->flags &= ~MEM_Ephem;
     }
   }
 
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 3a5b961a25..f2ca656752 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -56,6 +56,9 @@ typedef struct VdbeSorter VdbeSorter;
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;
 
+/* A cache of large TEXT or BLOB values in a VdbeCursor */
+typedef struct VdbeTxtBlbCache VdbeTxtBlbCache;
+
 /* Types of VDBE cursors */
 #define CURTYPE_BTREE       0
 #define CURTYPE_SORTER      1
@@ -87,6 +90,7 @@ struct VdbeCursor {
   Bool useRandomRowid:1;  /* Generate new record numbers semi-randomly */
   Bool isOrdered:1;       /* True if the table is not BTREE_UNORDERED */
   Bool noReuse:1;         /* OpenEphemeral may not reuse this cursor */
+  Bool colCache:1;        /* pCache pointer is initialized and non-NULL */
   u16 seekHit;            /* See the OP_SeekHit and OP_IfNoHope opcodes */
   union {                 /* pBtx for isEphermeral.  pAltMap otherwise */
     Btree *pBtx;            /* Separate file holding temporary table */
@@ -127,6 +131,7 @@ struct VdbeCursor {
 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
   u64 maskUsed;           /* Mask of columns used by this cursor */
 #endif
+  VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */
 
   /* 2*nField extra array elements allocated for aType[], beyond the one
   ** static element declared in the structure.  nField total array slots for
@@ -139,12 +144,23 @@ struct VdbeCursor {
 #define IsNullCursor(P) \
   ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0)
 
-
 /*
 ** A value for VdbeCursor.cacheStatus that means the cache is always invalid.
 */
 #define CACHE_STALE 0
 
+/*
+** Large TEXT or BLOB values can be slow to load, so we want to avoid
+** loading them more than once.  For that reason, large TEXT and BLOB values
+** can be stored in a cache defined by this object, and attached to the
+** VdbeCursor using the pCache field.
+*/
+struct VdbeTxtBlbCache {
+  char *pCValue;        /* A RCStr buffer to hold the value */
+  i64 iOffset;          /* File offset of the row being cached */
+  int iCol;             /* Column for which the cache is valid */
+};
+
 /*
 ** When a sub-program is executed (OP_Program), a structure of this type
 ** is allocated to store the current value of the program counter, as
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index a0eff155dd..52d3c9e063 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1498,7 +1498,6 @@ void sqlite3VdbeReleaseRegisters(
 }
 #endif /* SQLITE_DEBUG */
 
-
 /*
 ** Change the value of the P4 operand for a specific instruction.
 ** This routine is useful when a large program is loaded from a
@@ -2723,7 +2722,23 @@ void sqlite3VdbeMakeReady(
 void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
   if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx);
 }
+static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){
+  VdbeTxtBlbCache *pCache = pCx->pCache;
+  assert( pCx->colCache );
+  pCx->colCache = 0;
+  pCx->pCache = 0;
+  if( pCache->pCValue ){
+    sqlite3RCStrUnref(pCache->pCValue);
+    pCache->pCValue = 0;
+  }
+  sqlite3DbFree(p->db, pCache);
+  sqlite3VdbeFreeCursorNN(p, pCx);
+}
 void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){
+  if( pCx->colCache ){
+    freeCursorWithCache(p, pCx);
+    return;
+  }
   switch( pCx->eCurType ){
     case CURTYPE_SORTER: {
       sqlite3VdbeSorterClose(p->db, pCx);

From ea6bccaa4487fb3675cb0d9ff98ca603f4f2b9cd Mon Sep 17 00:00:00 2001
From: dan <Dan Kennedy>
Date: Thu, 27 Jul 2023 20:08:44 +0000
Subject: [PATCH 59/66] Fix a couple of compiler warnings in fts5_index.c.

FossilOrigin-Name: bf71faa2a1d29ea762c4d2485522d6f4f8a5a7166981a92d3ba9c96ccbbe1213
---
 ext/fts5/fts5_index.c |  5 ++---
 manifest              | 13 ++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 5e20da2b04..e46840c2e8 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -403,8 +403,8 @@ struct Fts5StructureSegment {
   u64 iOrigin1;
   u64 iOrigin2;
   int nPgTombstone;               /* Number of tombstone hash table pages */
-  i64 nEntryTombstone;            /* Number of tombstone entries that "count" */
-  i64 nEntry;                     /* Number of rows in this segment */
+  u64 nEntryTombstone;            /* Number of tombstone entries that "count" */
+  u64 nEntry;                     /* Number of rows in this segment */
 };
 struct Fts5StructureLevel {
   int nMerge;                     /* Number of segments in incr-merge */
@@ -4779,7 +4779,6 @@ static int fts5IndexMerge(
   int nRem = nPg;
   int bRet = 0;
   Fts5Structure *pStruct = *ppStruct;
-  int bTombstone = 0;
   while( nRem>0 && p->rc==SQLITE_OK ){
     int iLvl;                   /* To iterate through levels */
     int iBestLvl = 0;           /* Level offering the most input segments */
diff --git a/manifest b/manifest
index 9173c248a5..a1ea1f26ec 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\scontentless_delete=1\soption\sto\sfts5.\sFor\screating\scontentless\stables\sthat\ssupport\sDELETE\sand\sREPLACE\sstatements.
-D 2023-07-27T19:13:35.663
+C Fix\sa\scouple\sof\scompiler\swarnings\sin\sfts5_index.c.
+D 2023-07-27T20:08:44.624
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c 182cf576bae17682adf2fe8cefb8bc3b46bea27e6a227e444859b3c065322687
+F ext/fts5/fts5_index.c a4e35cd126c19df66887801db15ccae7290b51129add3d4cdd31e2d9ea240398
 F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2049,9 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c 719973d7f5a47b110e9919fcb96d21feab1e41356dbb3ec674c1116c17bbb778
-R aea8a90a4ab6b65ce2d69f610973d78d
-T +closed 719973d7f5a47b110e9919fcb96d21feab1e41356dbb3ec674c1116c17bbb778
+P d66b182d2bc6ce0772e69401b7affe1adbc1b128c4631cb3c17f98dde72af00a
+R 67e5566856c21efabeaa9f30d61da2e8
 U dan
-Z dc969c62dd2115415f4e48c1e3d95206
+Z 82b2d14bd5c4ac2218712be1054b30ff
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3150e0134d..c1365ea15a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d66b182d2bc6ce0772e69401b7affe1adbc1b128c4631cb3c17f98dde72af00a
\ No newline at end of file
+bf71faa2a1d29ea762c4d2485522d6f4f8a5a7166981a92d3ba9c96ccbbe1213
\ No newline at end of file

From 6bc4baf313d6ebb137679a8f9bbb73e77f52fc18 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 27 Jul 2023 20:28:29 +0000
Subject: [PATCH 60/66] If the input JSON to a json function that uses cache
 comes from an RCStr value, then use that RCStr value in the parse rather than
 making a copy.

FossilOrigin-Name: 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/json.c      | 26 ++++++++++++++++++++------
 src/sqliteInt.h |  1 +
 src/vdbemem.c   | 18 ++++++++++++++++++
 5 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 3ed2cf51e8..8cc0fe016d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sOP_Column\sopcode\scaches\slarge\scolumn\svalues\scoming\sfrom\soverflow\spages.
-D 2023-07-27T19:39:53.698
+C If\sthe\sinput\sJSON\sto\sa\sjson\sfunction\sthat\suses\scache\scomes\sfrom\san\sRCStr\nvalue,\sthen\suse\sthat\sRCStr\svalue\sin\sthe\sparse\srather\sthan\smaking\sa\scopy.
+D 2023-07-27T20:28:29.957
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c c992d2a87a6888bf16a5c7270ef523eb838cf49e6643420c3cf917124f41f97f
+F src/json.c 481dffa457276d850836953bbd2a61a00a98f8cf09c64c29c6f05ac02a84affd
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -643,7 +643,7 @@ F src/shell.c.in 62b58d7be1e44ec3eb729411278b079c39949e683853445323fd3f250b674a2
 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h 30d7b0d586a4d03a384dcb60088c81b6fc6f74ce85cc3a0b3242eedc3cc24dbd
+F src/sqliteInt.h 6140eb4058d995f3004ee2732fa098b884f26eb497a7fbb3a51c781ec824d14b
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -714,7 +714,7 @@ F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
 F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d
+F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144
-R 84bfc9450f26b6c2dd62e6fd9e08f614
+P ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888
+R bc4f27407c20176b7c3faf6538598e39
 U drh
-Z 46d23d17a391909f5bac31a5a474dfd0
+Z 559a7376c279a69632a349e06eee6ec2
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1d4873e515..9223cf7c8e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888
\ No newline at end of file
+509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 493a1b99c4..24a68d37cd 100644
--- a/src/json.c
+++ b/src/json.c
@@ -174,6 +174,7 @@ struct JsonParse {
   u16 iDepth;        /* Nesting depth */
   u8 nErr;           /* Number of errors seen */
   u8 oom;            /* Set to true if out of memory */
+  u8 bJsonIsRCStr;   /* True if zJson is an RCStr */
   u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   u8 useMod;         /* Actually use the edits contain inside aNode */
   u8 hasMod;         /* aNode contains edits from the original zJson */
@@ -619,6 +620,11 @@ static void jsonParseReset(JsonParse *pParse){
     sqlite3_free(pParse->aUp);
     pParse->aUp = 0;
   }
+  if( pParse->bJsonIsRCStr ){
+    sqlite3RCStrUnref(pParse->zJson);
+    pParse->zJson = 0;
+    pParse->bJsonIsRCStr = 0;
+  }
   if( pParse->zAlt ){
     sqlite3RCStrUnref(pParse->zAlt);
     pParse->zAlt = 0;
@@ -1884,6 +1890,8 @@ static JsonParse *jsonParseCached(
   int iMinKey = 0;
   u32 iMinHold = 0xffffffff;
   u32 iMaxHold = 0;
+  int bJsonRCStr;
+  int rc;
 
   if( zJson==0 ) return 0;
   for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
@@ -1895,7 +1903,7 @@ static JsonParse *jsonParseCached(
     if( pMatch==0
      && p->nJson==nJson
      && (p->hasMod==0 || bUnedited==0)
-     && memcmp(p->zJson,zJson,nJson)==0
+     && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0)
     ){
       p->nErr = 0;
       p->useMod = 0;
@@ -1930,16 +1938,22 @@ static JsonParse *jsonParseCached(
   /* The input JSON was not found anywhere in the cache.  We will need
   ** to parse it ourselves and generate a new JsonParse object.
   */
-  p = sqlite3_malloc64( sizeof(*p) + nJson + 1 );
+  bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref);
+  p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) );
   if( p==0 ){
     sqlite3_result_error_nomem(pCtx);
     return 0;
   }
   memset(p, 0, sizeof(*p));
-  p->zJson = (char*)&p[1];
-  memcpy(p->zJson, zJson, nJson);
-  p->zJson[nJson] = 0;
-  if( jsonParse(p, pErrCtx, p->zJson) ){
+  if( bJsonRCStr ){
+    p->zJson = sqlite3RCStrRef(zJson);
+  }else{
+    p->zJson = (char*)&p[1];
+    memcpy(p->zJson, zJson, nJson+1);
+  }
+  rc = jsonParse(p, pErrCtx, p->zJson);
+  p->bJsonIsRCStr = bJsonRCStr;
+  if( rc ){
     if( pErrCtx==0 ){
       p->nErr = 1;
       assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index a19a16d50d..78ae7ad69e 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -5200,6 +5200,7 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
+int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
diff --git a/src/vdbemem.c b/src/vdbemem.c
index c73e59c362..87dfbbebd8 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1368,6 +1368,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
+/* Return true if sqlit3_value object pVal is a string or blob value
+** that uses the destructor specified in the second argument.
+**
+** TODO:  Maybe someday promote this interface into a published API so
+** that third-party extensions can get access to it?
+*/
+int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
+  if( ALWAYS(pVal!=0)
+   && (pVal->flags & (MEM_Str|MEM_Blob))!=0
+   && (pVal->flags & MEM_Dyn)!=0
+   && pVal->xDel==xFree
+  ){
+    return 1;
+  }else{
+    return 0;
+  }
+}
+
 /*
 ** Create a new sqlite3_value object.
 */

From caf7e26c4f38d1cb894109e84c27727eaf72478f Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 27 Jul 2023 23:51:36 +0000
Subject: [PATCH 61/66] Switch to using jsonParseCached() for json_patch().

FossilOrigin-Name: 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3
---
 manifest      | 12 ++++-----
 manifest.uuid |  2 +-
 src/json.c    | 75 +++++++++++++++++++++++----------------------------
 3 files changed, 41 insertions(+), 48 deletions(-)

diff --git a/manifest b/manifest
index 8cc0fe016d..b519533d15 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sthe\sinput\sJSON\sto\sa\sjson\sfunction\sthat\suses\scache\scomes\sfrom\san\sRCStr\nvalue,\sthen\suse\sthat\sRCStr\svalue\sin\sthe\sparse\srather\sthan\smaking\sa\scopy.
-D 2023-07-27T20:28:29.957
+C Switch\sto\susing\sjsonParseCached()\sfor\sjson_patch().
+D 2023-07-27T23:51:36.392
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 481dffa457276d850836953bbd2a61a00a98f8cf09c64c29c6f05ac02a84affd
+F src/json.c 60b6b0815ddaf546396e3c7328065a114b581721dc1fa20697a52610fb2259aa
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465
 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888
-R bc4f27407c20176b7c3faf6538598e39
+P 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821
+R a667d9301618252314cdf4d506cbebf3
 U drh
-Z 559a7376c279a69632a349e06eee6ec2
+Z 152f86b69395ef2af8d236118a3bf25d
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 9223cf7c8e..56d469bf1c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821
\ No newline at end of file
+2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3
\ No newline at end of file
diff --git a/src/json.c b/src/json.c
index 24a68d37cd..cc39264c05 100644
--- a/src/json.c
+++ b/src/json.c
@@ -1751,26 +1751,15 @@ json_parse_restart:
 ** are any errors.  If an error occurs, free all memory held by pParse,
 ** but not pParse itself.
 **
-** pParse is uninitialized when this routine is called.
-**
-** pParse->nJPRef set to 1.  The caller becomes the owner of the
-** the JsonParse object.
-**
-** pParse->bOwnsJson is set to bTakeJson.  If bTakeJson is 1, the newly
-** initialized JsonParse object will become the owner of the zJson input
-** string.  If bTakeJson is 0, then the caller is responsible for
-** preserving zJson for the lifetime of the JsonParse object.
+** pParse must be initialized to an empty parse object prior to calling
+** this routine.
 */
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
-  sqlite3_context *pCtx,       /* Report errors here */
-  char *zJson                  /* Input JSON text to be parsed */
+  sqlite3_context *pCtx        /* Report errors here */
 ){
   int i;
-  memset(pParse, 0, sizeof(*pParse));
-  if( zJson==0 ) return 1;
-  pParse->zJson = zJson;
-  pParse->nJPRef = 1;
+  const char *zJson = pParse->zJson;
   i = jsonParseValue(pParse, 0);
   if( pParse->oom ) i = -1;
   if( i>0 ){
@@ -1891,7 +1880,6 @@ static JsonParse *jsonParseCached(
   u32 iMinHold = 0xffffffff;
   u32 iMaxHold = 0;
   int bJsonRCStr;
-  int rc;
 
   if( zJson==0 ) return 0;
   for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
@@ -1947,13 +1935,13 @@ static JsonParse *jsonParseCached(
   memset(p, 0, sizeof(*p));
   if( bJsonRCStr ){
     p->zJson = sqlite3RCStrRef(zJson);
+    p->bJsonIsRCStr = 1;
   }else{
     p->zJson = (char*)&p[1];
     memcpy(p->zJson, zJson, nJson+1);
   }
-  rc = jsonParse(p, pErrCtx, p->zJson);
-  p->bJsonIsRCStr = bJsonRCStr;
-  if( rc ){
+  p->nJPRef = 1;
+  if( jsonParse(p, pErrCtx) ){
     if( pErrCtx==0 ){
       p->nErr = 1;
       assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */
@@ -2677,29 +2665,27 @@ static void jsonPatchFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;     /* The JSON that is being patched */
-  JsonParse y;     /* The patch */
+  JsonParse *pX;     /* The JSON that is being patched */
+  JsonParse *pY;     /* The patch */
   JsonNode *pResult;   /* The result of the merge */
 
   UNUSED_PARAMETER(argc);
-  if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return;
-  if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){
-    jsonParseReset(&x);
-    return;
-  }
-  x.useMod = 1;
-  y.useMod = 1;
-  pResult = jsonMergePatch(&x, 0, y.aNode);
-  assert( pResult!=0 || x.oom );
-  if( pResult && x.oom==0 ){
-    jsonDebugPrintParse(&x);
+  pX = jsonParseCached(ctx, argv[0], ctx, 1);
+  if( pX==0 ) return;
+  pY = jsonParseCached(ctx, argv[1], ctx, 1);
+  if( pY==0 ) return;
+  pX->useMod = 1;
+  pX->hasMod = 1;
+  pY->useMod = 1;
+  pResult = jsonMergePatch(pX, 0, pY->aNode);
+  assert( pResult!=0 || pX->oom );
+  if( pResult && pX->oom==0 ){
+    jsonDebugPrintParse(pX);
     jsonDebugPrintNode(pResult);
-    jsonReturnJson(&x, pResult, ctx, 0);
+    jsonReturnJson(pX, pResult, ctx, 0);
   }else{
     sqlite3_result_error_nomem(ctx);
   }
-  jsonParseReset(&x);
-  jsonParseReset(&y);
 }
 
 
@@ -3326,7 +3312,6 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
 /* Reset a JsonEachCursor back to its original state.  Free any memory
 ** held. */
 static void jsonEachCursorReset(JsonEachCursor *p){
-  sqlite3_free(p->zJson);
   sqlite3_free(p->zRoot);
   jsonParseReset(&p->sParse);
   p->iRowid = 0;
@@ -3646,11 +3631,19 @@ static int jsonEachFilter(
   if( idxNum==0 ) return SQLITE_OK;
   z = (const char*)sqlite3_value_text(argv[0]);
   if( z==0 ) return SQLITE_OK;
-  n = sqlite3_value_bytes(argv[0]);
-  p->zJson = sqlite3_malloc64( n+1 );
-  if( p->zJson==0 ) return SQLITE_NOMEM;
-  memcpy(p->zJson, z, (size_t)n+1);
-  if( jsonParse(&p->sParse, 0, p->zJson) ){
+  memset(&p->sParse, 0, sizeof(p->sParse));
+  p->sParse.nJPRef = 1;
+  if( sqlite3ValueIsOfClass(argv[0], (void(*)(void*))sqlite3RCStrUnref) ){
+    p->sParse.zJson = sqlite3RCStrRef((char*)z);
+  }else{
+    n = sqlite3_value_bytes(argv[0]);
+    p->sParse.zJson = sqlite3RCStrNew( n+1 );
+    if( p->sParse.zJson==0 ) return SQLITE_NOMEM;
+    memcpy(p->sParse.zJson, z, (size_t)n+1);
+  }
+  p->sParse.bJsonIsRCStr = 1;
+  p->zJson = p->sParse.zJson;
+  if( jsonParse(&p->sParse, 0) ){
     int rc = SQLITE_NOMEM;
     if( p->sParse.oom==0 ){
       sqlite3_free(cur->pVtab->zErrMsg);

From 93269a9e411ffe393e1e331f160377c8feae69f0 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 28 Jul 2023 00:54:38 +0000
Subject: [PATCH 62/66] Remove some unnecessary and incorrect code that was
 mistakenly added in the previous check-in.

FossilOrigin-Name: d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    |  4 ----
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index b519533d15..6797755a7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Switch\sto\susing\sjsonParseCached()\sfor\sjson_patch().
-D 2023-07-27T23:51:36.392
+C Remove\ssome\sunnecessary\sand\sincorrect\scode\sthat\swas\smistakenly\sadded\sin\sthe\nprevious\scheck-in.
+D 2023-07-28T00:54:38.146
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -708,7 +708,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 2465f86f43892f173be761f05a4e6ba381119d6e8f5df7d172cc9ab123c9037a
+F src/vdbe.c cd0396758da6a95e6c4e3bec7c3d6e767c3b39930c295c2425f83bb086cdc6ba
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
 F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821
-R a667d9301618252314cdf4d506cbebf3
+P 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3
+R 865864ade75427cea7a0ed386e21a96e
 U drh
-Z 152f86b69395ef2af8d236118a3bf25d
+Z 7b84fe17156a3402b4d2c891447b8313
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 56d469bf1c..cd8eaaa0af 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3
\ No newline at end of file
+d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index f479440dab..b8ee05e415 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2979,10 +2979,6 @@ op_column_restart:
       ** dynamically allocated. */
       pC->aRow = 0;
       pC->szRow = 0;
-      if( pC->colCache && pC->pCache && pC->pCache->pCValue ){
-        sqlite3RCStrUnref(pC->pCache->pCValue);
-        pC->pCache->pCValue = 0;
-      }
 
       /* Make sure a corrupt database has not given us an oversize header.
       ** Do this now to avoid an oversize memory allocation.

From b1926192fa1a8c0cdb3faf9dbe111a57b4ab9e5a Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 28 Jul 2023 11:30:07 +0000
Subject: [PATCH 63/66] Mark an unreachable branch as ALWAYS().

FossilOrigin-Name: 8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 6797755a7c..f0021c4b83 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\ssome\sunnecessary\sand\sincorrect\scode\sthat\swas\smistakenly\sadded\sin\sthe\nprevious\scheck-in.
-D 2023-07-28T00:54:38.146
+C Mark\san\sunreachable\sbranch\sas\sALWAYS().
+D 2023-07-28T11:30:07.088
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -714,7 +714,7 @@ F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47
 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7
 F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
-F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35
+F src/vdbemem.c adb5877dc54330865e45e0fba50f6455c1d9d8b83434c59f0e3d2d2a0b9f0e58
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
 F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3
-R 865864ade75427cea7a0ed386e21a96e
+P d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45
+R 7d9c43b78a4f1c2315778caf36cecb44
 U drh
-Z 7b84fe17156a3402b4d2c891447b8313
+Z c4502aa3c9cdb29d4cc267ca47a361d2
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index cd8eaaa0af..326a691f5a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45
\ No newline at end of file
+8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 87dfbbebd8..07fb8feee8 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1376,7 +1376,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
 */
 int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
   if( ALWAYS(pVal!=0)
-   && (pVal->flags & (MEM_Str|MEM_Blob))!=0
+   && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0)
    && (pVal->flags & MEM_Dyn)!=0
    && pVal->xDel==xFree
   ){

From ab12b750ded006bbbbc6aaea7655faadff3bec3a Mon Sep 17 00:00:00 2001
From: stephan <stephan@noemail.net>
Date: Fri, 28 Jul 2023 11:52:58 +0000
Subject: [PATCH 64/66] Correct a recently-introduced falsehood in
 ext/wasm/README.md regarding ssh port forwarding.

FossilOrigin-Name: 2a3f3a9cf28849c99d83c256f813405a2f47f84532e1192d67bb7aa655dd98a3
---
 ext/wasm/README.md |  2 +-
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ext/wasm/README.md b/ext/wasm/README.md
index 769fe9edca..0c328310d9 100644
--- a/ext/wasm/README.md
+++ b/ext/wasm/README.md
@@ -90,7 +90,7 @@ features in the apps which use them.
   *  Remote: Install the SQLite source tree.  CD to ext/wasm
   *  Remote: "`make`" to build WASM
   *  Remote: `althttpd --enable-sab --port 8080 --popup`
-  *  Local:  `ssh -L 8180:remote:8080 remote`
+  *  Local:  `ssh -L 8180:localhost:8080 remote`
   *  Local:  Point your web-browser at http://localhost:8180/index.html
 
 In order to enable [SharedArrayBuffer][], the web-browser requires
diff --git a/manifest b/manifest
index a1ea1f26ec..5c60fc2782 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scouple\sof\scompiler\swarnings\sin\sfts5_index.c.
-D 2023-07-27T20:08:44.624
+C Correct\sa\srecently-introduced\sfalsehood\sin\sext/wasm/README.md\sregarding\sssh\sport\sforwarding.
+D 2023-07-28T11:52:58.874
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -489,7 +489,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
 F ext/wasm/GNUmakefile ddf1aede4275e404c7eda782462c33b6406fcd2dd327241f6b22c0f7b80938e4
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
-F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e
+F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
@@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d66b182d2bc6ce0772e69401b7affe1adbc1b128c4631cb3c17f98dde72af00a
-R 67e5566856c21efabeaa9f30d61da2e8
-U dan
-Z 82b2d14bd5c4ac2218712be1054b30ff
+P bf71faa2a1d29ea762c4d2485522d6f4f8a5a7166981a92d3ba9c96ccbbe1213
+R 74b11dad85afdc2fb1c15f83a8016991
+U stephan
+Z 786f34221ec4345651338e5a3b199da2
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index c1365ea15a..80aa70205d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf71faa2a1d29ea762c4d2485522d6f4f8a5a7166981a92d3ba9c96ccbbe1213
\ No newline at end of file
+2a3f3a9cf28849c99d83c256f813405a2f47f84532e1192d67bb7aa655dd98a3
\ No newline at end of file

From 8a056d486f71aad7df110737b26d38d1bc1ed26e Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 28 Jul 2023 14:20:31 +0000
Subject: [PATCH 65/66] Enhancements to the JSON performance testing scripts
 and instructions to cover the recent optimizations involving updates to large
 JSON strings that indexed.

FossilOrigin-Name: f9213289d09adfb0461d9644e80c7e889f4bae51563ae2e575a2c95336052bcb
---
 manifest              | 15 +++++++--------
 manifest.uuid         |  2 +-
 test/json/README.md   |  4 ++--
 test/json/json-q1.txt | 20 ++++++++++++++++++++
 4 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 03b960da65..bd4c3e8b62 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C JSON\sperformance\simprovements:\s\s(1)\sAdd\sa\scache\sfor\slarge\sstring\svalues\non\sOP_Column\sto\savoid\shaving\sto\sextract\sthe\sstring\smultiple\stimes.\n(2)\sInternal\sJSON\scaching\simprovements.
-D 2023-07-28T13:52:08.133
+C Enhancements\sto\sthe\sJSON\sperformance\stesting\sscripts\sand\sinstructions\sto\ncover\sthe\srecent\soptimizations\sinvolving\supdates\sto\slarge\sJSON\sstrings\nthat\sindexed.
+D 2023-07-28T14:20:31.825
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -1228,9 +1228,9 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff
 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json/README.md 0992b8ccbecd8424b84c1173f9ac56bcc0ae96d49912fd2d86c04eb1f3cba7af
+F test/json/README.md 3761f858e8069d006d9d863fee523f1cfaac5dd7181b196e16cabb6565bfe530
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
-F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d
+F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
 F test/json101.test 94126d4291d4a00e45f6988ce885c410de69243490e46e70e9946cb6e6f9ea02
 F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8
@@ -2049,9 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2a3f3a9cf28849c99d83c256f813405a2f47f84532e1192d67bb7aa655dd98a3 8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae
-R c22600634c62262906bbc544e7e5c805
-T +closed 8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae
+P 771fe35074b50b8d4a6583e61c53871b2445f7a58c82f3fc3bf6776e562e63af
+R 77beac32848c1c5c4f785514dd4faeff
 U drh
-Z 8d623546c0abbe473f1746e112ae16b4
+Z 2c88551e17349d1f41ec2f4976d62002
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 8ee91ce04e..56a5471731 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-771fe35074b50b8d4a6583e61c53871b2445f7a58c82f3fc3bf6776e562e63af
\ No newline at end of file
+f9213289d09adfb0461d9644e80c7e889f4bae51563ae2e575a2c95336052bcb
\ No newline at end of file
diff --git a/test/json/README.md b/test/json/README.md
index a100287b05..217b232eee 100644
--- a/test/json/README.md
+++ b/test/json/README.md
@@ -16,14 +16,14 @@ of the SQLite JSON parser.
 
   2.  Build the baseline sqlite3.c file.  ("`make sqlite3.c`")
 
-  3.  Run "`sh json-speed-check-1.sh trunk`".   This creates the baseline
+  3.  Run "`sh json-speed-check.sh trunk`".   This creates the baseline
       profile in "jout-trunk.txt".
 
 # 3.0 Testing
 
   1.  Build the sqlite3.c to be tested.
 
-  2.  Run "`sh json-speed-check-1.sh x1`".  The profile output will appear
+  2.  Run "`sh json-speed-check.sh x1`".  The profile output will appear
       in jout-x1.txt.  Substitute any label you want in place of "x1".
 
   3.  Run the script shown below in the CLI.
diff --git a/test/json/json-q1.txt b/test/json/json-q1.txt
index 0395f0c061..d122a2d826 100644
--- a/test/json/json-q1.txt
+++ b/test/json/json-q1.txt
@@ -2,3 +2,23 @@
 .timer on
 .param set $label 'q87'
 SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
+
+CREATE TEMP TABLE t2(x JSON TEXT);
+WITH RECURSIVE
+  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000),
+  array1(y) AS (
+    SELECT json_group_array(
+             json_object('x',x,'y',random(),'z',hex(randomblob(50)))
+           )
+      FROM c
+  ),
+  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
+INSERT INTO t2(x)
+  SELECT json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
+CREATE INDEX t2x1 ON t2(x->>'a');
+CREATE INDEX t2x2 ON t2(x->>'b');
+CREATE INDEX t2x3 ON t2(x->>'e');
+CREATE INDEX t2x4 ON t2(x->>'f');
+UPDATE t2 SET x=json_replace(x,'$.f',(x->>'f')+1);
+UPDATE t2 SET x=json_set(x,'$.e',(x->>'f')-1);
+UPDATE t2 SET x=json_remove(x,'$.d');

From 58bf53d5730d327d17446caeda31a926b9cb66f3 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 28 Jul 2023 16:12:39 +0000
Subject: [PATCH 66/66] Minor tweaks to the JSON performance measurement
 documentation.

FossilOrigin-Name: 0bed957e46aa3bf6a70292ae100de0459486c1469dd03de61207a708cc59a594
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 test/json/README.md | 10 +++++++---
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index bd4c3e8b62..e4ac2c7516 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhancements\sto\sthe\sJSON\sperformance\stesting\sscripts\sand\sinstructions\sto\ncover\sthe\srecent\soptimizations\sinvolving\supdates\sto\slarge\sJSON\sstrings\nthat\sindexed.
-D 2023-07-28T14:20:31.825
+C Minor\stweaks\sto\sthe\sJSON\sperformance\smeasurement\sdocumentation.
+D 2023-07-28T16:12:39.671
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -1228,7 +1228,7 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff
 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json/README.md 3761f858e8069d006d9d863fee523f1cfaac5dd7181b196e16cabb6565bfe530
+F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
@@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 771fe35074b50b8d4a6583e61c53871b2445f7a58c82f3fc3bf6776e562e63af
-R 77beac32848c1c5c4f785514dd4faeff
+P f9213289d09adfb0461d9644e80c7e889f4bae51563ae2e575a2c95336052bcb
+R fb3fd77e80d1642b3cf0d0d40a08f14e
 U drh
-Z 2c88551e17349d1f41ec2f4976d62002
+Z 4e3e52a362d42e532a27b931dd696fd5
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 56a5471731..5394eee03f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f9213289d09adfb0461d9644e80c7e889f4bae51563ae2e575a2c95336052bcb
\ No newline at end of file
+0bed957e46aa3bf6a70292ae100de0459486c1469dd03de61207a708cc59a594
\ No newline at end of file
diff --git a/test/json/README.md b/test/json/README.md
index 217b232eee..55044ffca7 100644
--- a/test/json/README.md
+++ b/test/json/README.md
@@ -1,5 +1,6 @@
 The files in this subdirectory are used to help measure the performance
-of the SQLite JSON parser.
+of the SQLite JSON functions, especially in relation to handling large
+JSON inputs.
 
 # 1.0 Prerequisites
 
@@ -7,6 +8,8 @@ of the SQLite JSON parser.
 
   2.  Fossil
 
+  3.  tclsh
+
 # 2.0 Setup
 
   1.  Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create
@@ -14,14 +17,15 @@ of the SQLite JSON parser.
       file lands in the directory from which you will run tests, not in
       the test/json subdirectory of the source tree.
 
-  2.  Build the baseline sqlite3.c file.  ("`make sqlite3.c`")
+  2.  Build the baseline sqlite3.c file with sqlite3.h and shell.c.
+      ("`CFLAGS='-Os -g' make -e clean sqlite3.c`")
 
   3.  Run "`sh json-speed-check.sh trunk`".   This creates the baseline
       profile in "jout-trunk.txt".
 
 # 3.0 Testing
 
-  1.  Build the sqlite3.c to be tested.
+  1.  Build the sqlite3.c (with sqlite3.h and shell.c) to be tested.
 
   2.  Run "`sh json-speed-check.sh x1`".  The profile output will appear
       in jout-x1.txt.  Substitute any label you want in place of "x1".