Revamp the QRF insert-mode optimization of [659ff6ab55802507]. The

control value is now nMultiInsert. Multiple rows are added to each INSERT statement until the number of bytes in that statement exceeds the nMultiInsert value. The CLI uses a default value of 3000, which provides a good balance between speed and prepare-statement size. The output from ".dump" now loads nearly 2x faster in some cases. FossilOrigin-Name: 55b0ce9f93d68b17a18423f0f751b778573127743a2bac67abfea074ac32d41a
2026-10-23 21:49:02 +03:00 · 2026-03-10 18:13:13 +00:00
parent bd10a18f00
commit abc82c49c2
9 changed files with 50 additions and 45 deletions
--- a/ext/qrf/qrf.c
+++ b/ext/qrf/qrf.c
@@ -66,7 +66,7 @@ struct Qrf {
      int iIndent;              /* Current slot */
      int *aiIndent;            /* Indentation for each opcode */
    } sExpln;
-    unsigned int nTuple;      /* Tuples issued under QRF_STYLE_Insert */
+    unsigned int nIns;        /* Bytes used for current INSERT stmt */
  } u;
  sqlite3_int64 nRow;         /* Number of rows handled so far */
  int *actualWidth;           /* Actual width of each column */
@@ -2564,12 +2564,13 @@ static void qrfOneSimpleRow(Qrf *p){
      break;
    }
    case QRF_STYLE_Insert: {
-      int mxTuple = p->spec.iVersion>=2 ? p->spec.mxTuple : 1;
-      if( p->u.nTuple==0 || p->u.nTuple>=mxTuple ){
-        if( p->u.nTuple ){
+      int mxIns = p->spec.iVersion>=2 ? p->spec.nMultiInsert : 0;
+      int szStart = sqlite3_str_length(p->pOut);
+      if( p->u.nIns==0 || p->u.nIns>=mxIns ){
+        if( p->u.nIns ){
          sqlite3_str_append(p->pOut, ";\n", 2);
+          p->u.nIns = 0;
        }
-        p->u.nTuple = 0;
        if( qrf_need_quote(p->spec.zTableName) ){
          sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName);
        }else{
@@ -2596,10 +2597,10 @@ static void qrfOneSimpleRow(Qrf *p){
        if( i>0 ) sqlite3_str_append(p->pOut, ",", 1);
        qrfRenderValue(p, p->pOut, i);
      }
-      p->u.nTuple++;
-      if( p->u.nTuple>=mxTuple ){
+      p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart;
+      if( p->u.nIns>=mxIns ){
        sqlite3_str_append(p->pOut, ");\n", 3);
-        p->u.nTuple = 0;
+        p->u.nIns = 0;
      }else{
        sqlite3_str_append(p->pOut, ")", 1);
      }
@@ -2771,7 +2772,7 @@ qrf_reinit:
      if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){
        p->spec.zTableName = "tab";
      }
-      p->u.nTuple = 0;
+      p->u.nIns = 0;
      break;
    }
    case QRF_STYLE_Line: {
@@ -2885,7 +2886,7 @@ static void qrfFinalize(Qrf *p){
      break;
    }
    case QRF_STYLE_Insert: {
-      if( p->u.nTuple ){
+      if( p->u.nIns ){
        sqlite3_str_append(p->pOut, ";\n", 2);
      }
      break;
--- a/ext/qrf/qrf.h
+++ b/ext/qrf/qrf.h
@@ -57,7 +57,7 @@ struct sqlite3_qrf_spec {
  void *pWriteArg;            /* First argument to the xWrite callback */
  char **pzOutput;            /* Storage location for output string */
  /* Fields below are only available if iVersion>=2 */
-  unsigned int mxTuple;       /* Tuples per INSERT in QRF_STYLE_Insert */
+  unsigned int nMultiInsert;  /* Add rows to one INSERT until size exceeds */
  /* Additional fields may be added in the future */
 };

--- a/24
+++ b/24
@@ -1,5 +1,5 @@
-C Bug\sfix\sto\sthe\s".dump"\scommand\sin\sthe\sCLI:\s\sTemporarily\sdisable\scharacter\nlimit\sto\sQRF\sso\sas\snot\sto\struncate\svalues\sin\sthe\sgenerated\soutput.
-D 2026-03-10T15:43:57.174
+C Revamp\sthe\sQRF\sinsert-mode\soptimization\sof\s[659ff6ab55802507].\s\sThe\ncontrol\svalue\sis\snow\snMultiInsert.\s\sMultiple\srows\sare\sadded\sto\seach\sINSERT\nstatement\suntil\sthe\snumber\sof\sbytes\sin\sthat\sstatement\sexceeds\sthe\nnMultiInsert\svalue.\s\sThe\sCLI\suses\sa\sdefault\svalue\sof\s3000,\swhich\sprovides\na\sgood\sbalance\sbetween\sspeed\sand\sprepare-statement\ssize.\s\sThe\soutput\nfrom\s".dump"\snow\sloads\snearly\s2x\sfaster\sin\ssome\scases.
+D 2026-03-10T18:13:13.382
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -420,8 +420,8 @@ F ext/misc/zipfile.c c8ee04e1b349270b5df401ad732f5d7c387146e69b33c02fa90322760cc
 F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee
 F ext/qrf/README.md e6e0ce2700acf6fd06312b42726a8f08ca240f30e1b122bff87c71c602046352
 F ext/qrf/dev-notes.md e68a6d91ce4c7eb296ef2daadc2bb79c95c317ad15b9fafe40850c67b29c2430
-F ext/qrf/qrf.c d9f2e9fe7f34e700cd4245d39b1272bb62fc360ba95aa3de264bc6521ad8dfa0
-F ext/qrf/qrf.h bdd1711b954c2a3714f22d22693e3d76eb8de3db5efa6afa00c55ed1d7f61028
+F ext/qrf/qrf.c d7ed1d2e2460637f970b796e9f04774cc96dbd6c7a85cfff4c78746c2f6c51fe
+F ext/qrf/qrf.h 2b27cb8079131ac3af2134e9e2ee9621b41d97bcb546fead41b9e1a12a3567a7
 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8
 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255
 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363
@@ -733,7 +733,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 928ff887f2a7c64275182060d94d06fdddbe32226c569781cf7e7edc6f58d7fd
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c ffe199f025a0dd74670d2a77232bdea364a4d7b36f32c64a6572d39ba6a11576
-F src/shell.c.in 339f8e92a8531375deb8dc649d60948149a297c55623e63f544f22d0f2700145
+F src/shell.c.in dc19fd2fddbfec60f52852edbb4a4e3723e23e00c63b2ea289800fa4d37b7a6e
 F src/sqlite.h.in 4d657846d68a58b028f0c4c331b9d3b4a79306f25c3b0d04fb56060343f73d85
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 1b7a0ee438bb5c2896d0609c537e917d8057b3340f6ad004d2de44f03e3d3cca
@@ -741,7 +741,7 @@ F src/sqliteInt.h 9716721fb57e32938a1d30a84560ce7633c63860a2209e188c87afad15d4b4
 F src/sqliteLimit.h 904a3f520362c7065c18165aaabd504fb13cc1b76cb411f38bd41ac219e4af1e
 F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
-F src/tclsqlite.c bebd32b937ec1542234cc63228720ecee7be1a6cd682fb6b55378d7ae01b8bc0
+F src/tclsqlite.c 71230da35d81b7234c5e4dc3747ca2302390884b051052956deb3dbf680a6ed4
 F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a
 F src/test1.c 3e3b013f59ffcb57dce00c90d55907072d71d4e970cb0a590cb261efe11bae9c
 F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff
@@ -1055,7 +1055,7 @@ F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014
 F test/distinct2.test 072f33e1348b5cae4156e7ca4c124d21053f77d96d5d960a1ba21806416074ab
 F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f
-F test/dotcmd01.sql 0388a778912ed08436ae5c80e03389d8bd347fa724611193257a18c69692019d
+F test/dotcmd01.sql dd5b0475df723fdb9f91031a7b9684abb279f6a686178f0d8e3b5f1da93c27c2
 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52
 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075
 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50
@@ -1445,7 +1445,7 @@ F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93
 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465
 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3
 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3
-F test/modeA.sql 24417bc672b9e3dff3243b6678595ecbf77ef30827b5e24f9faaec9edc30b650
+F test/modeA.sql fc64f646b0a1d0806af122fad6db2c89de63c51106655c09d44a652052a14d05
 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08
 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a
 F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7
@@ -1515,7 +1515,7 @@ F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224c
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
 F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd
-F test/qrf01.test d4bf258413592b8beb43d838f89c9f57882fc74efef556d2a201b7372ec543a7
+F test/qrf01.test 09caa00e6b4deea5fcd8958b4062233fcef5ef1b354312ec956ec4bc3a09572a
 F test/qrf02.test 39b4afdc000bedccdafc0aecf17638df67a67aaa2d2942865ae6abcc48ba0e92
 F test/qrf03.test e7efe46d204671726b4707585126cd78d107368de4a7d0c7b8d5157cdd8624ed
 F test/qrf04.test 0894692c998d2401dcc33449c02051b503ecce0c94217be54fb007c82d2d1379
@@ -2192,8 +2192,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
-P 659ff6ab5580250707908af003ecd093bd5313f03f21f0efd000a7aff6638b3c
-R c7258df5c286f69cf6ad0fcffe82369d
+P f6787e4b50af50663d8fa00ebe0694440354a7ad0509837a6b249e090e6cffca
+R d860bc66dd6bb7267fd1bfd3a3708814
 U drh
-Z ec10a6a2386acbe38f4be26bb414e44b
+Z d618cb096ff3c243141ddcd38eae46e6
 # Remove this line to create a well-formed Fossil manifest.
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f6787e4b50af50663d8fa00ebe0694440354a7ad0509837a6b249e090e6cffca
+55b0ce9f93d68b17a18423f0f751b778573127743a2bac67abfea074ac32d41a
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1375,6 +1375,9 @@ static const ModeInfo aModeInfo[] = {
 #ifndef DFLT_TITLE_LIMIT
 # define DFLT_TITLE_LIMIT 20
 #endif
+#ifndef DFLT_MULTI_INSERT
+# define DFLT_MULTI_INSERT 3000
+#endif

 /*
 ** Limit input nesting via .read or any other input redirect.
@@ -1677,6 +1680,7 @@ static void modeChange(ShellState *p, unsigned char eMode){
    p->mode.spec.nLineLimit = DFLT_LINE_LIMIT;
    p->mode.spec.bTextJsonb = QRF_Yes;
    p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT;
+    p->mode.spec.nMultiInsert = DFLT_MULTI_INSERT;
    p->mode.mFlags = mFlags;
  }
 }
@@ -7921,9 +7925,8 @@ static int modeTitleDsply(ShellState *p, int bAll){
 **                            can also be "off" to mean "0,0,0" or "on" to
 **                            mean "5,300,20".
 **   --list                   List available modes
-**   --mxtuple N              In "insert" mode, try to put as many as N
-**                            comma-separated tuples after the VALUES in each
-**                            INSERT statement.
+**   --multiinsert N          In "insert" mode, put multiple rows on a single
+**                            INSERT statement until the size exceeds N bytes.
 **   --null STRING            Render SQL NULL values as the given string
 **   --once                   Setting changes to the right are reverted after
 **                            the next SQL command.
@@ -8040,7 +8043,7 @@ static int dotCmdMode(ShellState *p){
      }
      chng = 1;
    }else if( 0<=(k=pickStr(z,0,
-                  "-charlimit","-linelimit","-titlelimit","-mxtuple","")) ){
+                  "-charlimit","-linelimit","-titlelimit","-multiinsert","")) ){
      int w;   /*  0            1            2             3 */
      if( i+1>=nArg ){
        dotCmdError(p, i, "missing argument", 0);
@@ -8048,10 +8051,10 @@ static int dotCmdMode(ShellState *p){
      }
      w = integerValue(azArg[++i]);
      switch( k ){
-        case 0:   p->mode.spec.nCharLimit = w;   break;
-        case 1:   p->mode.spec.nLineLimit = w;   break;
-        case 2:   p->mode.spec.nTitleLimit = w;  break;
-        default:  p->mode.spec.mxTuple = w;      break;
+        case 0:   p->mode.spec.nCharLimit = w;    break;
+        case 1:   p->mode.spec.nLineLimit = w;    break;
+        case 2:   p->mode.spec.nTitleLimit = w;   break;
+        default:  p->mode.spec.nMultiInsert = w;  break;
      }
      chng = 1;
    }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){
@@ -8395,9 +8398,10 @@ static int dotCmdMode(ShellState *p){
      }
    }
    if( bAll
-     || (p->mode.spec.mxTuple>1 && p->mode.spec.eStyle==QRF_STYLE_Insert)
+     || (p->mode.spec.nMultiInsert && p->mode.spec.eStyle==QRF_STYLE_Insert)
    ){
-      sqlite3_str_appendf(pDesc, " --mxtuple %u", p->mode.spec.mxTuple);
+      sqlite3_str_appendf(pDesc, " --multiinsert %u",
+                          p->mode.spec.nMultiInsert);
    }
    zSetting = aModeStr[pI->eNull];
    if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -2071,7 +2071,7 @@ static void DbHookCmd(
 **     -linelimit NUMBER                       Max lines for any cell
 **     -charlimit NUMBER                       Content truncated to this size
 **     -titlelimit NUMBER                      Max width of column titles
-**     -mxtuple NUMBER                         Max tuples per INSERT
+**     -multiinsert NUMBER                     Multi-row INSERT byte size
 **     -align LIST-OF-ALIGNMENT                Alignment of columns
 **     -widths LIST-OF-NUMBERS                 Widths for individual columns
 **     -columnsep TEXT                         Column separator text
@@ -2100,7 +2100,7 @@ static void DbHookCmd(
 **     -linelimit        nLineLimit
 **     -charlimit        nCharLimit
 **     -titlelimit       nTitleLimit
-**     -mxtuple          mxTuple
+**     -multiinsert      nMultiInsert
 **     -align            nAlign, aAlign
 **     -widths           nWidth, aWidth
 **     -columnsep        zColumnSep
@@ -2303,12 +2303,12 @@ static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){
      if( v<0 ) v = 0;
      qrf.nCharLimit = v;
      i++;
-    }else if( strcmp(zArg,"-mxtuple")==0 ){
+    }else if( strcmp(zArg,"-multiinsert")==0 ){
      int v = 0;
      rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v);
      if( rc ) goto format_failed;
      if( v<0 ) v = 0;
-      qrf.mxTuple = v;
+      qrf.nMultiInsert = v;
      i++;
    }else if( strcmp(zArg,"-align")==0 ){
      Tcl_Size n = 0;
--- a/test/dotcmd01.sql
+++ b/test/dotcmd01.sql
@@ -40,8 +40,8 @@ CREATE TABLE t1(a,b,c);
 CREATE INDEX t1a ON t1(a);
 CREATE INDEX t1b ON t1(b);
 ANALYZE sqlite_schema;
-INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10');
-INSERT INTO sqlite_stat1 VALUES('t1','t1a','300 30');
+INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'),
+  ('t1','t1a','300 30');
 ANALYZE sqlite_schema;
 END

--- a/test/modeA.sql
+++ b/test/modeA.sql
@@ -87,7 +87,7 @@ END
 .testcase 140
 .mode -v
 .check <<END
-.mode qbox --align "" --border on --blob-quote auto --colsep "" --escape auto --limits on --mxtuple 0 --null "NULL" --quote relaxed --rowsep "" --sw auto --tablename "" --textjsonb on --titles on --widths "" --wordwrap off --wrap 10
+.mode qbox --align "" --border on --blob-quote auto --colsep "" --escape auto --limits on --multiinsert 3000 --null "NULL" --quote relaxed --rowsep "" --sw auto --tablename "" --textjsonb on --titles on --widths "" --wordwrap off --wrap 10
 END
 .testcase 150 --error-prefix "Error:"
 .mode foo
@@ -146,7 +146,7 @@ END
 .mode --limits 0,0,0
 .mode -v
 .check <<END
-.mode box --align "" --border on --blob-quote auto --colsep "" --escape auto --limits off --mxtuple 0 --null "" --quote off --rowsep "" --sw 0 --tablename "" --textjsonb off --titles on --widths "" --wordwrap off
+.mode box --align "" --border on --blob-quote auto --colsep "" --escape auto --limits off --multiinsert 0 --null "" --quote off --rowsep "" --sw 0 --tablename "" --textjsonb off --titles on --widths "" --wordwrap off
 END

 .testcase 400
@@ -291,7 +291,7 @@ SELECT * FROM t1;
 .testcase 700
 CREATE TABLE tbl1(one,two);
 INSERT INTO tbl1 VALUES('hello!',10),('goodbye',20);
-.mode insert new_table
+.mode insert new_table --multiinsert 0
 SELECT * FROM tbl1;
 .check <<END
 INSERT INTO new_table VALUES('hello!',10);
--- a/test/qrf01.test
+++ b/test/qrf01.test
@@ -267,7 +267,7 @@ INSERT INTO "drop"("a-b",b,"123") VALUES(1,2.5,'three');
 INSERT INTO "drop"("a-b",b,"123") VALUES(x'424c4f42',NULL,'Ἀμήν');
 }
 do_test 1.84 {
-  set result "\n[db format -style insert {SELECT * FROM t1} -mxtuple 2]"
+  set result "\n[db format -style insert {SELECT * FROM t1} -multiinsert 2000]"
 } {
 INSERT INTO tab VALUES(1,2.5,'three'),
  (x'424c4f42',NULL,'Ἀμήν');
@@ -275,7 +275,7 @@ INSERT INTO tab VALUES(1,2.5,'three'),
 do_test 1.85 {
  set result "\n[db format -style insert {
     WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7)
-       SELECT n, n*10 AS m FROM c;} -mxtuple 5]"
+       SELECT n, n*10 AS m FROM c;} -multiinsert 70]"
 } {
 INSERT INTO tab VALUES(1,10),
  (2,20),