Add support for sqlite3_stmt_scanstatus_v2() profiling of GROUP BY clauses that use a temp b-tree.

FossilOrigin-Name: 796eadcc50e7ea9ec348b1f7e2d6a5a3de78c675a07f665078309e6d81479e7d

manifest

@@ -1,5 +1,5 @@
-C Add\sextra\sdebugging\scode\sfor\ssqlite3_stmt_scanstatus_v2()\sto\stest1.c.
+C Add\ssupport\sfor\ssqlite3_stmt_scanstatus_v2()\sprofiling\sof\sGROUP\sBY\sclauses\sthat\suse\sa\stemp\sb-tree.
-D 2023-06-30T16:16:51.838
+D 2023-06-30T17:14:36.762
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,7 +637,7 @@ F src/printf.c a87473be34fa2acafa27692b8ae078275c7e23360956c93c07ff22f5d609cbd7
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 383b9dba12493c365ee2036bcadd73013b7c0f7d2afcda0c378317c335d60ac2
+F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88
 F src/shell.c.in 2c02c819349de410d63fcc0217763dfe5a42dbe58f2d68046d4ea8a376d12c26
 F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -707,11 +707,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c 4264102045fdb36e9af3ff361e390a5f7a76342a2bd7069e55d8ad332026d6b5
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c c993304c609326cf625b4ad30cbb0e15a3f64c941cf2c9713d0c360b4abbaa98
+F src/vdbe.c 74282a947234513872a83b0bab1b8c644ece64b3e27b053ef17677c8ff9c81e0
 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae
 F src/vdbeapi.c de9703f8705afc393cc2864669ce28cf9516983c8331d59aa2b978de01634365
-F src/vdbeaux.c 4d5e68a3850d0b193a692eca6442d7afe35252aaf29728a67adcb542ecabd9ce
+F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339
 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce
 F src/vdbemem.c aed58a560caab12540f7c14c43ee188636017814e21247a97902f78de2d43117
 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015
@@ -1425,7 +1425,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa
 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e
-F test/scanstatus2.test 9a00becb1d60d168944f8e2f3585d44d46123aa95c5c7c25563309e1f15f924d
+F test/scanstatus2.test 97ebdac082f9d313aaf0f8ff25d4ea0552b2c2928c00006f12858d72157b927c
 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce
@@ -2041,8 +2041,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 85be05b66ef975f02a3e7b2984bcab97d9280c7f3b6ee1e16718de549f240d46
+P f936f101406069b29218c89a36581b4497226fb61906782ea368f12d943c901c
-R 433a85e7cf1b4578b68ff0f431f95f42
+R 67483c89b8d8a571047f048e0b9782e8
+T *branch * scanstatus-exp
+T *sym-scanstatus-exp *
+T -sym-trunk *
 U dan
-Z 30a28df351ab36975e3c1ccdb855302c
+Z aa893d30c01b934263e5d84c28eae25f
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-f936f101406069b29218c89a36581b4497226fb61906782ea368f12d943c901c
+796eadcc50e7ea9ec348b1f7e2d6a5a3de78c675a07f665078309e6d81479e7d

src/select.c

@@ -8009,9 +8009,13 @@ int sqlite3Select(
         int nCol;
         int nGroupBy;
 
-        explainTempTable(pParse,
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+        int addrExp;              /* Address of OP_Explain instruction */
+#endif
+        ExplainQueryPlan2(addrExp, (pParse, 0, "USE TEMP B-TREE FOR %s",
             (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ?
-                    "DISTINCT" : "GROUP BY");
+                    "DISTINCT" : "GROUP BY"
+        ));
 
         groupBySort = 1;
         nGroupBy = pGroupBy->nExpr;
@@ -8036,18 +8040,23 @@ int sqlite3Select(
         }
         pAggInfo->directMode = 0;
         regRecord = sqlite3GetTempReg(pParse);
+        sqlite3VdbeScanStatusCounters(v, addrExp, 0, sqlite3VdbeCurrentAddr(v));
         sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord);
         sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord);
+        sqlite3VdbeScanStatusRange(v, addrExp, sqlite3VdbeCurrentAddr(v)-2, -1);
         sqlite3ReleaseTempReg(pParse, regRecord);
         sqlite3ReleaseTempRange(pParse, regBase, nCol);
         TREETRACE(0x2,pParse,p,("WhereEnd\n"));
         sqlite3WhereEnd(pWInfo);
         pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++;
         sortOut = sqlite3GetTempReg(pParse);
+        sqlite3VdbeScanStatusCounters(v, addrExp, sqlite3VdbeCurrentAddr(v), 0);
         sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol);
         sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd);
         VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v);
         pAggInfo->useSortingIdx = 1;
+        sqlite3VdbeScanStatusRange(v, addrExp, -1, sortPTab);
+        sqlite3VdbeScanStatusRange(v, addrExp, -1, pAggInfo->sortingIdx);
       }
 
       /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions

src/vdbe.c

@@ -5851,7 +5851,7 @@ case OP_SorterCompare: {
 ** parameter P3.  Clearing the P3 column cache as part of this opcode saves
 ** us from having to issue a separate NullRow instruction to clear that cache.
 */
-case OP_SorterData: {
+case OP_SorterData: {       /* ncycle */
   VdbeCursor *pC;
 
   pOut = &aMem[pOp->p2];
@@ -6126,8 +6126,8 @@ case OP_IfSmaller: {        /* jump */
 ** regression tests can determine whether or not the optimizer is
 ** correctly optimizing out sorts.
 */
-case OP_SorterSort:    /* jump */
+case OP_SorterSort:    /* jump ncycle */
-case OP_Sort: {        /* jump */
+case OP_Sort: {        /* jump ncycle */
 #ifdef SQLITE_TEST
   sqlite3_sort_count++;
   sqlite3_search_count--;

src/vdbeaux.c

@@ -538,7 +538,7 @@ int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
     if( bPush){
       pParse->addrExplain = iThis;
     }
-    sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0);
+    sqlite3VdbeScanStatus(v, iThis, -1, -1, 0, 0);
   }
   return addr;
 }
@@ -1250,8 +1250,8 @@ void sqlite3VdbeScanStatusCounters(
       pScan = 0;
     }
     if( pScan ){
-      pScan->addrLoop = addrLoop;
+      if( addrLoop>0 ) pScan->addrLoop = addrLoop;
-      pScan->addrVisit = addrVisit;
+      if( addrVisit>0 ) pScan->addrVisit = addrVisit;
     }
   }
 }

test/scanstatus2.test

@@ -55,11 +55,12 @@ proc get_cycles {stmt} {
   dict get $r nCycle
 }
 
-proc foreach_scan {varname stmt body} {
+proc foreach_scan {varname stmt body {debug 0}} {
   upvar $varname var
-
   for {set ii 0} {1} {incr ii} {
-    set r [sqlite3_stmt_scanstatus -flags complex $stmt $ii]
+    set f "complex"
+    if {$debug} { set f "complex debug" }
+    set r [sqlite3_stmt_scanstatus -flags $f $stmt $ii]
     if {[llength $r]==0} break
     array set var $r
     uplevel $body
@@ -102,6 +103,15 @@ proc puts_graph {sql} {
   puts [string trim [get_graph $stmt]]
 }
 
+proc puts_debug_info {sql} {
+  db eval $sql
+  set stmt [db version -last-stmt-ptr]
+  foreach_scan X $stmt {
+    puts -nonewline "$X(debug_explain) $X(zExplain): "
+    puts -nonewline "loop=$X(debug_loop) visit=$X(debug_visit) "
+    puts            "csr=$X(debug_csr) range=$X(debug_range)"
+  } 1
+}
 
 do_zexplain_test 0 1.1 {
   SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2
@@ -235,7 +245,7 @@ do_graph_test 4.5 {
 QUERY (nCycle=nnn)
 --CO-ROUTINE v1
 ----SCAN rt2 (nCycle=nnn)
-----USE TEMP B-TREE FOR GROUP BY
+----USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
 --SCAN rt1 (nCycle=nnn)
 --CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn)
 --BLOOM FILTER ON v1 (x1=?)
@@ -270,6 +280,33 @@ ifcapable trace {
   } {{SCAN t1} {SCAN t1} {SCAN t1}}
 }
 
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_db_config db STMT_SCANSTATUS 1
+
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'one');
+  INSERT INTO t1 VALUES(2, 'two');
+  INSERT INTO t1 VALUES(3, 'three');
+  INSERT INTO t1 VALUES(4, 'four');
+  INSERT INTO t1 VALUES(5, 'five');
+  INSERT INTO t1 VALUES(6, 'six');
+  INSERT INTO t1 VALUES(7, 'seven');
+  INSERT INTO t1 VALUES(8, 'eight');
+}
+
+do_graph_test 6.1 {
+  SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1
+} {
+QUERY (nCycle=nnn)
+--SCAN t1 (nCycle=nnn)
+--USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
+}
+
+#explain_i { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 }
+#puts_debug_info { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 }
+
 finish_test