Experimental change to have sqlite3_vtab_distinct() return 3 to indicate that results should sorted and duplicates may be removed.

FossilOrigin-Name: 11f4508895c0a46d8623ca2f4f37b4f1b54b6d9022765f6124a9d42132f7d633

manifest

@@ -1,5 +1,5 @@
-C Make\sCLI\scomplain\sabout\sincomplete\sinput\sat\sEOF.\sFix\sfor\sregression\sreported\sat\shttps://sqlite.org/forum/forumpost/718f489a43be3197
-D 2022-03-15T17:57:42.959
+C Experimental\schange\sto\shave\ssqlite3_vtab_distinct()\sreturn\s3\sto\sindicate\sthat\sresults\sshould\ssorted\sand\sduplicates\smay\sbe\sremoved.
+D 2022-03-16T12:06:00.492
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -552,7 +552,7 @@ F src/printf.c 05d8dfd2018bc4fc3ddb8b37eb97ccef7abf985643fa1caebdcf2916ca90fa32
 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c
 F src/resolve.c ea935b87d6fb36c78b70cdc7b28561dc8f33f2ef37048389549c7b5ef9b0ba5e
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c 4890a3cfee0bc60ff231c3a44db37968859ab0be156983dbcc0c096109832cdd
+F src/select.c 16453eff42da7b9705f0e3e3cdf4a7251b27b7b88620d0697c15e4f656ff3e77
 F src/shell.c.in ec4b952f7d9c02919e8e8394be16bcbff55c105fd080895602d10fc8fa5328b8
 F src/sqlite.h.in 5845213799feca09cd69d18ff841a85fe0df31021f46aaa1797e703e80dc1d70
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -639,7 +639,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 7f7e6b5f0c017997a3a47988781de9481f9fa685febdaea779a9fd79ab17a43c
+F src/where.c 615ac2c6ea64baf48b023f27b33fa5b6f8f2bbd75d86d16c43ce7b0a2aa7a4fa
 F src/whereInt.h 15d2975c3b4c193c78c26674400a840da8647fe1777ae3b026e2d15937b38a03
 F src/wherecode.c 555f598a9ddad81761f084710fdb4f8733fe31bc14cd6b19f8ca4274a7eaa04c
 F src/whereexpr.c 2a71f5491798460c9590317329234d332d9eb1717cba4f3403122189a75c465e
@@ -732,6 +732,7 @@ F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb
 F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4
 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e
 F test/bestindex8.test abd0016fc04f19dc382976750b06df5463d2757e11e78a8ba7d7dc50671f3337
+F test/bestindex9.test bf2eb8556e8d5c00ef3ee18c521751cd03c1b55454b6e7683b4c6742e3131b23
 F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -1944,8 +1945,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fff1243b594c190d15f14b7ca4e60d23519cd15134f275991c685966fcc24145
-R 419d7db42dfecc265d3cd4cb01355313
-U larrybr
-Z e6c264c989ed1b7dd7a09ef4a9447bb1
+P 72029cf7cdb266703cc8716102dbba8e6f2666e1f47409f42c39528795757b73
+R 091535e0d89f14f4e262c52c3edd5ea6
+T *branch * vtab-distinct-ordered
+T *sym-vtab-distinct-ordered *
+T -sym-trunk *
+U dan
+Z e66666d32a77ed9604cb92699aaf4a49
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-72029cf7cdb266703cc8716102dbba8e6f2666e1f47409f42c39528795757b73
+11f4508895c0a46d8623ca2f4f37b4f1b54b6d9022765f6124a9d42132f7d633

src/select.c

@@ -6821,6 +6821,7 @@ int sqlite3Select(
     ** the sDistinct.isTnct is still set.  Hence, isTnct represents the
     ** original setting of the SF_Distinct flag, not the current setting */
     assert( sDistinct.isTnct );
+    sDistinct.isTnct = 2;
 
 #if SELECTTRACE_ENABLED
     if( sqlite3SelectTrace & 0x400 ){
@@ -7164,7 +7165,8 @@ int sqlite3Select(
       sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
       SELECTTRACE(1,pParse,p,("WhereBegin\n"));
       pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct,
-          0, (WHERE_GROUPBY|(orderByGrp ? WHERE_SORTBYGROUP : 0)|distFlag), 0
+          0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) 
+          |  (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0
       );
       if( pWInfo==0 ){
         sqlite3ExprListDelete(db, pDistinct);
@@ -7346,7 +7348,7 @@ int sqlite3Select(
       VdbeComment((v, "indicate accumulator empty"));
       sqlite3VdbeAddOp1(v, OP_Return, regReset);
 
-      if( eDist!=WHERE_DISTINCT_NOOP ){
+      if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){
         struct AggInfo_func *pF = &pAggInfo->aFunc[0];
         fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr);
       }

src/where.c

@@ -1212,8 +1212,10 @@ static sqlite3_index_info *allocateIndexInfo(
     }
     if( i==n ){
       nOrderBy = n;
-      if( (pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY)) ){
-        eDistinct = 1 + ((pWInfo->wctrlFlags & WHERE_DISTINCTBY)!=0);
+      if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){
+        eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
+      }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
+        eDistinct = 1;
       }
     }
   }
@@ -3771,7 +3773,9 @@ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
   HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
   assert( pHidden->eDistinct==0
        || pHidden->eDistinct==1
-       || pHidden->eDistinct==2 );
+       || pHidden->eDistinct==2 
+       || pHidden->eDistinct==3 
+  );
   return pHidden->eDistinct;
 }
 
@@ -4230,7 +4234,9 @@ static i8 wherePathSatisfiesOrderBy(
       pLoop = pLast;
     }
     if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
-      if( pLoop->u.vtab.isOrdered && (wctrlFlags & WHERE_DISTINCTBY)==0 ){
+      if( pLoop->u.vtab.isOrdered 
+       && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
+      ){
         obSat = obDone;
       }
       break;
@@ -4497,7 +4503,7 @@ static i8 wherePathSatisfiesOrderBy(
 **   SELECT * FROM t1 GROUP BY y,x ORDER BY y,x;   -- IsSorted()==0
 */
 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
-  assert( pWInfo->wctrlFlags & WHERE_GROUPBY );
+  assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
   assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
   return pWInfo->sorted;
 }
@@ -4898,12 +4904,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
   }
   pWInfo->bOrderedInnerLoop = 0;
   if( pWInfo->pOrderBy ){
+    pWInfo->nOBSat = pFrom->isOrdered;
     if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
       if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
         pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
       }
     }else{
-      pWInfo->nOBSat = pFrom->isOrdered;
       pWInfo->revMask = pFrom->revLoop;
       if( pWInfo->nOBSat<=0 ){
         pWInfo->nOBSat = 0;

test/bestindex9.test

@@ -0,0 +1,108 @@
+# 2020-01-29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex9
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return $::create_table
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set ::vtab_orderby [$hdl orderby]
+      set ::vtab_distinct [$hdl distinct]
+
+      return [list orderby 1]
+    }
+
+    xFilter {
+      return ""
+    }
+  }
+
+  return {}
+}
+
+proc do_bestindex9_test {tn create select orderby distinct} {
+  forcedelete test.db2
+  sqlite3 dbtest test.db2
+  register_tcl_module dbtest
+
+  set ::create_table $create
+  dbtest eval { CREATE VIRTUAL TABLE t1 USING tcl(vtab_command); }
+  dbtest eval $select
+
+  do_test $tn.orderby [list set {} $::vtab_orderby] $orderby
+  do_test $tn.distinct [list set {} $::vtab_distinct] $distinct
+
+  dbtest close
+}
+
+#-------------------------------------------------------------------------
+#
+do_bestindex9_test 1.0 {
+  CREATE TABLE x(k1, k2, v1, PRIMARY KEY(k1, k2))
+} {
+  SELECT DISTINCT k1, k2 FROM t1 
+} {{column 0 desc 0} {column 1 desc 0}} 2
+
+do_bestindex9_test 1.1 {
+  CREATE TABLE x(k1, k2, v1, PRIMARY KEY(k1, k2)) WITHOUT ROWID
+} {
+  SELECT DISTINCT k1, k2 FROM t1 
+} {} 0
+
+do_bestindex9_test 1.2 {
+  CREATE TABLE x(k1 NOT NULL, k2 NOT NULL, v1, PRIMARY KEY(k1, k2))
+} {
+  SELECT DISTINCT k1, k2 FROM t1 
+} {} 0
+
+#-------------------------------------------------------------------------
+#
+do_bestindex9_test 2 {
+  CREATE TABLE x(c1, c2, c3);
+} {
+  SELECT DISTINCT c1 FROM t1 ORDER BY c1
+} {{column 0 desc 0}} 3
+
+#-------------------------------------------------------------------------
+#
+do_bestindex9_test 3 {
+  CREATE TABLE x(c1, c2, c3);
+} {
+  SELECT DISTINCT c1 FROM t1 GROUP BY c1
+} {{column 0 desc 0}} 1
+
+do_bestindex9_test 4 {
+  CREATE TABLE x(c1, c2, c3);
+} {
+  CREATE TABLE t2(balls);
+  SELECT DISTINCT c1 FROM t1, t2
+} {{column 0 desc 0}} 2
+
+
+finish_test
+
+
+
+