diff --git a/manifest b/manifest index 511b9b2276..ea5c01f737 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sfix\sfor\sa\scorner-case\sin\ssqlite_offset()\s-\sthis\sone\shaving\sto\ndo\swith\scomputed\svirtual\scolumns\sin\sa\sWITHOUT\sROWID\stable. -D 2022-03-17T11:23:13.924 +C Add\sa\s4th\soutput\scode\s(value\s3)\sfrom\sthe\ssqlite3_vtab_distinct()\sinterface\nthat\sindicates\sthat\sSQLite\swants\sdistinct\sresults\sthat\sare\ssorted. +D 2022-03-17T22:09:17.044 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -552,9 +552,9 @@ F src/printf.c 05d8dfd2018bc4fc3ddb8b37eb97ccef7abf985643fa1caebdcf2916ca90fa32 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c ea935b87d6fb36c78b70cdc7b28561dc8f33f2ef37048389549c7b5ef9b0ba5e F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4890a3cfee0bc60ff231c3a44db37968859ab0be156983dbcc0c096109832cdd +F src/select.c c366c05e48e836ea04f8ecefb9c1225745dc250c3f01bdb39e9cbb0dc25e3610 F src/shell.c.in ec4b952f7d9c02919e8e8394be16bcbff55c105fd080895602d10fc8fa5328b8 -F src/sqlite.h.in 5845213799feca09cd69d18ff841a85fe0df31021f46aaa1797e703e80dc1d70 +F src/sqlite.h.in 2a35f62185eb5e7ecc64a2f68442b538ce9be74f80f28a00abc24837edcf1c17 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a95cb9ed106e3d39e2118e4dcc15a14faec3fa50d0093425083d340d9dfd96e6 F src/sqliteInt.h 2ce7d868630ccd70ffd4b15d46b59ccf7daf89198993b62ed6e4a165d3511280 @@ -639,7 +639,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 3f6926af089c0070dd627bef9e16f6890262b43341d5627097bc26fa121d0e12 +F src/where.c f661a49e6f100b03619eccc715d5d8b57a0564a96c8d1fa8597805bf9119316f F src/whereInt.h 15d2975c3b4c193c78c26674400a840da8647fe1777ae3b026e2d15937b38a03 F src/wherecode.c 555f598a9ddad81761f084710fdb4f8733fe31bc14cd6b19f8ca4274a7eaa04c F src/whereexpr.c 2a71f5491798460c9590317329234d332d9eb1717cba4f3403122189a75c465e @@ -731,7 +731,8 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test abd0016fc04f19dc382976750b06df5463d2757e11e78a8ba7d7dc50671f3337 +F test/bestindex8.test 333ad8c6a554b885a49b68c019166eda92b05f493a92b36b0acdf7f766d04dad +F test/bestindex9.test bf2eb8556e8d5c00ef3ee18c521751cd03c1b55454b6e7683b4c6742e3131b23 F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -859,7 +860,7 @@ F test/descidx1.test edc8adee58d491b06c7157c50364eaf1c3605c9c19f8093cb1ea2b6184f F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad78298 F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 3e4210ef9cd1985aeec44939ad912c4621fbea9bb4a9c565696cebfe184b2ec5 +F test/distinct.test a7687c2fb50c93f6a486936c51439a93221c6e1188f9bc7b27b3ec26f9c58b1e F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 F test/distinctagg.test d76ef2e91fe810630c176d6bd0a58c14d5851c3125f0a1d977db87ba76359639 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 @@ -1944,8 +1945,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 72029cf7cdb266703cc8716102dbba8e6f2666e1f47409f42c39528795757b73 -R daa2513edd6111fe3d506fefc3f7ebd4 +P 84ddd19bcec99f04b43b1a823477457758a2d93ea9beda43598e1234ea07776c 0c5be14aac07222b9cd2404ae485b6587f8cb2899e776bc45f1f1117bdd7e9b7 +R 3a06f6b09e46e5d74ffd1d941c48374e +T +closed 0c5be14aac07222b9cd2404ae485b6587f8cb2899e776bc45f1f1117bdd7e9b7 U drh -Z 44502c04520a4cca4d14e85f770454ce +Z b60aa89107b858dfaa16ff6d69f84def # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index da3c3c2eca..db4a1daef9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84ddd19bcec99f04b43b1a823477457758a2d93ea9beda43598e1234ea07776c \ No newline at end of file +135d8b97b01f9b5204f486f828c274c2d779c17723e79171ca61f738746adf8e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0c5ce55715..5165d6f600 100644 --- a/src/select.c +++ b/src/select.c @@ -21,7 +21,7 @@ */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -6821,6 +6821,7 @@ int sqlite3Select( ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + sDistinct.isTnct = 2; #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x400 ){ @@ -7164,7 +7165,8 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - 0, (WHERE_GROUPBY|(orderByGrp ? WHERE_SORTBYGROUP : 0)|distFlag), 0 + 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDistinct); @@ -7346,7 +7348,7 @@ int sqlite3Select( VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); - if( eDist!=WHERE_DISTINCT_NOOP ){ + if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ struct AggInfo_func *pF = &pAggInfo->aFunc[0]; fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9f77e01c69..9c4df2292d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9555,8 +9555,8 @@ SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** of a [virtual table] implementation. The result of calling this ** interface from outside of xBestIndex() is undefined and probably harmful. ** -** ^The sqlite3_vtab_distinct() interface returns an integer that is -** either 0, 1, or 2. The integer returned by sqlite3_vtab_distinct() +** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and +** 3. The integer returned by sqlite3_vtab_distinct() ** gives the virtual table additional information about how the query ** planner wants the output to be ordered. As long as the virtual table ** can meet the ordering requirements of the query planner, it may set @@ -9588,6 +9588,13 @@ SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** that have the same value for all columns identified by "aOrderBy". ** ^However omitting the extra rows is optional. ** This mode is used for a DISTINCT query. +**
+** ^(If the sqlite3_vtab_distinct() interface returns 3, that means +** that the query planner needs only distinct rows but it does need the +** rows to be sorted.)^ ^The virtual table implementation is free to omit +** rows that are identical in all aOrderBy columns, if it wants to, but +** it is not required to omit any rows. This mode is used for queries +** that have both DISTINCT and ORDER BY clauses. ** ** ** ^For the purposes of comparing virtual table output values to see if the diff --git a/src/where.c b/src/where.c index 83bd63fc30..bfe9980a18 100644 --- a/src/where.c +++ b/src/where.c @@ -1212,8 +1212,10 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n ){ nOrderBy = n; - if( (pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY)) ){ - eDistinct = 1 + ((pWInfo->wctrlFlags & WHERE_DISTINCTBY)!=0); + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ + eDistinct = 1; } } } @@ -3769,9 +3771,7 @@ int sqlite3_vtab_rhs_value( */ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; - assert( pHidden->eDistinct==0 - || pHidden->eDistinct==1 - || pHidden->eDistinct==2 ); + assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 ); return pHidden->eDistinct; } @@ -4230,7 +4230,9 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered && (wctrlFlags & WHERE_DISTINCTBY)==0 ){ + if( pLoop->u.vtab.isOrdered + && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) + ){ obSat = obDone; } break; @@ -4497,7 +4499,7 @@ static i8 wherePathSatisfiesOrderBy( ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 */ int sqlite3WhereIsSorted(WhereInfo *pWInfo){ - assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) ); assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); return pWInfo->sorted; } @@ -4898,12 +4900,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } pWInfo->bOrderedInnerLoop = 0; if( pWInfo->pOrderBy ){ + pWInfo->nOBSat = pFrom->isOrdered; if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } }else{ - pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ pWInfo->nOBSat = 0; diff --git a/test/bestindex8.test b/test/bestindex8.test index 6816400771..e95c3c6dc2 100644 --- a/test/bestindex8.test +++ b/test/bestindex8.test @@ -85,7 +85,7 @@ foreach {tn sql bDistinct idxinsert bConsumed res} { 4 "SELECT DISTINCT b FROM vt1" 2 1 0 {b d} 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 0 1 1 {b d} 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 0 1 1 {1 0} - 7 "SELECT DISTINCT a, b FROM vt1 ORDER BY a, b" 1 0 1 {a b c d} + 7 "SELECT DISTINCT a, b FROM vt1 ORDER BY a, b" 3 0 1 {a b c d} 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 0 1 1 {a b c d} 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 0 1 1 {a c} diff --git a/test/bestindex9.test b/test/bestindex9.test new file mode 100644 index 0000000000..94b9da0d38 --- /dev/null +++ b/test/bestindex9.test @@ -0,0 +1,108 @@ +# 2020-01-29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindex9 + +ifcapable !vtab { + finish_test + return +} + + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return $::create_table + } + + xBestIndex { + set hdl [lindex $args 0] + set ::vtab_orderby [$hdl orderby] + set ::vtab_distinct [$hdl distinct] + + return [list orderby 1] + } + + xFilter { + return "" + } + } + + return {} +} + +proc do_bestindex9_test {tn create select orderby distinct} { + forcedelete test.db2 + sqlite3 dbtest test.db2 + register_tcl_module dbtest + + set ::create_table $create + dbtest eval { CREATE VIRTUAL TABLE t1 USING tcl(vtab_command); } + dbtest eval $select + + do_test $tn.orderby [list set {} $::vtab_orderby] $orderby + do_test $tn.distinct [list set {} $::vtab_distinct] $distinct + + dbtest close +} + +#------------------------------------------------------------------------- +# +do_bestindex9_test 1.0 { + CREATE TABLE x(k1, k2, v1, PRIMARY KEY(k1, k2)) +} { + SELECT DISTINCT k1, k2 FROM t1 +} {{column 0 desc 0} {column 1 desc 0}} 2 + +do_bestindex9_test 1.1 { + CREATE TABLE x(k1, k2, v1, PRIMARY KEY(k1, k2)) WITHOUT ROWID +} { + SELECT DISTINCT k1, k2 FROM t1 +} {} 0 + +do_bestindex9_test 1.2 { + CREATE TABLE x(k1 NOT NULL, k2 NOT NULL, v1, PRIMARY KEY(k1, k2)) +} { + SELECT DISTINCT k1, k2 FROM t1 +} {} 0 + +#------------------------------------------------------------------------- +# +do_bestindex9_test 2 { + CREATE TABLE x(c1, c2, c3); +} { + SELECT DISTINCT c1 FROM t1 ORDER BY c1 +} {{column 0 desc 0}} 3 + +#------------------------------------------------------------------------- +# +do_bestindex9_test 3 { + CREATE TABLE x(c1, c2, c3); +} { + SELECT DISTINCT c1 FROM t1 GROUP BY c1 +} {{column 0 desc 0}} 1 + +do_bestindex9_test 4 { + CREATE TABLE x(c1, c2, c3); +} { + CREATE TABLE t2(balls); + SELECT DISTINCT c1 FROM t1, t2 +} {{column 0 desc 0}} 2 + + +finish_test + + + + diff --git a/test/distinct.test b/test/distinct.test index a3cb1afc8a..446f85bb8e 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -301,4 +301,52 @@ do_execsql_test 8.0 { SELECT DISTINCT pid FROM person where pid = 10; } {10} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES('a', 'a'); + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('a', 'c'); + + INSERT INTO t1 VALUES('b', 'a'); + INSERT INTO t1 VALUES('b', 'b'); + INSERT INTO t1 VALUES('b', 'c'); + + INSERT INTO t1 VALUES('a', 'a'); + INSERT INTO t1 VALUES('b', 'b'); + + INSERT INTO t1 VALUES('A', 'A'); + INSERT INTO t1 VALUES('B', 'B'); +} + +foreach {tn idx} { + 1 { } + 2 { CREATE INDEX i1 ON t1(a, b); } + 3 { CREATE INDEX i1 ON t1(b, a); } + 4 { CREATE INDEX i1 ON t1(a COLLATE nocase, b COLLATE nocase); } + 5 { CREATE INDEX i1 ON t1(b COLLATE nocase, a COLLATE nocase); } +} { + + execsql { DROP INDEX IF EXISTS i1 } + execsql $idx + + do_execsql_test 9.$tn.1 { + SELECT DISTINCT a, b FROM t1 ORDER BY a, b + } { + A A B B + a a a b a c + b a b b b c + } + + do_execsql_test 9.$tn.1 { + SELECT DISTINCT a COLLATE nocase, b COLLATE nocase FROM t1 + ORDER BY a COLLATE nocase, b COLLATE nocase + } { + a a a b a c + b a b b b c + } +} + + finish_test