From ec778d27241d54be6967a2d1eb44cf549507f728 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jan 2022 00:18:01 +0000 Subject: [PATCH 1/4] An initial attempt to implement sqlite3_vtab_distinct(). FossilOrigin-Name: d571262d2345bb11e71bef395cf078e5d7303b974b38b4e319adda6194ccc1c5 --- ext/misc/qpvtab.c | 2 ++ manifest | 24 ++++++++++++++---------- manifest.uuid | 2 +- src/loadext.c | 1 + src/sqlite.h.in | 22 ++++++++++++++++++++++ src/sqlite3ext.h | 2 ++ src/where.c | 28 +++++++++++++++++++++++----- test/vtabdistinct.test | 36 ++++++++++++++++++++++++++++++++++++ 8 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 test/vtabdistinct.test diff --git a/ext/misc/qpvtab.c b/ext/misc/qpvtab.c index 6c145ffd17..828a5ba1b8 100644 --- a/ext/misc/qpvtab.c +++ b/ext/misc/qpvtab.c @@ -372,6 +372,8 @@ static int qpvtabBestIndex( pIdxInfo->aOrderBy[i].desc ); } + sqlite3_str_appendf(pStr, "sqlite3_vtab_distinct,%d,,,,\n", + sqlite3_vtab_distinct(pIdxInfo)); sqlite3_str_appendf(pStr, "idxFlags,%d,,,,\n", pIdxInfo->idxFlags); sqlite3_str_appendf(pStr, "colUsed,%d,,,,\n", (int)pIdxInfo->colUsed); pIdxInfo->estimatedCost = (double)10; diff --git a/manifest b/manifest index 577199e3ba..cc18282832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sevidence\smarks\sfor\sthe\slatest\schanges\sto\sthe\sdocumentation. -D 2022-01-21T19:26:18.123 +C An\sinitial\sattempt\sto\simplement\ssqlite3_vtab_distinct(). +D 2022-01-22T00:18:01.617 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -316,7 +316,7 @@ F ext/misc/noop.c 81efe4cad9ec740e64388b14281cb983e6e2c223fed43eb77ab3e34946e0c1 F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 -F ext/misc/qpvtab.c d40b07a8c341d16629010c3e8d8801f1477f70d6257e052973a8ddf14c07cd8d +F ext/misc/qpvtab.c c662fa0a452ad286e49b6c83ac917600656b2eb47d2225ff6185c56bf80cf8d2 F ext/misc/regexp.c b267fd05ff8d38b22f4c2809d7b7a2c61d522e9faf2feb928dbb9662e4a3a386 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c @@ -515,7 +515,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c e528416ff5d86fc5d656ea6a26f03fde39836b6175f93048c32a03cb2ee16743 F src/json.c 78fdec9af3a8bfb5ae685707b2701276fec1942b8f5f26689b2701debe32bcd2 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 9a693eb89575af5d54b025c1e927b2ea8114bbeee3db346e8abc81676dd82160 +F src/loadext.c 657534339585ac234839e5187aa51d8802f292e0771c4f874b3af1f1223f81e2 F src/main.c 2b6b0dbfeb14d4bb57e368604b0736b2aa42b51b00339d399b01d6b1fc9b4960 F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -554,9 +554,9 @@ F src/resolve.c 359bc0e445d427583d2ab6110433a5dc777f64a0ecdf8d24826d8b475233ead9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab5717255420972e69b9b9ce4d1c4730fe82cfbdc14b7743e389a8bdb79ca027 F src/shell.c.in 4690f216dc4da0c104a8fd9f9e12bec0483242e630324aa7a3ccd155922e346e -F src/sqlite.h.in 67e49a50bb29e3af86d2b3606af6b3fe20a0d09cf326cc574d632d878c0696d6 +F src/sqlite.h.in 1335c91f118ed8490ab8661becf8ad7d29fe531149464d72ccc095290c872e86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h 234b5ff5c20512a116b14d6d08e23caeb68667749f8a94117779a9d38afc7e5c +F src/sqlite3ext.h 5d54cf13d3406d8eb65d921a0d3c349de6126b732e695e79ecd4830ce86b4f8a F src/sqliteInt.h 21a31abf60222f50c1d654cdc27ad9d4040249f0341129dd8286b8b5b32bcd30 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 @@ -639,7 +639,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c eb54405957c295e0706cbaea74f9e7cbb3d3dfdefe35d243440d2b4cbf601f0c +F src/where.c 7df767249645ba428b8a217278319028c679f48f2daecac8461ae6e2d281cd6d F src/whereInt.h 91865afa4a3540bb3bd643619acc56fbceff7defeb8f249b8e157fd5325d88be F src/wherecode.c a0a5138b28550dd95916435283c507aa6bf24607ef38a0a18011e6626684330a F src/whereexpr.c 9f64c39e53070584e99e4d20c1dd3397e125fabbae8fd414ffec574c410ac7d3 @@ -1724,6 +1724,7 @@ F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3af F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad +F test/vtabdistinct.test e35a8c91cd9e2a1130eb708376e33aa38511b880fb7c1691387d3e9ec57d9157 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test c138346be341916ecc9d918dcfc2657d27bce211a350a82b01d62d224b167b56 F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 @@ -1940,8 +1941,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2f5dc7a9eed89baf6814e9e123354b262c806c853dee1243c93286c564b9aba8 -R fe7b9dd51502fbf713e5aec593679235 +P 19247e919fab9748cae561cb12c4c3c106064390a37e32e724d9a9066cfaff8e +R dfc70e19d986a79e444a3389314d1aa6 +T *branch * sqlite3_vtab_distinct +T *sym-sqlite3_vtab_distinct * +T -sym-trunk * U drh -Z 6c89130f8a98285b65b9e81e57d8e5fd +Z 502f02fc5994424f0d3cd19482ecaf51 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7a58964939..7744a710a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19247e919fab9748cae561cb12c4c3c106064390a37e32e724d9a9066cfaff8e \ No newline at end of file +d571262d2345bb11e71bef395cf078e5d7303b974b38b4e319adda6194ccc1c5 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index c917e11c8e..a4aad7e748 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -488,6 +488,7 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.38.0 and later */ sqlite3_error_offset, sqlite3_vtab_rhs_value, + sqlite3_vtab_distinct, }; /* True if x is the directory separator character diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ca27742cb1..1e78911129 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9503,6 +9503,28 @@ int sqlite3_vtab_nochange(sqlite3_context*); */ SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +/* +** CAPI3REF: Determine if virtual table query is DISTINCT +** METHOD: sqlite3_index_info +** +** This API may only be used from within an xBestIndex() callback. The +** results of calling it from outside of an xBestIndex() callback are +** undefined and probably harmful. +** +** ^The sqlite3_vtab_distinct() returns an integer that is either 0, 1, or +** 2. ^The sqlite3_vtab_distinct() interface returns 0 if SQLite needs the +** output rows sorted as defined by the nOrderBy and aOrderBy fields of the +** [sqlite3_index_info] object P. ^The sqlite3_vtab_distinct() interface +** returns 1 if SQLite needs all rows to be returned and for equivalent rows +** to be adjacent to one another, but if it does not require that the rows +** be sorted. ^The sqlite3_vtab_distinct() interface returns 2 if SQLite only +** needs the virtual table to return rows that are distinct for the columns +** identified by the nOrderBy and aOrderBy fields of the [sqlite3_index_info] +** object. If rows are returned that differ in any of the columns identified +** by the nOrderBy and aOrderBy fields, then all such rows should be adjacent. +*/ +int sqlite3_vtab_distinct(sqlite3_index_info*); + /* ** CAPI3REF: Constraint values in xBestIndex() ** METHOD: sqlite3_index_info diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 580078b6d0..88010b9d8d 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -347,6 +347,7 @@ struct sqlite3_api_routines { /* Version 3.38.0 and later */ int (*error_offset)(sqlite3*); int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**); + int (*vtab_distinct)(sqlite3_index_info*); }; /* @@ -661,6 +662,7 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.38.0 and later */ #define sqlite3_error_offset sqlite3_api->error_offset #define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value +#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/where.c b/src/where.c index 3fa7a921de..fe262b36e1 100644 --- a/src/where.c +++ b/src/where.c @@ -32,6 +32,7 @@ typedef struct HiddenIndexInfo HiddenIndexInfo; struct HiddenIndexInfo { WhereClause *pWC; /* The Where clause being analyzed */ Parse *pParse; /* The parsing context */ + int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST ** because extra space is allocated to hold up ** to nTerm such values */ @@ -1101,15 +1102,15 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( ** by passing the pointer returned by this function to freeIndexInfo(). */ static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, /* The parsing context */ + WhereInfo *pWInfo, /* The WHERE clause */ WhereClause *pWC, /* The WHERE clause being analyzed */ Bitmask mUnusable, /* Ignore terms with these prereqs */ SrcItem *pSrc, /* The FROM clause term that is the vtab */ - ExprList *pOrderBy, /* The ORDER BY clause */ u16 *pmNoOmit /* Mask of terms not to omit */ ){ int i, j; int nTerm; + Parse *pParse = pWInfo->pParse; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_orderby *pIdxOrderBy; struct sqlite3_index_constraint_usage *pUsage; @@ -1119,7 +1120,9 @@ static sqlite3_index_info *allocateIndexInfo( sqlite3_index_info *pIdxInfo; u16 mNoOmit = 0; const Table *pTab; - + int eDistinct = 0; + ExprList *pOrderBy = pWInfo->pOrderBy; + assert( pSrc!=0 ); pTab = pSrc->pTab; assert( pTab!=0 ); @@ -1203,6 +1206,9 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n){ nOrderBy = n; + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + eDistinct = 1; + } } } @@ -1225,6 +1231,7 @@ static sqlite3_index_info *allocateIndexInfo( pIdxInfo->aConstraintUsage = pUsage; pHidden->pWC = pWC; pHidden->pParse = pParse; + pHidden->eDistinct = eDistinct; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ u16 op; if( (pTerm->wtFlags & TERM_OK)==0 ) continue; @@ -3691,6 +3698,18 @@ int sqlite3_vtab_rhs_value( return rc; } + +/* +** Return true if ORDER BY clause may be handled as DISTINCT. +*/ +int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + assert( pHidden->eDistinct==0 + || pHidden->eDistinct==1 + || pHidden->eDistinct==2 ); + return pHidden->eDistinct; +} + /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. @@ -3740,8 +3759,7 @@ static int whereLoopAddVirtual( pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; assert( IsVirtual(pSrc->pTab) ); - p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, - &mNoOmit); + p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; diff --git a/test/vtabdistinct.test b/test/vtabdistinct.test new file mode 100644 index 0000000000..63fa1569d3 --- /dev/null +++ b/test/vtabdistinct.test @@ -0,0 +1,36 @@ +# 2022-01-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for sqlite3_vtab_distinct() interface. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix vtabdistinct + +ifcapable !vtab { + finish_test + return +} +load_static_extension db qpvtab + +do_execsql_test 1.1 { + SELECT ix FROM qpvtab WHERE vn='sqlite3_vtab_distinct'; +} {0} +do_execsql_test 1.2 { + SELECT DISTINCT ix FROM qpvtab WHERE vn='sqlite3_vtab_distinct'; +} {1} +do_execsql_test 1.3 { + SELECT distinct vn, ix FROM qpvtab(3) + WHERE +vn IN ('sqlite3_vtab_distinct','nOrderBy'); +} {nOrderBy 2 sqlite3_vtab_distinct 1} + +finish_test From 57c58b4de8a9e0b41a683e9a1d8b6bb3fc31bac2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jan 2022 02:52:22 +0000 Subject: [PATCH 2/4] Omit the WhereLoopBuilder.pOrderBy field, which is no longer needed. FossilOrigin-Name: a13afc909c8bb643aa154b39ba8c023bae7352d3cd7cfb96be3891fa0e4bc045 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 2 -- src/whereInt.h | 1 - 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index cc18282832..b4574239e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\sinitial\sattempt\sto\simplement\ssqlite3_vtab_distinct(). -D 2022-01-22T00:18:01.617 +C Omit\sthe\sWhereLoopBuilder.pOrderBy\sfield,\swhich\sis\sno\slonger\sneeded. +D 2022-01-22T02:52:22.316 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -639,8 +639,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 7df767249645ba428b8a217278319028c679f48f2daecac8461ae6e2d281cd6d -F src/whereInt.h 91865afa4a3540bb3bd643619acc56fbceff7defeb8f249b8e157fd5325d88be +F src/where.c bf15b922a134948a56a078e0a01d1e63f9f8ff59f6503cba241855f8cb961b4f +F src/whereInt.h 8a215acde0f833a4dea3d30a7bbed9f48b4b547b5d5e34cd02acee366476ab80 F src/wherecode.c a0a5138b28550dd95916435283c507aa6bf24607ef38a0a18011e6626684330a F src/whereexpr.c 9f64c39e53070584e99e4d20c1dd3397e125fabbae8fd414ffec574c410ac7d3 F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e @@ -1941,11 +1941,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 19247e919fab9748cae561cb12c4c3c106064390a37e32e724d9a9066cfaff8e -R dfc70e19d986a79e444a3389314d1aa6 -T *branch * sqlite3_vtab_distinct -T *sym-sqlite3_vtab_distinct * -T -sym-trunk * +P d571262d2345bb11e71bef395cf078e5d7303b974b38b4e319adda6194ccc1c5 +R c8f2432bbe6ecf01b6e5ac0aa57d1f94 U drh -Z 502f02fc5994424f0d3cd19482ecaf51 +Z 0649a152fd9646f6591062a2384e3001 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7744a710a7..c378259f5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d571262d2345bb11e71bef395cf078e5d7303b974b38b4e319adda6194ccc1c5 \ No newline at end of file +a13afc909c8bb643aa154b39ba8c023bae7352d3cd7cfb96be3891fa0e4bc045 \ No newline at end of file diff --git a/src/where.c b/src/where.c index fe262b36e1..c8e7d906b9 100644 --- a/src/where.c +++ b/src/where.c @@ -3891,7 +3891,6 @@ static int whereLoopAddOr( int i, j; sSubBuild = *pBuilder; - sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); @@ -5283,7 +5282,6 @@ WhereInfo *sqlite3WhereBegin( /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; - sWLB.pOrderBy = pOrderBy; /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask diff --git a/src/whereInt.h b/src/whereInt.h index a97b4afc69..1304a40a77 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -386,7 +386,6 @@ struct WhereMaskSet { struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ WhereClause *pWC; /* WHERE clause terms */ - ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ #ifdef SQLITE_ENABLE_STAT4 From 9b9fc74f2eafeb8a280198ee7209c8ccefe6a108 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jan 2022 19:19:35 +0000 Subject: [PATCH 3/4] Iimproved documentation for sqlite3_vtab_distinct(). No changes to code. FossilOrigin-Name: 7af03f02940b5380ee7375672ca8d0ff68c5f741d0ea206911631f3eb5a78555 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/sqlite.h.in | 72 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index b4574239e7..00218ad384 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sWhereLoopBuilder.pOrderBy\sfield,\swhich\sis\sno\slonger\sneeded. -D 2022-01-22T02:52:22.316 +C Iimproved\sdocumentation\sfor\ssqlite3_vtab_distinct().\s\sNo\schanges\sto\scode. +D 2022-01-22T19:19:35.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -554,7 +554,7 @@ F src/resolve.c 359bc0e445d427583d2ab6110433a5dc777f64a0ecdf8d24826d8b475233ead9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab5717255420972e69b9b9ce4d1c4730fe82cfbdc14b7743e389a8bdb79ca027 F src/shell.c.in 4690f216dc4da0c104a8fd9f9e12bec0483242e630324aa7a3ccd155922e346e -F src/sqlite.h.in 1335c91f118ed8490ab8661becf8ad7d29fe531149464d72ccc095290c872e86 +F src/sqlite.h.in 31c2c8d737814369bd3b71f3849c4a97ef7ede0aa3ce976ecb11632fa5f1f863 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 5d54cf13d3406d8eb65d921a0d3c349de6126b732e695e79ecd4830ce86b4f8a F src/sqliteInt.h 21a31abf60222f50c1d654cdc27ad9d4040249f0341129dd8286b8b5b32bcd30 @@ -1941,8 +1941,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d571262d2345bb11e71bef395cf078e5d7303b974b38b4e319adda6194ccc1c5 -R c8f2432bbe6ecf01b6e5ac0aa57d1f94 +P a13afc909c8bb643aa154b39ba8c023bae7352d3cd7cfb96be3891fa0e4bc045 +R ee7e69afa00103e3a528aaa0afa4a945 U drh -Z 0649a152fd9646f6591062a2384e3001 +Z f9808520490668283a6ca50de433adb3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c378259f5f..fb902954b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13afc909c8bb643aa154b39ba8c023bae7352d3cd7cfb96be3891fa0e4bc045 \ No newline at end of file +7af03f02940b5380ee7375672ca8d0ff68c5f741d0ea206911631f3eb5a78555 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1e78911129..4b36233b37 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9471,13 +9471,14 @@ int sqlite3_vtab_nochange(sqlite3_context*); /* ** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** METHOD: sqlite3_index_info ** ** This function may only be called from within a call to the [xBestIndex] ** method of a [virtual table]. This function returns a pointer to a string ** that is the name of the appropriate collation sequence to use for text ** comparisons on the constraint identified by its arguments. ** -** The first argument must be the pointer to the sqlite3_index_info object +** The first argument must be the pointer to the [sqlite3_index_info] object ** that is the first parameter to the xBestIndex() method. The second argument ** must be an index into the aConstraint[] array belonging to the ** sqlite3_index_info structure passed to xBestIndex. @@ -9485,7 +9486,7 @@ int sqlite3_vtab_nochange(sqlite3_context*); ** Important: ** The first parameter must be the same pointer that is passed into the ** xBestMethod() method. The first parameter may not be a pointer to a -** different sqlite3_index_info object, even an exact copy. +** different [sqlite3_index_info] object, even an exact copy. ** ** The return value is computed as follows: ** @@ -9504,7 +9505,7 @@ int sqlite3_vtab_nochange(sqlite3_context*); SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); /* -** CAPI3REF: Determine if virtual table query is DISTINCT +** CAPI3REF: Determine if a virtual table query is DISTINCT ** METHOD: sqlite3_index_info ** ** This API may only be used from within an xBestIndex() callback. The @@ -9512,16 +9513,59 @@ SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** undefined and probably harmful. ** ** ^The sqlite3_vtab_distinct() returns an integer that is either 0, 1, or -** 2. ^The sqlite3_vtab_distinct() interface returns 0 if SQLite needs the -** output rows sorted as defined by the nOrderBy and aOrderBy fields of the -** [sqlite3_index_info] object P. ^The sqlite3_vtab_distinct() interface -** returns 1 if SQLite needs all rows to be returned and for equivalent rows -** to be adjacent to one another, but if it does not require that the rows -** be sorted. ^The sqlite3_vtab_distinct() interface returns 2 if SQLite only -** needs the virtual table to return rows that are distinct for the columns -** identified by the nOrderBy and aOrderBy fields of the [sqlite3_index_info] -** object. If rows are returned that differ in any of the columns identified -** by the nOrderBy and aOrderBy fields, then all such rows should be adjacent. +** 2. The integer returned by sqlite3_vtab_distinct() gives the virtual table +** additional information about how the query planner wants the output to be +** ordered. As long as the virtual table can meet the ordering requirements +** of the query planner, it may set the "orderByConsumed" flag. +** +**

  1. +** ^If the sqlite3_vtab_distinct() interface returns 0, that means +** that the query planner needs the virtual table to return all rows in the +** sort order defined by the "nOrderBy" and "aOrderBy" fields of the +** [sqlite3_index_info] object. This is the default expectation. If the +** virtual table outputs all rows in sorted order, then it is always safe for +** the xBestIndex method to set the "orderByConsumed" flag, regardless of +** what the return value from sqlite3_vtab_distinct(). +**

  2. +** ^(If the sqlite3_vtab_distinct() interface returns 1, that means +** that the query planner does not need the rows to be returned in sorted order +** as long as all rows with the same values in all columns identified by the +** "aOrderBy" field are adjacent.)^ This mode is used when the query planner +** is doing a GROUP BY. +**

  3. +** ^(If the sqlite3_vtab_distinct() interface returns 2, that means +** that the query planner does not need the rows returned in any particular +** order, as long as rows with the same values in all "aOrderBy" columns +** are adjacent.)^ ^(Furthermore, only a single row for each particular +** combination of values in the columns identified by the "aOrderBy" field +** needs to be returned.)^ ^It is ok always for two or more rows with the same +** values in all "aOrderBy" columns to be returned, as long as all such rows +** are adjacent. ^The virtual table may, if it chooses, omit extra rows +** that have the same value for all columns identified by "aOrderBy". +** ^However omitting the extra rows is optional. +** This mode is used for a DISTINCT query. +**

+** +** ^For the purposes of comparing virtual table output values to see if the +** values are same value for sorting purposes, two NULL values are considered +** to be the same. In other words, the comparison operator is "IS" +** (or "IS NOT DISTINCT FROM") and not "==". +** +** If a virtual table implementation is unable to meet the requirements +** specified above, then it must not set the "orderByConsumed" flag in the +** [sqlite3_index_info] object or an incorrect answer may result. +** +** ^A virtual table implementation is always free to return rows in any order +** it wants, as long as the "orderByConsumed" flag is not set. ^When the +** the "orderByConsumed" flag is unset, the query planner will add extra +** [bytecode] to ensure that the final results returned by the SQL query are +** ordered correctly. The use of the "orderByConsumed" flag and the +** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful +** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed" +** flag might help queries against a virtual table to run faster. Being +** overly aggressive and setting the "orderByConsumed" flag when it is not +** valid to do so, on the other hand, might cause SQLite to return incorrect +** results. */ int sqlite3_vtab_distinct(sqlite3_index_info*); @@ -9535,7 +9579,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info*); ** ** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within ** the [xBestIndex] method of a [virtual table] implementation, with P being -** a copy of the sqlite3_index_info object pointer passed into xBestIndex and +** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and ** J being a 0-based index into P->aConstraint[], then this routine ** attempts to set *V to be the value on the right-hand side of ** that constraint if the right-hand side is a known constant. ^If the From 68dc8151e092019baba5a190e195a982252b925f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jan 2022 20:45:57 +0000 Subject: [PATCH 4/4] Bring sqlite3_vtab_distinct() up to spec so that it works as described in the documentation. FossilOrigin-Name: 4289edf3c5e32a05b51f232020099b33f6f5e79b0ceca2b96baf1186168d9af6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 4 ++-- test/vtabdistinct.test | 9 +++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 00218ad384..882cfbc89f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Iimproved\sdocumentation\sfor\ssqlite3_vtab_distinct().\s\sNo\schanges\sto\scode. -D 2022-01-22T19:19:35.721 +C Bring\ssqlite3_vtab_distinct()\sup\sto\sspec\sso\sthat\sit\sworks\sas\sdescribed\sin\sthe\ndocumentation. +D 2022-01-22T20:45:57.210 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -639,7 +639,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c bf15b922a134948a56a078e0a01d1e63f9f8ff59f6503cba241855f8cb961b4f +F src/where.c 9824d15f2a92cbe4d6566e2a97fa4466ce6297e68693322e64281db5e942fa92 F src/whereInt.h 8a215acde0f833a4dea3d30a7bbed9f48b4b547b5d5e34cd02acee366476ab80 F src/wherecode.c a0a5138b28550dd95916435283c507aa6bf24607ef38a0a18011e6626684330a F src/whereexpr.c 9f64c39e53070584e99e4d20c1dd3397e125fabbae8fd414ffec574c410ac7d3 @@ -1724,7 +1724,7 @@ F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3af F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad -F test/vtabdistinct.test e35a8c91cd9e2a1130eb708376e33aa38511b880fb7c1691387d3e9ec57d9157 +F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test c138346be341916ecc9d918dcfc2657d27bce211a350a82b01d62d224b167b56 F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 @@ -1941,8 +1941,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a13afc909c8bb643aa154b39ba8c023bae7352d3cd7cfb96be3891fa0e4bc045 -R ee7e69afa00103e3a528aaa0afa4a945 +P 7af03f02940b5380ee7375672ca8d0ff68c5f741d0ea206911631f3eb5a78555 +R 8c92b45cd50ac9ababc3056da5861966 U drh -Z f9808520490668283a6ca50de433adb3 +Z 0f40ebbc8d49180e2de4a825f9a960d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb902954b0..e96d220661 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7af03f02940b5380ee7375672ca8d0ff68c5f741d0ea206911631f3eb5a78555 \ No newline at end of file +4289edf3c5e32a05b51f232020099b33f6f5e79b0ceca2b96baf1186168d9af6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c8e7d906b9..57c4808906 100644 --- a/src/where.c +++ b/src/where.c @@ -1206,8 +1206,8 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n){ nOrderBy = n; - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ - eDistinct = 1; + if( (pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY)) ){ + eDistinct = 1 + ((pWInfo->wctrlFlags & WHERE_DISTINCTBY)!=0); } } } diff --git a/test/vtabdistinct.test b/test/vtabdistinct.test index 63fa1569d3..e18e121e50 100644 --- a/test/vtabdistinct.test +++ b/test/vtabdistinct.test @@ -27,10 +27,15 @@ do_execsql_test 1.1 { } {0} do_execsql_test 1.2 { SELECT DISTINCT ix FROM qpvtab WHERE vn='sqlite3_vtab_distinct'; -} {1} +} {2} do_execsql_test 1.3 { SELECT distinct vn, ix FROM qpvtab(3) WHERE +vn IN ('sqlite3_vtab_distinct','nOrderBy'); -} {nOrderBy 2 sqlite3_vtab_distinct 1} +} {nOrderBy 2 sqlite3_vtab_distinct 2} +do_execsql_test 1.4 { + SELECT vn, ix FROM qpvtab + GROUP BY vn + HAVING vn='sqlite3_vtab_distinct'; +} {sqlite3_vtab_distinct 1} finish_test