database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-11-18 10:21:03 +03:00
Code Activity

Add support for the sqlite3_vtab_distinct() interface. Virtual table

Browse Source 
implementations can use this API to determine more detail about the ordering
requirements needed by the query plan and perhaps reduce the amount of
work required to compute a correct answer.  This is an optimization
opportunity for the virtual table implementation.  The correct answer should
still be obtained (though perhaps more slowly) even if sqlite3_vtab_distinct()
is ignored.

FossilOrigin-Name: e4caf1e3932b1bd0dea072df7fc9458aed98c84ea397b6948b89292603949c41
This commit is contained in:
drh
2022-01-22 22:28:32 +00:00
parent 4d7f335e8b 68dc8151e0
commit bcba3655ee
9 changed files with 152 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ext/misc/qpvtab.c
View File

@@ -372,6 +372,8 @@ static int qpvtabBestIndex(
pIdxInfo->aOrderBy[i].desc
);
}
sqlite3_str_appendf(pStr, "sqlite3_vtab_distinct,%d,,,,\n",
sqlite3_vtab_distinct(pIdxInfo));
sqlite3_str_appendf(pStr, "idxFlags,%d,,,,\n", pIdxInfo->idxFlags);
sqlite3_str_appendf(pStr, "colUsed,%d,,,,\n", (int)pIdxInfo->colUsed);
pIdxInfo->estimatedCost = (double)10;

24
manifest
View File

@@ -1,5 +1,5 @@
C Update\sevidence\smarks\sfor\sthe\slatest\schanges\sto\sthe\sdocumentation.
D 2022-01-21T19:26:18.123
C Add\ssupport\sfor\sthe\ssqlite3_vtab_distinct()\sinterface.\s\sVirtual\stable\nimplementations\scan\suse\sthis\sAPI\sto\sdetermine\smore\sdetail\sabout\sthe\sordering\nrequirements\sneeded\sby\sthe\squery\splan\sand\sperhaps\sreduce\sthe\samount\sof\nwork\srequired\sto\scompute\sa\scorrect\sanswer.\s\sThis\sis\san\soptimization\nopportunity\sfor\sthe\svirtual\stable\simplementation.\s\sThe\scorrect\sanswer\sshould\nstill\sbe\sobtained\s(though\sperhaps\smore\sslowly)\seven\sif\ssqlite3_vtab_distinct()\nis\signored.
D 2022-01-22T22:28:32.461
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -316,7 +316,7 @@ F ext/misc/noop.c 81efe4cad9ec740e64388b14281cb983e6e2c223fed43eb77ab3e34946e0c1
F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f
F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691
F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196
F ext/misc/qpvtab.c d40b07a8c341d16629010c3e8d8801f1477f70d6257e052973a8ddf14c07cd8d
F ext/misc/qpvtab.c c662fa0a452ad286e49b6c83ac917600656b2eb47d2225ff6185c56bf80cf8d2
F ext/misc/regexp.c b267fd05ff8d38b22f4c2809d7b7a2c61d522e9faf2feb928dbb9662e4a3a386
F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c
@@ -515,7 +515,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c e528416ff5d86fc5d656ea6a26f03fde39836b6175f93048c32a03cb2ee16743
F src/json.c 78fdec9af3a8bfb5ae685707b2701276fec1942b8f5f26689b2701debe32bcd2
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 9a693eb89575af5d54b025c1e927b2ea8114bbeee3db346e8abc81676dd82160
F src/loadext.c 657534339585ac234839e5187aa51d8802f292e0771c4f874b3af1f1223f81e2
F src/main.c 2b6b0dbfeb14d4bb57e368604b0736b2aa42b51b00339d399b01d6b1fc9b4960
F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -554,9 +554,9 @@ F src/resolve.c 359bc0e445d427583d2ab6110433a5dc777f64a0ecdf8d24826d8b475233ead9
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c ab5717255420972e69b9b9ce4d1c4730fe82cfbdc14b7743e389a8bdb79ca027
F src/shell.c.in 4690f216dc4da0c104a8fd9f9e12bec0483242e630324aa7a3ccd155922e346e
F src/sqlite.h.in 67e49a50bb29e3af86d2b3606af6b3fe20a0d09cf326cc574d632d878c0696d6
F src/sqlite.h.in 31c2c8d737814369bd3b71f3849c4a97ef7ede0aa3ce976ecb11632fa5f1f863
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 234b5ff5c20512a116b14d6d08e23caeb68667749f8a94117779a9d38afc7e5c
F src/sqlite3ext.h 5d54cf13d3406d8eb65d921a0d3c349de6126b732e695e79ecd4830ce86b4f8a
F src/sqliteInt.h 21a31abf60222f50c1d654cdc27ad9d4040249f0341129dd8286b8b5b32bcd30
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
@@ -639,8 +639,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
F src/where.c eb54405957c295e0706cbaea74f9e7cbb3d3dfdefe35d243440d2b4cbf601f0c
F src/whereInt.h 91865afa4a3540bb3bd643619acc56fbceff7defeb8f249b8e157fd5325d88be
F src/where.c 9824d15f2a92cbe4d6566e2a97fa4466ce6297e68693322e64281db5e942fa92
F src/whereInt.h 8a215acde0f833a4dea3d30a7bbed9f48b4b547b5d5e34cd02acee366476ab80
F src/wherecode.c a0a5138b28550dd95916435283c507aa6bf24607ef38a0a18011e6626684330a
F src/whereexpr.c 9f64c39e53070584e99e4d20c1dd3397e125fabbae8fd414ffec574c410ac7d3
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
@@ -1724,6 +1724,7 @@ F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3af
F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783
F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65
F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072
F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12
F test/vtabrhs1.test c138346be341916ecc9d918dcfc2657d27bce211a350a82b01d62d224b167b56
F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089
@@ -1940,8 +1941,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 2f5dc7a9eed89baf6814e9e123354b262c806c853dee1243c93286c564b9aba8
R fe7b9dd51502fbf713e5aec593679235
P 19247e919fab9748cae561cb12c4c3c106064390a37e32e724d9a9066cfaff8e 4289edf3c5e32a05b51f232020099b33f6f5e79b0ceca2b96baf1186168d9af6
R 8c92b45cd50ac9ababc3056da5861966
T +closed 4289edf3c5e32a05b51f232020099b33f6f5e79b0ceca2b96baf1186168d9af6
U drh
Z 6c89130f8a98285b65b9e81e57d8e5fd
Z 550b4c112b6ad484a8ff339f5683365a
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@@ -1 +1 @@
19247e919fab9748cae561cb12c4c3c106064390a37e32e724d9a9066cfaff8e
e4caf1e3932b1bd0dea072df7fc9458aed98c84ea397b6948b89292603949c41

1
src/loadext.c
View File

@@ -488,6 +488,7 @@ static const sqlite3_api_routines sqlite3Apis = {
/* Version 3.38.0 and later */
sqlite3_error_offset,
sqlite3_vtab_rhs_value,
sqlite3_vtab_distinct,
};
/* True if x is the directory separator character

72
src/sqlite.h.in
View File

@@ -9471,13 +9471,14 @@ int sqlite3_vtab_nochange(sqlite3_context*);
/*
** CAPI3REF: Determine The Collation For a Virtual Table Constraint
** METHOD: sqlite3_index_info
**
** This function may only be called from within a call to the [xBestIndex]
** method of a [virtual table]. This function returns a pointer to a string
** that is the name of the appropriate collation sequence to use for text
** comparisons on the constraint identified by its arguments.
**
** The first argument must be the pointer to the sqlite3_index_info object
** The first argument must be the pointer to the [sqlite3_index_info] object
** that is the first parameter to the xBestIndex() method. The second argument
** must be an index into the aConstraint[] array belonging to the
** sqlite3_index_info structure passed to xBestIndex.
@@ -9485,7 +9486,7 @@ int sqlite3_vtab_nochange(sqlite3_context*);
** Important:
** The first parameter must be the same pointer that is passed into the
** xBestMethod() method. The first parameter may not be a pointer to a
** different sqlite3_index_info object, even an exact copy.
** different [sqlite3_index_info] object, even an exact copy.
**
** The return value is computed as follows:
**
@@ -9503,6 +9504,71 @@ int sqlite3_vtab_nochange(sqlite3_context*);
*/
SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
/*
** CAPI3REF: Determine if a virtual table query is DISTINCT
** METHOD: sqlite3_index_info
**
** This API may only be used from within an xBestIndex() callback. The
** results of calling it from outside of an xBestIndex() callback are
** undefined and probably harmful.
**
** ^The sqlite3_vtab_distinct() returns an integer that is either 0, 1, or
** 2. The integer returned by sqlite3_vtab_distinct() gives the virtual table
** additional information about how the query planner wants the output to be
** ordered. As long as the virtual table can meet the ordering requirements
** of the query planner, it may set the "orderByConsumed" flag.
**
** <ol><li value="0"><p>
** ^If the sqlite3_vtab_distinct() interface returns 0, that means
** that the query planner needs the virtual table to return all rows in the
** sort order defined by the "nOrderBy" and "aOrderBy" fields of the
** [sqlite3_index_info] object. This is the default expectation. If the
** virtual table outputs all rows in sorted order, then it is always safe for
** the xBestIndex method to set the "orderByConsumed" flag, regardless of
** what the return value from sqlite3_vtab_distinct().
** <li value="1"><p>
** ^(If the sqlite3_vtab_distinct() interface returns 1, that means
** that the query planner does not need the rows to be returned in sorted order
** as long as all rows with the same values in all columns identified by the
** "aOrderBy" field are adjacent.)^ This mode is used when the query planner
** is doing a GROUP BY.
** <li value="2"><p>
** ^(If the sqlite3_vtab_distinct() interface returns 2, that means
** that the query planner does not need the rows returned in any particular
** order, as long as rows with the same values in all "aOrderBy" columns
** are adjacent.)^ ^(Furthermore, only a single row for each particular
** combination of values in the columns identified by the "aOrderBy" field
** needs to be returned.)^ ^It is ok always for two or more rows with the same
** values in all "aOrderBy" columns to be returned, as long as all such rows
** are adjacent. ^The virtual table may, if it chooses, omit extra rows
** that have the same value for all columns identified by "aOrderBy".
** ^However omitting the extra rows is optional.
** This mode is used for a DISTINCT query.
** </ol>
**
** ^For the purposes of comparing virtual table output values to see if the
** values are same value for sorting purposes, two NULL values are considered
** to be the same. In other words, the comparison operator is "IS"
** (or "IS NOT DISTINCT FROM") and not "==".
**
** If a virtual table implementation is unable to meet the requirements
** specified above, then it must not set the "orderByConsumed" flag in the
** [sqlite3_index_info] object or an incorrect answer may result.
**
** ^A virtual table implementation is always free to return rows in any order
** it wants, as long as the "orderByConsumed" flag is not set. ^When the
** the "orderByConsumed" flag is unset, the query planner will add extra
** [bytecode] to ensure that the final results returned by the SQL query are
** ordered correctly. The use of the "orderByConsumed" flag and the
** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful
** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed"
** flag might help queries against a virtual table to run faster. Being
** overly aggressive and setting the "orderByConsumed" flag when it is not
** valid to do so, on the other hand, might cause SQLite to return incorrect
** results.
*/
int sqlite3_vtab_distinct(sqlite3_index_info*);
/*
** CAPI3REF: Constraint values in xBestIndex()
** METHOD: sqlite3_index_info
@@ -9513,7 +9579,7 @@ SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
**
** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within
** the [xBestIndex] method of a [virtual table] implementation, with P being
** a copy of the sqlite3_index_info object pointer passed into xBestIndex and
** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and
** J being a 0-based index into P->aConstraint[], then this routine
** attempts to set *V to be the value on the right-hand side of
** that constraint if the right-hand side is a known constant. ^If the

2
src/sqlite3ext.h
View File

@@ -347,6 +347,7 @@ struct sqlite3_api_routines {
/* Version 3.38.0 and later */
int (*error_offset)(sqlite3*);
int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
int (*vtab_distinct)(sqlite3_index_info*);
};
/*
@@ -661,6 +662,7 @@ typedef int (*sqlite3_loadext_entry)(
/* Version 3.38.0 and later */
#define sqlite3_error_offset sqlite3_api->error_offset
#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value
#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

30
src/where.c
View File

@@ -32,6 +32,7 @@ typedef struct HiddenIndexInfo HiddenIndexInfo;
struct HiddenIndexInfo {
WhereClause *pWC; /* The Where clause being analyzed */
Parse *pParse; /* The parsing context */
int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
** because extra space is allocated to hold up
** to nTerm such values */
@@ -1101,15 +1102,15 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
** by passing the pointer returned by this function to freeIndexInfo().
*/
static sqlite3_index_info *allocateIndexInfo(
Parse *pParse, /* The parsing context */
WhereInfo *pWInfo, /* The WHERE clause */
WhereClause *pWC, /* The WHERE clause being analyzed */
Bitmask mUnusable, /* Ignore terms with these prereqs */
SrcItem *pSrc, /* The FROM clause term that is the vtab */
ExprList *pOrderBy, /* The ORDER BY clause */
u16 *pmNoOmit /* Mask of terms not to omit */
){
int i, j;
int nTerm;
Parse *pParse = pWInfo->pParse;
struct sqlite3_index_constraint *pIdxCons;
struct sqlite3_index_orderby *pIdxOrderBy;
struct sqlite3_index_constraint_usage *pUsage;
@@ -1119,7 +1120,9 @@ static sqlite3_index_info *allocateIndexInfo(
sqlite3_index_info *pIdxInfo;
u16 mNoOmit = 0;
const Table *pTab;
int eDistinct = 0;
ExprList *pOrderBy = pWInfo->pOrderBy;
assert( pSrc!=0 );
pTab = pSrc->pTab;
assert( pTab!=0 );
@@ -1203,6 +1206,9 @@ static sqlite3_index_info *allocateIndexInfo(
}
if( i==n){
nOrderBy = n;
if( (pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY)) ){
eDistinct = 1 + ((pWInfo->wctrlFlags & WHERE_DISTINCTBY)!=0);
}
}
}
@@ -1225,6 +1231,7 @@ static sqlite3_index_info *allocateIndexInfo(
pIdxInfo->aConstraintUsage = pUsage;
pHidden->pWC = pWC;
pHidden->pParse = pParse;
pHidden->eDistinct = eDistinct;
for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
u16 op;
if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
@@ -3691,6 +3698,18 @@ int sqlite3_vtab_rhs_value(
return rc;
}
/*
** Return true if ORDER BY clause may be handled as DISTINCT.
*/
int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
assert( pHidden->eDistinct==0
|| pHidden->eDistinct==1
|| pHidden->eDistinct==2 );
return pHidden->eDistinct;
}
/*
** Add all WhereLoop objects for a table of the join identified by
** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
@@ -3740,8 +3759,7 @@ static int whereLoopAddVirtual(
pNew = pBuilder->pNew;
pSrc = &pWInfo->pTabList->a[pNew->iTab];
assert( IsVirtual(pSrc->pTab) );
p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy,
&mNoOmit);
p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
if( p==0 ) return SQLITE_NOMEM_BKPT;
pNew->rSetup = 0;
pNew->wsFlags = WHERE_VIRTUALTABLE;
@@ -3873,7 +3891,6 @@ static int whereLoopAddOr(
int i, j;
sSubBuild = *pBuilder;
sSubBuild.pOrderBy = 0;
sSubBuild.pOrSet = &sCur;
WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm));
@@ -5265,7 +5282,6 @@ WhereInfo *sqlite3WhereBegin(
/* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
sWLB.pOrderBy = pOrderBy;
/* The number of tables in the FROM clause is limited by the number of
** bits in a Bitmask

1
src/whereInt.h
View File

@@ -386,7 +386,6 @@ struct WhereMaskSet {
struct WhereLoopBuilder {
WhereInfo *pWInfo; /* Information about this WHERE */
WhereClause *pWC; /* WHERE clause terms */
ExprList *pOrderBy; /* ORDER BY clause */
WhereLoop *pNew; /* Template WhereLoop */
WhereOrSet *pOrSet; /* Record best loops here, if not NULL */
#ifdef SQLITE_ENABLE_STAT4

41
test/vtabdistinct.test Normal file
View File

@@ -0,0 +1,41 @@
# 2022-01-21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file implements tests for sqlite3_vtab_distinct() interface.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix vtabdistinct
ifcapable !vtab {
finish_test
return
}
load_static_extension db qpvtab
do_execsql_test 1.1 {
SELECT ix FROM qpvtab WHERE vn='sqlite3_vtab_distinct';
} {0}
do_execsql_test 1.2 {
SELECT DISTINCT ix FROM qpvtab WHERE vn='sqlite3_vtab_distinct';
} {2}
do_execsql_test 1.3 {
SELECT distinct vn, ix FROM qpvtab(3)
WHERE +vn IN ('sqlite3_vtab_distinct','nOrderBy');
} {nOrderBy 2 sqlite3_vtab_distinct 2}
do_execsql_test 1.4 {
SELECT vn, ix FROM qpvtab
GROUP BY vn
HAVING vn='sqlite3_vtab_distinct';
} {sqlite3_vtab_distinct 1}
finish_test