database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-11-11 01:42:22 +03:00
Code Activity

Run as many Bloom filters as possible before index lookups.

Browse Source 
FossilOrigin-Name: 06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119
This commit is contained in:
drh
2021-12-05 20:19:47 +00:00
parent 35685d3e5e
commit 6ae49e67cc
6 changed files with 158 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
manifest
View File

@@ -1,5 +1,5 @@
C Try\sto\srun\sall\sBloom\sfilters\sbefore\sany\sSeeks.\s\sThis\sgives\sa\ssmall\sperformance\ngain\son\sthe\sStar-Schema\sBenchmark.
D 2021-12-05T00:45:55.528
C Run\sas\smany\sBloom\sfilters\sas\spossible\sbefore\sindex\slookups.
D 2021-12-05T20:19:47.744
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -555,7 +555,7 @@ F src/shell.c.in e7ee6517544d075d9f06ee2571567026b89cf9fbeef16a74918019b1cb42576
F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
F src/sqliteInt.h 4a19ff088cc3bb1467cca94011eaf7e8432476124bbb72c77053d9f1e6b0a6c7
F src/sqliteInt.h 178eb899c1edc08dcddf37e79dfaa39404a1f5d44a1d512509cd5d41867aa836
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
F src/where.c 3d29b27e345a28f76ecb20b0c789a224e9dc74534363c3ddbc44c283eb4cb708
F src/whereInt.h 3c634e184de128d99c2b017f899f2309c77330de54193ff970597e9acdeaff3b
F src/wherecode.c 48b456c910d552e1edf098071546c523837356ba0e3703d7b8fd9fc819da1c9b
F src/where.c b07c5eefecffa1b69b91c366a83c69d01a83f1c900b9d9b1ffb6eb5ab59902a1
F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
R 8ad2d75d69a2ffc06bdf1b52eb7537e1
P 5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
R da3cb867f9ab99abba060de93457c8d9
U drh
Z 42400fd1219d022ab29b577e6d964b89
Z d87a509afa829d1cf21b5a6dcadef441

2
manifest.uuid
View File

@@ -1 +1 @@
5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119

1
src/sqliteInt.h
View File

@@ -1762,6 +1762,7 @@ struct sqlite3 {
#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */
/* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */
#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */
#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
/*

67
src/where.c
View File

@@ -966,30 +966,52 @@ end_auto_index_create:
#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
/*
** Create a Bloom filter for the WhereLevel in the parameter.
** Generate bytecode that will initialize a Bloom filter that is appropriate
** for pLevel.
**
** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
** flag set, initialize a Bloomfilter for them as well. Except don't do
** this recursive initialization if the SQLITE_BloomPulldown optimization has
** been turned off.
**
** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
** from the loop, but the regFilter value is set to a register that implements
** the Bloom filter. When regFilter is positive, the
** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
** and skip the subsequence B-Tree seek if the Bloom filter indicates that
** no matching rows exist.
**
** This routine may only be called if it has previously been determined that
** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
** is set.
*/
SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
const WhereInfo *pWInfo, /* The WHERE clause */
static SQLITE_NOINLINE void constructBloomFilter(
WhereInfo *pWInfo, /* The WHERE clause */
int iLevel, /* Index in pWInfo->a[] that is pLevel */
WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
){
int addrTop;
int addrCont;
const WhereTerm *pTerm;
const WhereTerm *pWCEnd;
Parse *pParse = pWInfo->pParse;
Vdbe *v = pParse->pVdbe;
const WhereLoop *pLoop = pLevel->pWLoop;
int iCur;
int addrOnce; /* Address of opening OP_Once */
int addrTop; /* Address of OP_Rewind */
int addrCont; /* Jump here to skip a row */
const WhereTerm *pTerm; /* For looping over WHERE clause terms */
const WhereTerm *pWCEnd; /* Last WHERE clause term */
Parse *pParse = pWInfo->pParse; /* Parsing context */
Vdbe *v = pParse->pVdbe; /* VDBE under construction */
WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
int iCur; /* Cursor for table getting the filter */
assert( pLoop!=0 );
assert( v!=0 );
iCur = pLevel->iTabCur;
assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
do{
sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
addrCont = sqlite3VdbeMakeLabel(pParse);
addrTop = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
iCur = pLevel->iTabCur;
pLevel->regFilter = ++pParse->nMem;
sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
Expr *pExpr = pTerm->pExpr;
@@ -1017,10 +1039,19 @@ SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
sqlite3ReleaseTempRange(pParse, r1, n);
}
sqlite3VdbeResolveLabel(v, addrCont);
sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
VdbeCoverage(v);
sqlite3VdbeJumpHere(v, addrTop);
sqlite3VdbeJumpHere(v, addrTop+2);
pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
while( iLevel < pWInfo->nLevel ){
iLevel++;
pLevel = &pWInfo->a[iLevel];
pLoop = pLevel->pWLoop;
if( pLoop && pLoop->wsFlags & WHERE_BLOOMFILTER ) break;
}
}while( iLevel < pWInfo->nLevel );
sqlite3VdbeJumpHere(v, addrOnce);
}
@@ -5535,7 +5566,7 @@ WhereInfo *sqlite3WhereBegin(
&pTabList->a[pLevel->iFrom], notReady, pLevel);
#endif
}else{
sqlite3ConstructBloomFilter(pWInfo, pLevel);
constructBloomFilter(pWInfo, ii, pLevel);
}
if( db->mallocFailed ) goto whereBeginError;
}

7
src/whereInt.h
View File

@@ -484,7 +484,6 @@ struct WhereInfo {
** where.c:
*/
Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
void sqlite3ConstructBloomFilter(const WhereInfo*, WhereLevel*);
#ifdef WHERETRACE_ENABLED
void sqlite3WhereClausePrint(WhereClause *pWC);
void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm);
@@ -507,8 +506,14 @@ int sqlite3WhereExplainOneScan(
WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
);
int sqlite3WhereExplainBloomFilter(
const Parse *pParse, /* Parse context */
const WhereInfo *pWInfo, /* WHERE clause */
const WhereLevel *pLevel /* Bloom filter on this level */
);
#else
# define sqlite3WhereExplainOneScan(u,v,w,x) 0
# define sqlite3WhereExplainBloomFilter(u,v,w) 0
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
void sqlite3WhereAddScanStatus(

77
src/wherecode.c
View File

@@ -196,9 +196,6 @@ int sqlite3WhereExplainOneScan(
pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
}
#endif
if( flags & WHERE_BLOOMFILTER ){
sqlite3_str_appendf(&str, " WITH BLOOM FILTER");
}
#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS
if( pLoop->nOut>=10 ){
sqlite3_str_appendf(&str, " (~%llu rows)",
@@ -214,6 +211,52 @@ int sqlite3WhereExplainOneScan(
}
return ret;
}
/*
** Add a single OP_Explain opcode that describes a Bloom filter.
**
** Or if not processing EXPLAIN QUERY PLAN and not in a SQLITE_DEBUG and/or
** SQLITE_ENABLE_STMT_SCANSTATUS build, then OP_Explain opcodes are not
** required and this routine is a no-op.
**
** If an OP_Explain opcode is added to the VM, its address is returned.
** Otherwise, if no OP_Explain is coded, zero is returned.
*/
int sqlite3WhereExplainBloomFilter(
const Parse *pParse, /* Parse context */
const WhereInfo *pWInfo, /* WHERE clause */
const WhereLevel *pLevel /* Bloom filter on this level */
){
int ret = 0;
#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
if( sqlite3ParseToplevel(pParse)->explain==2 )
#endif
{
SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom];
Vdbe *v = pParse->pVdbe; /* VM being constructed */
sqlite3 *db = pParse->db; /* Database handle */
char *zMsg; /* Text to add to EQP output */
int i; /* Loop counter */
WhereLoop *pLoop; /* The where loop */
StrAccum str; /* EQP output string */
char zBuf[100]; /* Initial space for EQP output string */
sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
str.printfFlags = SQLITE_PRINTF_INTERNAL;
sqlite3_str_appendf(&str, "BLOOM FILTER ON %S(", pItem);
pLoop = pLevel->pWLoop;
for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
const char *z = pItem->pTab->aCol[i].zCnName;
if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
sqlite3_str_appendf(&str, "%s=?", z);
}
sqlite3_str_append(&str, ")", 1);
zMsg = sqlite3StrAccumFinish(&str);
ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
}
return ret;
}
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -1304,14 +1347,20 @@ static void whereApplyPartialIndexConstraints(
}
}
#if 1
/*
** An OP_Filter has just been generated, but the corresponding
** index search has not yet been performed. This routine
** checks to see if there are additional WHERE_BLOOMFILTER in
** inner loops that can be evaluated right away, and if there are,
** it evaluates those filters as well, and removes the WHERE_BLOOMFILTER
** tag.
** This routine is called right after An OP_Filter has been generated and
** before the corresponding index search has been performed. This routine
** checks to see if there are additional Bloom filters in inner loops that
** can be checked prior to doing the index lookup. If there are available
** inner-loop Bloom filters, then evaluate those filters now, before the
** index lookup. The idea is that a Bloom filter check is way faster than
** an index lookup, and the Bloom filter might return false, meaning that
** the index lookup can be skipped.
**
** We know that an inner loop uses a Bloom filter because it has the
** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked,
** then clear the WhereLoeve.regFilter value to prevent the Bloom filter
** from being checked a second time when the inner loop is evaluated.
*/
static SQLITE_NOINLINE void filterPullDown(
Parse *pParse, /* Parsing context */
@@ -1323,9 +1372,8 @@ static SQLITE_NOINLINE void filterPullDown(
while( ++iLevel < pWInfo->nLevel ){
WhereLevel *pLevel = &pWInfo->a[iLevel];
WhereLoop *pLoop = pLevel->pWLoop;
if( (pLoop->wsFlags & WHERE_BLOOMFILTER)==0 ) continue;
if( pLevel->regFilter==0 ) continue;
if( pLoop->prereq & notReady ) continue;
sqlite3ConstructBloomFilter(pWInfo, &pWInfo->a[iLevel]);
if( pLoop->wsFlags & WHERE_IPK ){
WhereTerm *pTerm = pLoop->aLTerm[0];
int r1, regRowid;
@@ -1351,12 +1399,9 @@ static SQLITE_NOINLINE void filterPullDown(
addrNxt, r1, nEq);
VdbeCoverage(pParse->pVdbe);
}
pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
pLevel->regFilter = 0;
}
}
#else
#define filterPullDown(A,B,C,D,E)
#endif
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause