diff --git a/manifest b/manifest index 7113b8ffd7..720b8aa23c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\squery\splanner,\sadd\sa\sheuristic\sthat\swill\sreduce\sthe\scost\sof\sa\sfull\ntable\sscan\sfor\sa\smaterialized\sview\sor\ssubquery\sif\sthe\sfull\sscan\sis\sthe\nouter-most\sloop.\s\sThis\sis\sshown\sto\sspeed\sup\ssome\squeries. -D 2022-09-01T10:41:24.845 +C If\sa\squery\suses\san\sindex\swhere\sone\sor\smore\sof\sthe\scolumns\sof\sthe\sindex\sis\nan\sexpression\sand\sif\sthe\scorresponding\sexpression\sis\nused\selsewhere\sin\sthe\squery,\sthen\sstrive\sto\sread\sthe\svalue\sof\sthe\sexpression\nout\sof\sthe\sindex,\srather\sthan\srecomputing\sit.\s\sThis\sis\sthe\n"Indexed\sExpression\sOptimizations". +D 2022-10-19T11:22:21.760 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -462,7 +462,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 47276c5c150c3d99d4b72e3d296a4c16e38a8531a3e8b3ece59fdf48208950d9 F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c 04bc5a6b6331a30348e59222ab132ecde7cf5dc04c0915a2182b0609d1ab3df0 +F src/build.c 9756d8b0bfd11b5050290ecc4cc34ef6ea45d7b4e8d430c8aa511ba6b7ee31c8 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -470,7 +470,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c c12833de69cb655751487d2c5a59607e36be1c58ba1f4bd536609909ad47b319 F src/delete.c 2bee826a5e1c2b2018895084850d69a7f60269ae6fa5e8c247e2a4e9faf2ccad -F src/expr.c e100212835d20498780e7c6d2bdb16c677ecc04350fb75db3bf192a86ba48c92 +F src/expr.c dc17ca9523293e15abe61f3d0002556d765b0fc3b985e67c07ec3de9dd36e636 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c bd0138acdc008c1845ccf92f8e73787880562de649471804801c06fed814c765 F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee @@ -479,7 +479,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c fc3cf5c371f9a400144e8c2f148ab29cd3f67f7da7eaf47e6a6959f8255fd92c +F src/insert.c 1b71223966e6e35ad23a6b69deba4fa502b2ee3c6511ca99933b1e39db523045 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 22afc33c3a61b4fd80a60a54f1882688371e6bc64685df2696b008fce65a999c F src/main.c 49657b103545c36b743d1384dbd03c26d49c69cba954a4abab320d5c1fe060b7 @@ -513,7 +513,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c be64b2f3908a7f97c56c963676eb12f0d6254c95b28cdc1d73a186eff213219d F src/pragma.c af67dedaad8bafe9a5f9adcec32a0da6dd118617dd8220ad1d118f5a6bf83a02 F src/pragma.h a776bb9c915207e9d1117b5754743ddf1bf6a39cc092a4a44e74e6cb5fab1177 -F src/prepare.c f739feb4cf0dfe22b5729c3610c80e2261c816731c7884641552e12aa9f7346d +F src/prepare.c 695a19948348b0202741a650af577c35ef2c8daad271bf818e3d0e4a5881ef3f F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 @@ -523,7 +523,7 @@ F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c18679681 F src/sqlite.h.in 59f5e145b8d7a915ca29c6bf4a1f00e3112c1605c9ac5c627c45060110332ba2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 -F src/sqliteInt.h 3d599ca87796deba6ebc5cab518262661c855ae31aa63eed0504223ec2339d13 +F src/sqliteInt.h d748774587502dcf0e77f54ddde94b3fdff32bdea91f2905b72a290d1a891fb9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -604,10 +604,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 -F src/where.c 742ced4550d1df8d683482b11f2432bb60a93f6f50b532ba9c3a32720fc3b78e +F src/where.c 5f54a44d4aa05c592ed8e19bad656ebb1a9d05157488edfa991e12b3098290f1 F src/whereInt.h ffebbbad9359cc602c9cbb24d926f73fc1bf696f0edb4ff896afa32018aad690 F src/wherecode.c 5e0b6dec8591e13f1f0af828d350e4a5dd2e3518b63d328f21bb38e2456dfeb7 -F src/whereexpr.c 90859652920f153d2c03f075488744be2926625ebd36911bcbcb17d0d29c891c +F src/whereexpr.c ca55a11c2443700fe084a1e039660688d7733c594a37697ee4bd99462e2c2f6a F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1819,9 +1819,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba6bf331476d0217f4132b73cb3da559e75bfb21856ec94f82c0f0150a53592b -Q +609fbb94b8f01d6792e5941ab23ce041313d359f6788c4dde6b1ca749ab49137 -R 7b7f582c65e5dee0ef3f194b93a7fa92 +P e3754cc18824aad4113ad6d81e33e5c763beb9705fc6af88d8f8c870a03c731d +Q +2435112867fbd7b6ebb7f2c2b9da57cdf1e23fab6c2869870b66133a9f9faedc +R ffc6c9dee666f2afa446429523ea6435 U drh -Z e7f00c4ae02183e078488febebe7b6ef +Z cbff34b53dc91f70749ae3ac7b08f467 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f4b0c049a..722622687f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3754cc18824aad4113ad6d81e33e5c763beb9705fc6af88d8f8c870a03c731d \ No newline at end of file +3da1032878bdc93f69b02926fb7243b31fe6b1a0ee93af68df52b203b0603dad \ No newline at end of file diff --git a/src/build.c b/src/build.c index b4434d080f..0abbcf7fe4 100644 --- a/src/build.c +++ b/src/build.c @@ -3406,6 +3406,7 @@ void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); diff --git a/src/expr.c b/src/expr.c index 9a5e0345a0..f092294fdf 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3418,6 +3418,53 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ return iResult; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxExpr; p; p=p->pIENext){ + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxExpr; + pParse->pIdxExpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxExpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -3449,6 +3496,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxExpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ op = pExpr->op; } @@ -4814,7 +4866,13 @@ int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTab){ if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ if( pA->op==TK_FUNCTION ){ diff --git a/src/insert.c b/src/insert.c index ee63eeda56..46e2849e23 100644 --- a/src/insert.c +++ b/src/insert.c @@ -95,6 +95,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ }else{ char aff; assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); if( aff==0 ) aff = SQLITE_AFF_BLOB; diff --git a/src/prepare.c b/src/prepare.c index 62abc9f6c3..2c34de9b1d 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -524,6 +524,12 @@ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); + while( pParse->pIdxExpr!=0 ){ + IndexedExpr *p = pParse->pIdxExpr; + pParse->pIdxExpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } if( db ){ assert( db->lookaside.bDisable >= pParse->disableLookaside ); db->lookaside.bDisable -= pParse->disableLookaside; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 04814da670..f274866b76 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1078,6 +1078,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -2255,6 +2256,8 @@ struct Index { unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -3048,6 +3051,28 @@ struct TriggerPrg { # define DbMaskNonZero(M) (M)!=0 #endif +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ +#endif +}; + /* ** An SQL parser context. A copy of this structure is passed through ** the parser and down into all the parser action routine in order to @@ -3091,6 +3116,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ diff --git a/src/where.c b/src/where.c index abe29279be..115f4e25aa 100644 --- a/src/where.c +++ b/src/where.c @@ -4519,6 +4519,50 @@ static int exprIsDeterministic(Expr *p){ return w.eCode; } +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + struct SrcList_item *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + assert( pIdx->bHasExpr ); + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + }else{ + continue; + } + if( sqlite3ExprIsConstant(pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxExpr; + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = (pTabItem->fg.jointype & JT_LEFT)!=0; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxExpr = p; + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -5067,6 +5111,9 @@ WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } } pLevel->iIdxCur = iIndexCur; assert( pIx->pSchema==pTab->pSchema ); @@ -5363,6 +5410,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxExpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ diff --git a/src/whereexpr.c b/src/whereexpr.c index 961495c584..7f1dbf785d 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -972,6 +972,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR;