From ff5e4eccedbfc2294a8072888de596a8825adee0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 13:33:58 +0000 Subject: [PATCH 01/29] This is the first in what is anticipated to be a long sequence of incremental changes aimed at improving aggregate query processing, and in particular helping aggregate queries take better advantage of indexes on expression. The end goal is to resolve ticket [99378177930f87bd], though it remains to be seen whether or not I can get there with this approach. FossilOrigin-Name: cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/select.c | 10 ++++------ src/sqliteInt.h | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f5beaaa03b..bf95667970 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. -D 2022-11-22T12:47:32.742 +C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. +D 2022-11-22T13:33:58.528 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d +F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d +F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 -R 6f72c126be40c707db8479e3d23315a3 +P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 +R 8a6d83ef799149939ad05ed6225df7c1 +T *branch * agg-with-indexed-expr +T *sym-agg-with-indexed-expr * +T -sym-trunk * U drh -Z 9806e837fcd3094e05cfafc89f1e72fe +Z c8ad55df8af287c23ae8c2fae9e521d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02e6c3a494..3f8ba1bdb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file +cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f996ac0852..5306eb399b 100644 --- a/src/select.c +++ b/src/select.c @@ -6223,18 +6223,17 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pParse->nErr ) return; #ifdef SQLITE_DEBUG /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ for(i=0; inColumn; i++){ assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aCol[i].iMemmnReg+nReg ); } for(i=0; inFunc; i++){ assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aFunc[i].iMemmnReg+nReg ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -7448,7 +7447,6 @@ int sqlite3Select( #endif sNC.ncFlags &= ~NC_InAggFunc; } - pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x400 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6ff49ab5d0..d4ffaa2251 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,7 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 42b78237cdd11b79e4c37d0373a338c1409f8283 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:10:22 +0000 Subject: [PATCH 02/29] Factor out the allocation of registers for aggregates into a separate subroutine. FossilOrigin-Name: 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 21 ++++++++++++++++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bf95667970..7f4b62ade6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. -D 2022-11-22T13:33:58.528 +C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. +D 2022-11-22T14:10:22.456 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 +F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc +F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,11 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 -R 8a6d83ef799149939ad05ed6225df7c1 -T *branch * agg-with-indexed-expr -T *sym-agg-with-indexed-expr * -T -sym-trunk * +P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 +R 8ec59d9575fc141fe0aff25c195ea672 U drh -Z c8ad55df8af287c23ae8c2fae9e521d0 +Z dd7dd4863c6318a4016da919a253840a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f8ba1bdb5..d932c72af1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file +4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fb718eb23e..f6b065208b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4441,6 +4441,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ + assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); return pInfo->aFunc[pExpr->iAgg].iMem; } break; @@ -4730,6 +4731,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ + assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); inReg = pAggInfo->aCol[pExpr->iAgg].iMem; break; } @@ -6294,7 +6296,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pCol->pTab = pExpr->y.pTab; pCol->iTable = pExpr->iTable; pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pCExpr = pExpr; if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ @@ -6357,7 +6358,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, diff --git a/src/select.c b/src/select.c index 5306eb399b..0872a95454 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,6 +6204,21 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +/* +** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] +** entries. +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + int i, m; + assert( pAggInfo!=0 ); + assert( pAggInfo->mnReg==0 ); + m = pParse->nMem; + pAggInfo->mnReg = m+1; + for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; + for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; + pParse->nMem = m; +} + /* ** Reset the aggregate accumulator. ** @@ -6219,6 +6234,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; #ifdef SQLITE_DEBUG @@ -7413,7 +7429,6 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -7534,6 +7549,7 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -7772,6 +7788,8 @@ int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } + assignAggregateRegisters(pParse, pAggInfo); + assert( pAggInfo->aFunc[0].iMem>=0 ); sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); @@ -7808,6 +7826,7 @@ int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row From 89e5dfac01904ef55be9f2e5c007b25d57bc0263 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:31:13 +0000 Subject: [PATCH 03/29] Omit the unnecessary AggInfo.mnReg field. FossilOrigin-Name: d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 26 ++++++++++++++++---------- src/sqliteInt.h | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 7f4b62ade6..e95765ddfb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. -D 2022-11-22T14:10:22.456 +C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. +D 2022-11-22T14:31:13.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a +F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 +F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 -R 8ec59d9575fc141fe0aff25c195ea672 +P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d +R 5936ef480bfa81b2caa13a6a172023d0 U drh -Z dd7dd4863c6318a4016da919a253840a +Z 8bcded315dd8c6f4a10b877ff07bc1b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d932c72af1..8897d99aa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file +d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0872a95454..dd02265958 100644 --- a/src/select.c +++ b/src/select.c @@ -6211,9 +6211,7 @@ void sqlite3SelectPrep( static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ int i, m; assert( pAggInfo!=0 ); - assert( pAggInfo->mnReg==0 ); m = pParse->nMem; - pAggInfo->mnReg = m+1; for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; pParse->nMem = m; @@ -6230,26 +6228,34 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; + int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); - assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; + if( pAggInfo->nColumn==0 ){ + iFirstReg = pAggInfo->aFunc[0].iMem; + }else{ + iFirstReg = pAggInfo->aCol[0].iMem; + } #ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ + /* Verify that all AggInfo register numbers have been assigned and that + ** they are all sequential. */ + assert( iFirstReg>0 ); for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMemmnReg+nReg ); + assert( pAggInfo->aCol[i].iMem>=iFirstReg ); + assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); } for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMemmnReg+nReg ); + assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); + assert( i>0 || pAggInfo->nColumn==0 + || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); + assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d4ffaa2251..68dbb3ab9f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,6 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 3c8e438583c617eea850ff52bec56f5db5a8d099 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 15:43:16 +0000 Subject: [PATCH 04/29] Since the memory registers used by the columns and functions of an AggInfo object are sequential, it is not neecessary to remember each register separately. We can simply remember the first one and do the math when others are needed. FossilOrigin-Name: dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 9 +++------ src/select.c | 49 ++++++++++++++----------------------------------- src/sqliteInt.h | 11 ++++++++--- 5 files changed, 34 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index e95765ddfb..8e8fab603a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. -D 2022-11-22T14:31:13.165 +C Since\sthe\smemory\sregisters\sused\sby\sthe\scolumns\sand\sfunctions\sof\san\nAggInfo\sobject\sare\ssequential,\sit\sis\snot\sneecessary\sto\sremember\seach\sregister\nseparately.\s\sWe\scan\ssimply\sremember\sthe\sfirst\sone\sand\sdo\sthe\smath\swhen\nothers\sare\sneeded. +D 2022-11-22T15:43:16.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc +F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 +F src/select.c cf17743a695b3f4ff2dbd06d2ca8152997d37e3b3fd951efe0a67966740bd2d7 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 +F src/sqliteInt.h fd605195929a3ce7f20da20b6a4da32045f9d4b343b51d94a0c7fdbad6207f24 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d -R 5936ef480bfa81b2caa13a6a172023d0 +P d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b +R a8a65a86f26eb6db314813261a6e0287 U drh -Z 8bcded315dd8c6f4a10b877ff07bc1b3 +Z 3374f8495add90924d12780ade06b13a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8897d99aa9..b43b5e65eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file +dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f6b065208b..42099536ed 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4128,8 +4128,7 @@ expr_code_doover: assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, @@ -4441,8 +4440,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -4731,8 +4729,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ - assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); - inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); break; } if( pExpr->pAggInfo->useSortingIdx ){ diff --git a/src/select.c b/src/select.c index dd02265958..e40de38a59 100644 --- a/src/select.c +++ b/src/select.c @@ -6209,12 +6209,10 @@ void sqlite3SelectPrep( ** entries. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ - int i, m; assert( pAggInfo!=0 ); - m = pParse->nMem; - for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; - for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; - pParse->nMem = m; + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; } /* @@ -6228,34 +6226,15 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; - int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; + assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; - if( pAggInfo->nColumn==0 ){ - iFirstReg = pAggInfo->aFunc[0].iMem; - }else{ - iFirstReg = pAggInfo->aCol[0].iMem; - } -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo register numbers have been assigned and that - ** they are all sequential. */ - assert( iFirstReg>0 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=iFirstReg ); - assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); - assert( i>0 || pAggInfo->nColumn==0 - || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); - assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -6287,7 +6266,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } @@ -6375,7 +6355,7 @@ static void updateAccumulator( if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -6390,7 +6370,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -6662,13 +6642,13 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), pCol->iSorterColumn); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } } @@ -7795,8 +7775,7 @@ int sqlite3Select( sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } assignAggregateRegisters(pParse, pAggInfo); - assert( pAggInfo->aFunc[0].iMem>=0 ); - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 68dbb3ab9f..45075cc361 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2716,15 +2716,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -2735,7 +2735,6 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; @@ -2743,6 +2742,12 @@ struct AggInfo { u32 selId; /* Select to which this AggInfo belongs */ }; +/* +** Macros to compute aCol[] and aFunc[] register numbers: +*/ +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater From b23f1572ab9c11e524023678a5347152aae48b04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 20:37:41 +0000 Subject: [PATCH 05/29] Add the stub function: optimizeAggregateUsingIndexedExpr(). The hope is that we can fill this in with a routine that does useful optimizations. FossilOrigin-Name: d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8586f2587b..e809fe26ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sand\senhancements\sinto\sthe\sagg-with-indexed-expr\sbranch. -D 2022-11-22T19:51:16.237 +C Add\sthe\sstub\sfunction:\soptimizeAggregateUsingIndexedExpr().\s\sThe\shope\sis\sthat\nwe\scan\sfill\sthis\sin\swith\sa\sroutine\sthat\sdoes\suseful\soptimizations. +D 2022-11-22T20:37:41.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 35c2b1b4d82190f202887620f99070c0d414c3e030e466032797792cffdbd9f9 +F src/select.c dfc504356e15bb8585d8535a927a72512d1d7a01f9074824964f1781e286231b F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd -R e6ec32714f5519d3f42f7a87b92d7eba +P 070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 +R 5bcfbb263a340b6c57659d80015cbb33 U drh -Z 31a3ccb601c45d898a857131ff2f396b +Z 28092ed1d737fad311ecc59737148e5d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1556ed2eb3..0a7b42af87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 \ No newline at end of file +d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 946649ad0b..ecd1791060 100644 --- a/src/select.c +++ b/src/select.c @@ -6654,6 +6654,34 @@ static void printAggInfo(AggInfo *pAggInfo){ } #endif /* TREETRACE_ENABLED */ +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static int optimizeAggregateUsingIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT being coded */ + AggInfo *pAggInfo /* The aggregate info */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x1000000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("cur=%d\n", pIEpr->iDataCur); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#endif + return 0; +} + + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7535,6 +7563,9 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); From c6138e970e4afca027635e97d0516a58e5c026a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 14:13:39 +0000 Subject: [PATCH 06/29] Further foundation prep work prior to starting to flesh-out the optimizeAggregateUseOfIndexedExpr() routine. FossilOrigin-Name: 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e --- manifest | 12 ++-- manifest.uuid | 2 +- src/select.c | 171 +++++++++++++++++++++++++++++--------------------- 3 files changed, 108 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 768c50fa57..1ffe8b4bb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sand\sfixes\sfrom\strunk. -D 2022-11-22T20:58:18.661 +C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. +D 2022-11-23T14:13:39.215 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6f5403b8f1849d2a8ba152dd790ec189bba18242157904ddca77707e209265d4 +F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 55f08c65335ec615b6caa516a7697c9d +P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 +R e49e6155bccbafba580b14baaea3db86 U drh -Z dbc39758acc6c225892e0cac1e391b2a +Z b6500d1e8d8b2720badce4287d0e1594 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c7d27e3fe..f88bcea3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 \ No newline at end of file +23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file diff --git a/src/select.c b/src/select.c index f71fa3197e..8f2517ad4f 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,9 +6204,106 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED /* -** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] -** entries. +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + Parse *pParse, + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x80000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + (void)pSelect; /* Suppress unused-parameter warnings */ +#endif + pAggInfo->nColumn = pAggInfo->nAccumulator; + analyzeAggFuncArgs(pParse, pAggInfo, pNC); +} + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6630,60 +6727,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } -#if TREETRACE_ENABLED -/* -** Display all information about an AggInfo object -*/ -static void printAggInfo(AggInfo *pAggInfo){ - int ii; - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d %s\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), - pCol->iSorterColumn, - ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", - ii, AggInfoFuncReg(pAggInfo,ii)); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } -} -#endif /* TREETRACE_ENABLED */ - -/* -** An index on expressions is being used in the inner loop of an -** aggregate query with a GROUP BY clause. This routine attempts -** to adjust the AggInfo object to take advantage of index and to -** perhaps use the index as a covering index. -** -*/ -static int optimizeAggregateUsingIndexedExpr( - Parse *pParse, /* Parsing context */ - Select *pSelect, /* The SELECT being coded */ - AggInfo *pAggInfo /* The aggregate info */ -){ -#if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ - IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); - for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ - printf("data-cursor=%d index={%d,%d}\n", - pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); - sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); - } - printAggInfo(pAggInfo); - } -#endif - return 0; -} - - /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7465,19 +7508,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } + analyzeAggFuncArgs(pParse, pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ @@ -7566,7 +7597,7 @@ int sqlite3Select( goto select_end; } if( pParse->pIdxEpr ){ - optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); From ad1285c5c0be7eb92cc44a3357be71507c3c07f2 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 16:39:07 +0000 Subject: [PATCH 07/29] Initial infrastructure for adding a mode to the OPFS VFS which causes implicit locks to be released ASAP, which increases concurrency at the cost of performance. FossilOrigin-Name: c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 --- ext/wasm/api/sqlite3-api-oo1.js | 5 +- ext/wasm/api/sqlite3-api-opfs.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 123 ++++++++++++++++------ ext/wasm/tests/opfs/concurrency/worker.js | 6 +- manifest | 21 ++-- manifest.uuid | 2 +- 6 files changed, 112 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..bc32ec1067 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -200,9 +200,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ const arg = {}; - if(1===arguments.length && 'object'===typeof arguments[0]){ - const x = arguments[0]; - Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){ + Object.assign(arg, arguments[0]); if(undefined===arg.flags) arg.flags = 'c'; if(undefined===arg.vfs) arg.vfs = null; if(undefined===arg.filename) arg.filename = ':memory:'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb4c923ab..53f68a1d58 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -348,6 +348,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_NOTFOUND', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', 'SQLITE_OPEN_READONLY' ].forEach((k)=>{ if(undefined === (state.sq3Codes[k] = capi[k])){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index c208932e17..cbe0549b3e 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -105,7 +105,7 @@ metrics.dump = ()=>{ */ const __openFiles = Object.create(null); /** - __autoLocks is a Set of sqlite3_file pointers (integers) which were + __implicitLocks is a Set of sqlite3_file pointers (integers) which were "auto-locked". i.e. those for which we obtained a sync access handle without an explicit xLock() call. Such locks will be released during db connection idle time, whereas a sync access @@ -117,7 +117,7 @@ const __openFiles = Object.create(null); penalty: speedtest1 benchmarks take up to 4x as long. By delaying the lock release until idle time, the hit is negligible. */ -const __autoLocks = new Set(); +const __implicitLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." @@ -166,7 +166,7 @@ const closeSyncHandle = async (fh)=>{ const h = fh.syncHandle; delete fh.syncHandle; delete fh.xLock; - __autoLocks.delete(fh.fid); + __implicitLocks.delete(fh.fid); return h.close(); } }; @@ -190,10 +190,10 @@ const closeSyncHandleNoThrow = async (fh)=>{ }; /* Release all auto-locks. */ -const closeAutoLocks = async ()=>{ - if(__autoLocks.size){ +const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ /* Release all auto-locks. */ - for(const fid of __autoLocks){ + for(const fid of __implicitLocks){ const fh = __openFiles[fid]; await closeSyncHandleNoThrow(fh); log("Auto-unlocked",fid,fh.filenameAbs); @@ -201,6 +201,32 @@ const closeAutoLocks = async ()=>{ } }; +/** + If true, any routine which implicitly acquires a sync access handle + (i.e. an OPFS lock) will release that locks at the end of the call + which acquires it. If false, such "autolocks" are not released + until the VFS is idle for some brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). +*/ +state.defaultReleaseImplicitLocks = false; + +/** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. +*/ +const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } +}; + /** An error class specifically for use with getSyncHandle(), the goal of which is to eventually be able to distinguish unambiguously @@ -246,7 +272,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{ still fails at that point it will give up and propagate the exception. */ -const getSyncHandle = async (fh)=>{ +const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -262,20 +288,21 @@ const getSyncHandle = async (fh)=>{ }catch(e){ if(i === maxTries){ throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, + e, "Error getting sync handle for",opName+"().",maxTries, "attempts failed.",fh.filenameAbs ); } - warn("Error getting sync handle. Waiting",ms, + warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - await closeAutoLocks(); + //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; @@ -409,7 +436,7 @@ const vfsAsyncImpls = { xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); - __autoLocks.delete(fid); + __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; wTimeStart(opName); @@ -474,13 +501,14 @@ const vfsAsyncImpls = { wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh)).getSize(); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); rc = 0; }catch(e){ state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xFileSize', rc); mTimeEnd(); @@ -495,8 +523,8 @@ const vfsAsyncImpls = { if( !fh.syncHandle ){ wTimeStart('xLock'); try { - await getSyncHandle(fh); - __autoLocks.delete(fid); + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); @@ -511,7 +539,6 @@ const vfsAsyncImpls = { flags/*SQLITE_OPEN_...*/){ const opName = 'xOpen'; mTimeStart(opName); - const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); wTimeStart('xOpen'); try{ @@ -526,14 +553,8 @@ const vfsAsyncImpls = { return; } const hFile = await hDir.getFileHandle(filenamePart, {create}); - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the syncHandle prop of the file state - object, but only for certain cases and it's unclear why it - places that limitation on it. - */ wTimeEnd(); - __openFiles[fid] = Object.assign(Object.create(null),{ + const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, @@ -542,8 +563,47 @@ const vfsAsyncImpls = { sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: deleteOnClose + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); + fh.releaseImplicitLocks = + state.defaultReleaseImplicitLocks + /* TODO: check URI flags for "opfs-auto-unlock". First we need to + reshape the API a bit to be able to pass those on to here + from the other half of the proxy. */; + /*if(fh.releaseImplicitLocks){ + console.warn("releaseImplicitLocks is ON for",fh); + }*/ + if(0 /* this block is modelled after something wa-sqlite + does but it leads to horrible contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + Regarding "immutable": that flag is not _really_ applicable + here. It's intended for use on read-only media. If, + however, a file is opened with that flag but changes later + (which can happen if we _don't_ grab a sync handle here) + then sqlite may misbehave. + + Regarding "nolock": ironically, the nolock flag forces us + to lock the file up front. "nolock" tells sqlite to _not_ + use its locking API, but OPFS requires a lock to perform + most of the operations performed in this file. If we don't + grab that lock up front, another handle could end up grabbing + it and mutating the database out from under our nolocked'd + handle. In the interest of preventing corruption, at the cost + of decreased concurrency, we have to lock it for the duration + of this file handle. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "atOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ wTimeEnd(); @@ -560,7 +620,7 @@ const vfsAsyncImpls = { try{ affirmLocked('xRead',fh); wTimeStart('xRead'); - nRead = (await getSyncHandle(fh)).read( + nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); @@ -575,6 +635,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } + await releaseImplicitLock(fh); storeAndNotify('xRead',rc); mTimeEnd(); }, @@ -603,12 +664,13 @@ const vfsAsyncImpls = { try{ affirmLocked('xTruncate',fh); affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh)).truncate(size); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xTruncate',rc); mTimeEnd(); @@ -640,7 +702,7 @@ const vfsAsyncImpls = { affirmLocked('xWrite',fh); affirmNotRO('xWrite', fh); rc = ( - n === (await getSyncHandle(fh)) + n === (await getSyncHandle(fh,'xWrite')) .write(fh.sabView.subarray(0, n), {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; @@ -649,6 +711,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xWrite',rc); mTimeEnd(); @@ -783,7 +846,7 @@ const waitLoop = async function f(){ if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - await closeAutoLocks(); + await releaseImplicitLocks(); continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 19b0a068e7..9b4898f4c3 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -43,7 +43,11 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb(dbName,'c'); + db = new sqlite3.oo1.DB({ + filename: 'file:'+dbName, + flags: 'c', + vfs: 'opfs' + }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ diff --git a/manifest b/manifest index 3ecb281109..d6e43cfa43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. +D 2022-11-23T16:39:07.866 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,12 +501,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b +F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 +F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de +F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R 5ffa2a11b06c4a44d92d27455a2fc3e8 +U stephan +Z 70f5adf76d87ddf8a488ff2bcd290afb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..54a2776a7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file From 590f013a87609ea33abc718aab71c8785a7213f9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 17:56:00 +0000 Subject: [PATCH 08/29] This attempt at modifying AggInfo to make use of indexed expressions does not work. It gets an incorrect answer for the test case shown in the ticket. FossilOrigin-Name: 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 --- manifest | 16 ++--- manifest.uuid | 2 +- src/expr.c | 157 ++++++++++++++++++++++++++++++------------------ src/select.c | 21 +++++-- src/sqliteInt.h | 1 - 5 files changed, 122 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index 1ffe8b4bb7..294477b047 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. -D 2022-11-23T14:13:39.215 +C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. +D 2022-11-23T17:56:00.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 +F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 +F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 894f7d98b1104fecb2ca2f457a4c598944250d9462eb433f12bd42b1080d525f +F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 -R e49e6155bccbafba580b14baaea3db86 +P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e +R f3df5c5148120a1bf76a131c870fdc09 U drh -Z b6500d1e8d8b2720badce4287d0e1594 +Z 97820e8b433aee285f0d7a5d438ada65 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f88bcea3d3..c7c9044f2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file +84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 42099536ed..db81983e4a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,7 +4135,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( ALWAYS(pTab!=0) ){ + }else if( pTab!=0 ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -6241,6 +6241,71 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ return i; } +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates @@ -6255,6 +6320,37 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( pNC->ncFlags & NC_UAggInfo ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( !ExprUseYTab(pExpr) ) break; + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { @@ -6266,66 +6362,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable - && pCol->iColumn==pExpr->iColumn - && pExpr->op!=TK_IF_NULL_ROW - ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN - && pE->iTable==pExpr->iTable - && pE->iColumn==pExpr->iColumn - ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ){ - pExpr->op = TK_AGG_COLUMN; - } - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ diff --git a/src/select.c b/src/select.c index 8f2517ad4f..49bef86bab 100644 --- a/src/select.c +++ b/src/select.c @@ -6281,11 +6281,22 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( pAggInfo->nSortingColumn>0 ){ + if( pAggInfo->nColumn==0 ){ + pAggInfo->nSortingColumn = 0; + }else{ + pAggInfo->nSortingColumn = + pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + } + } + analyzeAggFuncArgs(pParse, pAggInfo, pNC); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ + if( sqlite3TreeTrace & 0x20 ){ IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ printf("data-cursor=%d index={%d,%d}\n", pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); @@ -6296,8 +6307,6 @@ static void optimizeAggregateUseOfIndexedExpr( #else (void)pSelect; /* Suppress unused-parameter warnings */ #endif - pAggInfo->nColumn = pAggInfo->nAccumulator; - analyzeAggFuncArgs(pParse, pAggInfo, pNC); } /* @@ -7959,7 +7968,7 @@ select_end: if( pAggInfo && !db->mallocFailed ){ for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 74c997c6dd..3ecdb35fee 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1059,7 +1059,6 @@ extern u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated -** 0x00080000 Optimize Aggregates using indexed expressions */ /* From 2e30d95fb63e73cbae3cf07b0ac57d8d7e0321ba Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 18:51:04 +0000 Subject: [PATCH 09/29] Aggregates with GROUP BY now make use of expressions on indexes. This code works and gets the correct answer for the test case in the ticket. Lots more testing and documentation is needed, however. FossilOrigin-Name: 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 6 ++++-- src/select.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 294477b047..5a5586e90e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. -D 2022-11-23T17:56:00.136 +C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. +D 2022-11-23T18:51:04.363 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 +F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 +F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e -R f3df5c5148120a1bf76a131c870fdc09 +P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 +R 2344df85f68d9987b55d50d38bac47c1 U drh -Z 97820e8b433aee285f0d7a5d438ada65 +Z 4c924f435822db601e489fbe4dcd4a9e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7c9044f2a..56c84edce5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file +8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index db81983e4a..c34a28dc47 100644 --- a/src/expr.c +++ b/src/expr.c @@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){ } op = pExpr->op; if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); @@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ int j; assert( ExprUseYTab(p) ); assert( p->y.pTab!=0 ); diff --git a/src/select.c b/src/select.c index 49bef86bab..dcb878dde9 100644 --- a/src/select.c +++ b/src/select.c @@ -6222,7 +6222,7 @@ static void printAggInfo(AggInfo *pAggInfo){ sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } @@ -6309,6 +6309,40 @@ static void optimizeAggregateUseOfIndexedExpr( #endif } +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + /* ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first @@ -7669,6 +7703,18 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } + /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. From e79cb67c3575d8254b36eba3f35dba2d62124e07 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 19:03:22 +0000 Subject: [PATCH 10/29] Add an experimental OPFS VFS-specific URI flag, opfs-unlock-asap, which tells the VFS to release implicit locks ASAP. This permits higher concurrency but hurts performance considerably. This may or may not be obsoleted by other concurrency-related experimentation. FossilOrigin-Name: d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 --- ext/wasm/api/sqlite3-api-opfs.js | 27 ++++++++++++++++++++++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 25 +++++---------------- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 16 +++++++------- manifest.uuid | 2 +- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 53f68a1d58..deb8bf04e8 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -355,6 +355,25 @@ const installOpfsVfs = function callee(options){ toss("Maintenance required: not found:",k); } }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). "opfs-unlock-asap=1" enables + this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that locks at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); /** Runs the given operation (by name) in the async worker @@ -845,9 +864,15 @@ const installOpfsVfs = function callee(options){ //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); + let opfsFlags = 0; if(0===zName){ zName = randomFilename(); }else if('number'===typeof zName){ + if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ + /* -----------------------^^^^^ MUST pass the untranslated + C-string here. */ + opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; + } zName = wasm.cstringToJs(zName); } const fh = Object.create(null); @@ -855,7 +880,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - const rc = opRun('xOpen', pFile, zName, flags); + const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index cbe0549b3e..b14494a0c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -201,18 +201,6 @@ const releaseImplicitLocks = async ()=>{ } }; -/** - If true, any routine which implicitly acquires a sync access handle - (i.e. an OPFS lock) will release that locks at the end of the call - which acquires it. If false, such "autolocks" are not released - until the VFS is idle for some brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). -*/ -state.defaultReleaseImplicitLocks = false; - /** An experiment in improving concurrency by freeing up implicit locks sooner. This is known to impact performance dramatically but it has @@ -536,7 +524,8 @@ const vfsAsyncImpls = { mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/){ + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); @@ -566,13 +555,8 @@ const vfsAsyncImpls = { deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); fh.releaseImplicitLocks = - state.defaultReleaseImplicitLocks - /* TODO: check URI flags for "opfs-auto-unlock". First we need to - reshape the API a bit to be able to pass those on to here - from the other half of the proxy. */; - /*if(fh.releaseImplicitLocks){ - console.warn("releaseImplicitLocks is ON for",fh); - }*/ + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite does but it leads to horrible contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ @@ -887,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){ state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.opIds = opt.opIds; state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9b4898f4c3..1a896e714e 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -44,7 +44,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName, + filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, flags: 'c', vfs: 'opfs' }); diff --git a/manifest b/manifest index d6e43cfa43..d9cc7cb9d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. -D 2022-11-23T16:39:07.866 +C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. +D 2022-11-23T19:03:22.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 +F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee +F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 +F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R 5ffa2a11b06c4a44d92d27455a2fc3e8 +P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 +R 37d44560e41f4eab685876e5b8c66c33 U stephan -Z 70f5adf76d87ddf8a488ff2bcd290afb +Z 1cab17b5c5d56b08d5be52571107da20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 54a2776a7a..8365ac9b1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file +d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file From 056a71562f192c7fcd4ca73a3133b60dcf5a83fd Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 20:49:08 +0000 Subject: [PATCH 11/29] OPFS concurrency test: add a URL flag to enable/disable unlock-asap mode. FossilOrigin-Name: 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 --- ext/wasm/tests/opfs/concurrency/test.js | 6 +++++- ext/wasm/tests/opfs/concurrency/worker.js | 11 +++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 27bc47b19d..cb9d4275be 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -63,6 +63,9 @@ options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 ) || 750; + options.unlockAsap = ( + urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 + ) || 0; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -92,12 +95,13 @@ } }; - stdout("Launching",options.workerCount,"workers..."); + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 1a896e714e..91aa0fa6fa 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -3,9 +3,12 @@ importScripts( ); self.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(self.location.href).searchParams; - const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000); + const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + }; const wPost = (type,...payload)=>{ - postMessage({type, worker: wName, payload}); + postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); @@ -44,7 +47,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c', vfs: 'opfs' }); @@ -66,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ try{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [wName, new Date().getTime()] + bind: [options.workerName, new Date().getTime()] }); //stdout("Set",prefix); }catch(e){ diff --git a/manifest b/manifest index d9cc7cb9d0..a6b0539bab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. -D 2022-11-23T19:03:22.450 +C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. +D 2022-11-23T20:49:08.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 +F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 +F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 -R 37d44560e41f4eab685876e5b8c66c33 +P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 +R ff8c3df82102d217683515f7779a95c4 U stephan -Z 1cab17b5c5d56b08d5be52571107da20 +Z 269769e48cdf286c8fb19455cdae3a52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8365ac9b1a..bb0d09842b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file +1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file From 9b80cb5f39af66075762057df127ffbbb2c6ef2a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 01:40:20 +0000 Subject: [PATCH 12/29] Add explanatory comment to the new optimization. And add a test case. FossilOrigin-Name: e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/select.c | 5 +++ test/tkt-99378177930f87bd.test | 69 ++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 test/tkt-99378177930f87bd.test diff --git a/manifest b/manifest index 5a5586e90e..9476f82d3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. -D 2022-11-23T18:51:04.363 +C Add\sexplanatory\scomment\sto\sthe\snew\soptimization.\s\sAnd\sadd\sa\stest\scase. +D 2022-11-24T01:40:20.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 +F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1628,6 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2059,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 -R 2344df85f68d9987b55d50d38bac47c1 +P 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f +R 9274a8d11a500d33c0e780bbd7f350f0 U drh -Z 4c924f435822db601e489fbe4dcd4a9e +Z b2115b9d996407ff47bc262d3bc9ad3e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 56c84edce5..cdcf19b4bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file +e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 \ No newline at end of file diff --git a/src/select.c b/src/select.c index dcb878dde9..dcf899e4bb 100644 --- a/src/select.c +++ b/src/select.c @@ -7703,6 +7703,11 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + /* If there entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ if( pParse->pIdxEpr ){ aggregateConvertIndexedExprRefToColumn(pAggInfo); #if TREETRACE_ENABLED diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test new file mode 100644 index 0000000000..aad0a6dc21 --- /dev/null +++ b/test/tkt-99378177930f87bd.test @@ -0,0 +1,69 @@ +# 2022-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that the enhancement +# request documented by ticket 99378177930f87bd is working. +# +# The enhancement is that if an aggregate query with a GROUP BY clause +# uses subexpressions in the arguments to aggregate functions that are +# also columns of an index, then the values are pulled from the index +# rather than being recomputed. This has the potential to make some +# indexed queries works as if the index were covering. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test tkt-99378-100 { + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} {} +do_execsql_test tkt-99378-110 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} + +# The proof that the index on the expression is being used is in the +# fact that the byte code contains no "Function" opcodes. In other words, +# the ->> operator (which is implemented by a function) is never invoked. +# Instead, the b->>'x' value is pulled out of the index. +# +do_execsql_test tkt-99378-120 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + +finish_test From b669bb5e2bd130b2a8d8a87304449cfb3b27f9d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 13:19:25 +0000 Subject: [PATCH 13/29] New test cases. Fix the logic so that it works for GROUP BY aggregates that do not require sorting. FossilOrigin-Name: ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef --- manifest | 14 +++--- manifest.uuid | 2 +- src/expr.c | 5 ++ test/tkt-99378177930f87bd.test | 85 ++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 19b2b20356..7f64f61129 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sagg-with-indexed-expr\sbranch\sto\nsimplify\sdiffs. -D 2022-11-24T01:41:44.524 +C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. +D 2022-11-24T13:19:25.140 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 +F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -1628,7 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 +F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 -R beb6109a205ba56e52e9073d9db7b007 +P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 +R fe9857b5384581f396eba27025a5561f U drh -Z 9965e8c3c2c486ad5a3aae007bdf6965 +Z d9a080fe6c44b0ca440456485b55fa7a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6fe2b265a2..016d1fad45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 \ No newline at end of file +ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c34a28dc47..2e909dd8ca 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4145,6 +4145,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index aad0a6dc21..b21be3f67f 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -66,4 +66,89 @@ do_execsql_test tkt-99378-120 { GROUP BY a; } {~/Function/} + +do_execsql_test tkt-99378-130 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-140 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} {~/Function/} + +do_execsql_test tkt-99378-200 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(a, d, b->>'x', c); +} +do_execsql_test tkt-99378-210 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-220 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} +do_execsql_test tkt-99378-230 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-240 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + + + finish_test From c25f5ea6e838dabbd0721947a432c4d9717bda74 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 15:04:23 +0000 Subject: [PATCH 14/29] Add NEVER() and ALWAYS() macros on branches that are believed to be unreachable. FossilOrigin-Name: 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f64f61129..c36f686e3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. -D 2022-11-24T13:19:25.140 +C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. +D 2022-11-24T15:04:23.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f +F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 +F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 -R fe9857b5384581f396eba27025a5561f +P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef +R eb963117dcb4b03d7c27aa3277dea1bb U drh -Z d9a080fe6c44b0ca440456485b55fa7a +Z 65f0cab32e4b61b60d1aa8b06e205831 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 016d1fad45..17ef3a10aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file +3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 2e909dd8ca..dc918e0bd6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6335,14 +6335,14 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( pParse->iSelfTab ){ + if( NEVER(pParse->iSelfTab) ){ if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; iDataCur = -1; } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; - if( !ExprUseYTab(pExpr) ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; /* If we reach this point, it means that expression pExpr can be ** translated into a reference to an index column as described by diff --git a/src/select.c b/src/select.c index dcf899e4bb..e9b6296713 100644 --- a/src/select.c +++ b/src/select.c @@ -6282,7 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( NameContext *pNC /* Name context used to resolve agg-func args */ ){ pAggInfo->nColumn = pAggInfo->nAccumulator; - if( pAggInfo->nSortingColumn>0 ){ + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ pAggInfo->nSortingColumn = 0; }else{ From df5d06d03eca407aa84f12fce477a0c210bc4375 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 17:53:09 +0000 Subject: [PATCH 15/29] More work on the OPFS concurrency testing app. FossilOrigin-Name: c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 --- ext/wasm/api/sqlite3-api-opfs.js | 3 +++ ext/wasm/api/sqlite3-opfs-async-proxy.js | 28 +++++----------------- ext/wasm/tests/opfs/concurrency/index.html | 11 +++++---- ext/wasm/tests/opfs/concurrency/test.js | 28 ++++++++++++++++++---- ext/wasm/tests/opfs/concurrency/worker.js | 8 +++---- manifest | 20 ++++++++-------- manifest.uuid | 2 +- 7 files changed, 54 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb8bf04e8..5537df89fb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1171,6 +1171,9 @@ const installOpfsVfs = function callee(options){ //TODO to support fiddle and worker1 db upload: //opfsUtil.createFile = function(absName, content=undefined){...} + //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this + //purpose but its interface and name are still under + //consideration. if(sqlite3.oo1){ opfsUtil.OpfsDb = function(...args){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index b14494a0c9..8a3db9c648 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -558,33 +558,17 @@ const vfsAsyncImpls = { (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to horrible contention on journal files. */ + does but it leads to immediate contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS lock. - Regarding "immutable": that flag is not _really_ applicable - here. It's intended for use on read-only media. If, - however, a file is opened with that flag but changes later - (which can happen if we _don't_ grab a sync handle here) - then sqlite may misbehave. - - Regarding "nolock": ironically, the nolock flag forces us - to lock the file up front. "nolock" tells sqlite to _not_ - use its locking API, but OPFS requires a lock to perform - most of the operations performed in this file. If we don't - grab that lock up front, another handle could end up grabbing - it and mutating the database out from under our nolocked'd - handle. In the interest of preventing corruption, at the cost - of decreased concurrency, we have to lock it for the duration - of this file handle. - https://www.sqlite.org/uri.html */ - fh.xLock = "atOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; await getSyncHandle(fh,'xOpen'); } __openFiles[fid] = fh; @@ -824,7 +808,7 @@ const waitLoop = async function f(){ to do other things. If this is too high (e.g. 500ms) then even two workers/tabs can easily run into locking errors. */ - const waitTime = 150; + const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index a082dfe997..e19f6a8da6 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -24,10 +24,13 @@

URL flags: pass a number of workers using - the workers=N URL flag and the worker work interval - as interval=N (milliseconds). Enable OPFS VFS - verbosity with verbose=1-3 (output goes to the - dev console). + the workers=N URL flag. Set the time between each + workload with interval=N (milliseconds). Set the + number of worker iterations with iterations=N. + Enable OPFS VFS verbosity with verbose=1-3 (output + goes to the dev console). Enable/disable "unlock ASAP" mode + (higher concurrency, lower speed) + with unlock-asap=0-1.

Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome often fails with "cannot allocate diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index cb9d4275be..044d343745 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -56,13 +56,16 @@ options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 3; + ) || 4; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 - ) || 750; + ) || 1000; + options.iterations = ( + urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 + ) || 10; options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; @@ -70,15 +73,25 @@ workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; - workers.loadedCount = 0; + workers.counts = {loaded: 0, passed: 0, failed: 0}; + const checkFinished = function(){ + if(workers.counts.passed + workers.counts.failed !== workers.length){ + return; + } + if(workers.counts.failed>0){ + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + }else{ + logCss('tests-pass',"All",workers.length,"workers finished."); + } + }; workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ case 'loaded': stdout(prefix,"loaded"); - if(++workers.loadedCount === workers.length){ - stdout("All workers loaded. Telling them to run..."); + if(++workers.counts.loaded === workers.length){ + stdout("All",workers.length,"workers loaded. Telling them to run..."); workers.post('run'); } break; @@ -86,10 +99,14 @@ case 'stderr': stderr(prefix,...msg.payload); break; case 'error': stderr(prefix,"ERROR:",...msg.payload); break; case 'finished': + ++workers.counts.passed; logCss('tests-pass',prefix,...msg.payload); + checkFinished(); break; case 'failed': + ++workers.counts.failed; logCss('tests-fail',prefix,"FAILED:",...msg.payload); + checkFinished(); break; default: logCss('error',"Unhandled message type:",msg); break; } @@ -100,6 +117,7 @@ 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose + '&opfs-unlock-asap='+options.unlockAsap ); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 91aa0fa6fa..95fefa1954 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,10 +46,9 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.DB({ + db = new sqlite3.opfs.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c', - vfs: 'opfs' + flags: 'c' }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ @@ -59,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){ ]); }); - const maxIterations = 10; + const maxIterations = + urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); diff --git a/manifest b/manifest index a6b0539bab..602f6943ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. -D 2022-11-23T20:49:08.427 +C More\swork\son\sthe\sOPFS\sconcurrency\stesting\sapp. +D 2022-11-24T17:53:09.937 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 +F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 +F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 -F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 +F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a +F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 -R ff8c3df82102d217683515f7779a95c4 +P 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 +R b591bd74ceba5bd36575b86f818be346 U stephan -Z 269769e48cdf286c8fb19455cdae3a52 +Z f64db814b4972f9f3353d092900f9ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb0d09842b..11f565569e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file +c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 \ No newline at end of file From 594357f2e100fb0b504a9fc1625d8f739c75b1d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 18:45:56 +0000 Subject: [PATCH 16/29] Change a NEVER() into an assert(). FossilOrigin-Name: a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c36f686e3e..e69a4d81dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. -D 2022-11-24T15:04:23.564 +C Change\sa\sNEVER()\sinto\san\sassert(). +D 2022-11-24T18:45:56.655 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 +F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef -R eb963117dcb4b03d7c27aa3277dea1bb +P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 +R fd9fef17e074036cfa72866de0a2e639 U drh -Z 65f0cab32e4b61b60d1aa8b06e205831 +Z bd1dc748d429f9c2f3da3ec49c4b6323 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17ef3a10aa..91406edc13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file +a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dc918e0bd6..f5587395a7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6330,15 +6330,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ default: { IndexedExpr *pIEpr; Expr tmp; + assert( pParse->iSelfTab==0 ); if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; if( pParse->pIdxEpr==0 ) break; for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( NEVER(pParse->iSelfTab) ){ - if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; - iDataCur = -1; - } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; From e644f7c24bd97b24a0c1be9a82b11c13a316fafd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 19:02:40 +0000 Subject: [PATCH 17/29] Take care not to try to add comments to a TK_AGG_COLUMN opcode that does not have an associated Table object because it is a reference to an indexed expression. FossilOrigin-Name: 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e69a4d81dd..9d1dae173c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sNEVER()\sinto\san\sassert(). -D 2022-11-24T18:45:56.655 +C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. +D 2022-11-24T19:02:40.883 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 +F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 -R fd9fef17e074036cfa72866de0a2e639 +P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f +R 54c8f77b091e9c38ba79b231dfbd47a4 U drh -Z bd1dc748d429f9c2f3da3ec49c4b6323 +Z df0c8e1baf68d2353d611c7c44676419 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91406edc13..979e5c34b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file +5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f5587395a7..d7f8cfa917 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,9 +4135,11 @@ expr_code_doover: Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( pTab!=0 ){ + }else{ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ From 8683c0928111707903654278e6c87634cbee5d8c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 23:35:27 +0000 Subject: [PATCH 18/29] Correctly deal with IF-NULL-ROW operators when dealing with indexed expressions in aggregates. FossilOrigin-Name: 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9d1dae173c..b0b6894258 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. -D 2022-11-24T19:02:40.883 +C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. +D 2022-11-24T23:35:27.975 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb +F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f -R 54c8f77b091e9c38ba79b231dfbd47a4 +P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b +R 9699b80135f783c01dc948dbd9568fdf U drh -Z df0c8e1baf68d2353d611c7c44676419 +Z b4118428f98b75714904c3dd7e4b5c40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 979e5c34b9..5c7c74dae1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file +939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e9b6296713..2e6ad45614 100644 --- a/src/select.c +++ b/src/select.c @@ -6318,6 +6318,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->pAggInfo==0 ) return WRC_Continue; if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; From 7960da03466d96209f53defe088fc16916ba26fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:08:20 +0000 Subject: [PATCH 19/29] Improved comments. Add assert()s to verify that the AggInfo structure is unchanged after registers have been assigned. FossilOrigin-Name: 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/select.c | 16 ++++++++++++++++ src/sqliteInt.h | 11 ++++++++--- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b0b6894258..77a2dfc4d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. -D 2022-11-24T23:35:27.975 +C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. +D 2022-11-25T13:08:20.992 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e +F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 +F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 +F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b -R 9699b80135f783c01dc948dbd9568fdf +P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 +R 85b4d1494a033b09f8257ada7ad96c3c U drh -Z b4118428f98b75714904c3dd7e4b5c40 +Z f7018e8b2be8d601c79b6c13f2f54a26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c7c74dae1..07bc523f18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file +5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d7f8cfa917..d30ae97668 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6265,6 +6265,7 @@ static void findOrCreateAggInfoColumn( struct AggInfo_col *pCol; int k; + assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ if( pCol->iTable==pExpr->iTable @@ -6328,6 +6329,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ default: { IndexedExpr *pIEpr; diff --git a/src/select.c b/src/select.c index 2e6ad45614..e76198772d 100644 --- a/src/select.c +++ b/src/select.c @@ -6253,6 +6253,7 @@ static void analyzeAggFuncArgs( ){ int i; assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); pNC->ncFlags |= NC_InAggFunc; for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; @@ -6281,6 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + assert( pAggInfo->iFirstReg==0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ @@ -6348,6 +6350,18 @@ static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first ** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6369,6 +6383,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; @@ -6436,6 +6451,7 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ecdb35fee..db65082bbc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2766,10 +2766,15 @@ struct AggInfo { }; /* -** Macros to compute aCol[] and aFunc[] register numbers: +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. */ -#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) -#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. From f5d0656f167b4b9b7730e0af0caefcb7893b4432 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:15:48 +0000 Subject: [PATCH 20/29] Improve the robustness of the updateAccumulator() routine against OOM. FossilOrigin-Name: d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 77a2dfc4d6..d326feb02b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. -D 2022-11-25T13:08:20.992 +C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. +D 2022-11-25T13:15:48.547 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 +F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 -R 85b4d1494a033b09f8257ada7ad96c3c +P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 +R d91874ddf96958a005f34b5d280b17b3 U drh -Z f7018e8b2be8d601c79b6c13f2f54a26 +Z b1915dcd60d8f6304cf98a15b81e844e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07bc523f18..7938fd8387 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file +d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e76198772d..215acd6a01 100644 --- a/src/select.c +++ b/src/select.c @@ -6430,8 +6430,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator @@ -6452,6 +6452,7 @@ static void updateAccumulator( struct AggInfo_col *pC; assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; From bedbb1bfabc0ba78206aa7897f874d168aa0af13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:26:51 +0000 Subject: [PATCH 21/29] Remove a redundant assert() statement. FossilOrigin-Name: e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d326feb02b..728b14b688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. -D 2022-11-25T13:15:48.547 +C Remove\sa\sredundant\sassert()\sstatement. +D 2022-11-25T13:26:51.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c +F src/select.c f29c90b07067263139fa08f266860ba0877f7a1398775572c3996d3b67c2fe6e F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 -R d91874ddf96958a005f34b5d280b17b3 +P d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 +R b674915df2317c83a8051b3fae0cee5f U drh -Z b1915dcd60d8f6304cf98a15b81e844e +Z 8a9f5f62db508168a49cc0d82c6f6183 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7938fd8387..d164374f87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file +e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 215acd6a01..187beac7fb 100644 --- a/src/select.c +++ b/src/select.c @@ -6387,7 +6387,6 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; - assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, pAggInfo->iFirstReg+nReg-1); From 23f61a4ba80695ee6e9d88e56294d0dae24d5a41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 15:52:00 +0000 Subject: [PATCH 22/29] Add restriction (9) to the push-down optimization: If the subquery is a compound then all arms of the compound must have the same affinity. dbsqlfuzz 3a548de406a50e896c1bf7142692d35d339d697f. FossilOrigin-Name: 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f --- manifest | 16 +++++++------- manifest.uuid | 2 +- src/select.c | 53 ++++++++++++++++++++++++++++++++++------------ test/pushdown.test | 27 ++++++++++++++++++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index b3aa3b73da..64d80c2f59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sin\sfts3expr4.test\sto\saccount\sfor\sdifferent\slocales. -D 2022-11-24T17:58:55.503 +C Add\srestriction\s(9)\sto\sthe\spush-down\soptimization:\s\sIf\sthe\ssubquery\sis\na\scompound\sthen\sall\sarms\sof\sthe\scompound\smust\shave\sthe\ssame\saffinity.\ndbsqlfuzz\s3a548de406a50e896c1bf7142692d35d339d697f. +D 2022-11-25T15:52:00.241 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9 +F src/select.c bafe6424e942aad558b2d4be8dbcf5e35ce427ce3cf66d7f0c0ac37c366b00c6 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1403,7 +1403,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc +F test/pushdown.test c69f0970ea17e0afc674b89741f60c172cb6f761d81665fc71015f674f0f66ba F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 -R e3089d3062e617db155d3fe3a44f9c32 -U dan -Z c38105809dd01fc34e6f6bf40716c164 +P a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 +R 9c0acbc69bb427be679e111e4ec839ec +U drh +Z ae1fb8eb841aaf444344209ddd2c7ea7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bea5150fe5..4e3e4aa9d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 \ No newline at end of file +1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f \ No newline at end of file diff --git a/src/select.c b/src/select.c index 4d5bde9a31..65a0a06c33 100644 --- a/src/select.c +++ b/src/select.c @@ -4050,6 +4050,34 @@ static ExprList *findLeftmostExprlist(Select *pSel){ return pSel->pEList; } +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -4153,7 +4181,8 @@ static ExprList *findLeftmostExprlist(Select *pSel){ ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. +** compound must have the same affinity. (See restriction (9) +** on the push-down optimization.) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -4372,19 +4401,7 @@ static int flattenSubquery( if( (p->selFlags & SF_Recursive) ) return 0; /* Restriction (17h) */ - for(ii=0; iipEList->nExpr; ii++){ - char aff; - assert( pSub->pEList->a[ii].pExpr!=0 ); - aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); - for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ - assert( pSub1->pEList!=0 ); - assert( pSub1->pEList->nExpr>ii ); - assert( pSub1->pEList->a[ii].pExpr!=0 ); - if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ - return 0; - } - } - } + if( compoundHasDifferentAffinities(pSub) ) return 0; if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; @@ -5036,6 +5053,11 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** But it is a lot of work to check that case for an obscure and ** minor optimization, so we omit it for now.) ** +** (9) If the subquery is a compound, then all arms of the compound must +** have the same affinity. (This is the same as restriction (17h) +** for query flattening.) +** +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5061,6 +5083,9 @@ static int pushDownWhereTerms( if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ if( pSel->pWin ) return 0; /* restriction (6b) */ } + if( compoundHasDifferentAffinities(pSubq) ){ + return 0; /* restriction (9) */ + } }else{ if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; } diff --git a/test/pushdown.test b/test/pushdown.test index af5162495b..7c9b107841 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -86,6 +86,31 @@ do_test 2.2 { set L } {three} - +# 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f +# Disable the push-down optimization for compound subqueries if any +# arm of the compound has an incompatible affinity. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); --,(4,'four'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); --,(5,'five'); + CREATE VIEW v0 AS SELECT CAST(t0.c0 AS INTEGER) AS c0 FROM t0; + CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, 0 FROM t1_b; + SELECT t1.a, quote(t1.b), t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1; +} { + 1 'one' 0 + 2 '0' 0 +} +do_execsql_test 3.2 { + SELECT a, quote(b), cd FROM ( + SELECT t1.a, t1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1 + ) WHERE a=2 AND b='0' AND cd=0; +} { + 2 '0' 0 +} finish_test From 49d400b0912fe99f5d6c0bc9e361f132c4fa338e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 16:32:59 +0000 Subject: [PATCH 23/29] Fix #ifdefs so that restrictions (8) and (9) of the push-down optimization are still enforced even if compiled with SQLITE_OMIT_WINDOWFUNC. This fixes a bug introduced by check-in [346a3b12b861ce7b]. FossilOrigin-Name: 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 76e28676a0..8852b478ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Immprove\sthe\squery\splanner\ssuch\sthat\sit\sis\sable\sto\smake\suse\sof\sindexed\nexpressions\swithin\san\saggregate\squery\swith\sGROUP\sBY.\s\sThis\simplements\nenhancement\srequest\s[99378177930f87bd]. -D 2022-11-25T16:10:48.068 +C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T16:32:59.610 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7dcab394efe24c6016c6ed7958d426bfcdef57f50c2520dcccda27d9df28fdb6 +F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 -R eb048b7302ca70762f9994708c895609 -T +closed e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 +P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 +R ba0c31d84a1008c246f9efb185042449 U drh -Z 6fdb43be076439dd6a7f7f9247393df9 +Z 98b0ad17c9260369ac5243663e0dc610 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff7ccc702c..1de8ddd5c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 \ No newline at end of file +09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 067c98b4a3..bc9ae25ce8 100644 --- a/src/select.c +++ b/src/select.c @@ -5073,7 +5073,6 @@ static int pushDownWhereTerms( if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ @@ -5081,15 +5080,18 @@ static int pushDownWhereTerms( assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif } if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make From 902ea8392536b729c4c4553613dae9f65a4c1bf3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 17:05:55 +0000 Subject: [PATCH 24/29] Relax restriction (8) on the push-down optimization so that it only applies if one or more columns uses a collating sequence other than BINARY. See [forum:/forumpost/3824ced748baa808|forum post 3824ced748baa808] and check-in [346a3b12b861ce7b]. FossilOrigin-Name: adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 8852b478ad..3a6ab9396c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T16:32:59.610 +C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T17:05:55.692 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 +F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 -R ba0c31d84a1008c246f9efb185042449 +P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 +R 051f7bebeb695c93dfdb45030ad5b532 U drh -Z 98b0ad17c9260369ac5243663e0dc610 +Z 0a4a218169578e69e7b7469760b002aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1de8ddd5c0..76e9e654c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file +adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file diff --git a/src/select.c b/src/select.c index bc9ae25ce8..2bedfb9da0 100644 --- a/src/select.c +++ b/src/select.c @@ -5046,12 +5046,9 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** -** (8) The subquery may not be a compound that uses UNION, INTERSECT, -** or EXCEPT. (We could, perhaps, relax this restriction to allow -** this case if none of the comparisons operators between left and -** right arms of the compound use a collation other than BINARY. -** But it is a lot of work to check that case for an obscure and -** minor optimization, so we omit it for now.) +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. ** ** (9) If the subquery is a compound, then all arms of the compound must ** have the same affinity. (This is the same as restriction (17h) @@ -5075,11 +5072,14 @@ static int pushDownWhereTerms( if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ u8 op = pSel->op; assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); - if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ #endif @@ -5087,6 +5087,22 @@ static int pushDownWhereTerms( if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } + } }else{ #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; From 743c9ec4a5f3f00a85ad99c582b012c189a0d34f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:12:02 +0000 Subject: [PATCH 25/29] Fix an incorrect assertion-fault in the "TREETRACE" debugging logic on select.c. The problem does not affect production builds. FossilOrigin-Name: f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f14f6424d5..a2d2f52121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sadd\sthe\sopfs-unlock-asap=1\sURI\sflag\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP\sinstead\sof\sduring\sVFS\sidle\stime.\sThis\simproves\sconcurrency\snotably\sin\sthe\stest\sapp\sbut\sbrings\sa\ssignificant\sperformance\spenalty\sin\sspeedtest1\s(roughly\s4x\sslowdown).\sThis\sis\snot\sthe\sfinal\sword\sin\sOPFS\sconcurrency,\sbut\sgets\sus\sa\sstep\sfurther. -D 2022-11-26T15:24:58.092 +C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:12:02.257 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 +F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 -R ef0be46e6ac8ee603582dd679fae7434 -T +closed c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 Closed\sby\sintegrate-merge. -U stephan -Z 3b8060cf3f2afe36aab99fa36bba89db +P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 +R 581898efb4aefcef8555315298dbd7f7 +U drh +Z f6493a79b992a3696ede08a261d497d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e97840e0fc..d4e96fca06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 \ No newline at end of file +f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2bedfb9da0..10498310d4 100644 --- a/src/select.c +++ b/src/select.c @@ -6259,7 +6259,7 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d %s\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, pCol->iSorterColumn, ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); From 217e77d24008f5bd813c719e4a08da93500a2218 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:48:56 +0000 Subject: [PATCH 26/29] Show which AggInfo.aCol an Expr node represents in the tree trace debugging logic. Does not affect production builds. FossilOrigin-Name: b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/treeview.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a2d2f52121..54ca01a2ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:12:02.257 +C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:48:56.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 -R 581898efb4aefcef8555315298dbd7f7 +P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e +R 77aba841340d0be9ea0f0036dfeb9ac7 U drh -Z f6493a79b992a3696ede08a261d497d4 +Z cf507f961f29b58691309417f6fc3d94 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4e96fca06..4df4cac51e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file +b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; From ca9309cc565da99c6a8e8dea84852b9c4ee478a4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Nov 2022 00:57:56 +0000 Subject: [PATCH 27/29] Document an OPFS API change in Chrome v108 which does not break our code but does change several formerly async methods to synchronous. No code changes. FossilOrigin-Name: e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 16 ++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8a3db9c648..8e60969bc6 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -29,6 +29,22 @@ This file represents an implementation detail of a larger piece of code, and not a public interface. Its details may change at any time and are not intended to be used by any client-level code. + + 2022-11-27: Chrome v108 changes some async methods to synchronous, as + documented at: + + https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + + We cannot change to the sync forms at this point without breaking + clients who use Chrome v104-ish or higher. truncate(), getSize(), + flush(), and close() are now (as of v108) synchronous. Calling them + with an "await", as we have to for the async forms, is still legal + with the sync forms but is superfluous. Calling the async forms with + theFunc().then(...) is not compatible with the change to + synchronous, but we do do not use those APIs that way. i.e. we don't + _need_ to change anything for this, but at some point (after Chrome + versions (approximately) 104-107 are extinct) should change our + usage of those methods to remove the "await". */ "use strict"; const toss = function(...args){throw new Error(args.join(' '))}; diff --git a/manifest b/manifest index 54ca01a2ed..37ec0eef0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:48:56.633 +C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. +D 2022-11-27T00:57:56.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R 77aba841340d0be9ea0f0036dfeb9ac7 -U drh -Z cf507f961f29b58691309417f6fc3d94 +P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 +R 4c0998f1bfd61979c1006462c1fac102 +U stephan +Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4df4cac51e..852ac7c5b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file +e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file From c1ca183abfeb54a5735f4f3bd4c0e1994645c3f7 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 02:28:44 +0000 Subject: [PATCH 28/29] Conform CLI .trace arg handling to its help. FossilOrigin-Name: 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 2 +- test/shell4.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 37ec0eef0c..107bf9f15b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. -D 2022-11-27T00:57:56.426 +C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. +D 2022-11-28T02:28:44.889 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 -F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 +F src/shell.c.in 334b1661abb6453e4f5d1778ce9c80dddef83470867f1fba07e5ee400792adbe F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1497,7 +1497,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 7dc8a515705bc093d8ffe381670e8fa7a969661e8ed177c35c847e3c6dfc35e2 +F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 -R 4c0998f1bfd61979c1006462c1fac102 -U stephan -Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 +P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 +R ce11368d09e97cb20c99cf53926be781 +U larrybr +Z 2b24f85901c30dde8df5f7afa27a353b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 852ac7c5b3..2d0d9deb13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file +31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e1f0ecfcd4..ab47dba8c9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10658,7 +10658,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ diff --git a/test/shell4.test b/test/shell4.test index 068072202d..193dc8b69b 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -125,6 +125,12 @@ SELECT * FROM t1;}} do_test shell4-2.5 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;" } {0 {SELECT * FROM t1;}} +do_test shell4-2.6 { + catchcmd ":memory:" { +CREATE TABLE t1(x); +.trace --stmt stdout +SELECT * FROM t1;} +} {0 {SELECT * FROM t1;}} } do_test shell4-3.1 { From abbfa7aa17307f31011751072454850e6da4d435 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 13:47:27 +0000 Subject: [PATCH 29/29] Ensure the RETURNING clause is honoured when a row of a temp table is updated by an ON CONFLICT clause. FossilOrigin-Name: a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/returning1.test | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 107bf9f15b..91afe6c7c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. -D 2022-11-28T02:28:44.889 +C Ensure\sthe\sRETURNING\sclause\sis\shonoured\swhen\sa\srow\sof\sa\stemp\stable\sis\supdated\sby\san\sON\sCONFLICT\sclause. +D 2022-11-28T13:47:27.194 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 -F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 +F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1421,7 +1421,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0 F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test c43b8370a351f77aec6d71f4a2cde59b849369ed1933261a2c2c69e23e34ff5e +F test/returning1.test 1366e04566cfe1a082d17b1e0f195ec64473c79374b3a5d4ae00c43d885dea31 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 -R ce11368d09e97cb20c99cf53926be781 -U larrybr -Z 2b24f85901c30dde8df5f7afa27a353b +P 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c +R cab683454f2ed9db2c50c1dc1f9667b9 +U dan +Z f330cc4347042764fbf120efeed28ded # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d0d9deb13..0bd7375f33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file +a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index ca64d6145b..02d8540237 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -61,7 +61,7 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; diff --git a/test/returning1.test b/test/returning1.test index fb058a6438..92e10ee9f4 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -376,4 +376,28 @@ do_execsql_test 16.1 { SELECT * FROM t2; } {1 2 3 a b c} + +foreach {tn temp} { + 1 "" + 2 TEMP +} { + reset_db + do_execsql_test 17.$tn.0 " + CREATE $temp TABLE foo ( + fooid INTEGER PRIMARY KEY, + fooval INTEGER NOT NULL UNIQUE, + refcnt INTEGER NOT NULL DEFAULT 1 + ); + " + do_execsql_test 17.$tn.1 { + INSERT INTO foo (fooval) VALUES (17), (4711), (17) + ON CONFLICT DO + UPDATE SET refcnt = refcnt+1 + RETURNING fooid; + } { + 1 2 1 + } +} + + finish_test