From 4fcb30b7d7147cddb05adab2191ae0f1a8268b8a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Mar 2021 16:06:25 +0000 Subject: [PATCH 1/6] Experimental optimization for distinct aggregates (e.g. "SELECT count(DISTINCT nExpr; + Vdbe *v = pParse->pVdbe; + + switch( eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + int i; + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. */ + sqlite3VdbeChangeToNoop(v, iTabAddr); + pOp = sqlite3VdbeGetOp(v, iTabAddr); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ia[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + sqlite3VdbeChangeToNoop(v, iTabAddr); + break; + } + + default: { + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + } +} #ifdef SQLITE_ENABLE_SORTER_REFERENCES /* @@ -1013,59 +1086,9 @@ static void selectInnerLoop( ** part of the result. */ if( hasDistinct ){ - switch( pDistinct->eTnctType ){ - case WHERE_DISTINCT_ORDERED: { - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - int iJump; /* Jump destination */ - int regPrev; /* Previous row content */ - - /* Allocate space for the previous row */ - regPrev = pParse->nMem+1; - pParse->nMem += nResultCol; - - /* Change the OP_OpenEphemeral coded earlier to an OP_Null - ** sets the MEM_Cleared bit on the first register of the - ** previous value. This will cause the OP_Ne below to always - ** fail on the first iteration of the loop even if the first - ** row is all NULLs. - */ - sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); - pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); - pOp->opcode = OP_Null; - pOp->p1 = 1; - pOp->p2 = regPrev; - pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ - - iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; - for(i=0; ipEList->a[i].pExpr); - if( idb->mallocFailed ); - sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); - break; - } - - case WHERE_DISTINCT_UNIQUE: { - sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); - break; - } - - default: { - assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, - regResult); - break; - } - } + assert( nResultCol==p->pEList->nExpr ); + codeDistinct2(pParse, pDistinct->eTnctType, pDistinct->tabTnct, + pDistinct->addrTnct, iContinue, p->pEList, regResult); if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } @@ -5635,8 +5658,8 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ pFunc->iDistinct = -1; }else{ KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO); + pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } } } @@ -5668,7 +5691,12 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. */ -static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ +static void updateAccumulator( + Parse *pParse, + int regAcc, + AggInfo *pAggInfo, + int eDistinctType +){ Vdbe *v = pParse->pVdbe; int i; int regHit = 0; @@ -5714,13 +5742,14 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ nArg = 0; regAgg = 0; } - if( pF->iDistinct>=0 ){ + if( pF->iDistinct>=0 && pList ){ if( addrNext==0 ){ addrNext = sqlite3VdbeMakeLabel(pParse); } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ - codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); + codeDistinct2(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, + addrNext, pList, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl = 0; @@ -6905,7 +6934,7 @@ int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, iUseFlag, pAggInfo); + updateAccumulator(pParse, iUseFlag, pAggInfo, WHERE_DISTINCT_UNORDERED); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -7025,6 +7054,8 @@ int sqlite3Select( explainSimpleCount(pParse, pTab, pBest); }else{ int regAcc = 0; /* "populate accumulators" flag */ + ExprList *pDistinct = 0; + u16 distFlag = 0; /* If there are accumulator registers but no min() or max() functions ** without FILTER clauses, allocate register regAcc. Register regAcc @@ -7048,6 +7079,9 @@ int sqlite3Select( regAcc = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); } + }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){ + pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; + distFlag = pDistinct ? WHERE_WANT_DISTINCT : 0; } /* This case runs if the aggregate has no GROUP BY clause. The @@ -7067,12 +7101,14 @@ int sqlite3Select( SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, - 0, minMaxFlag, 0); + pDistinct, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); - updateAccumulator(pParse, regAcc, pAggInfo); + updateAccumulator( + pParse, regAcc, pAggInfo, sqlite3WhereIsDistinct(pWInfo) + ); if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 913de54ddc..7b451b5573 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2598,6 +2598,7 @@ struct AggInfo { FuncDef *pFunc; /* The aggregate function implementation */ int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ + int iDistAddr; /* Addres of OP_OpenEphemeral */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ diff --git a/src/where.c b/src/where.c index 75b3ceff87..8e44e25765 100644 --- a/src/where.c +++ b/src/where.c @@ -492,7 +492,7 @@ static int findIndexCol( for(i=0; inExpr; i++){ Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); if( ALWAYS(p!=0) - && p->op==TK_COLUMN + && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ @@ -557,7 +557,8 @@ static int isDistinctRedundant( for(i=0; inExpr; i++){ Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); if( NEVER(p==0) ) continue; - if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue; + if( p->iTable==iBase && p->iColumn<0 ) return 1; } /* Loop through all indices on the table, checking each to see if it makes @@ -3822,7 +3823,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); if( NEVER(pOBExpr==0) ) continue; - if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, ~ready, eqOpMask, 0); @@ -3951,7 +3952,7 @@ static i8 wherePathSatisfiesOrderBy( if( NEVER(pOBExpr==0) ) continue; if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; if( iColumn>=XN_ROWID ){ - if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ From 96ecb7b00425ae7f809dd356b17f72e6689eb99a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Mar 2021 16:47:33 +0000 Subject: [PATCH 2/6] Add missing comment to new routine in select.c. FossilOrigin-Name: ef2f0cf21ba61bdd29e09cf41b012a2d757683f524a252f0af7dfee7df1a1a0f --- manifest | 15 ++++++------- manifest.uuid | 2 +- src/select.c | 59 ++++++++++++++++++++++++++------------------------- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 9be753e63b..1e2167b501 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\soptimization\sfor\sdistinct\saggregates\s(e.g.\s"SELECT\scount(DISTINCT\s) ..."). Parameter iTab is +** the cursor number of an ephemeral table opened by instruction iTabAddr for +** the code generated by this routine to use. There are three strategies, based +** on the value of parameter eTnctType: ** -** A jump to addrRepeat is made and the N+1 values are popped from the -** stack if the top N elements are not distinct. +** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: +** The ephemeral cursor table is queries for a record identical to the +** record formed by the current array of registers. If one is found, +** jump to VM address addrRepeat. Otherwise, insert a new record into +** the ephemeral cursor and proceed. +** +** WHERE_DISTINCT_ORDERED: +** In this case rows are being delivered sorted order sorted. The ephermal +** table is not required in this case. Instead, the current set of +** registers are compared to previous row. If they match, the new row +** is not distinct and control jumps to VM address addrRepeat. Otherwise, +** the VM program proceeds with processing the new row. +** +** WHERE_DISTINCT_UNIQUE: +** In this case it has already been determined that the rows are distinct. +** No special action is required. +** +** Parameter pEList is the list of expressions used to generated the +** contents of each row. It is used by this routine to determine (a) +** how many elements there are in the array of registers and (b) the +** collation sequences that should be used for the comparisons if +** eTnctType is WHERE_DISTINCT_ORDERED. */ static void codeDistinct( - Parse *pParse, /* Parsing and code generating context */ - int iTab, /* A sorting index used to test for distinctness */ - int addrRepeat, /* Jump to here if not distinct */ - int N, /* Number of elements */ - int iMem /* First element */ -){ - Vdbe *v; - int r1; - - v = pParse->pVdbe; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3ReleaseTempReg(pParse, r1); -} -#endif - -static void codeDistinct2( Parse *pParse, /* Parsing and code generating context */ int eTnctType, /* WHERE_DISTINCT_* value */ int iTab, /* A sorting index used to test for distinctness */ @@ -1087,7 +1088,7 @@ static void selectInnerLoop( */ if( hasDistinct ){ assert( nResultCol==p->pEList->nExpr ); - codeDistinct2(pParse, pDistinct->eTnctType, pDistinct->tabTnct, + codeDistinct(pParse, pDistinct->eTnctType, pDistinct->tabTnct, pDistinct->addrTnct, iContinue, p->pEList, regResult); if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); @@ -5748,7 +5749,7 @@ static void updateAccumulator( } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ - codeDistinct2(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, + codeDistinct(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, addrNext, pList, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ From 5383db714e111b9c72179a3bd9c3c93583108161 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Mar 2021 18:24:31 +0000 Subject: [PATCH 3/6] Attempt to use an index for DISTINCT aggregate queries that have GROUP BY clauses. FossilOrigin-Name: 3bca003cd2b2cb38d4a4e2e5f673ee0ac05bfe31247ec09e7bd379b77a31b44c --- manifest | 14 +++--- manifest.uuid | 2 +- src/select.c | 98 ++++++++++++++++++++++++++------------ test/distinctagg.test | 108 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 184 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 1e2167b501..ae9cc305fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\scomment\sto\snew\sroutine\sin\sselect.c. -D 2021-03-09T16:47:33.281 +C Attempt\sto\suse\san\sindex\sfor\sDISTINCT\saggregate\squeries\sthat\shave\sGROUP\sBY\sclauses. +D 2021-03-12T18:24:31.455 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -542,7 +542,7 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 688070848f0a0c41bcc545a4b4b052921d9abc29ba3102985d3d6f7595d9637c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7741bb3b315bd9d36c01275fb7a0b319dc30b70054f46a3521acdd71e8615d07 +F src/select.c be01e5eab0a452687a3c01b810f34b2e64816e91bc7965ecdf5a9ef714ab466f F src/shell.c.in af18a2e980aabe739a8188266464866fe7947b100674e07480e7ba3e37595947 F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -844,7 +844,7 @@ F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a199 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test e7d0cf371944dd0cbedff86420744e2f1ea2b528156451c97eb6ff41a99b9236 F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 -F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/distinctagg.test bfdd84af7919687d416997bc8f156b11d693c2cc4d002bdb44c5884ec9eb757a F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d F test/e_blobopen.test e95e1d40f995056f6f322cd5e1a1b83a27e1a145 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22 -R 9199f92495fd5d3674624db01699c96b +P ef2f0cf21ba61bdd29e09cf41b012a2d757683f524a252f0af7dfee7df1a1a0f +R 7b3b84df059d1804d41c3dfed8389b28 U dan -Z a3e310de94fb22aed527b8a42451f5c8 +Z 2285f2ee6ec9cbb99c3f7d9e2d1b3385 diff --git a/manifest.uuid b/manifest.uuid index b05f544f27..49c33aa997 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef2f0cf21ba61bdd29e09cf41b012a2d757683f524a252f0af7dfee7df1a1a0f \ No newline at end of file +3bca003cd2b2cb38d4a4e2e5f673ee0ac05bfe31247ec09e7bd379b77a31b44c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 65345c839b..3b1697db13 100644 --- a/src/select.c +++ b/src/select.c @@ -749,7 +749,7 @@ static void codeOffset( ** on the value of parameter eTnctType: ** ** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: -** The ephemeral cursor table is queries for a record identical to the +** The ephemeral cursor table is queried for a record identical to the ** record formed by the current array of registers. If one is found, ** jump to VM address addrRepeat. Otherwise, insert a new record into ** the ephemeral cursor and proceed. @@ -771,41 +771,28 @@ static void codeOffset( ** collation sequences that should be used for the comparisons if ** eTnctType is WHERE_DISTINCT_ORDERED. */ -static void codeDistinct( +static int codeDistinct( Parse *pParse, /* Parsing and code generating context */ int eTnctType, /* WHERE_DISTINCT_* value */ int iTab, /* A sorting index used to test for distinctness */ - int iTabAddr, /* Address of OP_OpenEphemeral instruction for iTab */ int addrRepeat, /* Jump to here if not distinct */ ExprList *pEList, /* Expression for each element */ int regElem /* First element */ ){ + int iRet = 0; int nResultCol = pEList->nExpr; Vdbe *v = pParse->pVdbe; switch( eTnctType ){ case WHERE_DISTINCT_ORDERED: { int i; - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ int iJump; /* Jump destination */ int regPrev; /* Previous row content */ /* Allocate space for the previous row */ - regPrev = pParse->nMem+1; + iRet = regPrev = pParse->nMem+1; pParse->nMem += nResultCol; - /* Change the OP_OpenEphemeral coded earlier to an OP_Null - ** sets the MEM_Cleared bit on the first register of the - ** previous value. This will cause the OP_Ne below to always - ** fail on the first iteration of the loop even if the first - ** row is all NULLs. */ - sqlite3VdbeChangeToNoop(v, iTabAddr); - pOp = sqlite3VdbeGetOp(v, iTabAddr); - pOp->opcode = OP_Null; - pOp->p1 = 1; - pOp->p2 = regPrev; - pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ - iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; for(i=0; ia[i].pExpr); @@ -825,7 +812,7 @@ static void codeDistinct( } case WHERE_DISTINCT_UNIQUE: { - sqlite3VdbeChangeToNoop(v, iTabAddr); + /* nothing to do */ break; } @@ -837,9 +824,34 @@ static void codeDistinct( sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, r1); + iRet = iTab; break; } } + + return iRet; +} + +static void fixDistinctOpenEph( + Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ + int iVal, /* Value returned by codeDistinct() */ + int iTabAddr /* Address of OP_OpenEphemeral instruction for iTab */ +){ + if( eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeChangeToNoop(v, iTabAddr); + if( eTnctType==WHERE_DISTINCT_ORDERED ){ + /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared + ** bit on the first register of the previous value. This will cause the + ** OP_Ne added in codeDistinct() to always fail on the first iteration of + ** the loop even if the first row is all NULLs. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iTabAddr); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = iVal; + } + } } #ifdef SQLITE_ENABLE_SORTER_REFERENCES @@ -1087,9 +1099,11 @@ static void selectInnerLoop( ** part of the result. */ if( hasDistinct ){ + int eType = pDistinct->eTnctType; + int iTab = pDistinct->tabTnct; assert( nResultCol==p->pEList->nExpr ); - codeDistinct(pParse, pDistinct->eTnctType, pDistinct->tabTnct, - pDistinct->addrTnct, iContinue, p->pEList, regResult); + iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult); + fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct); if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } @@ -5749,8 +5763,8 @@ static void updateAccumulator( } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ - codeDistinct(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, - addrNext, pList, regAgg); + pF->iDistinct = codeDistinct(pParse, eDistinctType, + pF->iDistinct, addrNext, pList, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl = 0; @@ -6778,6 +6792,20 @@ int sqlite3Select( int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ int addrReset; /* Subroutine for resetting the accumulator */ int regReset; /* Return address register for reset subroutine */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist = WHERE_DISTINCT_NOOP; + + if( pAggInfo->nFunc==1 + && pAggInfo->aFunc[0].iDistinct>=0 + && pAggInfo->aFunc[0].pFExpr->x.pList + ){ + Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pDistinct = sqlite3ExprListDup(db, pGroupBy, 0); + pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr); + distFlag = pDistinct ? WHERE_WANT_DISTINCT : 0; + } /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out @@ -6814,8 +6842,8 @@ int sqlite3Select( */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); SELECTTRACE(1,pParse,p,("WhereBegin\n")); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, - WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ) goto select_end; SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); @@ -6935,7 +6963,8 @@ int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, iUseFlag, pAggInfo, WHERE_DISTINCT_UNORDERED); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -6991,7 +7020,13 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); - + + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + + sqlite3ExprListDelete(db, pDistinct); } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { Table *pTab; @@ -7057,6 +7092,7 @@ int sqlite3Select( int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; + int eDist; /* If there are accumulator registers but no min() or max() functions ** without FILTER clauses, allocate register regAcc. Register regAcc @@ -7107,9 +7143,13 @@ int sqlite3Select( goto select_end; } SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); - updateAccumulator( - pParse, regAcc, pAggInfo, sqlite3WhereIsDistinct(pWInfo) - ); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, regAcc, pAggInfo, eDist); + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); diff --git a/test/distinctagg.test b/test/distinctagg.test index 9b5dc21678..c090bf0275 100644 --- a/test/distinctagg.test +++ b/test/distinctagg.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix distinctagg do_test distinctagg-1.1 { execsql { @@ -31,7 +32,7 @@ do_test distinctagg-1.1 { } {1 2 3 3} do_test distinctagg-1.2 { execsql { - SELECT b, count(distinct c) FROM t1 GROUP BY b ORDER BY b + SELECT b, count(distinct c) FROM t1 GROUP BY b } } {2 1 3 2} do_test distinctagg-1.3 { @@ -59,4 +60,109 @@ do_test distinctagg-2.2 { } } {1 {DISTINCT aggregates must have exactly one argument}} +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(d, e, f); + + INSERT INTO t1 VALUES (1, 1, 1); + INSERT INTO t1 VALUES (2, 2, 2); + INSERT INTO t1 VALUES (3, 3, 3); + INSERT INTO t1 VALUES (4, 1, 4); + INSERT INTO t1 VALUES (5, 2, 1); + INSERT INTO t1 VALUES (5, 3, 2); + INSERT INTO t1 VALUES (4, 1, 3); + INSERT INTO t1 VALUES (3, 2, 4); + INSERT INTO t1 VALUES (2, 3, 1); + INSERT INTO t1 VALUES (1, 1, 2); + + INSERT INTO t2 VALUES('a', 'a', 'a'); + INSERT INTO t2 VALUES('b', 'b', 'b'); + INSERT INTO t2 VALUES('c', 'c', 'c'); + + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b, c); +} + +foreach {tn use_eph sql res} { + 1 0 "SELECT count(DISTINCT a) FROM t1" 5 + 2 0 "SELECT count(DISTINCT b) FROM t1" 3 + 3 1 "SELECT count(DISTINCT c) FROM t1" 4 + 4 0 "SELECT count(DISTINCT c) FROM t1 WHERE b=3" 3 + 5 0 "SELECT count(DISTINCT rowid) FROM t1" 10 + 6 0 "SELECT count(DISTINCT a) FROM t1, t2" 5 + 7 0 "SELECT count(DISTINCT a) FROM t2, t1" 5 + 8 1 "SELECT count(DISTINCT a+b) FROM t1, t2, t2, t2" 6 + 9 0 "SELECT count(DISTINCT c) FROM t1 WHERE c=2" 1 + 10 1 "SELECT count(DISTINCT t1.rowid) FROM t1, t2" 10 +} { + do_test 3.$tn.1 { + set prg [db eval "EXPLAIN $sql"] + set idx [lsearch $prg OpenEphemeral] + expr {$idx>=0} + } $use_eph + + do_execsql_test 3.$tn.2 $sql $res +} + +do_execsql_test 3.10 { + SELECT a, count(DISTINCT b) FROM t1 GROUP BY a; +} { + 1 1 2 2 3 2 4 1 5 2 +} + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b, c); + + INSERT INTO t1 VALUES(1, 'A', 1); + INSERT INTO t1 VALUES(1, 'A', 1); + INSERT INTO t1 VALUES(2, 'A', 2); + INSERT INTO t1 VALUES(2, 'A', 2); + INSERT INTO t1 VALUES(1, 'B', 1); + INSERT INTO t1 VALUES(2, 'B', 2); + INSERT INTO t1 VALUES(3, 'B', 3); + INSERT INTO t1 VALUES(NULL, 'B', NULL); + INSERT INTO t1 VALUES(NULL, 'C', NULL); + INSERT INTO t1 VALUES('d', 'D', 'd'); + + CREATE TABLE t2(d, e, f); + CREATE INDEX t2def ON t2(d, e, f); + + INSERT INTO t2 VALUES(1, 1, 'a'); + INSERT INTO t2 VALUES(1, 1, 'a'); + INSERT INTO t2 VALUES(1, 2, 'a'); + INSERT INTO t2 VALUES(1, 2, 'a'); + INSERT INTO t2 VALUES(1, 2, 'b'); + INSERT INTO t2 VALUES(1, 3, 'b'); + INSERT INTO t2 VALUES(1, 3, 'a'); + INSERT INTO t2 VALUES(1, 3, 'b'); + INSERT INTO t2 VALUES(2, 3, 'x'); + INSERT INTO t2 VALUES(2, 3, 'y'); + INSERT INTO t2 VALUES(2, 3, 'z'); +} + +foreach {tn use_eph sql res} { + 1 0 "SELECT count(DISTINCT c) FROM t1 GROUP BY b" {2 3 0 1} + 2 1 "SELECT count(DISTINCT a) FROM t1 GROUP BY b" {2 3 0 1} + 3 1 "SELECT count(DISTINCT a) FROM t1 GROUP BY b+c" {0 1 1 1 1} + + 4 0 "SELECT count(DISTINCT f) FROM t2 GROUP BY d, e" {1 2 2 3} + 5 1 "SELECT count(DISTINCT f) FROM t2 GROUP BY d" {2 3} + 6 0 "SELECT count(DISTINCT f) FROM t2 WHERE d IS 1 GROUP BY e" {1 2 2} +} { + do_test 4.$tn.1 { + set prg [db eval "EXPLAIN $sql"] + set idx [lsearch $prg OpenEphemeral] + expr {$idx>=0} + } $use_eph + + do_execsql_test 4.$tn.2 $sql $res +} + finish_test + From 76075807913521318e590a3bed40b14f136fb3c7 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Mar 2021 21:09:20 +0000 Subject: [PATCH 4/6] Fix a crash in new code on this branch. FossilOrigin-Name: c05ed2a8a9c4975c69bdfa733598ed90c1509d52b3c39dedbd829ab148777e1d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 47 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8dc150ac08..777c57add8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2021-03-12T18:33:55.409 +C Fix\sa\scrash\sin\snew\scode\son\sthis\sbranch. +D 2021-03-12T21:09:20.890 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -542,7 +542,7 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d95db73d3e6a5c689e5f6604b4d2521350e45f2a0f0f84f5a2dc2bfee56580a0 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c bc60b2b50953c73f743bf88e01c389a426a02f01c4744ffe21020476908a12e3 +F src/select.c 05c83f086a7da99478a53a5355ae0d9fd5edd3aeaa079c814205211ac58c21e1 F src/shell.c.in 35adf1212d759069b00e468a9304a05a67710c8f8f50e7312335e23cac985d8c F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3bca003cd2b2cb38d4a4e2e5f673ee0ac05bfe31247ec09e7bd379b77a31b44c acd63062eb06748bfe9e4886639e4f2b54ea6a496a83f10716abbaba4115500b -R 86fbe1c09f3de8e5e7b5e69edb87ebd0 +P 198bc510d64b5794559584ad5c9de41dc966dce4eb78be15b12adba43dfcb639 +R 6377899be744d0cc8507f339ce2d0263 U dan -Z b671b123ff2e4c1e64c0384e9dd5309a +Z baf0281e2435d9e0d6e51472154aad62 diff --git a/manifest.uuid b/manifest.uuid index 24c87ad4b9..4bc513c98f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -198bc510d64b5794559584ad5c9de41dc966dce4eb78be15b12adba43dfcb639 \ No newline at end of file +c05ed2a8a9c4975c69bdfa733598ed90c1509d52b3c39dedbd829ab148777e1d \ No newline at end of file diff --git a/src/select.c b/src/select.c index 43ad2784e9..a2a979b5ea 100644 --- a/src/select.c +++ b/src/select.c @@ -743,17 +743,20 @@ static void codeOffset( /* ** Add code that will check to make sure the array of registers starting at ** iMem form a distinct entry. This is used by both "SELECT DISTINCT ..." and -** distinct aggregates ("SELECT count(DISTINCT ) ..."). Parameter iTab is -** the cursor number of an ephemeral table opened by instruction iTabAddr for -** the code generated by this routine to use. There are three strategies, based -** on the value of parameter eTnctType: +** distinct aggregates ("SELECT count(DISTINCT ) ..."). Three strategies +** are available. Which is used depends on the value of parameter eTnctType, +** as follows: ** ** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: +** Parameter iTab is the cursor number of an ephemeral table that must +** be opened before the VM code generated by this routine is executed. ** The ephemeral cursor table is queried for a record identical to the ** record formed by the current array of registers. If one is found, ** jump to VM address addrRepeat. Otherwise, insert a new record into ** the ephemeral cursor and proceed. ** +** The returned value in this case is a copy of parameter iTab. +** ** WHERE_DISTINCT_ORDERED: ** In this case rows are being delivered sorted order sorted. The ephermal ** table is not required in this case. Instead, the current set of @@ -761,9 +764,14 @@ static void codeOffset( ** is not distinct and control jumps to VM address addrRepeat. Otherwise, ** the VM program proceeds with processing the new row. ** +** The returned value in this case is the register number of the first +** in an array of registers used to store the previous result row so that +** it can be compared to the next. The caller must ensure that this cell +** is initialized to NULL and has the "clear" flag set. +** ** WHERE_DISTINCT_UNIQUE: ** In this case it has already been determined that the rows are distinct. -** No special action is required. +** No special action is required. The return value is always zero. ** ** Parameter pEList is the list of expressions used to generated the ** contents of each row. It is used by this routine to determine (a) @@ -832,21 +840,42 @@ static int codeDistinct( return iRet; } +/* +** A call to this function must be made for each call to codeDistinct(). +** Parameter is passed the value returned by that call to codeDistinct(), +** and iOpenEphAddr the address of the instruction used to open the +** ephemeral table (that may be) used by codeDistinct(). +** +** Argument eTnctType is passed the strategy to be used for any DISTINCT +** operation, as returned by sqlite3WhereIsDistinct(). If the strategy +** is WHERE_DISTINCT_NOOP or WHERE_DISTINCT_UNORDERED, this function is +** a no-op. Otherwise: +** +** WHERE_DISTINCT_UNIQUE: +** In this case the ephemeral table is not required. So instruction +** iOpenEphAddr is replaced by an OP_Noop. +** +** WHERE_DISTINCT_ORDERED: +** In this case the ephemeral table is not required. The instruction +** iOpenEphAddr is replaced by an OP_Null instruction to set register +** iVal to a NULL value with the "clear" flag set (see comments above +** codeDistinct() for details). +*/ static void fixDistinctOpenEph( Parse *pParse, /* Parsing and code generating context */ int eTnctType, /* WHERE_DISTINCT_* value */ int iVal, /* Value returned by codeDistinct() */ - int iTabAddr /* Address of OP_OpenEphemeral instruction for iTab */ + int iOpenEphAddr /* Address of OP_OpenEphemeral instruction for iTab */ ){ if( eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeChangeToNoop(v, iTabAddr); + sqlite3VdbeChangeToNoop(v, iOpenEphAddr); if( eTnctType==WHERE_DISTINCT_ORDERED ){ /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared ** bit on the first register of the previous value. This will cause the ** OP_Ne added in codeDistinct() to always fail on the first iteration of ** the loop even if the first row is all NULLs. */ - VdbeOp *pOp = sqlite3VdbeGetOp(v, iTabAddr); + VdbeOp *pOp = sqlite3VdbeGetOp(v, iOpenEphAddr); pOp->opcode = OP_Null; pOp->p1 = 1; pOp->p2 = iVal; @@ -6848,6 +6877,7 @@ int sqlite3Select( WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ) goto select_end; + eDist = sqlite3WhereIsDistinct(pWInfo); SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so @@ -6965,7 +6995,6 @@ int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); From 9bfafa8917fd5e0aa1039ed3b74466cd5cb5eb3e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 13 Mar 2021 17:21:24 +0000 Subject: [PATCH 5/6] Fix typos in comments and add test cases. FossilOrigin-Name: 01312a3dbd92823af535dc618c68d95a2aa1cbee2501b0a9826eae3f09bec760 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- src/where.c | 4 +++- test/distinctagg.test | 26 ++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 777c57add8..47f8a93d45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sin\snew\scode\son\sthis\sbranch. -D 2021-03-12T21:09:20.890 +C Fix\stypos\sin\scomments\sand\sadd\stest\scases. +D 2021-03-13T17:21:24.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -547,7 +547,7 @@ F src/shell.c.in 35adf1212d759069b00e468a9304a05a67710c8f8f50e7312335e23cac985d8 F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h 312fc96be9050903b7d01dca7f80800b9188ffab6074506f1a73077e3f1cafec +F src/sqliteInt.h d10a181d069075fe335f5965829b498d17d66ea4afb2c0a04dea5633a62fdf79 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -629,7 +629,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d42d6c80ea363ef689a462e65eefcfe87deab924c50de5baa37ecb6af7d7ddaa -F src/where.c 6fcbfab409b4a60dd92abd4c5f3fca44f5df3f8ea24f689d2ea620f05da89d96 +F src/where.c 07d1232df6fc4705815b0ef5182fe3e56bdba6ac09ce52bd66ecb207626aac2b F src/whereInt.h 446e5e8018f83358ef917cf32d8e6a86dc8430113d0b17e720f1839d3faa44c4 F src/wherecode.c e57a8690311a75d06e723e8d379f9831de04aba300e07174d236e32a7f9c7a13 F src/whereexpr.c 53452fe2fb07be2f4cb17f55cc721416fae0092c00717f106faf289c990b6494 @@ -844,7 +844,7 @@ F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a199 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test e7d0cf371944dd0cbedff86420744e2f1ea2b528156451c97eb6ff41a99b9236 F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 -F test/distinctagg.test bfdd84af7919687d416997bc8f156b11d693c2cc4d002bdb44c5884ec9eb757a +F test/distinctagg.test fd74c3783700f1d99c67a5d906bba97fb6650f4559f3ea7b1873f793ffc601c9 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d F test/e_blobopen.test e95e1d40f995056f6f322cd5e1a1b83a27e1a145 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 198bc510d64b5794559584ad5c9de41dc966dce4eb78be15b12adba43dfcb639 -R 6377899be744d0cc8507f339ce2d0263 +P c05ed2a8a9c4975c69bdfa733598ed90c1509d52b3c39dedbd829ab148777e1d +R 457fb9da170354f72baa7acae0abf7b9 U dan -Z baf0281e2435d9e0d6e51472154aad62 +Z 04936f8ca7b90bfb2cf6a7dfa6671741 diff --git a/manifest.uuid b/manifest.uuid index 4bc513c98f..e951580dac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c05ed2a8a9c4975c69bdfa733598ed90c1509d52b3c39dedbd829ab148777e1d \ No newline at end of file +01312a3dbd92823af535dc618c68d95a2aa1cbee2501b0a9826eae3f09bec760 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c060e47b3b..43917efdc9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2603,7 +2603,7 @@ struct AggInfo { FuncDef *pFunc; /* The aggregate function implementation */ int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ - int iDistAddr; /* Addres of OP_OpenEphemeral */ + int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ diff --git a/src/where.c b/src/where.c index 8e44e25765..ff6220b83d 100644 --- a/src/where.c +++ b/src/where.c @@ -5025,7 +5025,9 @@ WhereInfo *sqlite3WhereBegin( /* Attempt to omit tables from the join that do not affect the result. ** For a table to not affect the result, the following must be true: ** - ** 1) The query must not be an aggregate. + ** 1) The query must not be an aggregate. Or it must be an aggregate + ** that contains only one aggregate function with the DISTINCT + ** qualifier. e.g. "SELECT count(DISTINCT ...) FROM". ** 2) The table must be the RHS of a LEFT JOIN. ** 3) Either the query must be DISTINCT, or else the ON or USING clause ** must contain a constraint that limits the scan of the table to diff --git a/test/distinctagg.test b/test/distinctagg.test index c090bf0275..af2c26e02d 100644 --- a/test/distinctagg.test +++ b/test/distinctagg.test @@ -144,6 +144,10 @@ do_execsql_test 3.0 { INSERT INTO t2 VALUES(2, 3, 'x'); INSERT INTO t2 VALUES(2, 3, 'y'); INSERT INTO t2 VALUES(2, 3, 'z'); + + CREATE TABLE t3(x, y, z); + INSERT INTO t3 VALUES(1,1,1); + INSERT INTO t3 VALUES(2,2,2); } foreach {tn use_eph sql res} { @@ -164,5 +168,27 @@ foreach {tn use_eph sql res} { do_execsql_test 4.$tn.2 $sql $res } + +set t3root [db one {SELECT rootpage FROM sqlite_schema WHERE name='t3'}] +foreach {tn use_t3 sql res} { + 1 1 "SELECT count(*) FROM t3" 2 + 2 0 "SELECT count(*) FROM t1" 10 + 2 1 "SELECT count(DISTINCT a) FROM t1, t3" 4 + 3 0 "SELECT count(DISTINCT a) FROM t1 LEFT JOIN t3" 4 + 4 1 "SELECT count(DISTINCT a) FROM t1 LEFT JOIN t3 WHERE t3.x=1" 4 + 5 1 "SELECT count(DISTINCT a) FROM t1 LEFT JOIN t3 WHERE t3.x=0" 0 + 6 0 "SELECT count(DISTINCT a) FROM t1 LEFT JOIN t3 ON (t3.x=0)" 4 +} { + do_test 5.$tn.1 { + set bUse 0 + db eval "EXPLAIN $sql" a { + if {$a(opcode)=="OpenRead" && $a(p2)==$t3root} {set bUse 1} + } + set bUse + } $use_t3 + + do_execsql_test 5.$tn.2 $sql $res +} + finish_test From b19a5812417f2b276689929a76c087bc54e6cf97 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 13 Mar 2021 18:23:30 +0000 Subject: [PATCH 6/6] Fix a memory leak in the new code on this branch. FossilOrigin-Name: 0817cf2ec08fdefd3c1d41790df7b5d6d490767757b44bb0229694023c8e40fc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 47f8a93d45..5340b580e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\scomments\sand\sadd\stest\scases. -D 2021-03-13T17:21:24.914 +C Fix\sa\smemory\sleak\sin\sthe\snew\scode\son\sthis\sbranch. +D 2021-03-13T18:23:30.834 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -542,7 +542,7 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d95db73d3e6a5c689e5f6604b4d2521350e45f2a0f0f84f5a2dc2bfee56580a0 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 05c83f086a7da99478a53a5355ae0d9fd5edd3aeaa079c814205211ac58c21e1 +F src/select.c 1a436834b6f8a3db9bafdca72f19d44b0b5a531e52e1394992fe3da009de8c7c F src/shell.c.in 35adf1212d759069b00e468a9304a05a67710c8f8f50e7312335e23cac985d8c F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c05ed2a8a9c4975c69bdfa733598ed90c1509d52b3c39dedbd829ab148777e1d -R 457fb9da170354f72baa7acae0abf7b9 +P 01312a3dbd92823af535dc618c68d95a2aa1cbee2501b0a9826eae3f09bec760 +R 58cc435e024217b2ab7db85c0988b918 U dan -Z 04936f8ca7b90bfb2cf6a7dfa6671741 +Z 17a70b170153158b9e1985af59afd0f7 diff --git a/manifest.uuid b/manifest.uuid index e951580dac..90583e1fd5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01312a3dbd92823af535dc618c68d95a2aa1cbee2501b0a9826eae3f09bec760 \ No newline at end of file +0817cf2ec08fdefd3c1d41790df7b5d6d490767757b44bb0229694023c8e40fc \ No newline at end of file diff --git a/src/select.c b/src/select.c index a2a979b5ea..7fc79c39e6 100644 --- a/src/select.c +++ b/src/select.c @@ -6876,6 +6876,7 @@ int sqlite3Select( pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); + sqlite3ExprListDelete(db, pDistinct); if( pWInfo==0 ) goto select_end; eDist = sqlite3WhereIsDistinct(pWInfo); SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); @@ -7056,8 +7057,6 @@ int sqlite3Select( struct AggInfo_func *pF = &pAggInfo->aFunc[0]; fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); } - - sqlite3ExprListDelete(db, pDistinct); } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { Table *pTab;