From 257c13fa008b2033960e60927d180f2ebbe1fdca Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Nov 2016 20:14:06 +0000 Subject: [PATCH 1/2] Avoid storing redundant fields in sorter records when the sort-key and data have fields in common (as in "SELECT a FROM t1 ORDER BY 1"). FossilOrigin-Name: 0af62fdbd8e2aab14718ff8bcb5934f05463c176 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/expr.c | 9 +++++++-- src/select.c | 42 +++++++++++++++++++++++++++++++++--------- src/sqliteInt.h | 1 + 5 files changed, 54 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index a18f153b6d..089494dd59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sthe\sCSV\sextension. -D 2016-11-09T01:46:13.326 +C Avoid\sstoring\sredundant\sfields\sin\ssorter\srecords\swhen\sthe\ssort-key\sand\sdata\shave\nfields\sin\scommon\s(as\sin\s"SELECT\sa\sFROM\st1\sORDER\sBY\s1"). +D 2016-11-10T20:14:06.787 F Makefile.in 6fd48ffcf7c2deea7499062d1f3747f986c19678 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e0217f2d35a0448abbe4b066132ae20136e8b408 @@ -340,7 +340,7 @@ F src/ctime.c a2a52d6e353f459d8ab0f07321f60fafa47d5421 F src/date.c 95c9a8d00767e7221a8e9a31f4e913fc8029bf6b F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c cb3f6300df24c26c609778b2731f82644b5532ec -F src/expr.c ce7110980fac6dfdfbe1e393443bdb79bad29339 +F src/expr.c 1fd53148e575de810ad10a9eb15f16f2e44ff1dc F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c b9ca262f6ad4d030a3cab737ebf9b0b3c8b4ac80 F src/func.c 7057bc2c105b82faa668d8e2ec85fad4540e5c51 @@ -387,12 +387,12 @@ F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 3fac1b2737ea5a724f20b921ac7e259c9be2100b F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c ea3af83e2d0f245fef81ea4cf04cb730ce67f722 +F src/select.c 64b2273747c6485638bfeb6790e3e774e0605d02 F src/shell.c 63e54cfa1c7ec5b70a4c9a86502bc10280c3d5a3 F src/sqlite.h.in 803f7050f69b2eea573fac219f3c92582c096027 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 37628fe30c464dc790bcee3bfd3d0caa8f222ed1 +F src/sqliteInt.h 603953faca895386d4f3a8b7046f3e4e6c071c53 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -1530,7 +1530,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 754ee844958bdc3b96acfd1f5395da5796e54a82 -R eb905f0421ecb200099f22c1d4910f11 -U mistachkin -Z a09941219c710c6bfaf84cdaf6c10994 +P b4889588246c33374ff3758e21ccc4ce246380b6 +R e8a5368c67459f65dc206ad3df15a284 +T *branch * sorter-opt +T *sym-sorter-opt * +T -sym-trunk * +U dan +Z 220ba0a8da96dcdf18555b62136e6fa1 diff --git a/manifest.uuid b/manifest.uuid index 952d3c0676..2523f32ffc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4889588246c33374ff3758e21ccc4ce246380b6 \ No newline at end of file +0af62fdbd8e2aab14718ff8bcb5934f05463c176 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c2b9c8fe46..165bb30f73 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4086,8 +4086,13 @@ int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ - sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ + if( flags & SQLITE_ECEL_OMITREF ){ + i--; + n--; + }else{ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ diff --git a/src/select.c b/src/select.c index 199e13f113..003d42e728 100644 --- a/src/select.c +++ b/src/select.c @@ -533,11 +533,11 @@ static void pushOntoSorter( iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, - SQLITE_ECEL_DUP|SQLITE_ECEL_REF); + SQLITE_ECEL_DUP); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } - if( nPrefixReg==0 ){ + if( nPrefixReg==0 && nData>0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -666,7 +666,7 @@ static void codeDistinct( ** If srcTab is negative, then the pEList expressions ** are evaluated in order to get the data for this row. If srcTab is ** zero or more, then data is pulled from srcTab and pEList is used only -** to get number columns and the datatype for each column. +** to get the number of columns and the collation sequence for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ @@ -734,7 +734,25 @@ static void selectInnerLoop( }else{ ecelFlags = 0; } - sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); + if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){ + /* For each expression in pEList that is a copy of an expression in + ** the ORDER BY clause (pSort->pOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY + ** expression within the sort-key that pushOntoSorter() will generate. + ** This allows the pEList field to be omitted from the sorted record, + ** saving space and CPU cycles. */ + ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ + int j; + if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ + pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; + } + } + + assert( eDest==SRT_Set || eDest==SRT_Mem + || eDest==SRT_Coroutine || eDest==SRT_Output ); + } + nResultCol = sqlite3ExprCodeExprList(pParse,pEList,regResult,0,ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -900,11 +918,12 @@ static void selectInnerLoop( ** memory cells and break out of the scan loop. */ case SRT_Mem: { - assert( nResultCol==pDest->nSdst ); if( pSort ){ + assert( nResultCol<=pDest->nSdst ); pushOntoSorter( pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); }else{ + assert( nResultCol==pDest->nSdst ); assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } @@ -1202,14 +1221,13 @@ static void generateSortTail( int iParm = pDest->iSDParm; int regRow; int regRowid; + int iCol; int nKey; int iSortTab; /* Sorter cursor to read from */ int nSortData; /* Trailing values to read from sorter */ int i; int bSeq; /* True if sorter record includes seq. no. */ -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; -#endif assert( addrBreak<0 ); if( pSort->labelBkOut ){ @@ -1247,8 +1265,14 @@ static void generateSortTail( iSortTab = iTab; bSeq = 1; } - for(i=0; i Date: Fri, 11 Nov 2016 18:08:59 +0000 Subject: [PATCH 2/2] Reenable the SQLITE_EXPR_REF optimization for "SELECT DISTINCT ... ORDER BY" queries. FossilOrigin-Name: 6e2e9d383f5fc4a0cbf05fe83ec7425812c0f556 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/select.c | 28 ++++++++++++++++++---------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 089494dd59..fb20daadba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sstoring\sredundant\sfields\sin\ssorter\srecords\swhen\sthe\ssort-key\sand\sdata\shave\nfields\sin\scommon\s(as\sin\s"SELECT\sa\sFROM\st1\sORDER\sBY\s1"). -D 2016-11-10T20:14:06.787 +C Reenable\sthe\sSQLITE_EXPR_REF\soptimization\sfor\s"SELECT\sDISTINCT\s...\sORDER\sBY"\nqueries. +D 2016-11-11T18:08:59.063 F Makefile.in 6fd48ffcf7c2deea7499062d1f3747f986c19678 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e0217f2d35a0448abbe4b066132ae20136e8b408 @@ -387,7 +387,7 @@ F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 3fac1b2737ea5a724f20b921ac7e259c9be2100b F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 64b2273747c6485638bfeb6790e3e774e0605d02 +F src/select.c 04fd717fb99aea2110b752e2b6186b966fa13cb4 F src/shell.c 63e54cfa1c7ec5b70a4c9a86502bc10280c3d5a3 F src/sqlite.h.in 803f7050f69b2eea573fac219f3c92582c096027 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1530,10 +1530,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4889588246c33374ff3758e21ccc4ce246380b6 -R e8a5368c67459f65dc206ad3df15a284 -T *branch * sorter-opt -T *sym-sorter-opt * -T -sym-trunk * +P 0af62fdbd8e2aab14718ff8bcb5934f05463c176 +R dccdf5cb21f8816d64d778a9f6efd1e8 U dan -Z 220ba0a8da96dcdf18555b62136e6fa1 +Z 74767a1cb4eda88af63e605e54888e92 diff --git a/manifest.uuid b/manifest.uuid index 2523f32ffc..ee5ae85add 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0af62fdbd8e2aab14718ff8bcb5934f05463c176 \ No newline at end of file +6e2e9d383f5fc4a0cbf05fe83ec7425812c0f556 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 003d42e728..b8ed890b3f 100644 --- a/src/select.c +++ b/src/select.c @@ -521,7 +521,7 @@ static void pushOntoSorter( int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); - assert( nData==1 || regData==regOrigData ); + assert( nData==1 || regData==regOrigData || regOrigData==0 ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -533,7 +533,7 @@ static void pushOntoSorter( iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, - SQLITE_ECEL_DUP); + SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } @@ -681,13 +681,20 @@ static void selectInnerLoop( ){ Vdbe *v = pParse->pVdbe; int i; - int hasDistinct; /* True if the DISTINCT keyword is present */ - int regResult; /* Start of memory holding result set */ + int hasDistinct; /* True if the DISTINCT keyword is present */ int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ int nPrefixReg = 0; /* Number of extra registers before regResult */ + /* Usually, regResult is the first cell in an array of memory cells + ** containing the current result row. In this case regOrig is set to the + ** same value. However, if the results are being sent to the sorter, the + ** values for any expressions that are also part of the sort-key are omitted + ** from this array. In this case regOrig is set to zero. */ + int regResult; /* Start of memory holding current results */ + int regOrig; /* Start of memory holding full result (or 0) */ + assert( v ); assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; @@ -718,7 +725,7 @@ static void selectInnerLoop( pParse->nMem += nResultCol; } pDest->nSdst = nResultCol; - regResult = pDest->iSdst; + regOrig = regResult = pDest->iSdst; if( srcTab>=0 ){ for(i=0; ipOrderBy), set the associated ** iOrderByCol value to one more than the index of the ORDER BY @@ -748,7 +756,7 @@ static void selectInnerLoop( pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; } } - + regOrig = 0; assert( eDest==SRT_Set || eDest==SRT_Mem || eDest==SRT_Coroutine || eDest==SRT_Output ); } @@ -892,7 +900,7 @@ static void selectInnerLoop( ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ pushOntoSorter( - pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); @@ -921,7 +929,7 @@ static void selectInnerLoop( if( pSort ){ assert( nResultCol<=pDest->nSdst ); pushOntoSorter( - pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else{ assert( nResultCol==pDest->nSdst ); assert( regResult==iParm ); @@ -936,7 +944,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);