database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-11-12 13:01:09 +03:00
Code Activity

Experiments with the optimization of ORDER BY and GROUP BY clauses.

Browse Source 
FossilOrigin-Name: b150902579d708b454efd5f8750e26a816f7f1a6
This commit is contained in:
drh
2014-03-18 15:30:27 +00:00
parent 0e682099a1
commit 0401acec91
4 changed files with 47 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
manifest
View File

@@ -1,5 +1,5 @@
C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses.
D 2014-03-17T15:06:57.359 D 2014-03-18T15:30:27.660
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2
F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36
R f35820126ad6a1c360258186873a2e79 R 8abedb680a805ab9b5e400a1ac18119b
T *branch * orderby-planning
T *sym-orderby-planning *
T -sym-trunk *
U drh U drh
Z b6382160ccdf74a44bc9653763bb82ef Z 6c19349f1c38f37cb3b2bffb6095d2a0

2
manifest.uuid
View File

@@ -1 +1 @@
ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 b150902579d708b454efd5f8750e26a816f7f1a6

72
src/where.c
View File

@@ -4506,8 +4506,8 @@ static int whereLoopAddVirtual(
pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
pIdxInfo->needToFreeIdxStr = 0; pIdxInfo->needToFreeIdxStr = 0;
pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.idxStr = pIdxInfo->idxStr;
pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
&& pIdxInfo->orderByConsumed); pIdxInfo->nOrderBy : 0);
pNew->rSetup = 0; pNew->rSetup = 0;
pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
@@ -4668,11 +4668,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
/* /*
** Examine a WherePath (with the addition of the extra WhereLoop of the 5th ** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
** parameters) to see if it outputs rows in the requested ORDER BY ** parameters) to see if it outputs rows in the requested ORDER BY
** (or GROUP BY) without requiring a separate sort operation. Return: ** (or GROUP BY) without requiring a separate sort operation. Return N:
** **
** 0: ORDER BY is not satisfied. Sorting required ** N>0: N terms of the ORDER BY clause are satisfied
** 1: ORDER BY is satisfied. Omit sorting ** N==0: No terms of the ORDER BY clause are satisfied
** -1: Unknown at this time ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
** **
** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
** strict. With GROUP BY and DISTINCT the only requirement is that ** strict. With GROUP BY and DISTINCT the only requirement is that
@@ -4682,7 +4682,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
** the pOrderBy terms can be matched in any order. With ORDER BY, the ** the pOrderBy terms can be matched in any order. With ORDER BY, the
** pOrderBy terms must be matched in strict left-to-right order. ** pOrderBy terms must be matched in strict left-to-right order.
*/ */
static int wherePathSatisfiesOrderBy( static i8 wherePathSatisfiesOrderBy(
WhereInfo *pWInfo, /* The WHERE clause */ WhereInfo *pWInfo, /* The WHERE clause */
ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
WherePath *pPath, /* The WherePath to check */ WherePath *pPath, /* The WherePath to check */
@@ -4915,7 +4915,7 @@ static int wherePathSatisfiesOrderBy(
} }
} }
} /* End the loop over all WhereLoops from outer-most down to inner-most */ } /* End the loop over all WhereLoops from outer-most down to inner-most */
if( obSat==obDone ) return 1; if( obSat==obDone ) return nOrderBy;
if( !isOrderDistinct ) return 0; if( !isOrderDistinct ) return 0;
return -1; return -1;
} }
@@ -5001,12 +5001,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
** to sqlite3WhereBegin() was concerned about sorting */ ** to sqlite3WhereBegin() was concerned about sorting */
rSortCost = 0; rSortCost = 0;
if( pWInfo->pOrderBy==0 || nRowEst==0 ){ if( pWInfo->pOrderBy==0 || nRowEst==0 ){
aFrom[0].isOrderedValid = 1; aFrom[0].isOrdered = 0;
}else{ }else{
/* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the
** number of output rows. The 48 is the expected size of a row to sort. ** number of output rows. The 48 is the expected size of a row to sort.
** FIXME: compute a better estimate of the 48 multiplier based on the ** FIXME: compute a better estimate of the 48 multiplier based on the
** result set expressions. */ ** result set expressions. */
aFrom[0].isOrdered = -1;
rSortCost = nRowEst + estLog(nRowEst); rSortCost = nRowEst + estLog(nRowEst);
WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost));
} }
@@ -5020,8 +5021,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
Bitmask maskNew; Bitmask maskNew;
Bitmask revMask = 0; Bitmask revMask = 0;
u8 isOrderedValid = pFrom->isOrderedValid; i8 isOrdered = pFrom->isOrdered;
u8 isOrdered = pFrom->isOrdered;
if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
/* At this point, pWLoop is a candidate to be the next loop. /* At this point, pWLoop is a candidate to be the next loop.
@@ -5030,21 +5030,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); rCost = sqlite3LogEstAdd(rCost, pFrom->rCost);
nOut = pFrom->nRow + pWLoop->nOut; nOut = pFrom->nRow + pWLoop->nOut;
maskNew = pFrom->maskLoop | pWLoop->maskSelf; maskNew = pFrom->maskLoop | pWLoop->maskSelf;
if( !isOrderedValid ){ if( isOrdered<0 ){
switch( wherePathSatisfiesOrderBy(pWInfo, isOrdered = wherePathSatisfiesOrderBy(pWInfo,
pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
iLoop, pWLoop, &revMask) ){ iLoop, pWLoop, &revMask);
case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ if( isOrdered==0 ){
isOrdered = 1; rCost = sqlite3LogEstAdd(rCost, rSortCost);
isOrderedValid = 1;
break;
case 0: /* No. pFrom+pWLoop will require a separate sort */
isOrdered = 0;
isOrderedValid = 1;
rCost = sqlite3LogEstAdd(rCost, rSortCost);
break;
default: /* Cannot tell yet. Try again on the next iteration */
break;
} }
}else{ }else{
revMask = pFrom->revLoop; revMask = pFrom->revLoop;
@@ -5052,7 +5043,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
/* Check to see if pWLoop should be added to the mxChoice best so far */ /* Check to see if pWLoop should be added to the mxChoice best so far */
for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){ for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
if( pTo->maskLoop==maskNew if( pTo->maskLoop==maskNew
&& pTo->isOrderedValid==isOrderedValid && ((pTo->isOrdered^isOrdered)&80)==0
&& ((pTo->rCost<=rCost && pTo->nRow<=nOut) || && ((pTo->rCost<=rCost && pTo->nRow<=nOut) ||
(pTo->rCost>=rCost && pTo->nRow>=nOut)) (pTo->rCost>=rCost && pTo->nRow>=nOut))
){ ){
@@ -5066,7 +5057,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
if( sqlite3WhereTrace&0x4 ){ if( sqlite3WhereTrace&0x4 ){
sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n",
wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); isOrdered>=0 ? isOrdered+'0' : '?');
} }
#endif #endif
continue; continue;
@@ -5084,7 +5075,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
if( sqlite3WhereTrace&0x4 ){ if( sqlite3WhereTrace&0x4 ){
sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n", sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n",
wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); isOrdered>=0 ? isOrdered+'0' : '?');
} }
#endif #endif
}else{ }else{
@@ -5094,10 +5085,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
sqlite3DebugPrintf( sqlite3DebugPrintf(
"Skip %s cost=%-3d,%3d order=%c", "Skip %s cost=%-3d,%3d order=%c",
wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); isOrdered>=0 ? isOrdered+'0' : '?');
sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n",
wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
} }
#endif #endif
testcase( pTo->rCost==rCost ); testcase( pTo->rCost==rCost );
@@ -5110,10 +5101,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
sqlite3DebugPrintf( sqlite3DebugPrintf(
"Update %s cost=%-3d,%3d order=%c", "Update %s cost=%-3d,%3d order=%c",
wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); isOrdered>=0 ? isOrdered+'0' : '?');
sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n",
wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
} }
#endif #endif
} }
@@ -5122,7 +5113,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
pTo->revLoop = revMask; pTo->revLoop = revMask;
pTo->nRow = nOut; pTo->nRow = nOut;
pTo->rCost = rCost; pTo->rCost = rCost;
pTo->isOrderedValid = isOrderedValid;
pTo->isOrdered = isOrdered; pTo->isOrdered = isOrdered;
memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
pTo->aLoop[iLoop] = pWLoop; pTo->aLoop[iLoop] = pWLoop;
@@ -5147,8 +5137,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
if( pTo->isOrderedValid && pTo->isOrdered ){ if( pTo->isOrdered>0 ){
sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
}else{ }else{
sqlite3DebugPrintf("\n"); sqlite3DebugPrintf("\n");
@@ -5191,9 +5181,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
Bitmask notUsed; Bitmask notUsed;
int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed); WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed);
if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; if( rc==pWInfo->pResultSet->nExpr ){
pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
}
} }
if( pFrom->isOrdered ){ if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
}else{ }else{
@@ -5386,7 +5378,7 @@ WhereInfo *sqlite3WhereBegin(
Parse *pParse, /* The parser context */ Parse *pParse, /* The parser context */
SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
Expr *pWhere, /* The WHERE clause */ Expr *pWhere, /* The WHERE clause */
ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
ExprList *pResultSet, /* Result set of the query */ ExprList *pResultSet, /* Result set of the query */
u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */
int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */
@@ -5408,6 +5400,10 @@ WhereInfo *sqlite3WhereBegin(
/* Variable initialization */ /* Variable initialization */
db = pParse->db; db = pParse->db;
memset(&sWLB, 0, sizeof(sWLB)); memset(&sWLB, 0, sizeof(sWLB));
/* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
sWLB.pOrderBy = pOrderBy; sWLB.pOrderBy = pOrderBy;
/* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via

5
src/whereInt.h
View File

@@ -121,7 +121,7 @@ struct WhereLoop {
struct { /* Information for virtual tables */ struct { /* Information for virtual tables */
int idxNum; /* Index number */ int idxNum; /* Index number */
u8 needFree; /* True if sqlite3_free(idxStr) is needed */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */
u8 isOrdered; /* True if satisfies ORDER BY */ i8 isOrdered; /* True if satisfies ORDER BY */
u16 omitMask; /* Terms that may be omitted */ u16 omitMask; /* Terms that may be omitted */
char *idxStr; /* Index identifier string */ char *idxStr; /* Index identifier string */
} vtab; } vtab;
@@ -183,8 +183,7 @@ struct WherePath {
Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */
LogEst nRow; /* Estimated number of rows generated by this path */ LogEst nRow; /* Estimated number of rows generated by this path */
LogEst rCost; /* Total cost of this path */ LogEst rCost; /* Total cost of this path */
u8 isOrdered; /* True if this path satisfies ORDER BY */ i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */
u8 isOrderedValid; /* True if the isOrdered field is valid */
WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */
}; };