diff --git a/manifest b/manifest index f791b128ae..07aaffea96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\stest\sscript\supsert4.test. -D 2018-04-18T19:45:14.101 +C Add\sthe\s"sorter-reference"\soptimization,\sallowing\sSQLite\sto\sbe\sconfigured\sso\nthat\ssome\srequired\svalues\smay\sbe\sloaded\sfrom\sthe\sdatabase\safter\sexternal\nsorting\soccurs\sfor\sSELECT\sstatements\swith\sORDER\sBY\sclauses\sthat\sare\snot\nsatisfied\sby\sdatabase\sindexes. +D 2018-04-18T19:56:14.234 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439 @@ -435,19 +435,19 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 9eb9531c65346bbfccf5325384b7db1849daf4db6601dcfe21ba5c5b20623b64 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 61320fb84034c24313de699f3385c6bfe093c925b4df2931c6eb63d7c94ec62a +F src/build.c 91d548027a54044851299e719d60d6a2b6eaf3e3274678098cb73a6ab8c0fa19 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c bd9da3f1ff21b432564a16ef0b154cff03585dc43742842e99c58907c6cb4bef +F src/ctime.c 849d4cebe008cfc6e4799b034a172b4eaf8856b100739632a852732ba66eee48 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 7c08754d68897488cf4d916d15680f7d8ef6ea3c7fd32a9c7d739b0b4dda9cf7 +F src/expr.c 53df437b3a4a404f039645919b2d6a56a2f59b9883e858940cd2a8858a25cd3d F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 -F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a +F src/global.c 9bf034fd560bdd514715170ed8460bb7f823cec113f0569ef3f18a20c7ccd128 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da @@ -455,7 +455,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 5fa74146492f5da33e42c35f5f58fb8d56e047c42746fdbd52c8ebdb21160e27 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302 -F src/main.c 1648fc7a9bcfdbfd9a9a04af96ff2796c3164b3f3c7e56ed63a3c51cd11d198d +F src/main.c 10e3897f5d78cef6bcbd1eedc8ccc3fe9e9783d07e052d9d70e57364ded19274 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -491,17 +491,17 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 6415381a0e9d22c0e7cba33ca4a53f81474190862f5d4838190f5eb5b0b47bc9 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 3e84cb869930aa8fcacd3acbb1a0ec2d82736c8479d6a4367a5f1a926fb8a763 -F src/shell.c.in bcde676be8aef998449e4fc8076c671257f41cbfe0cf39c87464b4558e6abf9a -F src/sqlite.h.in e0be726ea6e4e6571724d39d242472ecd8bd1ba6f84ade88e1641bde98a6d02b +F src/select.c 04ae9b4a30992bf428ec27027972ea9e8b31c59e4cbead8adebda1512be2e450 +F src/shell.c.in 8ab4687da814ddc4adf6ea0fcd43ea1eb2784ee6915674dd690759241b7a24b3 +F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 1d4317da12c4a555edf1630c02336708ae57bd6acc6effec73bd6ceca6c50afc +F src/sqliteInt.h a5c534cda7ec61f7180eee99804a63ee2e4a6412a9da6c4e2a77dee7f8caafb5 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 916a92de77ec5cbe27818ca194d8cf0c58aa7ad5b87527098f6aa5a6068800ce -F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828 +F src/test1.c b2df6d7ed8ecb53680f7040163ba92b50c471ed8104d128c88c8e969a107887f F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1142,7 +1142,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 8ada8c1dee071e0fc275bc8bc2db7de537d625cad949d2200664b99a0a89eac5 +F test/permutations.test 10793f1de89a226fa22dde9ba9398de22571fee1bfb53a935a11be4aa014704f F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1263,6 +1263,7 @@ F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c F test/sort5.test 6b43ae0e2169b5ceed441844492e55ba7f1ae0790528395ddf7888ab3094525d +F test/sorterref.test a13ed207a0eea3c7898f308f979bfb518f68c598ec737d2c494dfd3deaa83506 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1584,7 +1585,7 @@ F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test f520bcec2c3d12dc4615623b06b2aec7c2d67e94 -F test/where8.test 98eedca0d375fb400b8377269c4b4686582dfb45 +F test/where8.test 461ca40265ed996a6305da99bb024b0e41602bb586acf544c08f95922358e49f F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 6c6a420ca7d313242f9b1bd471dc80e4d0f8323700ba9c78df0bb843d4daa3b4 F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 @@ -1722,7 +1723,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61cb8a391a0c709340ac60ffd0c58f950567892a8404c2bec7b9b1f64b3cc5cf -R 6c33138f3c02d47a81dcde27f15e792f +P 0cb83c84d10b89ef7a5504862566a609307c63e7571dd711d8b9f995d29e5a8c 413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092 +R 4678faec743cd6b53d517fb16a4da20a +T +closed 413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092 U dan -Z 1740208f0524dae2e22fe373b2efc4e7 +Z 97f82718fcc75c5ef29e50fdc72006e4 diff --git a/manifest.uuid b/manifest.uuid index b4c6aa04e4..e1de99ec00 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cb83c84d10b89ef7a5504862566a609307c63e7571dd711d8b9f995d29e5a8c \ No newline at end of file +ef74090a40ceaef2fd93a7613ec99a191ce986811c852e96f4a19719f18af4f0 \ No newline at end of file diff --git a/src/build.c b/src/build.c index c286b4bbe7..b032ff9f2c 100644 --- a/src/build.c +++ b/src/build.c @@ -1095,15 +1095,20 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ if( pType->n==0 ){ /* If there is no type specified, columns have the default affinity - ** 'BLOB'. */ + ** 'BLOB' with a default size of 4 bytes. */ pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( 4>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif }else{ zType = z + sqlite3Strlen30(z) + 1; memcpy(zType, pType->z, pType->n); zType[pType->n] = 0; sqlite3Dequote(zType); - pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; @@ -1163,7 +1168,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ ** If none of the substrings in the above table are found, ** SQLITE_AFF_NUMERIC is returned. */ -char sqlite3AffinityType(const char *zIn, u8 *pszEst){ +char sqlite3AffinityType(const char *zIn, Column *pCol){ u32 h = 0; char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; @@ -1200,27 +1205,32 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ } } - /* If pszEst is not NULL, store an estimate of the field size. The + /* If pCol is not NULL, store an estimate of the field size. The ** estimate is scaled so that the size of an integer is 1. */ - if( pszEst ){ - *pszEst = 1; /* default size is approx 4 bytes */ + if( pCol ){ + int v = 0; /* default size is approx 4 bytes */ if( aff r=(k/4+1) */ sqlite3GetInt32(zChar, &v); - v = v/4 + 1; - if( v>255 ) v = 255; - *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */ break; } zChar++; } }else{ - *pszEst = 5; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ } } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( v>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif + v = v/4 + 1; + if( v>255 ) v = 255; + pCol->szEst = v; } return aff; } diff --git a/src/ctime.c b/src/ctime.c index 1877aee10a..da79f3a350 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -286,6 +286,9 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_ENABLE_SNAPSHOT "ENABLE_SNAPSHOT", #endif +#if SQLITE_ENABLE_SORTER_REFERENCES + "ENABLE_SORTER_REFERENCES", +#endif #if SQLITE_ENABLE_SQLLOG "ENABLE_SQLLOG", #endif diff --git a/src/expr.c b/src/expr.c index 534fdfe6bc..6997e8430f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1363,6 +1363,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pItem->sortOrder = pOldItem->sortOrder; pItem->done = 0; pItem->bSpanIsTab = pOldItem->bSpanIsTab; + pItem->bSorterRef = pOldItem->bSorterRef; pItem->u = pOldItem->u; } return pNew; @@ -4373,6 +4374,12 @@ int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pItem->bSorterRef ){ + i--; + n--; + }else +#endif if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ if( flags & SQLITE_ECEL_OMITREF ){ i--; diff --git a/src/global.c b/src/global.c index 04a3d185a8..f0362d1e04 100644 --- a/src/global.c +++ b/src/global.c @@ -240,7 +240,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xTestCallback */ #endif 0, /* bLocaltimeFault */ - 0x7ffffffe /* iOnceResetThreshold */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE /* szSorterRef */ }; /* diff --git a/src/main.c b/src/main.c index 5b6c867097..99e3e6f970 100644 --- a/src/main.c +++ b/src/main.c @@ -642,6 +642,17 @@ int sqlite3_config(int op, ...){ break; } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; + } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; + break; + } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + default: { rc = SQLITE_ERROR; break; diff --git a/src/select.c b/src/select.c index c2686e0fec..570b066ddd 100644 --- a/src/select.c +++ b/src/select.c @@ -44,6 +44,20 @@ struct DistinctCtx { /* ** An instance of the following object is used to record information about ** the ORDER BY (or GROUP BY) clause of query is being coded. +** +** The aDefer[] array is used by the sorter-references optimization. For +** example, assuming there is no index that can be used for the ORDER BY, +** for the query: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10; +** +** it may be more efficient to add just the "a" values to the sorter, and +** retrieve the associated "bigblob" values directly from table t1 as the +** 10 smallest "a" values are extracted from the sorter. +** +** When the sorter-reference optimization is used, there is one entry in the +** aDefer[] array for each database table that may be read as values are +** extracted from the sorter. */ typedef struct SortCtx SortCtx; struct SortCtx { @@ -56,6 +70,14 @@ struct SortCtx { int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + u8 nDefer; /* Number of valid entries in aDefer[] */ + struct DeferredCsr { + Table *pTab; /* Table definition */ + int iCsr; /* Cursor number for table */ + int nKey; /* Number of PK columns for table pTab (>=1) */ + } aDefer[4]; +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -678,6 +700,90 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES +/* +** This function is called as part of inner-loop generation for a SELECT +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference +** optimization should be used for. The sorter-reference optimization +** is used for SELECT queries like: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10 +** +** If the optimization is used for expression "bigblob", then instead of +** storing values read from that column in the sorter records, the PK of +** the row from table t1 is stored instead. Then, as records are extracted from +** the sorter to return to the user, the required value of bigblob is +** retrieved directly from table t1. If the values are very large, this +** can be more efficient than storing them directly in the sorter records. +** +** The ExprList_item.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. +** Additionally, the pSort->aDefer[] array is populated with entries +** for all cursors required to evaluate all selected expressions. Finally. +** output variable (*ppExtra) is set to an expression list containing +** expressions for all extra PK values that should be stored in the +** sorter records. +*/ +static void selectExprDefer( + Parse *pParse, /* Leave any error here */ + SortCtx *pSort, /* Sorter context */ + ExprList *pEList, /* Expressions destined for sorter */ + ExprList **ppExtra /* Expressions to append to sorter record */ +){ + int i; + int nDefer = 0; + ExprList *pExtra = 0; + for(i=0; inExpr; i++){ + struct ExprList_item *pItem = &pEList->a[i]; + if( pItem->u.x.iOrderByCol==0 ){ + Expr *pExpr = pItem->pExpr; + Table *pTab = pExpr->pTab; + if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab) + && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) +#if 0 + && pTab->pSchema && pTab->pSelect==0 && !IsVirtual(pTab) +#endif + ){ + int j; + for(j=0; jaDefer[j].iCsr==pExpr->iTable ) break; + } + if( j==nDefer ){ + if( nDefer==ArraySize(pSort->aDefer) ){ + continue; + }else{ + int nKey = 1; + int k; + Index *pPk = 0; + if( !HasRowid(pTab) ){ + pPk = sqlite3PrimaryKeyIndex(pTab); + nKey = pPk->nKeyCol; + } + for(k=0; kiTable = pExpr->iTable; + pNew->pTab = pExpr->pTab; + pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; + pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); + } + } + pSort->aDefer[nDefer].pTab = pExpr->pTab; + pSort->aDefer[nDefer].iCsr = pExpr->iTable; + pSort->aDefer[nDefer].nKey = nKey; + nDefer++; + } + } + pItem->bSorterRef = 1; + } + } + } + pSort->nDefer = (u8)nDefer; + *ppExtra = pExtra; +} +#endif + /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -750,6 +856,9 @@ static void selectInnerLoop( VdbeComment((v, "%s", p->pEList->a[i].zName)); } }else if( eDest!=SRT_Exists ){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra = 0; +#endif /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ @@ -773,12 +882,34 @@ static void selectInnerLoop( p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; } } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + selectExprDefer(pParse, pSort, p->pEList, &pExtra); + if( pExtra && pParse->db->mallocFailed==0 ){ + /* If there are any extra PK columns to add to the sorter records, + ** allocate extra memory cells and adjust the OpenEphemeral + ** instruction to account for the larger records. This is only + ** required if there are one or more WITHOUT ROWID tables with + ** composite primary keys in the SortCtx.aDefer[] array. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->p2 += (pExtra->nExpr - pSort->nDefer); + pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer); + pParse->nMem += pExtra->nExpr; + } +#endif regOrig = 0; assert( eDest==SRT_Set || eDest==SRT_Mem || eDest==SRT_Coroutine || eDest==SRT_Output ); } nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult, 0,ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pExtra ){ + nResultCol += sqlite3ExprCodeExprList( + pParse, pExtra, regResult + nResultCol, 0, 0 + ); + sqlite3ExprListDelete(pParse->db, pExtra); + } +#endif } /* If the DISTINCT keyword was present on the SELECT statement @@ -1236,7 +1367,7 @@ static void generateSortTail( Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ - int addr; + int addr; /* Top of output loop. Jump for Next. */ int addrOnce = 0; int iTab; ExprList *pOrderBy = pSort->pOrderBy; @@ -1245,10 +1376,11 @@ static void generateSortTail( int regRow; int regRowid; int iCol; - int nKey; + int nKey; /* Number of key columns in sorter record */ int iSortTab; /* Sorter cursor to read from */ int i; int bSeq; /* True if sorter record includes seq. no. */ + int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; assert( addrBreak<0 ); @@ -1257,6 +1389,17 @@ static void generateSortTail( sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + /* Open any cursors needed for sorter-reference expressions */ + for(i=0; inDefer; i++){ + Table *pTab = pSort->aDefer[i].pTab; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); + } +#endif + iTab = pSort->iECursor; if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ regRowid = 0; @@ -1272,7 +1415,8 @@ static void generateSortTail( if( pSort->labelBkOut ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } - sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nColumn); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + nKey+1+nColumn+nRefKey); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); @@ -1286,17 +1430,58 @@ static void generateSortTail( bSeq = 1; } for(i=0, iCol=nKey+bSeq-1; i=0; i--){ - int iRead; - if( aOutEx[i].u.x.iOrderByCol ){ - iRead = aOutEx[i].u.x.iOrderByCol-1; - }else{ - iRead = iCol--; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pSort->nDefer ){ + int iKey = iCol+1; + int regKey = sqlite3GetTempRange(pParse, nRefKey); + + for(i=0; inDefer; i++){ + int iCsr = pSort->aDefer[i].iCsr; + Table *pTab = pSort->aDefer[i].pTab; + int nKey = pSort->aDefer[i].nKey; + + sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeCurrentAddr(v)+1, regKey); + }else{ + int k; + int iJmp; + assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey ); + for(k=0; k=0; i--){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( aOutEx[i].bSorterRef ){ + sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); + }else +#endif + { + int iRead; + if( aOutEx[i].u.x.iOrderByCol ){ + iRead = aOutEx[i].u.x.iOrderByCol-1; + }else{ + iRead = iCol--; + } + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); + VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan)); } - sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); - VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); } switch( eDest ){ case SRT_Table: @@ -6073,6 +6258,7 @@ int sqlite3Select( if( sSort.pOrderBy ){ explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } diff --git a/src/shell.c.in b/src/shell.c.in index 87087ed45f..38e5c5cbe2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8074,6 +8074,9 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -separator SEP set output column separator. Default: '|'\n" +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + " -sorterref SIZE sorter references threshold size\n" +#endif " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" @@ -8343,6 +8346,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); +#endif }else if( strcmp(z,"-vfs")==0 ){ zVfs = cmdline_option_value(argc, argv, ++i); #ifdef SQLITE_HAVE_ZLIB @@ -8489,6 +8497,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ i+=2; }else if( strcmp(z,"-mmap")==0 ){ i++; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + i++; +#endif }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 202155df71..02dc8944b9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1930,6 +1930,22 @@ struct sqlite3_mem_methods { ** I/O required to support statement rollback. ** The default value for this setting is controlled by the ** [SQLITE_STMTJRNL_SPILL] compile-time option. +** +** [[SQLITE_CONFIG_SORTERREF_SIZE]] +**
SQLITE_CONFIG_SORTERREF_SIZE +**
The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted record and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behaviour. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1959,6 +1975,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ae703ae766..73cb6299d6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -637,6 +637,13 @@ # define SQLITE_DEFAULT_PCACHE_INITSZ 20 #endif +/* +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option. +*/ +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff +#endif + /* ** The compile-time options SQLITE_MMAP_READWRITE and ** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another. @@ -1760,6 +1767,7 @@ struct Column { #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ #define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ #define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -2499,6 +2507,7 @@ struct ExprList { unsigned done :1; /* A flag to indicate when processing is finished */ unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -3351,6 +3360,7 @@ struct Sqlite3Config { #endif int bLocaltimeFault; /* True to fail localtime() calls */ int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ }; /* @@ -4135,7 +4145,7 @@ void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); -char sqlite3AffinityType(const char*, u8*); +char sqlite3AffinityType(const char*, Column*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*); int sqlite3FindDb(sqlite3*, Token*); diff --git a/src/test1.c b/src/test1.c index bc8f389dbd..0f139ceba5 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2256,6 +2256,27 @@ static int SQLITE_TCLAPI test_config_sqllog( } #endif +/* +** Usage: sqlite3_config_sorterref +** +** Set the SQLITE_CONFIG_SORTERREF_SIZE configuration option +*/ +static int SQLITE_TCLAPI test_config_sorterref( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int iVal; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NBYTE"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &iVal) ) return TCL_ERROR; + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, iVal); + return TCL_OK; +} + /* ** Usage: vfs_current_time_int64 ** @@ -7751,6 +7772,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_delete_database", test_delete_database, 0 }, { "atomic_batch_write", test_atomic_batch_write, 0 }, { "sqlite3_mmap_warm", test_mmap_warm, 0 }, + { "sqlite3_config_sorterref", test_config_sorterref, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); diff --git a/test/permutations.test b/test/permutations.test index c1d28d4e09..52e2509fc1 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1073,6 +1073,26 @@ test_suite "prepare" -description { stmtvtab1.test index9.test ] +test_suite "sorterref" -prefix "" -description { + Run the "veryquick" test suite with SQLITE_CONFIG_SORTERREF_SIZE set + to 0 so that sorter-references are used whenever possible. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \ + *fts5corrupt* *fts5big* *fts5aj* +] -initialize { + catch {db close} + sqlite3_shutdown + sqlite3_config_sorterref 0 + sqlite3_initialize + autoinstall_test_functions +} -shutdown { + catch {db close} + sqlite3_shutdown + sqlite3_config_sorterref -1 + sqlite3_initialize + autoinstall_test_functions +} + # End of tests ############################################################################# diff --git a/test/sorterref.test b/test/sorterref.test new file mode 100644 index 0000000000..28445c6e72 --- /dev/null +++ b/test/sorterref.test @@ -0,0 +1,50 @@ +# 2018 April 14. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sorterref + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + ALTER TABLE t1 ADD COLUMN d DEFAULT 'string'; + INSERT INTO t1 VALUES(7, 8, 9, 'text'); +} + +do_execsql_test 1.1 { + SELECT * FROM t1 ORDER BY b; +} { + 1 2 3 string 4 5 6 string 7 8 9 text +} + +do_execsql_test 2.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d, PRIMARY KEY(c)) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(2, 3); + INSERT INTO t1 VALUES(3, 4); + + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(3, 'three'); +} + +do_execsql_test 2.1 { + SELECT * FROM t1 LEFT JOIN t2 ON (a=c) ORDER BY b; +} {1 2 1 one 2 3 {} {} 3 4 3 three} + + + +finish_test diff --git a/test/where8.test b/test/where8.test index 38214bc895..a8dbcfd9fe 100644 --- a/test/where8.test +++ b/test/where8.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +if {[permutation]=="sorterref"} { + finish_test + return +} + # Test organization: # # where8-1.*: Tests to demonstrate simple cases work with a single table