From 24e25d32ce2bb671d088c3cea8bf8ba400750377 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 14 Apr 2018 18:46:20 +0000 Subject: [PATCH 1/6] Experimental change to "SELECT * FROM ... ORDER BY" processing to load some column values from the db after sorting. FossilOrigin-Name: 9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d --- manifest | 23 +++-- manifest.uuid | 2 +- src/expr.c | 7 ++ src/select.c | 206 +++++++++++++++++++++++++++++++++++++++++--- src/sqliteInt.h | 1 + test/sorterref.test | 50 +++++++++++ 6 files changed, 268 insertions(+), 21 deletions(-) create mode 100644 test/sorterref.test diff --git a/manifest b/manifest index ef91e54a13..302de8f7e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\salways-true\sbranch\sfrom\sthe\spreupdate\shook\slogic. -D 2018-04-13T16:23:22.787 +C Experimental\schange\sto\s"SELECT\s*\sFROM\s...\sORDER\sBY"\sprocessing\sto\sload\ssome\ncolumn\svalues\sfrom\sthe\sdb\safter\ssorting. +D 2018-04-14T18:46:20.429 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -443,7 +443,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 6a41ceb27924dcfb6dc910a283ce74e136c9c305aba87a5acbfca32f5c49caa7 +F src/expr.c 2448a255ce627c4e772bd68cf5529877c2bdfb6b580803d5fadc8528bdf7c1ef F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -491,12 +491,12 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 41e857c107be5fb972b87a4cbb948c8b1191f339c1e32a65742581573932e119 +F src/select.c c87425f0d56340a4ccc5aac52d07264b73c348ca45103caef9f289b09cb0e052 F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7 F src/sqlite.h.in e0be726ea6e4e6571724d39d242472ecd8bd1ba6f84ade88e1641bde98a6d02b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h a4837c57f9a3e2af100bc59f4be60d16b823f18131f8cef6a6685440f775eebd +F src/sqliteInt.h 31a7f214a6b52d413f144d303d22c6e9a940059d9fb86e6f49e5ec7ad65636e9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1262,6 +1262,7 @@ F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c F test/sort5.test 6b43ae0e2169b5ceed441844492e55ba7f1ae0790528395ddf7888ab3094525d +F test/sorterref.test a13ed207a0eea3c7898f308f979bfb518f68c598ec737d2c494dfd3deaa83506 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1717,7 +1718,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7ec7b5970df9e542401f8299e098a5039d1875e2e3383dad06a0ec998589b6e -R f9cb5caefaf1e23db60ecde977cce853 -U drh -Z 2e05817b48e1d8380863fbfa532bc935 +P 0ab4518811b23bdb57feba55014cef07de66028f8fcbf8cf0831a712b2954b91 +R 58c773e5cd03f7f4dcbed950a8e0accb +T *branch * sorter-reference +T *sym-sorter-reference * +T +closed caee77d9a8e1da2551cef44ba7a11e2de36eb8c047273148b4126bfc1edd21da +T -sym-trunk * +U dan +Z 5d1ae4ddc55c798d695f83fc2835288d diff --git a/manifest.uuid b/manifest.uuid index 8a952e019d..3d40132909 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ab4518811b23bdb57feba55014cef07de66028f8fcbf8cf0831a712b2954b91 \ No newline at end of file +9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index eac5476f73..0adfc1dbb6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1363,6 +1363,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pItem->sortOrder = pOldItem->sortOrder; pItem->done = 0; pItem->bSpanIsTab = pOldItem->bSpanIsTab; + pItem->bSorterRef = pOldItem->bSorterRef; pItem->u = pOldItem->u; } return pNew; @@ -4371,6 +4372,12 @@ int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pItem->bSorterRef ){ + i--; + n--; + }else +#endif if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ if( flags & SQLITE_ECEL_OMITREF ){ i--; diff --git a/src/select.c b/src/select.c index 3e765c8464..eca67282e5 100644 --- a/src/select.c +++ b/src/select.c @@ -44,6 +44,20 @@ struct DistinctCtx { /* ** An instance of the following object is used to record information about ** the ORDER BY (or GROUP BY) clause of query is being coded. +** +** The aDefer[] array is used by the sorter-references optimization. For +** example, assuming there is no index that can be used for the ORDER BY, +** for the query: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10; +** +** it may be more efficient to add just the "a" values to the sorter, and +** retrieve the associated "bigblob" values directly from table t1 as the +** 10 smallest "a" values are extracted from the sorter. +** +** When the sorter-reference optimization is used, there is one entry in the +** aDefer[] array for each database table that may be read as values are +** extracted from the sorter. */ typedef struct SortCtx SortCtx; struct SortCtx { @@ -56,6 +70,14 @@ struct SortCtx { int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + u8 nDefer; /* Number of valid entries in aDefer[] */ + struct DeferredCsr { + Table *pTab; /* Table definition */ + int iCsr; /* Cursor number for table */ + int nKey; /* Number of PK columns for table pTab (>=1) */ + } aDefer[4]; +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -678,6 +700,87 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES +/* +** This function is called as part of inner-loop generation for a SELECT +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference +** optimization should be used for. The sorter-reference optimization +** is used for SELECT queries like: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10 +** +** If the optimization is used for expression "bigblob", then instead of +** storing values read from that column in the sorter records, the PK of +** the row from table t1 is stored instead. Then, as records are extracted from +** the sorter to return to the user, the required value of bigblob is +** retrieved directly from table t1. If the values are very large, this +** can be more efficient than storing them directly in the sorter records. +** +** The ExprList_item.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. +** Additionally, the pSort->aDefer[] array is populated with entries +** for all cursors required to evaluate all selected expressions. Finally. +** output variable (*ppExtra) is set to an expression list containing +** expressions for all extra PK values that should be stored in the +** sorter records. +*/ +static void selectExprDefer( + Parse *pParse, /* Leave any error here */ + SortCtx *pSort, /* Sorter context */ + ExprList *pEList, /* Expressions destined for sorter */ + ExprList **ppExtra /* Expressions to append to sorter record */ +){ + int i; + int nDefer = 0; + ExprList *pExtra = 0; + for(i=0; inExpr; i++){ + struct ExprList_item *pItem = &pEList->a[i]; + if( pItem->u.x.iOrderByCol==0 ){ + Expr *pExpr = pItem->pExpr; + Table *pTab = pExpr->pTab; + if( pExpr->op==TK_COLUMN && pTab && pTab->pSchema && pTab->pSelect==0 + && !IsVirtual(pTab) + ){ + int j; + for(j=0; jaDefer[j].iCsr==pExpr->iTable ) break; + } + if( j==nDefer ){ + if( nDefer==ArraySize(pSort->aDefer) ){ + continue; + }else{ + int nKey = 1; + int k; + Index *pPk = 0; + if( !HasRowid(pTab) ){ + pPk = sqlite3PrimaryKeyIndex(pTab); + nKey = pPk->nKeyCol; + } + for(k=0; kiTable = pExpr->iTable; + pNew->pTab = pExpr->pTab; + pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; + pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); + } + } + pSort->aDefer[nDefer].pTab = pExpr->pTab; + pSort->aDefer[nDefer].iCsr = pExpr->iTable; + pSort->aDefer[nDefer].nKey = nKey; + nDefer++; + } + } + pItem->bSorterRef = 1; + } + } + } + pSort->nDefer = (u8)nDefer; + *ppExtra = pExtra; +} +#endif + /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -750,6 +853,9 @@ static void selectInnerLoop( VdbeComment((v, "%s", p->pEList->a[i].zName)); } }else if( eDest!=SRT_Exists ){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra = 0; +#endif /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ @@ -773,12 +879,34 @@ static void selectInnerLoop( p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; } } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + selectExprDefer(pParse, pSort, p->pEList, &pExtra); + if( pExtra ){ + /* If there are any extra PK columns to add to the sorter records, + ** allocate extra memory cells and adjust the OpenEphemeral + ** instruction to account for the larger records. This is only + ** required if there are one or more WITHOUT ROWID tables with + ** composite primary keys in the SortCtx.aDefer[] array. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->p2 += (pExtra->nExpr - pSort->nDefer); + pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer); + pParse->nMem += pExtra->nExpr; + } +#endif regOrig = 0; assert( eDest==SRT_Set || eDest==SRT_Mem || eDest==SRT_Coroutine || eDest==SRT_Output ); } nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult, 0,ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pExtra ){ + nResultCol += sqlite3ExprCodeExprList( + pParse, pExtra, regResult + nResultCol, 0, 0 + ); + sqlite3ExprListDelete(pParse->db, pExtra); + } +#endif } /* If the DISTINCT keyword was present on the SELECT statement @@ -1236,7 +1364,7 @@ static void generateSortTail( Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ - int addr; + int addr; /* Top of output loop. Jump for Next. */ int addrOnce = 0; int iTab; ExprList *pOrderBy = pSort->pOrderBy; @@ -1245,10 +1373,11 @@ static void generateSortTail( int regRow; int regRowid; int iCol; - int nKey; + int nKey; /* Number of key columns in sorter record */ int iSortTab; /* Sorter cursor to read from */ int i; int bSeq; /* True if sorter record includes seq. no. */ + int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; assert( addrBreak<0 ); @@ -1257,6 +1386,17 @@ static void generateSortTail( sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + /* Open any cursors needed for sorter-reference expressions */ + for(i=0; inDefer; i++){ + Table *pTab = pSort->aDefer[i].pTab; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); + } +#endif + iTab = pSort->iECursor; if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ regRowid = 0; @@ -1272,7 +1412,8 @@ static void generateSortTail( if( pSort->labelBkOut ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } - sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nColumn); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + nKey+1+nColumn+nRefKey); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); @@ -1286,17 +1427,59 @@ static void generateSortTail( bSeq = 1; } for(i=0, iCol=nKey+bSeq-1; i=0; i--){ - int iRead; - if( aOutEx[i].u.x.iOrderByCol ){ - iRead = aOutEx[i].u.x.iOrderByCol-1; - }else{ - iRead = iCol--; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pSort->nDefer ){ + int iKey = iCol+1; + int regKey = sqlite3GetTempRange(pParse, nRefKey); + + for(i=0; inDefer; i++){ + int iCsr = pSort->aDefer[i].iCsr; + Table *pTab = pSort->aDefer[i].pTab; + int nKey = pSort->aDefer[i].nKey; + + sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeCurrentAddr(v)+1, regKey); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int k; + int iJmp; + assert( pPk->nKeyCol==nKey ); + for(k=0; k=0; i--){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( aOutEx[i].bSorterRef ){ + sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); + }else +#endif + { + int iRead; + if( aOutEx[i].u.x.iOrderByCol ){ + iRead = aOutEx[i].u.x.iOrderByCol-1; + }else{ + iRead = iCol--; + } + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); + VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan)); } - sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); - VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); } switch( eDest ){ case SRT_Table: @@ -6072,6 +6255,7 @@ int sqlite3Select( if( sSort.pOrderBy ){ explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d932ae87da..bc2840ef10 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2499,6 +2499,7 @@ struct ExprList { unsigned done :1; /* A flag to indicate when processing is finished */ unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ diff --git a/test/sorterref.test b/test/sorterref.test new file mode 100644 index 0000000000..28445c6e72 --- /dev/null +++ b/test/sorterref.test @@ -0,0 +1,50 @@ +# 2018 April 14. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sorterref + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + ALTER TABLE t1 ADD COLUMN d DEFAULT 'string'; + INSERT INTO t1 VALUES(7, 8, 9, 'text'); +} + +do_execsql_test 1.1 { + SELECT * FROM t1 ORDER BY b; +} { + 1 2 3 string 4 5 6 string 7 8 9 text +} + +do_execsql_test 2.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d, PRIMARY KEY(c)) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(2, 3); + INSERT INTO t1 VALUES(3, 4); + + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(3, 'three'); +} + +do_execsql_test 2.1 { + SELECT * FROM t1 LEFT JOIN t2 ON (a=c) ORDER BY b; +} {1 2 1 one 2 3 {} {} 3 4 3 three} + + + +finish_test From 2e3a5a810a8df7ee3220eca524b2b8a372f7ef35 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 16 Apr 2018 21:12:42 +0000 Subject: [PATCH 2/6] Add SQLITE_CONFIG_SORTERREF_SIZE configuration option. FossilOrigin-Name: b25a7bb769b8397a00736bd922bd24b1200b2f82d36d42ecb4c3eb17efb4b84d --- manifest | 30 +++++++++++++----------------- manifest.uuid | 2 +- src/build.c | 28 +++++++++++++++++----------- src/global.c | 3 ++- src/main.c | 11 +++++++++++ src/select.c | 7 +++++-- src/sqlite.h.in | 15 +++++++++++++++ src/sqliteInt.h | 11 ++++++++++- src/test1.c | 22 ++++++++++++++++++++++ test/permutations.test | 20 ++++++++++++++++++++ 10 files changed, 116 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 302de8f7e0..a13c8f9627 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schange\sto\s"SELECT\s*\sFROM\s...\sORDER\sBY"\sprocessing\sto\sload\ssome\ncolumn\svalues\sfrom\sthe\sdb\safter\ssorting. -D 2018-04-14T18:46:20.429 +C Add\sSQLITE_CONFIG_SORTERREF_SIZE\sconfiguration\soption. +D 2018-04-16T21:12:42.011 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -435,7 +435,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 9eb9531c65346bbfccf5325384b7db1849daf4db6601dcfe21ba5c5b20623b64 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 61320fb84034c24313de699f3385c6bfe093c925b4df2931c6eb63d7c94ec62a +F src/build.c 850d831d98ceaad4b8d1ed1f487697e008150804ad284eff4fb41cb067561d8f F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c bd9da3f1ff21b432564a16ef0b154cff03585dc43742842e99c58907c6cb4bef @@ -447,7 +447,7 @@ F src/expr.c 2448a255ce627c4e772bd68cf5529877c2bdfb6b580803d5fadc8528bdf7c1ef F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 -F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a +F src/global.c 9bf034fd560bdd514715170ed8460bb7f823cec113f0569ef3f18a20c7ccd128 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da @@ -455,7 +455,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 752740e4619416d4262f6e9e51cdb6af5965eb0c8e943832a5af77d41e2839c7 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302 -F src/main.c 1648fc7a9bcfdbfd9a9a04af96ff2796c3164b3f3c7e56ed63a3c51cd11d198d +F src/main.c 8f89ff60cd0c759bd6078fdf4db12a1fedcb93a214f13d545ba64c297429725e F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -491,17 +491,17 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c c87425f0d56340a4ccc5aac52d07264b73c348ca45103caef9f289b09cb0e052 +F src/select.c 875b63fd9658701cb468dfecb251d2f93c4f7106028fd34624559e44c171813a F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7 -F src/sqlite.h.in e0be726ea6e4e6571724d39d242472ecd8bd1ba6f84ade88e1641bde98a6d02b +F src/sqlite.h.in 51a98c2a112df85bd32eef1cbe0f89257180d64100efc85e6dac6fb1f93ff5cc F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 31a7f214a6b52d413f144d303d22c6e9a940059d9fb86e6f49e5ec7ad65636e9 +F src/sqliteInt.h ad14bfeab6c1ada3aa1181c6cc14a3e6a8f24b35ad96a2daa3d11100a5267236 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 916a92de77ec5cbe27818ca194d8cf0c58aa7ad5b87527098f6aa5a6068800ce -F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828 +F src/test1.c b2df6d7ed8ecb53680f7040163ba92b50c471ed8104d128c88c8e969a107887f F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1141,7 +1141,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 8ada8c1dee071e0fc275bc8bc2db7de537d625cad949d2200664b99a0a89eac5 +F test/permutations.test 10793f1de89a226fa22dde9ba9398de22571fee1bfb53a935a11be4aa014704f F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1718,11 +1718,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ab4518811b23bdb57feba55014cef07de66028f8fcbf8cf0831a712b2954b91 -R 58c773e5cd03f7f4dcbed950a8e0accb -T *branch * sorter-reference -T *sym-sorter-reference * -T +closed caee77d9a8e1da2551cef44ba7a11e2de36eb8c047273148b4126bfc1edd21da -T -sym-trunk * +P 9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d +R b51a7eb766ea2611d831b0e6f1e9c6dd U dan -Z 5d1ae4ddc55c798d695f83fc2835288d +Z 811f1b43e3ad404510c72b9e701b11d8 diff --git a/manifest.uuid b/manifest.uuid index 3d40132909..dcc5330e09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d \ No newline at end of file +b25a7bb769b8397a00736bd922bd24b1200b2f82d36d42ecb4c3eb17efb4b84d \ No newline at end of file diff --git a/src/build.c b/src/build.c index c286b4bbe7..1287dd129f 100644 --- a/src/build.c +++ b/src/build.c @@ -1095,15 +1095,18 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ if( pType->n==0 ){ /* If there is no type specified, columns have the default affinity - ** 'BLOB'. */ + ** 'BLOB' and a default size of 4 bytes. */ pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; + if( 4>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } }else{ zType = z + sqlite3Strlen30(z) + 1; memcpy(zType, pType->z, pType->n); zType[pType->n] = 0; sqlite3Dequote(zType); - pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; @@ -1163,7 +1166,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ ** If none of the substrings in the above table are found, ** SQLITE_AFF_NUMERIC is returned. */ -char sqlite3AffinityType(const char *zIn, u8 *pszEst){ +char sqlite3AffinityType(const char *zIn, Column *pCol){ u32 h = 0; char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; @@ -1200,27 +1203,30 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ } } - /* If pszEst is not NULL, store an estimate of the field size. The + /* If pCol is not NULL, store an estimate of the field size. The ** estimate is scaled so that the size of an integer is 1. */ - if( pszEst ){ - *pszEst = 1; /* default size is approx 4 bytes */ + if( pCol ){ + int v = 0; /* default size is approx 4 bytes */ if( aff r=(k/4+1) */ sqlite3GetInt32(zChar, &v); - v = v/4 + 1; - if( v>255 ) v = 255; - *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */ break; } zChar++; } }else{ - *pszEst = 5; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ } } + if( v>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } + v = v/4 + 1; + if( v>255 ) v = 255; + pCol->szEst = v; } return aff; } diff --git a/src/global.c b/src/global.c index 04a3d185a8..f0362d1e04 100644 --- a/src/global.c +++ b/src/global.c @@ -240,7 +240,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xTestCallback */ #endif 0, /* bLocaltimeFault */ - 0x7ffffffe /* iOnceResetThreshold */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE /* szSorterRef */ }; /* diff --git a/src/main.c b/src/main.c index 5b6c867097..7fd218cc6b 100644 --- a/src/main.c +++ b/src/main.c @@ -642,6 +642,17 @@ int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_SORTERREF_SIZE: { +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; + } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + break; + } + default: { rc = SQLITE_ERROR; break; diff --git a/src/select.c b/src/select.c index eca67282e5..38f58c04c0 100644 --- a/src/select.c +++ b/src/select.c @@ -739,8 +739,11 @@ static void selectExprDefer( if( pItem->u.x.iOrderByCol==0 ){ Expr *pExpr = pItem->pExpr; Table *pTab = pExpr->pTab; - if( pExpr->op==TK_COLUMN && pTab && pTab->pSchema && pTab->pSelect==0 - && !IsVirtual(pTab) + if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab) + && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) +#if 0 + && pTab->pSchema && pTab->pSelect==0 && !IsVirtual(pTab) +#endif ){ int j; for(j=0; jSQLITE_CONFIG_SORTERREF_SIZE +**
The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted records and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behaviour. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1959,6 +1973,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index bc2840ef10..ac75b0de27 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -637,6 +637,13 @@ # define SQLITE_DEFAULT_PCACHE_INITSZ 20 #endif +/* +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option. +*/ +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff +#endif + /* ** The compile-time options SQLITE_MMAP_READWRITE and ** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another. @@ -1759,6 +1766,7 @@ struct Column { #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ #define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ #define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -3321,6 +3329,7 @@ struct Sqlite3Config { #endif int bLocaltimeFault; /* True to fail localtime() calls */ int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ }; /* @@ -4103,7 +4112,7 @@ void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); -char sqlite3AffinityType(const char*, u8*); +char sqlite3AffinityType(const char*, Column*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*); int sqlite3FindDb(sqlite3*, Token*); diff --git a/src/test1.c b/src/test1.c index bc8f389dbd..0f139ceba5 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2256,6 +2256,27 @@ static int SQLITE_TCLAPI test_config_sqllog( } #endif +/* +** Usage: sqlite3_config_sorterref +** +** Set the SQLITE_CONFIG_SORTERREF_SIZE configuration option +*/ +static int SQLITE_TCLAPI test_config_sorterref( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int iVal; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NBYTE"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &iVal) ) return TCL_ERROR; + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, iVal); + return TCL_OK; +} + /* ** Usage: vfs_current_time_int64 ** @@ -7751,6 +7772,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_delete_database", test_delete_database, 0 }, { "atomic_batch_write", test_atomic_batch_write, 0 }, { "sqlite3_mmap_warm", test_mmap_warm, 0 }, + { "sqlite3_config_sorterref", test_config_sorterref, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); diff --git a/test/permutations.test b/test/permutations.test index c1d28d4e09..52e2509fc1 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1073,6 +1073,26 @@ test_suite "prepare" -description { stmtvtab1.test index9.test ] +test_suite "sorterref" -prefix "" -description { + Run the "veryquick" test suite with SQLITE_CONFIG_SORTERREF_SIZE set + to 0 so that sorter-references are used whenever possible. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \ + *fts5corrupt* *fts5big* *fts5aj* +] -initialize { + catch {db close} + sqlite3_shutdown + sqlite3_config_sorterref 0 + sqlite3_initialize + autoinstall_test_functions +} -shutdown { + catch {db close} + sqlite3_shutdown + sqlite3_config_sorterref -1 + sqlite3_initialize + autoinstall_test_functions +} + # End of tests ############################################################################# From 7590d093247c6503b8406da58da32411b6938435 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 18 Apr 2018 14:04:19 +0000 Subject: [PATCH 3/6] Remove an unused local variable. FossilOrigin-Name: 9afeb0c88b211e3ad90697778ec8ebb80f216fe7078c3a2f51651c35d3828746 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a13c8f9627..20ac700cb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_CONFIG_SORTERREF_SIZE\sconfiguration\soption. -D 2018-04-16T21:12:42.011 +C Remove\san\sunused\slocal\svariable. +D 2018-04-18T14:04:19.741 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -491,7 +491,7 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 875b63fd9658701cb468dfecb251d2f93c4f7106028fd34624559e44c171813a +F src/select.c 6cf4c11b2f258940acc289ea57eaae1fba16689ae62e3a09ebbc6487d440bf09 F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7 F src/sqlite.h.in 51a98c2a112df85bd32eef1cbe0f89257180d64100efc85e6dac6fb1f93ff5cc F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1718,7 +1718,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d -R b51a7eb766ea2611d831b0e6f1e9c6dd -U dan -Z 811f1b43e3ad404510c72b9e701b11d8 +P b25a7bb769b8397a00736bd922bd24b1200b2f82d36d42ecb4c3eb17efb4b84d +R eba9a240e0de928e991f348803303439 +U drh +Z 5d665bfe789699fa240c154efc829ec2 diff --git a/manifest.uuid b/manifest.uuid index dcc5330e09..fc722f046d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b25a7bb769b8397a00736bd922bd24b1200b2f82d36d42ecb4c3eb17efb4b84d \ No newline at end of file +9afeb0c88b211e3ad90697778ec8ebb80f216fe7078c3a2f51651c35d3828746 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 38f58c04c0..227e812ba1 100644 --- a/src/select.c +++ b/src/select.c @@ -1451,10 +1451,9 @@ static void generateSortTail( sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, sqlite3VdbeCurrentAddr(v)+1, regKey); }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); int k; int iJmp; - assert( pPk->nKeyCol==nKey ); + assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey ); for(k=0; k Date: Wed, 18 Apr 2018 14:48:08 +0000 Subject: [PATCH 4/6] Omit some code not used without SQLITE_ENABLE_SORTER_REFERENCES. Improvements to comments used for documentation. FossilOrigin-Name: f3596ab9eb1168dc75202e6f79e12c67d518b3a0659a0a629c707f43990fa7cf --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 6 +++++- src/ctime.c | 3 +++ src/main.c | 4 ++-- src/sqlite.h.in | 4 +++- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 20ac700cb5..4df9bedbe8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\slocal\svariable. -D 2018-04-18T14:04:19.741 +C Omit\ssome\scode\snot\sused\swithout\sSQLITE_ENABLE_SORTER_REFERENCES.\s\sImprovements\nto\scomments\sused\sfor\sdocumentation. +D 2018-04-18T14:48:08.814 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -435,10 +435,10 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 9eb9531c65346bbfccf5325384b7db1849daf4db6601dcfe21ba5c5b20623b64 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 850d831d98ceaad4b8d1ed1f487697e008150804ad284eff4fb41cb067561d8f +F src/build.c 91d548027a54044851299e719d60d6a2b6eaf3e3274678098cb73a6ab8c0fa19 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c bd9da3f1ff21b432564a16ef0b154cff03585dc43742842e99c58907c6cb4bef +F src/ctime.c 849d4cebe008cfc6e4799b034a172b4eaf8856b100739632a852732ba66eee48 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91 @@ -455,7 +455,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 752740e4619416d4262f6e9e51cdb6af5965eb0c8e943832a5af77d41e2839c7 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302 -F src/main.c 8f89ff60cd0c759bd6078fdf4db12a1fedcb93a214f13d545ba64c297429725e +F src/main.c 10e3897f5d78cef6bcbd1eedc8ccc3fe9e9783d07e052d9d70e57364ded19274 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -493,7 +493,7 @@ F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 6cf4c11b2f258940acc289ea57eaae1fba16689ae62e3a09ebbc6487d440bf09 F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7 -F src/sqlite.h.in 51a98c2a112df85bd32eef1cbe0f89257180d64100efc85e6dac6fb1f93ff5cc +F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d F src/sqliteInt.h ad14bfeab6c1ada3aa1181c6cc14a3e6a8f24b35ad96a2daa3d11100a5267236 @@ -1718,7 +1718,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b25a7bb769b8397a00736bd922bd24b1200b2f82d36d42ecb4c3eb17efb4b84d -R eba9a240e0de928e991f348803303439 +P 9afeb0c88b211e3ad90697778ec8ebb80f216fe7078c3a2f51651c35d3828746 +R 33406b12bca77020e0867e36a6f40300 U drh -Z 5d665bfe789699fa240c154efc829ec2 +Z 8662be91b6a3020051547b8fbdcb3aff diff --git a/manifest.uuid b/manifest.uuid index fc722f046d..5314eed86c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9afeb0c88b211e3ad90697778ec8ebb80f216fe7078c3a2f51651c35d3828746 \ No newline at end of file +f3596ab9eb1168dc75202e6f79e12c67d518b3a0659a0a629c707f43990fa7cf \ No newline at end of file diff --git a/src/build.c b/src/build.c index 1287dd129f..b032ff9f2c 100644 --- a/src/build.c +++ b/src/build.c @@ -1095,12 +1095,14 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ if( pType->n==0 ){ /* If there is no type specified, columns have the default affinity - ** 'BLOB' and a default size of 4 bytes. */ + ** 'BLOB' with a default size of 4 bytes. */ pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES if( 4>=sqlite3GlobalConfig.szSorterRef ){ pCol->colFlags |= COLFLAG_SORTERREF; } +#endif }else{ zType = z + sqlite3Strlen30(z) + 1; memcpy(zType, pType->z, pType->n); @@ -1221,9 +1223,11 @@ char sqlite3AffinityType(const char *zIn, Column *pCol){ v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ } } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES if( v>=sqlite3GlobalConfig.szSorterRef ){ pCol->colFlags |= COLFLAG_SORTERREF; } +#endif v = v/4 + 1; if( v>255 ) v = 255; pCol->szEst = v; diff --git a/src/ctime.c b/src/ctime.c index 1877aee10a..da79f3a350 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -286,6 +286,9 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_ENABLE_SNAPSHOT "ENABLE_SNAPSHOT", #endif +#if SQLITE_ENABLE_SORTER_REFERENCES + "ENABLE_SORTER_REFERENCES", +#endif #if SQLITE_ENABLE_SQLLOG "ENABLE_SQLLOG", #endif diff --git a/src/main.c b/src/main.c index 7fd218cc6b..99e3e6f970 100644 --- a/src/main.c +++ b/src/main.c @@ -642,16 +642,16 @@ int sqlite3_config(int op, ...){ break; } - case SQLITE_CONFIG_SORTERREF_SIZE: { #ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { int iVal = va_arg(ap, int); if( iVal<0 ){ iVal = SQLITE_DEFAULT_SORTERREF_SIZE; } sqlite3GlobalConfig.szSorterRef = (u32)iVal; -#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ break; } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ default: { rc = SQLITE_ERROR; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ed4cffeab2..02dc8944b9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1940,10 +1940,12 @@ struct sqlite3_mem_methods { ** sorted records. However, if SQLite determines based on the declared type ** of a table column that its values are likely to be very large - larger ** than the configured sorter-reference size threshold - then a reference -** is stored in each sorted records and the required column values loaded +** is stored in each sorted record and the required column values loaded ** from the database as records are returned in sorted order. The default ** value for this option is to never use this optimization. Specifying a ** negative value for this option restores the default behaviour. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ From a90d84fe211178c757f9d74430f8b589a0438fe8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 18 Apr 2018 15:21:13 +0000 Subject: [PATCH 5/6] Add the --sorterref N option to the CLI. FossilOrigin-Name: 902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4df9bedbe8..b89a9e6a13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\ssome\scode\snot\sused\swithout\sSQLITE_ENABLE_SORTER_REFERENCES.\s\sImprovements\nto\scomments\sused\sfor\sdocumentation. -D 2018-04-18T14:48:08.814 +C Add\sthe\s--sorterref\sN\soption\sto\sthe\sCLI. +D 2018-04-18T15:21:13.476 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -492,7 +492,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 6cf4c11b2f258940acc289ea57eaae1fba16689ae62e3a09ebbc6487d440bf09 -F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7 +F src/shell.c.in 6c9e2c1136f3697eb75f5dce010e7af005f62b4e1fda6d1066c3a473eb922889 F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1718,7 +1718,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9afeb0c88b211e3ad90697778ec8ebb80f216fe7078c3a2f51651c35d3828746 -R 33406b12bca77020e0867e36a6f40300 +P f3596ab9eb1168dc75202e6f79e12c67d518b3a0659a0a629c707f43990fa7cf +R 0443c59f405eaa117ee330ad90988b78 U drh -Z 8662be91b6a3020051547b8fbdcb3aff +Z 0415bd978b9c9d919a9a079609154b94 diff --git a/manifest.uuid b/manifest.uuid index 5314eed86c..038423a21c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3596ab9eb1168dc75202e6f79e12c67d518b3a0659a0a629c707f43990fa7cf \ No newline at end of file +902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index ddbf825951..9750cf2803 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8076,6 +8076,9 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -separator SEP set output column separator. Default: '|'\n" +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + " -sorterref SIZE sorter references threshold size\n" +#endif " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" @@ -8331,6 +8334,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); +#endif }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ @@ -8467,6 +8475,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ i+=2; }else if( strcmp(z,"-mmap")==0 ){ i++; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + i++; +#endif }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE From 5a2e65ed633df2d208c591e7520456ed79ad3385 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 18 Apr 2018 19:08:44 +0000 Subject: [PATCH 6/6] Avoid a NULL-pointer deref following OOM. FossilOrigin-Name: 413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b89a9e6a13..b1c72a522d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--sorterref\sN\soption\sto\sthe\sCLI. -D 2018-04-18T15:21:13.476 +C Avoid\sa\sNULL-pointer\sderef\sfollowing\sOOM. +D 2018-04-18T19:08:44.210 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -491,7 +491,7 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 6cf4c11b2f258940acc289ea57eaae1fba16689ae62e3a09ebbc6487d440bf09 +F src/select.c 9257a7f26e6c9da27deae819098d1005f4e1c37e19b8f029292bd2b1aab35721 F src/shell.c.in 6c9e2c1136f3697eb75f5dce010e7af005f62b4e1fda6d1066c3a473eb922889 F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1718,7 +1718,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3596ab9eb1168dc75202e6f79e12c67d518b3a0659a0a629c707f43990fa7cf -R 0443c59f405eaa117ee330ad90988b78 +P 902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce +R 4bf2a8a4f284f188f9cc406a1fead38f U drh -Z 0415bd978b9c9d919a9a079609154b94 +Z 857937520ddb8fedd9d522f195cdf4dc diff --git a/manifest.uuid b/manifest.uuid index 038423a21c..8036b78b15 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce \ No newline at end of file +413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 227e812ba1..20d11764c2 100644 --- a/src/select.c +++ b/src/select.c @@ -884,7 +884,7 @@ static void selectInnerLoop( } #ifdef SQLITE_ENABLE_SORTER_REFERENCES selectExprDefer(pParse, pSort, p->pEList, &pExtra); - if( pExtra ){ + if( pExtra && pParse->db->mallocFailed==0 ){ /* If there are any extra PK columns to add to the sorter records, ** allocate extra memory cells and adjust the OpenEphemeral ** instruction to account for the larger records. This is only