From f45f2326a28870f9533d5e75f8daa7562708b96a Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Mar 2014 17:45:03 +0000 Subject: [PATCH 01/15] Use only a single OP_MakeRecord instead of two when constructing entries to go onto a sorter. FossilOrigin-Name: d696cdedacd39075aa7fc407ab7c7e50f01d9f39 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/select.c | 82 ++++++++++++++++++++++++++----------------------- test/tester.tcl | 2 ++ 4 files changed, 53 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index d2ff62a057..1724c9322b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sOFFSET-on-query-without-FROM\sfix\sfrom\strunk. -D 2014-03-21T18:45:19.215 +C Use\sonly\sa\ssingle\sOP_MakeRecord\sinstead\sof\stwo\swhen\sconstructing\sentries\nto\sgo\sonto\sa\ssorter. +D 2014-03-23T17:45:03.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 596b8098be41e6256968f167d1d8ece2be08d082 +F src/select.c b1e0ac15d846c1d584c26f95bd92e887313b55df F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -845,7 +845,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744 +F test/tester.tcl bc0889a2f86d9c17307992ca1e70391794780265 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e70cfa28aa393661ccc742ecd5e672d807bdd0a9 179ef81648b0ad557df78b7712f216b876b6fb65 -R 654a6218c0c034c3e6462d9f9d0a06b2 +P 71e9ae72c272dc86720b2bfe719f57de437c400b +R d93a4c6f8aefee44f30ed73f4e05ae09 U drh -Z 7a7ab7eaa2946af0174662f0ca0e6009 +Z 5ef6faddb00682067ed8bb051e6b9892 diff --git a/manifest.uuid b/manifest.uuid index d1f85a0601..b62e4b84bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71e9ae72c272dc86720b2bfe719f57de437c400b \ No newline at end of file +d696cdedacd39075aa7fc407ab7c7e50f01d9f39 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f81a5ef3e7..6a3dff28a9 100644 --- a/src/select.c +++ b/src/select.c @@ -455,26 +455,29 @@ static KeyInfo *keyInfoFromExprList( ); /* -** Insert code into "v" that will push the record in register regData -** into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int nData /* Number of elements in the data array */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); - int nOBSat = pSort->nOBSat; - int op; + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ + int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -509,7 +512,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + sqlite3ReleaseTempRange(pParse, regBase, nBase); } if( pSelect->iLimit ){ int addr1, addr2; @@ -773,7 +776,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1); + pushOntoSorter(pParse, pSort, p, r1, 1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -799,7 +802,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -825,7 +828,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -839,10 +842,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pSort, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -1129,6 +1129,10 @@ static void generateSortTail( int regRow; int regRowid; int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + u8 p5; /* p5 parameter for 1st OP_Column */ + int i; if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1142,26 +1146,35 @@ static void generateSortTail( pseudoTab = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; - int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + iSortTab = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + p5 = OPFLAG_CLEARCACHE; }else{ if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + iSortTab = iTab; + p5 = 0; + } + for(i=0; iiSdst+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); @@ -1210,9 +1215,10 @@ static void generateSortTail( break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -1223,9 +1229,6 @@ static void generateSortTail( } if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); - } } /* @@ -4760,8 +4763,9 @@ int sqlite3Select( sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ sSort.addrSortIndex = -1; } diff --git a/test/tester.tcl b/test/tester.tcl index 1c4e93937c..4d55042b43 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1076,6 +1076,7 @@ proc explain_i {sql {db db}} { foreach opcode { Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind NoConflict Next Prev VNext VPrev VFilter + SorterSort SorterNext } { set color($opcode) $B } @@ -1098,6 +1099,7 @@ proc explain_i {sql {db db}} { if {$opcode=="Next" || $opcode=="Prev" || $opcode=="VNext" || $opcode=="VPrev" + || $opcode=="SorterNext" } { for {set i $p2} {$i<$addr} {incr i} { incr x($i) 2 From 70f624c3a9ed4a9d1efc61ddd3a6744d741be886 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 01:43:50 +0000 Subject: [PATCH 02/15] Further enhancements to geneverated VDBE code for ORDER BY. FossilOrigin-Name: e7188fad87ec82d36a39b80ccaf8006bf45a9bcd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 140d5c124c..979df5c0f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C merge\sfixes\sfrom\strunk -D 2014-03-23T18:47:00.384 +C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. +D 2014-03-24T01:43:50.641 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 09fef04ec0746d168ddcff37031ee804ac19dd0e +F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d696cdedacd39075aa7fc407ab7c7e50f01d9f39 641408a1395bfc911ca619ef9e5f073b913d856b -R e64080ab6e1cc6573ca05b14626c10fb +P faf7f9caf526ab33a6fdb9c89b45a0483510db21 +R 55d45b1a91856051dc771a7feb58050e U drh -Z 0df19c139e23d451d51cd0e8cf9cddd3 +Z 450407adee455925374bbdaef08e9b36 diff --git a/manifest.uuid b/manifest.uuid index 49e295d769..61b2c1f687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faf7f9caf526ab33a6fdb9c89b45a0483510db21 \ No newline at end of file +e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1acc6d1020..5ececf1540 100644 --- a/src/select.c +++ b/src/select.c @@ -473,8 +473,7 @@ static void pushOntoSorter( int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -1127,7 +1126,6 @@ static void generateSortTail( int addr; int addrOnce = 0; int iTab; - int pseudoTab = 0; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; @@ -1138,6 +1136,9 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1148,8 +1149,6 @@ static void generateSortTail( iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; @@ -1180,6 +1179,7 @@ static void generateSortTail( for(i=0; i Date: Mon, 24 Mar 2014 02:20:53 +0000 Subject: [PATCH 03/15] Remove a pointless OP_Once operation in ORDER BY clauses with LIMIT. FossilOrigin-Name: e6c59d23316c83b318b1a94d9b28a5d321737fa5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 979df5c0f1..e046917d18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. -D 2014-03-24T01:43:50.641 +C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. +D 2014-03-24T02:20:53.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b +F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P faf7f9caf526ab33a6fdb9c89b45a0483510db21 -R 55d45b1a91856051dc771a7feb58050e +P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd +R 7cf4844b34fb28597f11c358a9ceae37 U drh -Z 450407adee455925374bbdaef08e9b36 +Z 5144b0c0a9c74c80111921622c3f3ee8 diff --git a/manifest.uuid b/manifest.uuid index 61b2c1f687..0c4d7e566f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file +e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 5ececf1540..013feb019f 100644 --- a/src/select.c +++ b/src/select.c @@ -1144,7 +1144,6 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); @@ -1161,6 +1160,9 @@ static void generateSortTail( if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); @@ -1169,10 +1171,8 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; }else{ - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); iSortTab = iTab; p5 = 0; } From 3f802ebce2d497fce3ba9330fb8bc4c0ab1cf539 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 09:34:58 +0000 Subject: [PATCH 04/15] Remove an unnecessary temporary register allocation. FossilOrigin-Name: 5d506743f541b022cde04a9606baa4680cdfd70b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e046917d18..feb6340a70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. -D 2014-03-24T02:20:53.278 +C Remove\san\sunnecessary\stemporary\sregister\sallocation. +D 2014-03-24T09:34:58.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 +F src/select.c a088183774c4efae4105076355ac4010c62390a8 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd -R 7cf4844b34fb28597f11c358a9ceae37 -U drh -Z 5144b0c0a9c74c80111921622c3f3ee8 +P e6c59d23316c83b318b1a94d9b28a5d321737fa5 +R ab9aa4f9f7d5ea8c370ac77061403538 +U dan +Z 631d6a70f4aeb5d6a455a81761c8aefd diff --git a/manifest.uuid b/manifest.uuid index 0c4d7e566f..36b51c0a51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file +5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 013feb019f..641a54faa2 100644 --- a/src/select.c +++ b/src/select.c @@ -1146,7 +1146,6 @@ static void generateSortTail( sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ regRowid = 0; regRow = pDest->iSdst; From fd0a2f9756745401f68fc0e76b868c682194d023 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 18:08:15 +0000 Subject: [PATCH 05/15] Avoid unnecessary moving of content between registers during an ORDER BY. FossilOrigin-Name: 4f472accf072d9cb64f209923924b26f21b13d27 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 38 +++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index feb6340a70..a1b753f05e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\stemporary\sregister\sallocation. -D 2014-03-24T09:34:58.396 +C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. +D 2014-03-24T18:08:15.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a088183774c4efae4105076355ac4010c62390a8 +F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6c59d23316c83b318b1a94d9b28a5d321737fa5 -R ab9aa4f9f7d5ea8c370ac77061403538 -U dan -Z 631d6a70f4aeb5d6a455a81761c8aefd +P 5d506743f541b022cde04a9606baa4680cdfd70b +R edfc832bbac864fb9d0e2cb38ae7f098 +U drh +Z a04be0daeb9646c039a07b2f019c65a9 diff --git a/manifest.uuid b/manifest.uuid index 36b51c0a51..1c7384bd0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file +4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 641a54faa2..40fd59daab 100644 --- a/src/select.c +++ b/src/select.c @@ -463,19 +463,28 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ - int nData /* Number of elements in the data array */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + 1 + nData; /* Fields in sorter record */ - int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regBase; /* Regs for sorter record */ int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+1 ); + regBase = regData - nExpr - 1; + }else{ + regBase = sqlite3GetTempRange(pParse, nBase); + } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -515,7 +524,9 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nBase); + if( nPrefixReg==0 ){ + sqlite3ReleaseTempRange(pParse, regBase, nBase); + } } if( pSelect->iLimit ){ int addr1, addr2; @@ -631,6 +642,7 @@ static void selectInnerLoop( int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); assert( pEList!=0 ); @@ -646,6 +658,10 @@ static void selectInnerLoop( nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr + 1; + pParse->nMem += nPrefixReg; + } pDest->iSdst = pParse->nMem+1; pParse->nMem += nResultCol; }else if( pDest->iSdst+nResultCol > pParse->nMem ){ @@ -762,10 +778,10 @@ static void selectInnerLoop( case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -780,7 +796,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1, 1); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -788,7 +804,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -806,7 +822,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -832,7 +848,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -846,7 +862,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ From dd23c6bfb42cc87dc5919657a972d84afff2162a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 20:19:07 +0000 Subject: [PATCH 06/15] Omit the sequence value from sorter records used by GROUP BY queries that cannot use an index. FossilOrigin-Name: 3f90abddc31ac20739778c235a834c33f7057997 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 9 ++++----- src/vdbesort.c | 1 + 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a1b753f05e..63d2130c2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. -D 2014-03-24T18:08:15.960 +C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. +D 2014-03-24T20:19:07.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e +F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 +F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5d506743f541b022cde04a9606baa4680cdfd70b -R edfc832bbac864fb9d0e2cb38ae7f098 -U drh -Z a04be0daeb9646c039a07b2f019c65a9 +P 4f472accf072d9cb64f209923924b26f21b13d27 +R e1309a603202b4d83715c783fe89f3da +U dan +Z 8e3a9a2ddbbe8575a70cbc6b0dba434a diff --git a/manifest.uuid b/manifest.uuid index 1c7384bd0a..0ff8edcdd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file +3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 40fd59daab..7e0085dd36 100644 --- a/src/select.c +++ b/src/select.c @@ -4910,7 +4910,7 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); @@ -5002,8 +5002,8 @@ int sqlite3Select( groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -5013,8 +5013,7 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index d1b726b727..d55156a2c1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -480,6 +480,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + pSorter->pUnpacked->nField = pCsr->pKeyInfo->nField; if( !sqlite3TempInMemory(db) ){ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); From 78d5843245b2ef9c1fb92213d0c0e4bb49e6c70d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 15:04:07 +0000 Subject: [PATCH 07/15] Remove the sequence values from sorter records used by ORDER BY as well. FossilOrigin-Name: c3ae3697832a00d4d5758988a8766bdbb691e6b8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 40 +++++++++++++++++++++++++++------------- src/vdbe.c | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 63d2130c2b..2690ecef90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. -D 2014-03-24T20:19:07.793 +C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. +D 2014-03-25T15:04:07.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 +F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d +F src/vdbe.c 42177064bd02fc55984aabc3a241368fda01e8b4 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4f472accf072d9cb64f209923924b26f21b13d27 -R e1309a603202b4d83715c783fe89f3da +P 3f90abddc31ac20739778c235a834c33f7057997 +R 743d88a284437e5e0babc8e33ec37e61 U dan -Z 8e3a9a2ddbbe8575a70cbc6b0dba434a +Z 5024d9c780855d9abccbd4c6c23f3054 diff --git a/manifest.uuid b/manifest.uuid index 0ff8edcdd3..b187241ae4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file +c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7e0085dd36..7480eeb94b 100644 --- a/src/select.c +++ b/src/select.c @@ -467,24 +467,29 @@ static void pushOntoSorter( int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ - int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ - int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ - int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ - int op; /* Opcode to add sorter record to sorter */ + int regRecord = sqlite3GetTempReg(pParse); /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + assert( bSeq==0 || bSeq==1 ); if( nPrefixReg ){ - assert( nPrefixReg==nExpr+1 ); - regBase = regData - nExpr - 1; + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; }else{ regBase = sqlite3GetTempRange(pParse, nBase); } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); - sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - if( nPrefixReg==0 ){ - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+bSeq, nData); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -496,8 +501,13 @@ static void pushOntoSorter( regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; - nKey = nExpr - pSort->nOBSat + 1; - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; @@ -659,7 +669,8 @@ static void selectInnerLoop( if( pDest->iSdst==0 ){ if( pSort ){ - nPrefixReg = pSort->pOrderBy->nExpr + 1; + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; pParse->nMem += nPrefixReg; } pDest->iSdst = pParse->nMem+1; @@ -1152,6 +1163,7 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; + int bSeq; /* True if sorter record includes seq. no. */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; #endif @@ -1185,14 +1197,16 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; + bSeq = 0; }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; p5 = 0; + bSeq = 1; } for(i=0; ip1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC->pSorter ); + if( (pC->seqCount++)==0 ){ + pc = pOp->p2 - 1; + } + break; +} + /* Opcode: OpenPseudo P1 P2 P3 * * ** Synopsis: P3 columns in r[P2] ** From ab1dcc1a4be2d1636bb43685525ab60c2e2e6ee8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 17:07:48 +0000 Subject: [PATCH 08/15] Fix a problem in the code added by [707ea170b3] causing vdbesort.c to sort unstably. FossilOrigin-Name: d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 20 ++++++++++++++++---- test/sort.test | 23 +++++++++++++++++++++++ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2690ecef90..eb635d79a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. -D 2014-03-25T15:04:07.777 +C Fix\sa\sproblem\sin\sthe\scode\sadded\sby\s[707ea170b3]\scausing\svdbesort.c\sto\ssort\sunstably. +D 2014-03-25T17:07:48.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf +F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -816,7 +816,7 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 +F test/sort.test cb76a6e9db897b6871ef4dbc206ebc6dbc033bf4 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3f90abddc31ac20739778c235a834c33f7057997 -R 743d88a284437e5e0babc8e33ec37e61 +P c3ae3697832a00d4d5758988a8766bdbb691e6b8 +R eb749164f2115a0b3ef6dbb32fd74b6a U dan -Z 5024d9c780855d9abccbd4c6c23f3054 +Z ecf1eb93a5e3757595a49ddb23f9c671 diff --git a/manifest.uuid b/manifest.uuid index b187241ae4..bb3b1adce9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file +d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d55156a2c1..7b43554282 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -562,6 +562,9 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. +** +** In cases where key values are equal, keys from list p1 are considered +** to be smaller than list p2. */ static void vdbeSorterMerge( const VdbeCursor *pCsr, /* For pKeyInfo */ @@ -597,6 +600,11 @@ static void vdbeSorterMerge( ** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK ** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error ** occurs. +** +** The sort is required to be stable - if two elements compare as equal +** then the one added to the sorter first is considered the smaller. +** Currently, the list is sorted from newest to oldest - pSorter->pRecord +** points to the most recently added sort key. */ static int vdbeSorterSort(const VdbeCursor *pCsr){ int i; @@ -837,7 +845,7 @@ static int vdbeSorterInitMerge( int i; /* Used to iterator through aIter[] */ i64 nByte = 0; /* Total bytes in all opened PMAs */ - /* Initialize the iterators. */ + /* Initialize the iterators. Iterator 0 contains the oldest data. */ for(i=0; iaIter[i]; rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); @@ -1009,8 +1017,13 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() ** was actually called above, then pSorter->pUnpacked now contains ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. */ - if( iRes<=0 ){ + ** vdbeSorterCompare() from decoding pIter2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aIter[] array + ** is sorted from oldest to newest, so pIter1 contains older values + ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pSorter->aIter); pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; pKey2 = pIter2->aKey; @@ -1019,7 +1032,6 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; } - } *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); } diff --git a/test/sort.test b/test/sort.test index 08d496b259..b543ffad24 100644 --- a/test/sort.test +++ b/test/sort.test @@ -464,4 +464,27 @@ do_test sort-12.1 { } } {1 2 xxx 1 3 yyy 1 1 zzz} + +#------------------------------------------------------------------------- +# Check that the sorter in vdbesort.c sorts in a stable fashion. +# +do_execsql_test sort-13.0 { + CREATE TABLE t10(a, b); +} +do_test sort-13.1 { + db transaction { + for {set i 0} {$i < 100000} {incr i} { + execsql { INSERT INTO t10 VALUES( $i/10, $i%10 ) } + } + } +} {} +do_execsql_test sort-13.2 { + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +do_execsql_test sort-13.3 { + PRAGMA cache_size = 5; + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] + + finish_test From 382bdeabefc740398f8fd6e55d12a8d952965d85 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 14:05:38 +0000 Subject: [PATCH 09/15] Enhance the logest.c utility with new operators: "dup", "inv", "log", and "nlogn". Provide help on an invalid input. FossilOrigin-Name: b4bd2a062c4baf5f622d61b7411f00de5904ef56 --- manifest | 14 ++++++------ manifest.uuid | 2 +- tool/logest.c | 62 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index fed419c7cf..fc133ce7b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\stest\scase\sfor\sthe\spotential\sbuffer\soverread\spatched\sby\s[28ddecff04]. -D 2014-03-26T15:14:59.762 +C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. +D 2014-03-27T14:05:38.733 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1121,7 +1121,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 +F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b28e8d582cf10936fa1faca04a16ca2eeead66f -R 76c12b0a4284d7654cb2771b2fc9c8eb -U dan -Z 22c3e1f8cdae5a20dae420dc313d8edd +P f585f5d7a0f9bf8c590388654a3638231eba8892 +R fc449dde2f7cf3aac00c9e1a5da52aee +U drh +Z ebedee01c152a936b8dfbac0e8a85bc4 diff --git a/manifest.uuid b/manifest.uuid index bdc4496d4e..bf7192882e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f585f5d7a0f9bf8c590388654a3638231eba8892 \ No newline at end of file +b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 8dad6cc9b6..1ac337d36c 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -17,13 +17,7 @@ ** ** ./LogEst ARGS ** -** Arguments: -** -** 'x' Multiple the top two elements of the stack -** '+' Add the top two elements of the stack -** NUM Convert NUM from integer to LogEst and push onto the stack -** ^NUM Interpret NUM as a LogEst and push onto stack. -** +** See the showHelp() routine for a description of valid arguments. ** Examples: ** ** To convert 123 from LogEst to integer: @@ -97,12 +91,31 @@ static LogEst logEstFromDouble(double x){ return e*10; } +int isInteger(const char *z){ + while( z[0]>='0' && z[0]<='9' ) z++; + return z[0]==0; +} + int isFloat(const char *z){ - while( z[0] ){ - if( z[0]=='.' || z[0]=='E' || z[0]=='e' ) return 1; - z++; - } - return 0; + char c; + while( ((c=z[0])>='0' && c<='9') || c=='.' || c=='E' || c=='e' + || c=='+' || c=='-' ) z++; + return z[0]==0; +} + +static void showHelp(const char *zArgv0){ + printf("Usage: %s ARGS...\n", zArgv0); + printf("Arguments:\n" + " NUM Convert NUM from integer to LogEst and push onto the stack\n" + " ^NUM Interpret NUM as a LogEst and push onto stack\n" + " x Multiple the top two elements of the stack\n" + " + Add the top two elements of the stack\n" + " dup Dupliate the top element on the stack\n" + " inv Take the reciprocal of the top of stack. N = 1/N.\n" + " log Find the LogEst of the number on top of stack\n" + " nlogn Compute NlogN where N is the top of stack\n" + ); + exit(1); } int main(int argc, char **argv){ @@ -111,30 +124,43 @@ int main(int argc, char **argv){ LogEst a[100]; for(i=1; i=2 ){ a[n-2] = logEstAdd(a[n-2],a[n-1]); n--; } - }else if( z[0]=='x' ){ + }else if( strcmp(z,"x")==0 ){ if( n>=2 ){ a[n-2] = logEstMultiply(a[n-2],a[n-1]); n--; } + }else if( strcmp(z,"dup")==0 ){ + if( n>0 ){ + a[n] = a[n-1]; + n++; + } + }else if( strcmp(z,"log")==0 ){ + if( n>0 ) a[n-1] = logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"nlogn")==0 ){ + if( n>0 ) a[n-1] += logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"inv")==0 ){ + if( n>0 ) a[n-1] = -a[n-1]; }else if( z[0]=='^' ){ a[n++] = atoi(z+1); - }else if( isFloat(z) ){ + }else if( isInteger(z) ){ + a[n++] = logEstFromInteger(atoi(z)); + }else if( isFloat(z) && z[0]!='-' ){ a[n++] = logEstFromDouble(atof(z)); }else{ - a[n++] = logEstFromInteger(atoi(z)); + showHelp(argv[0]); } } for(i=n-1; i>=0; i--){ if( a[i]<0 ){ - printf("%d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; - printf("%d (%lld.%02lld)\n", a[i], x/100, x%100); + printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); } } return 0; From face0872125725b1d104b5a5d8e0f04079c9e7a6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 17:23:41 +0000 Subject: [PATCH 10/15] Use xFetch() to access temporary files in vdbesort.c. Use a single large allocation instead of many small allocations when accumulating records in vdbesort.c. This is an interim commit - it allocates a buffer the size of the page-cache every time data is sorted. FossilOrigin-Name: f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 --- manifest | 14 ++-- manifest.uuid | 2 +- src/vdbesort.c | 204 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 50b1f52a8e..dc6ca295c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sand\sthe\sfix\sfor\sthe\scrash\son\sa\scorrupt\ndatabase. -D 2014-03-27T00:09:00.185 +C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. +D 2014-03-27T17:23:41.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc +F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e005f2d6dd9faf38cc8fdb9428b5aa6192a6adae f585f5d7a0f9bf8c590388654a3638231eba8892 -R 9012cf758152dd3775ad665fff626f11 -U drh -Z ffbdc2f043e4deddac6c4b509aff14ac +P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc +R 8f417577dbf4ab5b5ce5a802872c70f0 +U dan +Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 diff --git a/manifest.uuid b/manifest.uuid index f8b9980e3d..853b60d3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b35346c32dba14963c85ec178f2b46aa2bbf6dc \ No newline at end of file +f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 7b43554282..2a15168fd2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,6 +105,8 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ + u8* aMemory; /* Block to allocate records from */ + int iMemory; /* Offset of free space in aMemory */ }; /* @@ -121,6 +123,7 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ u8 *aBuffer; /* Current read buffer */ int nBuffer; /* Size of read buffer in bytes */ + u8 *aMap; /* Pointer to mapping of pFile */ }; /* @@ -163,6 +166,7 @@ struct SorterRecord { static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ sqlite3DbFree(db, pIter->aAlloc); sqlite3DbFree(db, pIter->aBuffer); + if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); memset(pIter, 0, sizeof(VdbeSorterIter)); } @@ -183,6 +187,13 @@ static int vdbeSorterIterRead( ){ int iBuf; /* Offset within buffer to read from */ int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + assert( p->aBuffer ); /* If there is no more data to be read from the buffer, read the next @@ -264,18 +275,22 @@ static int vdbeSorterIterRead( static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ int iBuf; - iBuf = p->iReadOff % p->nBuffer; - if( iBuf && (p->nBuffer-iBuf)>=9 ){ - p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); }else{ - u8 aVarint[16], *a; - int i = 0, rc; - do{ - rc = vdbeSorterIterRead(db, p, 1, &a); - if( rc ) return rc; - aVarint[(i++)&0xf] = a[0]; - }while( (a[0]&0x80)!=0 ); - sqlite3GetVarint(aVarint, pnOut); + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbeSorterIterRead(db, p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } } return SQLITE_OK; @@ -323,6 +338,7 @@ static int vdbeSorterIterInit( ){ int rc = SQLITE_OK; int nBuf; + void *pMap; nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); @@ -333,33 +349,41 @@ static int vdbeSorterIterInit( pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; + /* See if this PMA can be read using xFetch. */ + rc = sqlite3OsFetch(pIter->pFile, 0, pSorter->iWriteOff, &pMap); + if( rc!=SQLITE_OK ) return rc; + if( pMap ){ + pIter->aMap = (u8*)pMap; }else{ - int iBuf; + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; + }else{ + int iBuf; + + iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pSorter->iWriteOff ){ + nRead = (int)(pSorter->iWriteOff - iStart); + } + rc = sqlite3OsRead( + pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } - rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); } + } - if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; - *pnByte += nByte; - } + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + pIter->iEof = pSorter->iWriteOff; + rc = vdbeSorterIterVarint(db, pIter, &nByte); + pIter->iEof = pIter->iReadOff + nByte; + *pnByte += nByte; } if( rc==SQLITE_OK ){ @@ -488,6 +512,10 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + + pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); + assert( pSorter->iMemory==0 ); + if( !pSorter->aMemory ) return SQLITE_NOMEM; } return SQLITE_OK; @@ -521,7 +549,9 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ sqlite3OsCloseFree(pSorter->pTemp1); pSorter->pTemp1 = 0; } - vdbeSorterRecordFree(db, pSorter->pRecord); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pSorter->pRecord); + } pSorter->pRecord = 0; pSorter->iWriteOff = 0; pSorter->iReadOff = 0; @@ -529,6 +559,7 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ pSorter->nTree = 0; pSorter->nPMA = 0; pSorter->aTree = 0; + pSorter->iMemory = 0; } @@ -540,6 +571,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3DbFree(db, pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -551,12 +583,17 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ - int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + int rc; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + } + return rc; } /* @@ -725,6 +762,29 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ fileWriterWrite(p, aByte, nByte); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ + int rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } + return rc; +} +#else +# define vdbeSorterExtendFile(x,y) SQLITE_OK +#endif + /* ** Write the current contents of the in-memory linked-list to a PMA. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. @@ -760,6 +820,13 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ assert( pSorter->nPMA==0 ); } + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + ); + } + if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; @@ -771,12 +838,14 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } + if( pSorter->aMemory ) pSorter->pRecord = 0; + assert( pSorter->pRecord==0 || rc!=SQLITE_OK ); return rc; } @@ -789,23 +858,37 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; + SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( pSorter->aMemory ){ + int nReq = sizeof(SorterRecord) + pVal->n; + if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ + pNew = &sRecord; + pNew->pVal = pVal->z; + }else{ + pNew = &pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + } }else{ - pNew->pVal = (void *)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } } + if( pNew!=&sRecord ){ + pNew->pVal = (void*)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + } + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + /* See if the contents of the sorter should now be written out. They ** are written out when either of the following are true: ** @@ -815,18 +898,22 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( - (pSorter->nInMemory>pSorter->mxPmaSize) - || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - )){ + if( pSorter->mxPmaSize>0 ){ + if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + (pSorter->nInMemory > pSorter->mxPmaSize) + || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ))){ #ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; #endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } } return rc; @@ -934,6 +1021,9 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( pTemp2==0 ){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + } } if( rc==SQLITE_OK ){ @@ -1039,7 +1129,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; pFree->pNext = 0; - vdbeSorterRecordFree(db, pFree); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pFree); + } *pbEof = !pSorter->pRecord; rc = SQLITE_OK; } From 27de5c5cb623fcdbfe4aa128a30a08a5efa268f4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 18:36:34 +0000 Subject: [PATCH 11/15] Minor cleanup of the code in the query planner that computes the costs estimates for the various plans. There are no changes to the costs at this time. But the code is slightly more readable now and that might facilitate future enhancements. FossilOrigin-Name: 9b4d7226bcee38be5ac68a54bee03b4179cb69fc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 44 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index fc133ce7b9..c8233bfa12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. -D 2014-03-27T14:05:38.733 +C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. +D 2014-03-27T18:36:34.321 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca +F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f585f5d7a0f9bf8c590388654a3638231eba8892 -R fc449dde2f7cf3aac00c9e1a5da52aee +P b4bd2a062c4baf5f622d61b7411f00de5904ef56 +R 96966a646967b8a2388d32a924ac0ae2 U drh -Z ebedee01c152a936b8dfbac0e8a85bc4 +Z 308d20d60c0618b1b3ae79dc397c2638 diff --git a/manifest.uuid b/manifest.uuid index bf7192882e..65bd2a013d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file +9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file diff --git a/src/where.c b/src/where.c index 15084f099a..16e7e40bc3 100644 --- a/src/where.c +++ b/src/where.c @@ -4328,18 +4328,34 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: The base cost of an index scan is N + log2(N). + ** The log2(N) is for the initial seek to the beginning and the N + ** is for the scan itself. */ + pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); if( m==0 ){ /* TUNING: Cost of a covering index scan is K*(N + log2(N)). ** + The extra factor K of between 1.1 and 3.0 that depends ** on the relative sizes of the table and the index. K ** is smaller for smaller indices, thus favoring them. + ** The upper bound on K (3.0) matches the penalty factor + ** on a full table scan that tries to encourage the use of + ** indexed lookups over full scans. */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 1 + - (15*pProbe->szIdxRow)/pTab->szTabRow; + pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; }else{ - /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) - ** which we will simplify to just N*log2(N) */ - pNew->rRun = rSize + rLogSize; + /* TUNING: The cost of scanning a non-covering index is multiplied + ** by log2(N) to account for the binary search of the main table + ** that must happen for each row of the index. + ** TODO: Should there be a multiplier here, analogous to the 3x + ** multiplier for a fulltable scan or covering index scan, to + ** further discourage the use of an index scan? Or is the log2(N) + ** term sufficient discouragement? + ** TODO: What if some or all of the WHERE clause terms can be + ** computed without reference to the original table. Then the + ** penality should reduce to logK where K is the number of output + ** rows. + */ + pNew->rRun += rLogSize; } whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); @@ -5041,11 +5057,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 ); + rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; + rSortCost = nRowEst + estLog(nRowEst) + rScale; /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ From 6971952c650cdf1d4e76e63fdb89165bac5c1af0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 19:25:02 +0000 Subject: [PATCH 12/15] Instead of allocating a single large buffer at the beginning of each sort operation, start with a small buffer and extend it using realloc() as required. FossilOrigin-Name: 81987c8ceb64f051528a6ca42673821d9ab7c0ff --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 191 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 134 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index dc6ca295c7..986addf1fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. -D 2014-03-27T17:23:41.403 +C Instead\sof\sallocating\sa\ssingle\slarge\sbuffer\sat\sthe\sbeginning\sof\seach\ssort\soperation,\sstart\swith\sa\ssmall\sbuffer\sand\sextend\sit\susing\srealloc()\sas\srequired. +D 2014-03-27T19:25:02.222 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf +F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc -R 8f417577dbf4ab5b5ce5a802872c70f0 +P f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 +R 5fefa7763a5f0d580046ba5219de21a5 U dan -Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 +Z 8f7301353335dd4530b24cac1ad9c37a diff --git a/manifest.uuid b/manifest.uuid index 853b60d3d3..cf2496b0b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file +81987c8ceb64f051528a6ca42673821d9ab7c0ff \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 2a15168fd2..f229b1f18b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -107,6 +107,7 @@ struct VdbeSorter { UnpackedRecord *pUnpacked; /* Used to unpack keys */ u8* aMemory; /* Block to allocate records from */ int iMemory; /* Offset of free space in aMemory */ + int nMemory; /* Current size of allocation at aMemory */ }; /* @@ -144,15 +145,37 @@ struct FileWriter { /* ** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord using the -** SorterRecord.pNext pointer. +** together into a linked list headed at VdbeSorter.pRecord. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. */ struct SorterRecord { - void *pVal; int nVal; - SorterRecord *pNext; + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; }; +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -513,9 +536,15 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ if( mxCachemxPmaSize = mxCache * pgsz; - pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); - assert( pSorter->iMemory==0 ); - if( !pSorter->aMemory ) return SQLITE_NOMEM; + /* If the application is using memsys3 or memsys5, use a separate + ** allocation for each sort-key in memory. Otherwise, use a single big + ** allocation at pSorter->aMemory for all sort-keys. */ + if( sqlite3GlobalConfig.pHeap==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->aMemory = (u8*)sqlite3Malloc(pSorter->nMemory); + if( !pSorter->aMemory ) return SQLITE_NOMEM; + } } return SQLITE_OK; @@ -528,7 +557,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *p; SorterRecord *pNext; for(p=pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; sqlite3DbFree(db, p); } } @@ -611,22 +640,22 @@ static void vdbeSorterMerge( ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? p2->pVal : 0; + void *pVal2 = p2 ? SRVAL(p2) : 0; while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pCsr, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); if( res<=0 ){ *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; + pp = &p1->u.pNext; + p1 = p1->u.pNext; pVal2 = 0; }else{ *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; + pp = &p2->u.pNext; + p2 = p2->u.pNext; if( p2==0 ) break; - pVal2 = p2->pVal; + pVal2 = SRVAL(p2); } } *pp = p1 ? p1 : p2; @@ -656,8 +685,18 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = pSorter->pRecord; while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; + SorterRecord *pNext; + if( pSorter->aMemory ){ + assert( p->u.iNextnMemory ); + if( (u8*)p==pSorter->aMemory ){ + pNext = 0; + }else{ + pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + p->u.pNext = 0; for(i=0; aSlot[i]; i++){ vdbeSorterMerge(pCsr, p, aSlot[i], &p); aSlot[i] = 0; @@ -835,9 +874,9 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pSorter->nPMA++; fileWriterWriteVarint(&writer, pSorter->nInMemory); for(p=pSorter->pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, p->pVal, p->nVal); + fileWriterWrite(&writer, SRVAL(p), p->nVal); if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; @@ -858,39 +897,24 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + assert( pSorter ); - pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - if( pSorter->aMemory ){ - int nReq = sizeof(SorterRecord) + pVal->n; - if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ - pNew = &sRecord; - pNew->pVal = pVal->z; - }else{ - pNew = &pSorter->aMemory[pSorter->iMemory]; - pSorter->iMemory += ROUND8(nReq); - } - }else{ - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); - if( pNew==0 ){ - return SQLITE_NOMEM; - } - } - - if( pNew!=&sRecord ){ - pNew->pVal = (void*)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - } - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - - /* See if the contents of the sorter should now be written out. They - ** are written out when either of the following are true: + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: ** ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or @@ -898,23 +922,62 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( pSorter->mxPmaSize>0 ){ - if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( (pSorter->nInMemory > pSorter->mxPmaSize) || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - ))){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; -#endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); - } + ); } + if( bFlush ){ +#ifdef SQLITE_DEBUG + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; +#endif + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } + + pSorter->nInMemory += nPMA; + + if( pSorter->aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + int nNew = pSorter->nMemory * 2; + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + + aNew = sqlite3Realloc(pSorter->aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM; + pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; + }else{ + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pNew->u.pNext = pSorter->pRecord; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->pRecord = pNew; return rc; } @@ -1127,8 +1190,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ } }else{ SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->pNext; - pFree->pNext = 0; + pSorter->pRecord = pFree->u.pNext; + pFree->u.pNext = 0; if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(db, pFree); } @@ -1154,7 +1217,7 @@ static void *vdbeSorterRowkey( pKey = pIter->aKey; }else{ *pnKey = pSorter->pRecord->nVal; - pKey = pSorter->pRecord->pVal; + pKey = SRVAL(pSorter->pRecord); } return pKey; } From a1f7c0a21c65431a3fbfeba25ab7f1e9ad827273 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 03:12:48 +0000 Subject: [PATCH 13/15] Enhance the sqlite3VdbeRecordCompare() routines so that if they encounter database corruption, they will set the UnpackedRecord.isCorrupt field and return 0. The sqlite3BtreeMovetoUnpacked() routine detects this and returns SQLITE_CORRUPT, causing the corruption to be reported back to the top-level. FossilOrigin-Name: 7fa85eaaaf6d211378620d728a759fdfe30a15b0 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 3 +++ src/sqliteInt.h | 1 + src/vdbe.h | 4 ++-- src/vdbeInt.h | 2 +- src/vdbeaux.c | 27 +++++++++++++++++++-------- test/corruptG.test | 11 +++-------- test/corruptI.test | 2 +- 9 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index c8233bfa12..48ca1421b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. -D 2014-03-27T18:36:34.321 +C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. +D 2014-03-28T03:12:48.763 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1 +F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -222,7 +222,7 @@ F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h afbf39e96736ceb85e1d896b281ba2406dd70aa0 +F src/sqliteInt.h cc9582a91b2910404ccda7b7e198815ea0f75948 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -279,10 +279,10 @@ F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 -F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 +F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 +F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd +F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -403,9 +403,9 @@ F test/corruptC.test 02405cf7ed0c1e989060e1aab6d02ffbc3906fbb F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 -F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d +F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff +F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b4bd2a062c4baf5f622d61b7411f00de5904ef56 -R 96966a646967b8a2388d32a924ac0ae2 +P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc +R e6ff5922142fb694f528da1c1b2a1baf U drh -Z 308d20d60c0618b1b3ae79dc397c2638 +Z 2ba581858b0e7a94365bc5b4fc44e179 diff --git a/manifest.uuid b/manifest.uuid index 65bd2a013d..ef58fd2f6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file +7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 43d41d67e9..c3055836c9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4588,6 +4588,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->isCorrupt = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4711,6 +4712,7 @@ int sqlite3BtreeMovetoUnpacked( c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } + assert( pIdxKey->isCorrupt==0 || c==0 ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4720,6 +4722,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; + if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5e0dd91ec9..0f67748fc8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1627,6 +1627,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ + u8 isCorrupt; /* Corruption detected by xRecordCompare() */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ diff --git a/src/vdbe.h b/src/vdbe.h index 8e300b88a6..10a4140ee1 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -211,10 +211,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,const UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,const UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b752478035..043b56da59 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -392,7 +392,7 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,const UnpackedRecord*,int*); +int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3VdbeExec(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f5e4b0a9fc..0ce21378d5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3405,10 +3405,13 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** Key1 and Key2 do not have to contain the same number of fields. If all ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. +** +** If database corruption is discovered, set pPKey2->isCorrupt to non-zero +** and return 0. */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ @@ -3434,7 +3437,10 @@ int sqlite3VdbeRecordCompare( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( d1>(unsigned)nKey1 ) return 1; /* Corruption */ + if( d1>(unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } i = 0; } @@ -3511,7 +3517,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; @@ -3537,7 +3544,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3596,7 +3604,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; @@ -3694,7 +3702,7 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip ){ const u8 *aKey1 = (const u8*)pKey1; @@ -3715,7 +3723,10 @@ static int vdbeRecordCompareString( int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; - if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ + if( (szHdr + nStr) > nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); @@ -3880,7 +3891,7 @@ idx_rowid_corruption: */ int sqlite3VdbeIdxKeyCompare( VdbeCursor *pC, /* The cursor to compare against */ - const UnpackedRecord *pUnpacked, /* Unpacked version of key */ + UnpackedRecord *pUnpacked, /* Unpacked version of key */ int *res /* Write the comparison result here */ ){ i64 nCellKey = 0; diff --git a/test/corruptG.test b/test/corruptG.test index 1ec5d6f6a9..af920edf41 100644 --- a/test/corruptG.test +++ b/test/corruptG.test @@ -47,12 +47,12 @@ do_test 1.2 { catchsql { SELECT c FROM t1 WHERE a>'abc'; } -} {0 {}} +} {1 {database disk image is malformed}} do_test 1.3 { catchsql { PRAGMA integrity_check } -} {0 ok} +} {1 {database disk image is malformed}} do_test 1.4 { catchsql { SELECT c FROM t1 ORDER BY a; @@ -71,11 +71,6 @@ do_test 2.1 { catchsql { SELECT rowid FROM t1 WHERE a='abc' and b='xyz123456789XYZ'; } - # The following test result is brittle. The point above is to try to - # force a buffer overread by a corrupt database file. If we get an - # incorrect answer from a corrupt database file, that is OK. If the - # result below changes, that just means that "undefined behavior" has - # changed. -} {/0 .*/} +} {1 {database disk image is malformed}} finish_test diff --git a/test/corruptI.test b/test/corruptI.test index 087a0f3b01..ed34c0f8c3 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -51,7 +51,7 @@ do_test 1.3 { hexio_write test.db $off FFFF7f02 sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } -} {0 {}} +} {1 {database disk image is malformed}} do_test 2.0 { execsql { From 36ed03428d501b8de6b86aeca09f27c9ec5869ad Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 12:56:57 +0000 Subject: [PATCH 14/15] Fix a harmless compiler warning. FossilOrigin-Name: a4e47150f32b3a4120b1f89ccc66d633d829e3bb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 48ca1421b2..3f74055a6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. -D 2014-03-28T03:12:48.763 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-03-28T12:56:57.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d +F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc -R e6ff5922142fb694f528da1c1b2a1baf +P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 +R 84c408a5c4c5e7a44048ef6100ea9043 U drh -Z 2ba581858b0e7a94365bc5b4fc44e179 +Z 645b10a99877639c66e2a5f7ed2d5a66 diff --git a/manifest.uuid b/manifest.uuid index ef58fd2f6f..b73eec43f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file +a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file diff --git a/src/where.c b/src/where.c index 16e7e40bc3..93ee8c59c3 100644 --- a/src/where.c +++ b/src/where.c @@ -4936,7 +4936,7 @@ static i8 wherePathSatisfiesOrderBy( } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return nOrderBy; + if( obSat==obDone ) return (i8)nOrderBy; if( !isOrderDistinct ){ for(i=nOrderBy-1; i>0; i--){ Bitmask m = MASKBIT(i) - 1; From 70c93204ea3e1b37750dd7f362dfe379101cba68 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 14:41:35 +0000 Subject: [PATCH 15/15] Disable the wal64k.test script for non-unix systems since it depends on unix-only features. FossilOrigin-Name: 27deb6e49bcc76714dbdc61b34748603155ac770 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal64k.test | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f74055a6f..52d212de1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2014-03-28T12:56:57.685 +C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. +D 2014-03-28T14:41:35.536 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1056,7 +1056,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e -F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698 +F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 -R 84c408a5c4c5e7a44048ef6100ea9043 +P a4e47150f32b3a4120b1f89ccc66d633d829e3bb +R 578568415288cd31fe0adba6128329da U drh -Z 645b10a99877639c66e2a5f7ed2d5a66 +Z 045e2748905f8bd05ecf1197b97d7f20 diff --git a/manifest.uuid b/manifest.uuid index b73eec43f2..d9c988d4d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file +27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file diff --git a/test/wal64k.test b/test/wal64k.test index afd820778e..e962da128e 100644 --- a/test/wal64k.test +++ b/test/wal64k.test @@ -19,6 +19,11 @@ set testprefix wal64k ifcapable !wal {finish_test ; return } +if {$tcl_platform(platform) != "unix"} { + finish_test + return +} + db close test_syscall pagesize 65536 sqlite3 db test.db @@ -44,4 +49,3 @@ integrity_check 1.3 db close test_syscall pagesize -1 finish_test -